We Keep Coding

Send an email with pytest

I have method that sends an email and I want to make parametrize pytest test, but when I run my test it doesn't run and in logs it shows no tests ran. What's wrong with my test?
pytest scenario:
import pytest
import new_version
#pytest.mark.parametrize("email", ['my_email#gmail.com'])
def send_email(email):
new_version.send_mail(email)
method that sends email:
# Imports
import smtplib
import time
# Language
error_title = "ERROR"
send_error = "I'm waiting {idle} and I will try again."
send_error_mail = "Mail invalid"
send_error_char = "Invalid character"
send_error_connection_1_2 = "Connection problem..."
send_error_connection_2_2 = "Gmail server is down or internet connection is instabil."
login_browser = "Please log in via your web browser and then try again."
login_browser_info = "That browser and this software shood have same IP connection first time."
# Gmaild ID fro login
fromMail = "myemail#gmail.com"
fromPass = "pass"
# Some configurations
mailDelay = 15
exceptionDelay = 180
# SEND MAILS
def send_mail(email, thisSubject="Just a subject",
thisMessage="This is just a simple message..."):
# To ho to send mails
mailTo = [
email
]
# If still have mails to send
while len(mailTo) != 0:
sendItTo = mailTo[0] # Memorise what mail will be send it (debug purpose)
try:
# Connect to the server
server = smtplib.SMTP("smtp.gmail.com:587")
server.ehlo()
server.starttls()
# Sign In
server.login(fromMail, fromPass)
# Set the message
message = f"Subject: {thisSubject}\n{thisMessage}"
# Send one mail
server.sendmail(fromMail, mailTo.pop(0), message)
# Sign Out
server.quit()
# If is a problem
except Exception as e:
# Convert error in a string for som checks
e = str(e)
# Show me if...
if "The recipient address" in e and "is not a valid" in e:
print(f"\n>>> {send_error_mail} [//> {sendItTo}\n")
elif "'ascii'" in e and "code can't encode characters" in e:
print(f"\n>>> {send_error_char} [//> {sendItTo}\n")
elif "Please" in e and "log in via your web browser" in e:
print(f"\n>>> {login_browser}\n>>> - {login_browser_info}")
break
elif "[WinError 10060]" in e:
if "{idle}" in send_error:
se = send_error.split("{idle}");
seMsg = f"{se[0]}{exceptionDelay} sec.{se[1]}"
else:
seMsg = send_error
print(f"\n>>> {send_error_connection_1_2}\n>>> {send_error_connection_2_2}")
print(f">>> {seMsg}\n")
# Wait 5 minutes
waitTime = exceptionDelay - mailDelay
if waitTime <= 0:
waitTime = exceptionDelay
time.sleep(waitTime)
else:
if "{idle}" in send_error:
se = send_error.split("{idle}");
seMsg = f"{se[0]}{exceptionDelay} sec.{se[1]}"
else:
seMsg = send_error
print(f">>> {error_title} <<<", e)
print(f">>> {seMsg}\n")
# Wait 5 minutes
time.sleep(exceptionDelay)
# If are still mails wait before to send another one
if len(mailTo) != 0:
time.sleep(mailDelay)
logs:
platform darwin -- Python 3.7.5, pytest-5.3.5, py-1.8.1, pluggy-0.13.1 -- /Users/nikolai/Documents/Python/gmail/venv/bin/python
cachedir: .pytest_cache
rootdir: /Users/nikolai/Documents/Python/gmail
collected 0 items

Unless you change the discovery prefixes for Pytest, test functions must be named test_*:
import pytest
import new_version
#pytest.mark.parametrize("email", ['my_email#gmail.com'])
def test_send_email(email):
new_version.send_mail(email)

How to invoke C-STORE when trying to query from SCP

I've been using the following code, which is from the pynetdicom library to query and store some images from a remote server SCP, on my machine(SCU).
"""
Query/Retrieve SCU AE example.
This demonstrates a simple application entity that support the Patient
Root Find and Move SOP Classes as SCU. In order to receive retrieved
datasets, this application entity must support the CT Image Storage
SOP Class as SCP as well. For this example to work, there must be an
SCP listening on the specified host and port.
For help on usage,
python qrscu.py -h
"""
import argparse
from netdicom.applicationentity import AE
from netdicom.SOPclass import *
from dicom.dataset import Dataset, FileDataset
from dicom.UID import ExplicitVRLittleEndian, ImplicitVRLittleEndian, \
ExplicitVRBigEndian
import netdicom
# netdicom.debug(True)
import tempfile
# parse commandline
parser = argparse.ArgumentParser(description='storage SCU example')
parser.add_argument('remotehost')
parser.add_argument('remoteport', type=int)
parser.add_argument('searchstring')
parser.add_argument('-p', help='local server port', type=int, default=9999)
parser.add_argument('-aet', help='calling AE title', default='PYNETDICOM')
parser.add_argument('-aec', help='called AE title', default='REMOTESCU')
parser.add_argument('-implicit', action='store_true',
help='negociate implicit transfer syntax only',
default=False)
parser.add_argument('-explicit', action='store_true',
help='negociate explicit transfer syntax only',
default=False)
args = parser.parse_args()
if args.implicit:
ts = [ImplicitVRLittleEndian]
elif args.explicit:
ts = [ExplicitVRLittleEndian]
else:
ts = [
ExplicitVRLittleEndian,
ImplicitVRLittleEndian,
ExplicitVRBigEndian
]
# call back
def OnAssociateResponse(association):
print "Association response received"
def OnAssociateRequest(association):
print "Association resquested"
return True
def OnReceiveStore(SOPClass, DS):
print("FINALLY ENTERED")
print "Received C-STORE", DS.PatientName
try:
# do something with dataset. For instance, store it.
file_meta = Dataset()
file_meta.MediaStorageSOPClassUID = '1.2.840.10008.5.1.4.1.1.2'
# !! Need valid UID here
file_meta.MediaStorageSOPInstanceUID = "1.2.3"
# !!! Need valid UIDs here
file_meta.ImplementationClassUID = "1.2.3.4"
filename = '%s/%s.dcm' % (tempfile.gettempdir(), DS.SOPInstanceUID)
ds = FileDataset(filename, {},
file_meta=file_meta, preamble="\0" * 128)
ds.update(DS)
#ds.is_little_endian = True
#ds.is_implicit_VR = True
ds.save_as(filename)
print "File %s written" % filename
except:
pass
# must return appropriate status
return SOPClass.Success
# create application entity
MyAE = AE(args.aet, args.p, [PatientRootFindSOPClass,
PatientRootMoveSOPClass,
VerificationSOPClass], [StorageSOPClass], ts)
MyAE.OnAssociateResponse = OnAssociateResponse
MyAE.OnAssociateRequest = OnAssociateRequest
MyAE.OnReceiveStore = OnReceiveStore
MyAE.start()
# remote application entity
RemoteAE = dict(Address=args.remotehost, Port=args.remoteport, AET=args.aec)
# create association with remote AE
print "Request association"
assoc = MyAE.RequestAssociation(RemoteAE)
# perform a DICOM ECHO
print "DICOM Echo ... ",
st = assoc.VerificationSOPClass.SCU(1)
print 'done with status "%s"' % st
print "DICOM FindSCU ... ",
d = Dataset()
d.PatientsName = args.searchstring
d.QueryRetrieveLevel = "PATIENT"
d.PatientID = "*"
st = assoc.PatientRootFindSOPClass.SCU(d, 1)
print 'done with status "%s"' % st
for ss in st:
if not ss[1]:
continue
# print ss[1]
try:
d.PatientID = ss[1].PatientID
except:
continue
print "Moving"
print d
assoc2 = MyAE.RequestAssociation(RemoteAE)
gen = assoc2.PatientRootMoveSOPClass.SCU(d, 'SAMTEST', 1)
for gg in gen:
print
print gg
assoc2.Release(0)
print "QWEQWE"
print "Release association"
assoc.Release(0)
# done
MyAE.Quit()
Running the program, I get the following output:
Request association
Association response received
DICOM Echo ... done with status "Success "
DICOM FindSCU ... done with status "<generator object SCU at 0x106014c30>"
Moving
(0008, 0052) Query/Retrieve Level CS: 'PATIENT'
(0010, 0010) Patient's Name PN: 'P*'
(0010, 0020) Patient ID LO: 'Pat00001563'
Association response received
QWEQWE
Moving
(0008, 0052) Query/Retrieve Level CS: 'PATIENT'
(0010, 0010) Patient's Name PN: 'P*'
(0010, 0020) Patient ID LO: 'Pat00002021'
Association response received
QWEQWE
Release association
Echo works so I know my associations are working and I am able to query and see the files on the server as suggested by the output. As you can see however, OnReceiveStore does not seem to be called. I am pretty new to DICOM and I was wondering what might be the case. Correct me if I am wrong but I think the line gen = assoc2.PatientRootMoveSOPClass.SCU(d, 'SAMTEST', 1) is supposed to invoke OnReceiveStore. If not, some insight on how C-STORE should be called is appreciated.

DICOM C-MOVE is a complicated operation, if You are not very familiar with DICOM.
You are not calling OnReceiveStore Yourself, it's called when the SCP starts sending instances to Your application. You issue a C-MOVE command to the SCP with Your own AE title, where You want to receive the images. In Your case SAMTEST. Since this is the only parameter for the C-MOVE command, then the SCP needs to be configured before-hand to know the IP and port of the SAMTEST AE. Have You done this?
I don't know, why the code is not outputting responses from the SCP to the C-MOVE command. It looks like it should. These could give a good indication, what is happening. You can also check the logs on the SCP side, to see, what's happening.
Also, here is some quite good reading about C-MOVE operation.

How to get http server response on a different device in LAN?

I am new to python and my networking logics are at the beginner level. I have an HTTP server running in a VM and when I curl it from a different terminal on the same machine, I get the expected response. I am looking for a functionality where I can get the same response on my mobile device when I type the ip and port in the browser. My mobile device is connected to the same WiFi. Here's the server code:
import socket
MAX_PACKET = 32768
def recv_all(sock):
r'''Receive everything from `sock`, until timeout occurs, meaning sender
is exhausted, return result as string.'''
# dirty hack to simplify this stuff - you should really use zero timeout,
# deal with async socket and implement finite automata to handle incoming data
prev_timeout = sock.gettimeout()
try:
sock.settimeout(0.01)
rdata = []
while True:
try:
rdata.append(sock.recv(MAX_PACKET))
except socket.timeout:
return ''.join(rdata)
# unreachable
finally:
sock.settimeout(prev_timeout)
def normalize_line_endings(s):
r'''Convert string containing various line endings like \n, \r or \r\n,
to uniform \n.'''
return ''.join((line + '\n') for line in s.splitlines())
def run():
r'''Main loop'''
# Create TCP socket listening on 10000 port for all connections,
# with connection queue of length 1
server_sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM, \
socket.IPPROTO_TCP)
server_sock.bind(('10.0.2.15',80))
server_sock.listen(1)
while True:
# accept connection
client_sock, client_addr = server_sock.accept()
# headers and body are divided with \n\n (or \r\n\r\n - that's why we
# normalize endings). In real application usage, you should handle
# all variations of line endings not to screw request body
request = normalize_line_endings(recv_all(client_sock)) # hack again
request_head, request_body = request.split('\n\n', 1)
# first line is request headline, and others are headers
request_head = request_head.splitlines()
request_headline = request_head[0]
# headers have their name up to first ': '. In real world uses, they
# could duplicate, and dict drops duplicates by default, so
# be aware of this.
request_headers = dict(x.split(': ', 1) for x in request_head[1:])
# headline has form of "POST /can/i/haz/requests HTTP/1.0"
request_method, request_uri, request_proto = request_headline.split(' ', 3)
response_body = [
'<html><body><h1>Hello, world!</h1>',
'<p>This page is in location %(request_uri)r, was requested ' % locals(),
'using %(request_method)r, and with %(request_proto)r.</p>' % locals(),
'<p>Request body is %(request_body)r</p>' % locals(),
'<p>Actual set of headers received:</p>',
'<ul>',
]
for request_header_name, request_header_value in request_headers.iteritems():
response_body.append('<li><b>%r</b> == %r</li>' % (request_header_name, \
request_header_value))
response_body.append('</ul></body></html>')
response_body_raw = ''.join(response_body)
# Clearly state that connection will be closed after this response,
# and specify length of response body
response_headers = {
'Content-Type': 'text/html; encoding=utf8',
'Content-Length': len(response_body_raw),
'Connection': 'close',
}
response_headers_raw = ''.join('%s: %s\n' % (k, v) for k, v in \
response_headers.iteritems())
# Reply as HTTP/1.1 server, saying "HTTP OK" (code 200).
response_proto = 'HTTP/1.1'
response_status = '200'
response_status_text = 'OK' # this can be random
# sending all this stuff
client_sock.send('%s %s %s' % (response_proto, response_status, \
response_status_text))
client_sock.send(response_headers_raw)
client_sock.send('\n') # to separate headers from body
client_sock.send(response_body_raw)
# and closing connection, as we stated before
client_sock.close()
run()
Here's the response when I run curl from a different terminal on the same VM.
I want to ping it from my mobile device connected to the same WiFi. Thank you!

http fuzzing with boofuzz

I was looking for a fuzzing library and I happened to see "boofuzz"
though there are no examples of how to use the library for http fuzzing.
This is the only code I see in their github page, but they say it was taken from sulley (an old fuzzing library):
import sys
sys.path.insert(0, '../')
from boofuzz.primitives import String, Static, Delim
class Group(object):
blocks = []
def __init__(self, name, definition=None):
self.name = name
if definition:
self.definition = definition
def add_definition(self, definition):
assert isinstance(definition, (list, tuple)), "Definition must be a list or a tuple!"
self.definition = definition
def render(self):
return "".join([x.value for x in self.definition])
def exhaust(self):
for item in self.definition:
while item.mutate():
current_value = item.value
self.log_send(current_value)
recv_data = self.send_buffer(current_value)
self.log_recv(recv_data)
def __repr__(self):
return '<%s [%s items]>' % (self.__class__.__name__, len(self.definition))
# noinspection PyMethodMayBeStatic
def send_buffer(self, current_value):
return "Sent %s!" % current_value
def log_send(self, current_value):
pass
def log_recv(self, recv_data):
pass
s_static = Static
s_delim = Delim
s_string = String
CloseHeader = Group(
"HTTP Close Header",
definition=[
# GET / HTTP/1.1\r\n
s_static("GET / HTTP/1.1\r\n"),
# Connection: close
s_static("Connection"), s_delim(":"), s_delim(" "), s_string("close"),
s_static("\r\n\r\n")
]
)
OpenHeader = Group(
"HTTP Open Header",
definition=[
# GET / HTTP/1.1\r\n
Static("GET / HTTP/1.1\r\n"),
# Connection: close
Static("Connection"), Delim(":"), Delim(" "), String("open"),
Static("\r\n\r\n")
]
)
# CloseHeader = Group("HTTP Close Header")
# CloseHeader.add_definition([
# # GET / HTTP/1.1\r\n
# s_static("GET / HTTP/1.1\r\n"),
# # Connection: close
# s_static("Connection"), s_delim(":"), s_delim(" "), s_string("close"),
# s_static("\r\n\r\n")
# ])
Why would they post it, if it's another's library code? And is there a good explanation of how to work with the boofuzz library?

If you Google "http protocol format", the first result right now is this HTTP tutorial. If you read a few pages there, you can get a pretty good description of the protocol format. Based on that, I wrote the following fuzz script, source code here:
#!/usr/bin/env python
# Designed for use with boofuzz v0.0.9
from boofuzz import *
def main():
session = Session(
target=Target(
connection=SocketConnection("127.0.0.1", 80, proto='tcp')
),
)
s_initialize(name="Request")
with s_block("Request-Line"):
s_group("Method", ['GET', 'HEAD', 'POST', 'PUT', 'DELETE', 'CONNECT', 'OPTIONS', 'TRACE'])
s_delim(" ", name='space-1')
s_string("/index.html", name='Request-URI')
s_delim(" ", name='space-2')
s_string('HTTP/1.1', name='HTTP-Version')
s_static("\r\n", name="Request-Line-CRLF")
s_static("\r\n", "Request-CRLF")
session.connect(s_get("Request"))
session.fuzz()
if __name__ == "__main__":
main()
Although I got tripped up for a while because I only had one CRLF. After checking RFC 2616 (Section 5), it's pretty clear that this example should end with two CRLFs.
Request = Request-Line ; Section 5.1
*(( general-header ; Section 4.5
| request-header ; Section 5.3
| entity-header ) CRLF) ; Section 7.1
CRLF
[ message-body ] ; Section 4.3
[...]
Request-Line = Method SP Request-URI SP HTTP-Version CRLF
Obviously, this fuzz script doesn't come close to covering the whole protocol. Just a few things that could be added:
HTTP Headers (there are a lot)
Specialized formats for each HTTP Method
A message body (e.g. on POST)
Some way to choose valid URIs for the particular target server
Report warnings based on server response (could get noisy, but server errors do tend to indicate... errors)

Convert Outlook PST to json using libpst

I have an Outlook PST file, and I'd like to get a json of the emails, e.g. something like
{"emails": [
{"from": "alice#example.com",
"to": "bob#example.com",
"bcc": "eve#example.com",
"subject": "mitm",
"content": "be careful!"
}, ...]}
I've thought using readpst to convert to MH format and then scan it in a ruby/python/bash script, is there a better way?
Unfortunately the ruby-msg gem doesn't work on my PST files (and looks like it wasn't updated since 2014).

I found a way to do it in 2 stages, first convert to mbox and then to json:
# requires installing libpst
pst2json my.pst
# or you can specify a custom output dir and an outlook mail folder,
# e.g. Inbox, Sent, etc.
pst2json -o email/ -f Inbox my.pst
Where pst2json is my script and mbox2json is slightly modified from Mining the Social Web.
pst2json:
#!/usr/bin/env bash
usage(){
echo "usage: $(basename $0) [-o <output-dir>] [-f <folder>] <pst-file>"
echo "default output-dir: email/mbox-all/<pst-file>"
echo "default folder: Inbox"
exit 1
}
which readpst || { echo "Error: libpst not installed"; exit 1; }
folder=Inbox
while (( $# > 0 )); do
[[ -n "$pst_file" ]] && usage
case "$1" in
-o)
if [[ -n "$2" ]]; then
out_dir="$2"
shift 2
else
usage
fi
;;
-f)
if [[ -n "$2" ]]; then
folder="$2"
shift 2
else
usage
fi
;;
*)
pst_file="$1"
shift
esac
done
default_out_dir="email/mbox-all/$(basename $pst_file)"
out_dir=${out_dir:-"$default_out_dir"}
mkdir -p "$out_dir"
readpst -o "$out_dir" "$pst_file"
[[ -f "$out_dir/$folder" ]] || { echo "Error: folder $folder is missing or empty."; exit 1; }
res="$out_dir"/"$folder".json
mbox2json "$out_dir/$folder" "$res" && echo "Success: result saved to $res"
mbox2json (python 2.7):
# -*- coding: utf-8 -*-
import sys
import mailbox
import email
import quopri
import json
from BeautifulSoup import BeautifulSoup
MBOX = sys.argv[1]
OUT_FILE = sys.argv[2]
SKIP_HTML=True
def cleanContent(msg):
# Decode message from "quoted printable" format
msg = quopri.decodestring(msg)
# Strip out HTML tags, if any are present
soup = BeautifulSoup(msg)
return ''.join(soup.findAll(text=True))
def jsonifyMessage(msg):
json_msg = {'parts': []}
for (k, v) in msg.items():
json_msg[k] = v.decode('utf-8', 'ignore')
# The To, CC, and Bcc fields, if present, could have multiple items
# Note that not all of these fields are necessarily defined
for k in ['To', 'Cc', 'Bcc']:
if not json_msg.get(k):
continue
json_msg[k] = json_msg[k].replace('\n', '').replace('\t', '').replace('\r'
, '').replace(' ', '').decode('utf-8', 'ignore').split(',')
try:
for part in msg.walk():
json_part = {}
if part.get_content_maintype() == 'multipart':
continue
type = part.get_content_type()
if SKIP_HTML and type == 'text/html':
continue
json_part['contentType'] = type
content = part.get_payload(decode=False).decode('utf-8', 'ignore')
json_part['content'] = cleanContent(content)
json_msg['parts'].append(json_part)
except Exception, e:
sys.stderr.write('Skipping message - error encountered (%s)\n' % (str(e), ))
finally:
return json_msg
# There's a lot of data to process, so use a generator to do it. See http://wiki.python.org/moin/Generators
# Using a generator requires a trivial custom encoder be passed to json for serialization of objects
class Encoder(json.JSONEncoder):
def default(self, o):
return {'emails': list(o)}
# The generator itself...
def gen_json_msgs(mb):
while 1:
msg = mb.next()
if msg is None:
break
yield jsonifyMessage(msg)
mbox = mailbox.UnixMailbox(open(MBOX, 'rb'), email.message_from_file)
json.dump(gen_json_msgs(mbox),open(OUT_FILE, 'wb'), indent=4, cls=Encoder)
Now, it's possible to process the file easily. E.g. to get just the contents of the emails:
jq '.emails[] | .parts[] | .content' < out/Inbox.json