Broken attachment encryption with python and GnuPG - python

I've taken all the forks of gpg-mailgate and put all the working parts together to get it almost totally working. The last issue I"m having is that attachments come through encrpted as filename.originalextension.pgp BUT are undecryptable.
Here is the full code of the mailgate plugin as I have it working.
#!/usr/bin/python
from ConfigParser import RawConfigParser
from email.mime.base import MIMEBase
import email
import email.message
import re
import GnuPG
import smtplib
import sys
# Read configuration from /etc/gpg-mailgate.conf
_cfg = RawConfigParser()
_cfg.read('/etc/gpg-mailgate.conf')
cfg = dict()
for sect in _cfg.sections():
cfg[sect] = dict()
for (name, value) in _cfg.items(sect):
cfg[sect][name] = value
# Read e-mail from stdin
raw = sys.stdin.read()
raw_message = email.message_from_string( raw )
from_addr = raw_message['From']
to_addrs = sys.argv[1:]
def send_msg( message, recipients = None ):
if recipients == None:
recipients = to_addrs
if cfg.has_key('logging') and cfg['logging'].has_key('file'):
log = open(cfg['logging']['file'], 'a')
log.write("Sending email to: <%s>\n" % '> <'.join( recipients ))
log.close()
relay = (cfg['relay']['host'], int(cfg['relay']['port']))
smtp = smtplib.SMTP(relay[0], relay[1])
smtp.sendmail( from_addr, recipients, message.as_string() )
def encrypt_payload( payload, gpg_to_cmdline ):
gpg = GnuPG.GPGEncryptor( cfg['gpg']['keyhome'], gpg_to_cmdline )
raw_payload = payload.get_payload(decode=True)
gpg.update( raw_payload )
if "-----BEGIN PGP MESSAGE-----" in raw_payload and "-----END PGP MESSAGE-----" in raw_payload:
return payload
payload.set_payload( gpg.encrypt() )
if payload['Content-Disposition']:
payload.replace_header( 'Content-Disposition', re.sub(r'filename="([^"]+)"', r'filename="\1.pgp"', payload['Content-Disposition']) )
if payload['Content-Type']:
payload.replace_header( 'Content-Type', re.sub(r'name="([^"]+)"', r'name="\1.pgp"', payload['Content-Type']) )
# if payload.get_content_type() != 'text/plain' and payload.get_content_type != 'text/html':
if 'name="' in payload['Content-Type']:
payload.replace_header( 'Content-Type', re.sub(r'^[a-z/]+;', r'application/octet-stream;', payload['Content-Type']) )
payload.set_payload( "\n".join( filter( lambda x:re.search(r'^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=|[A-Za-z0-9+/]{4})$',x), payload.get_payload().split("\n") ) ) )
return payload
def encrypt_all_payloads( payloads, gpg_to_cmdline ):
encrypted_payloads = list()
if type( payloads ) == str:
msg = email.message.Message()
msg.set_payload( payloads )
return encrypt_payload( msg, gpg_to_cmdline ).as_string()
for payload in payloads:
if( type( payload.get_payload() ) == list ):
encrypted_payloads.append( encrypt_all_payloads( payload.get_payload(), gpg_to_cmdline ) )
else:
encrypted_payloads.append( [encrypt_payload( payload, gpg_to_cmdline )] )
return sum(encrypted_payloads, [])
def get_msg( message ):
if not message.is_multipart():
return message.get_payload()
return '\n\n'.join( [str(m) for m in message.get_payload()] )
keys = GnuPG.public_keys( cfg['gpg']['keyhome'] )
gpg_to = list()
ungpg_to = list()
for to in to_addrs:
domain = to.split('#')[1]
if domain in cfg['default']['domains'].split(','):
if to in keys:
gpg_to.append( (to, to) )
elif cfg.has_key('keymap') and cfg['keymap'].has_key(to):
gpg_to.append( (to, cfg['keymap'][to]) )
else:
ungpg_to.append(to)
if gpg_to == list():
if cfg['default'].has_key('add_header') and cfg['default']['add_header'] == 'yes':
raw_message['X-GPG-Mailgate'] = 'Not encrypted, public key not found'
send_msg( raw_message )
exit()
if ungpg_to != list():
send_msg( raw_message, ungpg_to )
if cfg.has_key('logging') and cfg['logging'].has_key('file'):
log = open(cfg['logging']['file'], 'a')
log.write("Encrypting email to: %s\n" % ' '.join( map(lambda x: x[0], gpg_to) ))
log.close()
if cfg['default'].has_key('add_header') and cfg['default']['add_header'] == 'yes':
raw_message['X-GPG-Mailgate'] = 'Encrypted by GPG Mailgate'
gpg_to_cmdline = list()
gpg_to_smtp = list()
for rcpt in gpg_to:
gpg_to_smtp.append(rcpt[0])
gpg_to_cmdline.extend(rcpt[1].split(','))
encrypted_payloads = encrypt_all_payloads( raw_message.get_payload(), gpg_to_cmdline )
raw_message.set_payload( encrypted_payloads )
send_msg( raw_message, gpg_to_smtp )
My client ( both roundcube and k-9 ), do not know what to do with the file.
From command line if I do a gpg --decrypt filename.txt.pgp I get:
gpg: no valid OpenPGP data found.
gpg: decrypt_message failed: eof
Headers of email are:
User-Agent: K-9 Mail for Android
MIME-Version: 1.0
Content-Type: multipart/mixed; boundary="----YR49011GO0MWM753ETYWUA7CBOKGAV"
Subject: New with attachments
From: Bruce Markey <bruce#secryption.com>
Date: Fri, 13 Sep 2013 06:18:03 -0400
To: "bruce#packetaddiction.com" <bruce#packetaddiction.com>
Message-ID: <53178821-6d6c-4b7d-b9c7-5a49034da1ef#email.android.com>
X-GPG-Mailgate: Encrypted by GPG Mailgate
I'm not even sure "what" to debug since everything looks ok, and there is a total lack of errors.
If anyone has any direction I'd appreciate it.
Update:
I came across this Decrypt gpg file attached from email (file.pgp)
I decided to add a line to write raw to a log file.
MIME-Version: 1.0
X-Received: by 10.68.130.1 with SMTP id oa1mr18868651pbb.35.1379162867744;
Sat, 14 Sep 2013 05:47:47 -0700 (PDT)
Received: by 10.68.46.72 with HTTP; Sat, 14 Sep 2013 05:47:47 -0700 (PDT)
Date: Sat, 14 Sep 2013 08:47:47 -0400
Message-ID: <CACRtyey-L9Z5JGNG4bheYqJ7tVK+6qfigmanH9pTUk0ute5gEw#mail.gmail.com>
Subject: Test with attachment - Saturday
From: Bruce Markey <bmarkey#gmail.com>
To: bruce#packetaddiction.com
Content-Type: multipart/mixed; boundary=047d7b10ca15d1c7b904e65760eb
--047d7b10ca15d1c7b904e65760eb
Content-Type: text/plain; charset=ISO-8859-1
Just a simple test with txt attachment
--047d7b10ca15d1c7b904e65760eb
Content-Type: text/plain; charset=US-ASCII; name="TestAttach.txt"
Content-Disposition: attachment; filename="TestAttach.txt"
Content-Transfer-Encoding: base64
X-Attachment-Id: f_hlkty4930
VGhpcyBpcyBqdXN0IGEgdGVzdCBvZiB0aGUgYXR0YWNobWVudHMuIApUaGlzIGlzIGEgc2ltcGxl
IHRleHQgZmlsZS4gCgo=
--047d7b10ca15d1c7b904e65760eb--
Since this is being written raw it's pre encryption. So should I be decoding the base64 prior to encryption?
After staring at this for awhile I don't understand why this line is here.
if 'name="' in payload['Content-Type']:
payload.replace_header( 'Content-Type', re.sub(r'^[a-z/]+;', r'application/octet- stream;', payload['Content-Type']) )
payload.set_payload( "\n".join( filter( lambda x:re.search(r'^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=|[A-Za-z0-9+/]{4})$',x), payload.get_payload().split("\n") ) ) )
Why change to application/octet-stream?
UPDATE:
I think I got it working unless I'm doing something horribly wrong. I changed the following:
def get_msg( message ):
if not message.is_multipart():
return message.get_payload()
return '\n\n'.join( [base64.decodestring(str(m)) for m in message.get_payload()] )
This now allows me to actually run gpg --decrypt filename.txt .
( I assume that most attachments will come through as base64 although I'll probably add a test for all content-transfer-encoding types. )

The attachment needed to be decoded, content-transfer-encoding, prior to encrypting. See code in update of original question.

Related

How do I send a string + variable as sms using python?

I need to send a message when my program is done running, but I also want to include variables such as how long the program took to run in the text.
Here is my code for the texting:
import smtplib
carriers = {
'att': '#mms.att.net',
'tmobile': ' #tmomail.net',
'verizon': '#vtext.com',
'sprint': '#page.nextel.com'
}
def send(message):
# Replace the number with your own, or consider using an argument\dict for multiple people.
to_number = 'xxxxxxxxxx{}'.format(carriers['verizon'])
auth = ('xxxxx', 'xxxx')
# Establish a secure session with gmail's outgoing SMTP server using your gmail account
server = smtplib.SMTP("smtp.gmail.com", 587)
server.starttls()
server.login(auth[0], auth[1])
# Send text message through SMS gateway of destination number
server.sendmail(auth[0], to_number, message)
Obviously, I replaced my info with the xxx.
Now, to send my text I'm calling the function using:
found = 'The program is done!'
timetook = "Time took: %s (HOUR:MIN:SEC)" % timedelta(seconds=round(elapsed_time_secs))
send(found)
send(timetook)
It just sends blank texts for the timetook, but the program is done message works fine. How do I send the timetook?
The problem is that you aren't following the rules for SMTP. Below is the equivalent solution I wrote for my own use many years ago.
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""SMS (short message service) functions."""
import logging
import smtplib
import sys
import rate_limiting
SMTP_GATEWAY = 'localhost'
FROM_ADDR = 'me#domain.us'
PHONE_NUMBER = '4081234567'
# T-Mobile SMS email gateway
# TO_ADDR = PHONE_NUMBER + '#tmomail.net'
# Verizon SMS email gateway
TO_ADDR = PHONE_NUMBER + '#vtext.com'
# Allow only three SMS messages per minute and five per hour.
short_term_rate_limiter = rate_limiting.SimpleRateLimiter(3, 60)
long_term_rate_limiter = rate_limiting.SimpleRateLimiter(5, 60 * 60)
def SendSms(msg, why=''):
"""Send a SMS message."""
short_term_rate_exceeded = short_term_rate_limiter()
long_term_rate_exceeded = long_term_rate_limiter()
if short_term_rate_exceeded or long_term_rate_exceeded:
logging.warning('SMS message rate exceeded, dropping msg: %s', msg)
return
smtp_conn = smtplib.SMTP(SMTP_GATEWAY)
hdrs = 'From: {}\r\nTo: {}\r\n\r\n'.format(FROM_ADDR, TO_ADDR)
if why:
hdrs += 'Subject: {}\r\n'.format(why[:20])
hdrs += "\r\n\r\n"
max_msg_len = 140 - 3 - min(20, len(why))
msg = hdrs + msg[:max_msg_len]
# Make sure the message has only ASCII characters.
msg = msg.encode('ascii', errors='replace').decode('ascii')
smtp_conn.sendmail(FROM_ADDR, [TO_ADDR], msg)
smtp_conn.quit()
if __name__ == '__main__':
SendSms(' '.join(sys.argv[1:]))

python-messaging Failed to handle HTTP request

I am using the code below to try to send an MMS message with python-messaging https://github.com/pmarti/python-messaging/blob/master/doc/tutorial/mms.rst Although the connection seems to go smoothly I get the following response from the mmsc:
PROXY RESPONSE HTTP/1.0 200 OK
content-type: application/vnd.wap.mms-message
content-length: 59
Connection: close
Date: Sat, 05 Jan 2019 16:36:44 GMT
Server: Mavenir Web Application Server
���1234�����,�Failed to handle HTTP request in Mm1Server
Does, anyone have an idea on what the problem might be and how I can fix it? Here is my code:
from messaging.mms.message import MMSMessage, MMSMessagePage
mms = MMSMessage()
mms.headers['To'] = '+212XXXXXXX/TYPE=PLMN'
mms.headers['Message-Type'] = 'm-send-req'
mms.headers['Subject'] = 'Test python-messaging.mms'
slide1 = MMSMessagePage()
slide1.add_image('/home/richard/screensaver/TolleConscQte.jpg')
slide1.add_text('This first slide, is a step towards enlightenment.')
slide2 = MMSMessagePage()
slide2.set_duration(4500)
slide2.add_image('/home/richard/screensaver/TollePastALL.jpg', 1500)
slide2.add_text('This second slide is a second step towards enlightenment.', 500, 3500)
mms.add_page(slide1)
mms.add_page(slide2)
payload = mms.encode()
## sending the MMS
from cStringIO import StringIO
import socket
gw_host, gw_port = "10.188.239.143", 80 #ting
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect((gw_host, gw_port))
s.send("POST %s HTTP/1.0\r\n" % "http://wholesale.mmsmvno.com/mms/wapenc")
s.send("Content-Type: application/vnd.wap.mms-message\r\n")
s.send("Content-Length: %d\r\n\r\n" % len(payload))
s.sendall(payload)
buf = StringIO()
while True:
data = s.recv(4096)
if not data:
break
buf.write(data)
s.close()
data = buf.getvalue()
buf.close()
print "PROXY RESPONSE", data

Extracting some SIP headers

I am using this regular expression for SIP (Session Initiation Protocol) URIs to extract the different internal variables.
_syntax = re.compile('^(?P<scheme>[a-zA-Z][a-zA-Z0-9\+\-\.]*):' # scheme
+ '(?:(?:(?P<user>[a-zA-Z0-9\-\_\.\!\~\*\'\(\)&=\+\$,;\?\/\%]+)' # user
+ '(?::(?P<password>[^:#;\?]+))?)#)?' # password
+ '(?:(?:(?P<host>[^;\?:]*)(?::(?P<port>[\d]+))?))' # host, port
+ '(?:;(?P<params>[^\?]*))?' # parameters
+ '(?:\?(?P<headers>.*))?$') # headers
m = URI._syntax.match(value)
if m:
self.scheme, self.user, self.password, self.host, self.port, params, headers = m.groups()
and i want to extract specific header like the header via,branch,contact,callID or Cseq.
The general form of a sip message is:
OPTIONS sip:172.16.18.35:5060 SIP/2.0
Content-Length: 0
Via: SIP/2.0/UDP 172.16.18.90:5060
From: "fake" <sip:fake#172.16.18.90>
Supported: replaces, timer
User-Agent: SIPPing
To: <sip:172.16.18.35:5060>
Contact: <sip:fake#172.16.18.90:5060>
CSeq: 1 OPTIONS
Allow: INVITE, ACK, CANCEL, OPTIONS, BYE, REFER, SUBSCRIBE, NOTIFY, INFO, PUBLISH
Call-ID: fake-id#172.16.18.90
Date: Thu, 25 Apr 2013 003024 +0000
Max-Forwards: 70
I would suggest taking advantage of the intentional similarities between SIP header format and RFC822.
from email.parser import Parser
msg = Parser().parsestr(m.group('headers'))
...thereafter:
>>> msg.keys()
['Content-Length', 'Via', 'From', 'Supported', 'User-Agent', 'To', 'Contact', 'CSeq', 'Allow', 'Call-ID', 'Date', 'Max-Forwards']
>>> msg['To']
'<sip:172.16.18.35:5060>'
>>> msg['Date']
'Thu, 25 Apr 2013 003024 +0000'
...etc. See the documentation for the Python standard-library email module for more details.

Python regex re.match() not returning any results

I'm hoping this is just something simple. I'm trying to determine whether or not an email is already encrypted.
# Read e-mail from stdin
raw = sys.stdin.read()
raw_message = email.message_from_string( raw )
I took the example from http://docs.python.org/2/howto/regex.html on doing a simple test for match.
p = re.compile('-----BEGIN\sPGP\sMESSAGE-----')
m = p.match(raw)
if m:
log = open(cfg['logging']['file'], 'a')
log.write("THIS IS ENCRYPTED")
log.close()
else:
log = open(cfg['logging']['file'], 'a')
log.write("NOT ENCRYPTED:")
log.close()
The email is read. The log file is written to but it always comes back no match. I've written raw to a logfile and that string is present.
Not sure where to go next.
UPDATE:
Here is the output from a raw ( a simple test message )
Sending email to: <bruce#packetaddiction.com>
Received: from localhost (localhost [127.0.0.1])
by mail2.packetaddiction.com (Postfix) with ESMTP id 5FE5D22A65
for <bruce#packetaddiction.com>; Tue, 10 Sep 2013 16:19:12 +0000 (UTC)
X-Virus-Scanned: Debian amavisd-new at mail2.packetaddiction.com
Received: from mail2.packetaddiction.com ([127.0.0.1])
by localhost (mail2.packetaddiction.com [127.0.0.1]) (amavisd-new, port 10024)
with ESMTP id cc3zZ_izEb1j for <bruce#packetaddiction.com>;
Tue, 10 Sep 2013 16:19:06 +0000 (UTC)
Received: from mail.secryption.com (mail.secryption.com [178.18.24.223])
by mail2.packetaddiction.com (Postfix) with ESMTPS id 9CA3C22A5B
for <bruce#packetaddiction.com>; Tue, 10 Sep 2013 16:19:06 +0000 (UTC)
Received: from localhost (localhost.localdomain [127.0.0.1])
by mail.secryption.com (Postfix) with ESMTP id 9994E1421F81
for <bruce#packetaddiction.com>; Tue, 10 Sep 2013 12:19:19 -0400 (EDT)
X-Virus-Scanned: Debian amavisd-new at mail.secryption.com
Received: from mail.secryption.com ([127.0.0.1])
by localhost (mail.secryption.com [127.0.0.1]) (amavisd-new, port 10024)
with ESMTP id WbkVn_cowG6q for <bruce#packetaddiction.com>;
Tue, 10 Sep 2013 12:19:18 -0400 (EDT)
Received: from dennis.cng.int (mail.compassnetworkgroup.com [173.163.129.21])
(using TLSv1 with cipher RC4-MD5 (128/128 bits))
(No client certificate requested)
by mail.secryption.com (Postfix) with ESMTPSA id 5B4191421F80
for <bruce#packetaddiction.com>; Tue, 10 Sep 2013 12:19:18 -0400 (EDT)
User-Agent: K-9 Mail for Android
MIME-Version: 1.0
Content-Type: text/plain;
charset=UTF-8
Content-Transfer-Encoding: 8bit
Subject: Message
From: Bruce Markey <bruce#secryption.com>
Date: Tue, 10 Sep 2013 12:19:00 -0400
To: "bruce#packetaddiction.com" <bruce#packetaddiction.com>
Message-ID: <36615ed6-a1a9-49ac-ac85-31905916d478#email.android.com>
-----BEGIN PGP MESSAGE-----
Version: APG v1.0.8
hQEMAwPNxvNWsisWAQgAuOTLkiitYzhGJydOzN4sBoGjhRm9JeJMfmxKxKTKcV2W
ZBuN0z+nS1KxnXrIlahhwLtpiFvp5apI8wAyAiLC2BhFieFttOl1/xLVJbd1nI1o
KQE1RUXhPURejJ3eH9g/LmkhtFQcnsuHGTGnLi6dugBNhWLqgnLUBX+VLt6moz2C
84lDuQ1y7B/JFOctKRScUqmxDd8b2peZJOnVT/p0tSYNfN9QGH3W02FZShE4KKBl
HpezK8KC6cZdf34Eao+ep+fP5DuKx/4j3ksCbFKyQ3gd+yxK/xnhkijDsYCfFRiF
ElAGDvXu4RXqrKRpBxq1bRhU8YqS7j5593MTUViWitLAGgH1DV0UeA/B5LMUDRyz
4ZfDqd0kDYsPUy2Cg20HdXHaobkzdvHLzfqQq0Owc1nTcvu4nzCbIMhTAlZjn8ZA
aODTlKcvnFBWEtNERPm0x6nkbhMo3GeysejaJSRod3aGqhuhga4iIrrew1W03297
aalwY8RKeNoV15VItsyrbbT+HvDNSaFFCPUAs+KcLHCOez5/woozjlqKdBI6yHCe
gqpYJPP07qFsVviltfDO63xS48f2HCPe4iyXCy6Usp0+jM7zAzH7KH1O854GH46Q
r0A01DLo9REmDr4U
=pBQZ
-----END PGP MESSAGE-----
re.match will only find a match at the beginning of the string, as noted here. You want to use re.search
raw = """Sending email to: <bruce#packetaddiction.com>...
...
-----BEGIN PGP MESSAGE-----
...
"""
>>> p = re.compile('-----BEGIN\sPGP\sMESSAGE-----')
>>> m = p.search(raw)
>>> m
<_sre.SRE_Match object at 0x0000000002E02510>
>>> m.group()
'-----BEGIN PGP MESSAGE-----'
>>> m = p.match(raw)
>>> print m
None
Although, as noted, regex is likely overkill for this problem as the matching text is static.
Regular expressions are used when you want a "fuzzy" match - that is, you aren't sure if the string you are looking for will be identical every time.
In this case, the string you are looking for appears to be exactly -----BEGIN PGP MESSAGE----. In this case, the string.find function will be simpler to use and faster to boot.
>>> a = "This is a PGP encrypted email. -----BEGIN PGP MESSAGE----- !##$%^..."
>>> b = "This is not encrypted. My hovercraft is full of eels." #example strings
>>> a.find("-----BEGIN PGP MESSAGE-----")
30 # Return value '30' means that the search string was found at index 30 of source string
>>> b.find("-----BEGIN PGP MESSAGE-----")
-1 # -1 means 'not found in the source string'
>>>

Undoing "marked as read" status of emails fetched with imaplib

I wrote a python script to fetch all of my gmail. I have hundreds of thousands of old emails, of which about 10,000 were unread.
After successfully fetching all of my email, I find that gmail has marked all the fetched emails as "read". This is disastrous for me since I need to check all unread emails only.
How can I recover the information about which emails were unread? I dumped each mail object into files, the core of my code is shown below:
m = imaplib.IMAP4_SSL("imap.gmail.com")
m.login(user,pwd)
m.select("[Gmail]/All Mail")
resp, items = m.uid('search', None, 'ALL')
uids = items[0].split()
for uid in uids:
resp, data = m.uid('fetch', uid, "(RFC822)")
email_body = data[0][1]
mail = email.message_from_string(email_body)
dumbobj(uid, mail)
I am hoping there is either an option to undo this in gmail, or a member inside the stored mail objects reflecting the seen-state information.
For anyone looking to prevent this headache, consider this answer here. This does not work for me, however, since the damage has already been done.
Edit:
I have written the following function to recursively "grep" all strings in an object, and applied it to a dumped email object using the following keywords:
regex = "(?i)((marked)|(seen)|(unread)|(read)|(flag)|(delivered)|(status)|(sate))"
So far, no results (only an unrelated "Delivered-To"). Which other keywords could I try?
def grep_object (obj, regex , cycle = set(), matched = set()):
import re
if id(obj) in cycle:
return
cycle.update([id(obj)])
if isinstance(obj, basestring):
if re.search(regex, obj):
matched.update([obj])
def grep_dict (adict ):
try:
[ [ grep_object(a, regex, cycle, matched ) for a in ab ] for ab in adict.iteritems() ]
except:pass
grep_dict(obj)
try:grep_dict(obj.__dict__)
except:pass
try:
[ grep_object(elm, regex, cycle, matched ) for elm in obj ]
except: pass
return matched
grep_object(mail_object, regex)
I'm having a similar problem (not with gmail), and the biggest problem for me was to make a reproducible test case; and I finally managed to produce one (see below).
In terms of the Seen flag, I now gather it goes like this:
If a message is new/unseen, IMAP fetch for \Seen flag will return empty (i.e. it will not be present, as related to the email message).
If you do IMAP select on a mailbox (INBOX), you get a "flag" UNSEEN which contains a list of ids (or uids) of emails in that folder that are new (do not have the \Seen flag)
In my test case, if you fetch say headers for a message with BODY.PEEK, then \Seen on a message is not set; if you fetch them with BODY, then \Seen is set
In my test case, also fetching (RFC822) doesn't set \Seen (unlike your case with Gmail)
In the test case, I try to do pprint.pprint(inspect.getmembers(mail)) (in lieu of your dumpobj(uid, mail)) - but only after I'm certain \Seen has been set. The output I get is posted in mail_object_inspect.txt - and as far as I can see, there is no mention of 'new/read/seen' etc. in none of the readable fields; furthermore mail.as_string() prints:
'From: jesse#example.com\nTo: user#example.com\nSubject: This is a test message!\n\nHello. I am executive assistant to the director of\nBear Stearns, a failed investment Bank. I have\naccess to USD6,000,000. ...\n'
Even worse, there is no mention of "fields" anywhere in the imaplib code (below filenames are printed if they do not contain case-insensitive "field" anywhere):
$ grep -L -i field /usr/lib/python{2.7,3.2}/imaplib.py
/usr/lib/python2.7/imaplib.py
/usr/lib/python3.2/imaplib.py
... so I guess that information was not saved with your dumps.
Here is a bit on reconstructing the test case. The hardest was to find a small IMAP server, that can be quickly ran with some arbitrary users and emails, but without having to install a ton of stuff on your system. Finally I found one: trivial-server.pl, the example file of Perl's Net::IMAP::Server; tested on Ubuntu 11.04.
The test case is pasted in this gist, with two files (with many comments) that I'll try to post abridged:
trivial-serverB.pl - Perl (v5.10.1) Net::IMAP::Server server (has a terminal output paste at end of file with a telnet client session)
testimap.py - Python 2.7/3.2 imaplib
client (has a terminal output paste at end of file, of itself operating with the server)
trivial-serverB.pl
First, make sure you have Net::IMAP::Server - note, it has many dependencies, so the below command may take a while to install:
sudo perl -MCPAN -e 'install Net::IMAP::Server'
Then, in the directory where you got trivial-serverB.pl, create a subdirectory with SSL certificates:
mkdir certs
openssl req \
-x509 -nodes -days 365 \
-subj '/C=US/ST=Oregon/L=Portland/CN=localhost' \
-newkey rsa:1024 -keyout certs/server-key.pem -out certs/server-cert.pem
Finally run the server with administrative properties:
sudo perl trivial-serverB.pl
Note that the trivial-serverB.pl has a hack which will let a client to connect without SSL. Here is trivial-serverB.pl:
#!/usr/bin/perl
use v5.10.1;
use feature qw(say);
use Net::IMAP::Server;
package Demo::IMAP::Hack;
$INC{'Demo/IMAP/Hack.pm'} = 1;
sub capabilityb {
my $self = shift;
print STDERR "Capabilitin'\n";
my $base = $self->server->capability;
my #words = split " ", $base;
#words = grep {$_ ne "STARTTLS"} #words
if $self->is_encrypted;
unless ($self->auth) {
my $auth = $self->auth || $self->server->auth_class->new;
my #auth = $auth->sasl_provides;
# hack:
#unless ($self->is_encrypted) {
# # Lack of encrpytion makes us turn off all plaintext auth
# push #words, "LOGINDISABLED";
# #auth = grep {$_ ne "PLAIN"} #auth;
#}
push #words, map {"AUTH=$_"} #auth;
}
return join(" ", #words);
}
package Demo::IMAP::Auth;
$INC{'Demo/IMAP/Auth.pm'} = 1;
use base 'Net::IMAP::Server::DefaultAuth';
sub auth_plain {
my ( $self, $user, $pass ) = #_;
# XXX DO AUTH CHECK
$self->user($user);
return 1;
}
package Demo::IMAP::Model;
$INC{'Demo/IMAP/Model.pm'} = 1;
use base 'Net::IMAP::Server::DefaultModel';
sub init {
my $self = shift;
$self->root( Demo::IMAP::Mailbox->new() );
$self->root->add_child( name => "INBOX" );
}
###########################################
package Demo::IMAP::Mailbox;
use base qw/Net::IMAP::Server::Mailbox/;
use Data::Dumper;
my $data = <<'EOF';
From: jesse#example.com
To: user#example.com
Subject: This is a test message!
Hello. I am executive assistant to the director of
Bear Stearns, a failed investment Bank. I have
access to USD6,000,000. ...
EOF
my $msg = Net::IMAP::Server::Message->new($data);
sub load_data {
my $self = shift;
$self->add_message($msg);
}
my %ports = ( port => 143, ssl_port => 993 );
$ports{$_} *= 10 for grep {$> > 0} keys %ports;
$myserv = Net::IMAP::Server->new(
auth_class => "Demo::IMAP::Auth",
model_class => "Demo::IMAP::Model",
user => 'nobody',
log_level => 3, # at least 3 to output 'CONNECT TCP Peer: ...' message; 4 to output IMAP commands too
%ports,
);
# apparently, this overload MUST be after the new?! here:
{
no strict 'refs';
*Net::IMAP::Server::Connection::capability = \&Demo::IMAP::Hack::capabilityb;
}
# https://stackoverflow.com/questions/27206371/printing-addresses-of-perl-object-methods
say " -", $myserv->can('validate'), " -", $myserv->can('capability'), " -", \&Net::IMAP::Server::Connection::capability, " -", \&Demo::IMAP::Hack::capabilityb;
$myserv->run();
testimap.py
With the server above running in one terminal, in another terminal you can just do:
python testimap.py
The code will simply read fields and content from the one (and only) message the server above presents, and will eventually restore (remove) the \Seen field.
import sys
if sys.version_info[0] < 3: # python 2.7
def uttc(x):
return x
else: # python 3+
def uttc(x):
return x.decode("utf-8")
import imaplib
import email
import pprint,inspect
imap_user = 'nobody'
imap_password = 'whatever'
imap_server = 'localhost'
conn = imaplib.IMAP4(imap_server)
conn.debug = 3
try:
(retcode, capabilities) = conn.login(imap_user, imap_password)
except:
print(sys.exc_info()[1])
sys.exit(1)
# not conn.select(readonly=1), else we cannot modify the \Seen flag later
conn.select() # Select inbox or default namespace
(retcode, messages) = conn.search(None, '(UNSEEN)')
if retcode == 'OK':
for num in uttc(messages[0]).split(' '):
if not(num):
print("No messages available: num is `{0}`!".format(num))
break
print('Processing message: {0}'.format(num))
typ, data = conn.fetch(num,'(FLAGS)')
isSeen = ( "Seen" in uttc(data[0]) )
print('Got flags: {2}: {0} .. {1}'.format(typ,data, # NEW: OK .. ['1 (FLAGS ())']
"Seen" if isSeen else "NEW"))
print('Peeking headers, message: {0} '.format(num))
typ, data = conn.fetch(num,'(BODY.PEEK[HEADER])')
pprint.pprint(data)
typ, data = conn.fetch(num,'(FLAGS)')
isSeen = ( "Seen" in uttc(data[0]) )
print('Got flags: {2}: {0} .. {1}'.format(typ,data, # NEW: OK .. ['1 (FLAGS ())']
"Seen" if isSeen else "NEW"))
print('Get RFC822 body, message: {0} '.format(num))
typ, data = conn.fetch(num,'(RFC822)')
mail = email.message_from_string(uttc(data[0][1]))
#pprint.pprint(inspect.getmembers(mail))
typ, data = conn.fetch(num,'(FLAGS)')
isSeen = ( "Seen" in uttc(data[0]) )
print('Got flags: {2}: {0} .. {1}'.format(typ,data, # NEW: OK .. ['1 (FLAGS ())']
"Seen" if isSeen else "NEW"))
print('Get headers, message: {0} '.format(num))
typ, data = conn.fetch(num,'(BODY[HEADER])') # note, FLAGS (\\Seen) is now in data, even if not explicitly requested!
pprint.pprint(data)
print('Get RFC822 body, message: {0} '.format(num))
typ, data = conn.fetch(num,'(RFC822)')
mail = email.message_from_string(uttc(data[0][1]))
pprint.pprint(inspect.getmembers(mail)) # this is in mail_object_inspect.txt
pprint.pprint(mail.as_string())
typ, data = conn.fetch(num,'(FLAGS)')
isSeen = ( "Seen" in uttc(data[0]) )
print('Got flags: {2}: {0} .. {1}'.format(typ,data, # Seen: OK .. ['1 (FLAGS (\\Seen))']
"Seen" if isSeen else "NEW"))
conn.select() # select again, to see flags server side
# * OK [UNSEEN 0] # no more unseen messages (if there was only one msg in folder)
print('Restoring flag to unseen/new, message: {0} '.format(num))
ret, data = conn.store(num,'-FLAGS','\\Seen')
if ret == 'OK':
print("Set back to unseen; Got OK: {0}{1}{2}".format(data,'\n',30*'-'))
print(mail)
typ, data = conn.fetch(num,'(FLAGS)')
isSeen = ( "Seen" in uttc(data[0]) )
print('Got flags: {2}: {0} .. {1}'.format(typ,data, # NEW: OK .. [b'1 (FLAGS ())']
"Seen" if isSeen else "NEW"))
conn.close()
References
How do I mock an IMAP server in Python, despite extreme laziness?
Get only NEW Emails imaplib and python
Undoing "marked as read" status of emails fetched with imaplib
http://www.skytale.net/blog/archives/23-Manual-IMAP.html
IMAP FETCH Subject
https://mail.python.org/pipermail/python-list/2009-March/527020.html
http://www.thecodingforums.com/threads/re-imaplib-fetch-message-flags.673872/

Categories