We Keep Coding

Extract body from email message objects in Python

I have an .mbox file that represents many messages at location mbox_fname. In Python 3, I have already loaded each of the messages, which are objects of the class email.message.Message.
I'd like to get access to the body content of the message.
For instance, something like:
import mailbox
the_mailbox = mailbox.mbox(mbox_fname)
for message in the_mailbox:
subject = message["subject"]
content = <???>
How do I access the body of the message?

I made some progress modifying this answer. This is the best I have so far:
import email
def get_body(message: email.message.Message, encoding: str = "utf-8") -> str:
body_in_bytes = ""
if message.is_multipart():
for part in message.walk():
ctype = part.get_content_type()
cdispo = str(part.get("Content-Disposition"))
# skip any text/plain (txt) attachments
if ctype == "text/plain" and "attachment" not in cdispo:
body_in_bytes = part.get_payload(decode=True) # decode
break
# not multipart - i.e. plain text, no attachments, keeping fingers crossed
else:
body_in_bytes = message.get_payload(decode=True)
body = body_in_bytes.decode(encoding)
return body
So modifying the code in the original question, this gets called like the following:
for message in the_mailbox:
content = get_body(message)

IMAP message gets UnicodeDecodeError 'utf-8' codec can't decode

After 5 hours of trying, time to get some help. Sifted through all the stackoverflow questions related to this but couldn't find the answer.
The code is a gmail parser - works for most emails but some emails cause the UnicodeDecodeError. The problem is "raw_email.decode('utf-8')" but changing it (see comments) causes a different problem down below.
# Source: https://stackoverflow.com/questions/7314942/python-imaplib-to-get-gmail-inbox-subjects-titles-and-sender-name
import datetime
import time
import email
import imaplib
import mailbox
from vars import *
import re # to remove links from str
import string
EMAIL_ACCOUNT = 'gmail_login'
PASSWORD = 'gmail_psswd'
mail = imaplib.IMAP4_SSL('imap.gmail.com')
mail.login(EMAIL_ACCOUNT, PASSWORD)
mail.list()
mail.select('inbox')
result, data = mail.uid('search', None, "ALL") # (ALL/UNSEEN)
id_list = data[0].split()
email_rev = reversed(id_list) # Returns a type list.reverseiterator, which is not list
email_list = list(email_rev)
i = len(email_list)
todays_date = time.strftime("%m/%d/%Y")
for x in range(i):
latest_email_uid = email_list[x]
result, email_data = mail.uid('fetch', latest_email_uid, '(RFC822)')
raw_email = email_data[0][1] # Returns a byte
raw_email_str = raw_email.decode('utf-8') # Returns a str
#raw_email_str = base64.b64decode(raw_email_str1) # Tried this but didn't work.
#raw_email_str = raw_email.decode('utf-8', errors='ignore') # Tried this but caused a TypeError down where var subject is created because something there is expecting a str or byte-like
email_message = email.message_from_string(raw_email_str)
date_tuple = email.utils.parsedate_tz(email_message['Date'])
date_short = f'{date_tuple[1]}/{date_tuple[2]}/{date_tuple[0]}'
# Header Details
if date_short == '12/23/2019':
#if date_tuple:
# local_date = datetime.datetime.fromtimestamp(email.utils.mktime_tz(date_tuple))
# local_message_date = "%s" %(str(local_date.strftime("%a, %d %b %Y %H:%M:%S")))
email_from = str(email.header.make_header(email.header.decode_header(email_message['From'])))
subject = str(email.header.make_header(email.header.decode_header(email_message['Subject'])))
#print(subject)
if email_from.find('restaurants#uber.com') != -1:
print('yay')
# Body details
if email_from.find('restaurants#uber.com') != -1 and subject.find('Payment Summary') != -1:
for part in email_message.walk():
if part.get_content_type() == "text/plain":
body = part.get_payload(decode=True)
body = body.decode("utf-8") # Convert byte to str
body = body.replace("\r\n", " ")
text = re.sub(r'\w+:\/{2}[\d\w-]+(\.[\d\w-]+)*(?:(?:\/[^\s/]*))*', '', body) # removes url links
text2 = text.translate(str.maketrans('', '', string.punctuation))
body_list = re.sub("[^\w]", " ", text2).split()
print(body_list)
print(date_short)
else:
continue

Here is an example how to retrieve and read mail parts with imapclient and the email.* modules from the python standard libs:
from imapclient import IMAPClient
import email
from email import policy
def walk_parts(part, level=0):
print(' ' * 4 * level + part.get_content_type())
# do something with part content (applies encoding by default)
# part.get_content()
if part.is_multipart():
for part in part.get_payload():
get_parts(part, level + 1)
# context manager ensures the session is cleaned up
with IMAPClient(host="your_mail_host") as client:
client.login('user', 'password')
# select some folder
client.select_folder('INBOX')
# do something with folder, e.g. search & grab unseen mails
messages = client.search('UNSEEN')
for uid, message_data in client.fetch(messages, 'RFC822').items():
email_message = email.message_from_bytes(
message_data[b'RFC822'], policy=policy.default)
print(uid, email_message.get('From'), email_message.get('Subject'))
# alternatively search for specific mails
msgs = client.search(['SUBJECT', 'some subject'])
#
# do something with a specific mail:
#
# fetch a single mail with UID 12345
raw_mails = client.fetch([12345], 'RFC822')
# parse the mail (very expensive for big mails with attachments!)
mail = email.message_from_bytes(
raw_mails[12345][b'RFC822'], policy=policy.default)
# Now you have a python object representation of the mail and can dig
# into it. Since a mail can be composed of several subparts we have
# to walk the subparts.
# walk all parts at once
for part in mail.walk():
# do something with that part
print(part.get_content_type())
# or recurse yourself into sub parts until you find the interesting part
walk_parts(mail)
See the docs for email.message.EmailMessage. There you find all needed bits to read into a mail message.

use 'ISO 8859-1' instead of 'utf-8'

I had the same issue And after a lot of research I realized that I simply need to use, message_from_bytes function from email rather than using message_from_string
so for your code simply replace:
raw_email_str = raw_email.decode('utf-8')
email_message = email.message_from_string(raw_email_str)
to
email_message = email.message_from_bytes(raw_email)
should work like a charm :)

Python Message' object has no attribute 'get_body

I'm trying to search email body but facing some issues:
#!/usr/local/bin/python3
from email.message import EmailMessage
import email
import imaplib
import re
import sys
import logging
import base64
import os
logging.basicConfig(stream=sys.stdout, level=logging.INFO)
###########log in to mailbox########################
user = 'email#company.com'
pwd = 'pwd'
conn = imaplib.IMAP4_SSL("outlook.office365.com")
conn.login(user,pwd)
conn.select("test")
count = conn.select("test")
resp, items = conn.uid("search" ,None, '(OR (FROM "some#email) (FROM "some#email"))')
items = items[0].split()
for emailid in items:
resp, data = conn.uid("fetch",emailid, "(RFC822)")
if resp == 'OK':
email_body = data[0][1]#.decode('utf-8')
mail = email.message_from_bytes(email_body)
#get all emails with words "PA1" or "PA2" in subject
if mail["Subject"].find("PA1") > 0 or mail["Subject"].find("PA2") > 0:
print (mail)
I have issues in following line:
body = mail.get_body(preferencelist=('plain', 'html'))
getting:
AttributeError: 'Message' object has no attribute 'get_body'

To address the message:
AttributeError: 'Message' object has no attribute 'get_body'
When creating the Message object you need to specify a policy or you get the default email.policy.Compat32 policy. get_body() and several other methods did not exist in Python 3.2.
The line creating the mail object should be:
mail = email.message_from_bytes(data, policy=email.policy.default)
More information at:
https://docs.python.org/3/library/email.policy.html

You should not convert the MIME structure to a string and then feed that to message_from_string. Instead, keep it as a bytes object.
from email.policy import default as default_policy
...
items = items[0].split()
for emailid in items:
resp, data = conn.uid("fetch",emailid, "(RFC822)")
if resp == 'OK':
email_blob = data[0][1]
mail = email.message_from_bytes(email_blob, policy=default_policy)
if not any(x in mail['subject'] for x in ('PA1', 'PA2')):
continue
You are not showing how you are traversing the MIME structure so I sort of assume you are currently not doing that at all. Probably you want something like
# continuation for the above code
body = mail.get_body(preferencelist=('plain', 'html'))
for lines in body.split('\n'):
if line.startswith('MACHINE:'):
result = line[8:].strip()
break
It looks like you have an email body part encoded using Content-Transfer-Encoding: quoted-printable. The above code is robust against various encodings because the email library decodes the encapsulation transparently for you, which gets rid of any QP-escaped line breaks, like the one in your question. For the record, quoted-printable can break up a long line anywere, including in the middle of the value you are attempting to extract, so you really do want to decode before attempting to extract anything.

If it's acceptable for you to first remove all the line breaks =^M\n from the text, then it's quite simple:
import re
email_body = open("1.txt").read().replace("=^M\n", "")
matches = re.findall(r"(?<=MACHINE:)\s*(\w+)", email_body)
print(matches)
print(list(set(matches)))
Output:
['p1prog07', 'p2prog06', 'p2prog06', 'p1prog07', 'ldnv260']
['p2prog06', 'ldnv260', 'p1prog07']
The positive look-behind is a non-capturing group, so the only captured group in the regex is your desired string.

Python search imap email for a string

New to python, having some trouble getting past this.
Am getting back emails from gmail via imap (with starter code from https://yuji.wordpress.com/2011/06/22/python-imaplib-imap-example-with-gmail/) and want to search a specific email (which I am able to fetch) for a specific string. Something like this
ids = data[0]
id_list = ids.split()
ids = data[0]
id_list = ids.split()
latest_email_id = id_list[-1]
result, data = mail.fetch(latest_email_id, "(RFC822)")
raw_email = data[0][1]
def search_raw():
if 'gave' in raw_email:
done = 'yes'
else:
done = 'no'
and it always sets done to no. Here's the output for the email (for the body section of the email)
Content-Type multipart/related;boundary=1_56D8EAE1_29AD7EA0;type="text/html"
--1_56D8EAE1_29AD7EA0
Content-Type text/html;charset="UTF-8"
Content-Transfer-Encoding base64
PEhUTUw+CiAgICAgICAgPEhFQUQ+CiAgICAgICAgICAgICAgICA8VElUTEU+PC9USVRMRT4KICAg
ICAgICA8L0hFQUQ+CiAgICAgICAgPEJPRFk+CiAgICAgICAgICAgICAgICA8UCBhbGlnbj0ibGVm
dCI+PEZPTlQgZmFjZT0iVmVyZGFuYSIgY29sb3I9IiNjYzAwMDAiIHNpemU9IjIiPlNlbnQgZnJv
bSBteSBtb2JpbGUuCiAgICAgICAgICAgICAgICA8QlI+X19fX19fX19fX19fX19fX19fX19fX19f
X19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fXzwvRk9OVD48L1A+CgogICAgICAg
ICAgICAgICAgPFBSRT4KR2F2ZQoKPC9QUkU+CiAgICAgICAgPC9CT0RZPgo8L0hUTUw+Cg==
--1_56D8EAE1_29AD7EA0--
I know the issue is the html, but can't seem to figure out how to parse the email properly.
Thank you!

The text above is base64 encoding. Python has a module named base64 which gives you the ability to decode it.
import base64
import re
def has_gave(raw_email):
email_body = base64.b64decode(raw_email)
match = re.search(r'.*gave.*', email_body , re.IGNORECASE)
if match:
done = 'yes'
print 'match found for word ', match.group()
else:
done = 'no'
print 'no match found'
return done

Reading the mail content of an mbox file using python mailbox

I am trying to print the content of the mail ( Mail body) using Python mailbox.
import mailbox
mbox = mailbox.mbox('Inbox')
i=1
for message in mbox:
print i
print "from :",message['from']
print "subject:",message['subject']
print "message:",message['**messages**']
print "**************************************"
i+=1
But I feel message['messages'] is not the right one to print the mail content here. I could not understand it from the documentation

To get the message content, you want to use get_payload(). mailbox.Message is a subclass of email.message.Message. You'll also want to check is_multipart() because that will affect the return value of get_payload(). Example:
if message.is_multipart():
content = ''.join(part.get_payload(decode=True) for part in message.get_payload())
else:
content = message.get_payload(decode=True)

def getbody(message): #getting plain text 'email body'
body = None
if message.is_multipart():
for part in message.walk():
if part.is_multipart():
for subpart in part.walk():
if subpart.get_content_type() == 'text/plain':
body = subpart.get_payload(decode=True)
elif part.get_content_type() == 'text/plain':
body = part.get_payload(decode=True)
elif message.get_content_type() == 'text/plain':
body = message.get_payload(decode=True)
return body
this function can give you message body if the body is plain text.