We Keep Coding

How do I implement the imaplib search function?

import imaplib
import email
from email.header import decode_header
import webbrowser
import os
# account credentials
username = "example#stack.com"
password = "exapleforstack"
imap_server = "imap.one.com"
def clean(text):
# clean text for creating a folder
return "".join(c if c.isalnum() else "_" for c in text)
# create an IMAP4 class with SSL
imap = imaplib.IMAP4_SSL(imap_server)
# authenticate
imap.login(username, password)
status, messages = imap.select("INBOX")
imap.search(None, 'SUBJECT', '"exampleforstack"')
# number of top emails to fetch
N = 3
# total number of emails
messages = int(messages[0])
for i in range(messages, messages-N, -1):
# fetch the email message by ID
res, msg = imap.fetch(str(i), "(RFC822)")
for response in msg:
if isinstance(response, tuple):
# parse a bytes email into a message object
msg = email.message_from_bytes(response[1])
# decode the email subject
subject, encoding = decode_header(msg["Subject"])[0]
if isinstance(subject, bytes):
# if it's a bytes, decode to str
subject = subject.decode(encoding)
# decode email sender
From, encoding = decode_header(msg.get("From"))[0]
if isinstance(From, bytes):
From = From.decode(encoding)
print("Subject:", subject)
print("From:", From)
# if the email message is multipart
if msg.is_multipart():
# iterate over email parts
for part in msg.walk():
# extract content type of email
content_type = part.get_content_type()
content_disposition = str(part.get("Content-Disposition"))
try:
# get the email body
body = part.get_payload(decode=True).decode()
except:
pass
if content_type == "text/plain" and "attachment" not in content_disposition:
# print text/plain emails and skip attachments
print(body)
elif "attachment" in content_disposition:
# download attachment
filename = part.get_filename()
if filename:
folder_name = clean(subject)
if not os.path.isdir(folder_name):
# make a folder for this email (named after the subject)
os.mkdir(folder_name)
filepath = os.path.join(folder_name, filename)
# download attachment and save it
open(filepath, "wb").write(part.get_payload(decode=True))
else:
# extract content type of email
content_type = msg.get_content_type()
# get the email body
body = msg.get_payload(decode=True).decode()
if content_type == "text/plain":
# print only text email parts
print(body)
if content_type == "text/html":
# if it's HTML, create a new HTML file and open it in browser
folder_name = clean(subject)
if not os.path.isdir(folder_name):
# make a folder for this email (named after the subject)
os.mkdir(folder_name)
filename = "index.html"
filepath = os.path.join(folder_name, filename)
# write the file
open(filepath, "w").write(body)
# open in the default browser
webbrowser.open(filepath)
print("="*100)
# close the connection and logout
imap.close()
imap.logout()
I tried using the search method a couple different ways, like:
res, msg = imap.search(None, 'SUBJECT', "example")
and
res, msg = imap.search(None, 'SUBJECT, "example"')
but my code just gives an error and automatically fetches the most recent 3.
I've tried replacing the line:
res, msg = imap.fetch(str(i), "(RFC822)")
with
res, msg = imap.search(None, 'SUBJECT', '"example"')
but the program returns nothing at all.
how would I go about implementing the search I got this code from pythoncode and altered its credentials.
But I'm not sure why I can't implement the search function.

Revised AWS SES forwarder sample code (Python) unable to forward email body

I'm trying to get the AWS SES Lambda forwarder Python sample code to forward emails to an outside account unmodified. In (A) below, the message body will not forward properly unless the email is multipart. Most errors are tied to:
body = MIMEText(str(mailobject.get_payload(decode=True), 'UTF-8'))
Which should pass the email body to body as a string when MIMEmultipart is false if I'm understanding the python documentation correctly. Depending on how that line of code is done, it will send the body as shown below, or the compiler will return an error as shown in (B) or 'str' object has no attribute 'policy'",.
What an email looks like when I get it. The html formatting shows up as text.
<html>
<body>
<p>You have chosen to subscribe to the topic:
<br /><b>arn:aws:sns:I INTENTIONALLY REMOVED THIS</b></p>
<p>To confirm this subscription, click or visit the link below (If this was in error no action is necessary):
<br /><a href="https://sns.us-east-1.amazonaws.com/confirmation.html?TopicArn=arn:aws:sns:us-east-1:I INTENTIONALLY REMOVED THIS</a></p>
<p><small>Please do not reply directly to this email. If you wish to remove yourself from receiving all future SNS subscription confirmation requests please send an email to sns-opt-out</small></p>
</body>
</html>
(A) The code I'm using:
import os
import boto3
import email
import re
from botocore.exceptions import ClientError
from email.mime.multipart import MIMEMultipart
from email.mime.text import MIMEText
from email.mime.application import MIMEApplication
from datetime import datetime
region = os.environ['Region']
def get_time():
# datetime object containing current date and time
now = datetime.now()
# dd/mm/YY H:M:S
return now.strftime("%d/%m/%Y %H:%M:%S")
def get_message_from_s3(message_id):
incoming_email_bucket = os.environ['MailS3Bucket']
incoming_email_prefix = os.environ['MailS3Prefix']
if incoming_email_prefix:
object_path = (incoming_email_prefix + "/" + message_id)
else:
object_path = message_id
object_http_path = (f"http://s3.console.aws.amazon.com/s3/object/{incoming_email_bucket}/{object_path}?region={region}")
# Create a new S3 client.
client_s3 = boto3.client("s3")
# Get the email object from the S3 bucket.
object_s3 = client_s3.get_object(Bucket=incoming_email_bucket,
Key=object_path)
# Read the content of the message.
file = object_s3['Body'].read()
file_dict = {
"file": file,
"path": object_http_path
}
return file_dict
def create_message(file_dict):
stringMsg = file_dict['file'].decode('utf-8')
# Create a MIME container.
msg = MIMEMultipart('alternative')
sender = os.environ['MailSender']
recipient = os.environ['MailRecipient']
# Parse the email body.
mailobject = email.message_from_string(file_dict['file'].decode('utf-8'))
#print(mailobject.as_string())
# Get original sender for reply-to
from_original = mailobject['Return-Path']
from_original = from_original.replace('<', '');
from_original = from_original.replace('>', '');
print(from_original)
# Create a new subject line.
subject = mailobject['Subject']
body = ""
if mailobject.is_multipart():
print('IS multipart')
index = stringMsg.find('Content-Type: multipart/')
stringBody = stringMsg[index:]
#print(stringBody)
stringData = 'Subject: ' + subject + '\nTo: ' + sender + '\nreply-to: ' + from_original + '\n' + stringBody
message = {
"Source": sender,
"Destinations": recipient,
"Data": stringData
}
return message
for part in mailobject.walk():
ctype = part.get_content_type()
cdispo = str(part.get('Content-Disposition'))
# case for each common content type
if ctype == 'text/plain' and 'attachment' not in cdispo:
bodyPart = MIMEText(part.get_payload(decode=True), 'plain', part.get_content_charset())
msg.attach(bodyPart)
if ctype == 'text/html' and 'attachment' not in cdispo:
mt = MIMEText(part.get_payload(decode=True), 'html', part.get_content_charset())
email.encoders.encode_quopri(mt)
del mt['Content-Transfer-Encoding']
mt.add_header('Content-Transfer-Encoding', 'quoted-printable')
msg.attach(mt)
if 'attachment' in cdispo and 'image' in ctype:
mi = MIMEImage(part.get_payload(decode=True), ctype.replace('image/', ''))
del mi['Content-Type']
del mi['Content-Disposition']
mi.add_header('Content-Type', ctype)
mi.add_header('Content-Disposition', cdispo)
msg.attach(mi)
if 'attachment' in cdispo and 'application' in ctype:
ma = MIMEApplication(part.get_payload(decode=True), ctype.replace('application/', ''))
del ma['Content-Type']
del ma['Content-Disposition']
ma.add_header('Content-Type', ctype)
ma.add_header('Content-Disposition', cdispo)
msg.attach(ma)
# not multipart - i.e. plain text, no attachments, keeping fingers crossed
else:
body = MIMEText(str(mailobject.get_payload(decode=True), 'UTF-8'))
msg.attach(body)
# The file name to use for the attached message. Uses regex to remove all
# non-alphanumeric characters, and appends a file extension.
filename = re.sub('[^0-9a-zA-Z]+', '_', subject)
# Add subject, from and to lines.
msg['Subject'] = subject
msg['From'] = sender
msg['To'] = recipient
msg['reply-to'] = mailobject['Return-Path']
# Create a new MIME object.
att = MIMEApplication(file_dict["file"], filename)
att.add_header("Content-Disposition", 'attachment', filename=filename)
# Attach the file object to the message.
msg.attach(att)
message = {
"Source": sender,
"Destinations": recipient,
"Data": msg.as_string()
}
return message
def send_email(message):
aws_region = os.environ['Region']
# Create a new SES client.
client_ses = boto3.client('ses', region)
# Send the email.
try:
#Provide the contents of the email.
response = client_ses.send_raw_email(
Source=message['Source'],
Destinations=[
message['Destinations']
],
RawMessage={
'Data':message['Data']
}
)
# Display an error if something goes wrong.
except ClientError as e:
output = e.response['Error']['Message']
else:
output = "Email sent! Message ID: " + response['MessageId']
return output
def lambda_handler(event, context):
# Get the unique ID of the message. This corresponds to the name of the file
# in S3.
message_id = event['Records'][0]['ses']['mail']['messageId']
print(f"Received message ID {message_id}")
# Retrieve the file from the S3 bucket.
file_dict = get_message_from_s3(message_id)
# Create the message.
message = create_message(file_dict)
#this alone didn't fix it: message = create_message(str(file_dict))
# Send the email and print the result.
result = send_email(message)
print(result)
(B). The code example from this post fails at the 'body' assignment and what I've been trying to get to work in (A)...
body = MIMEText(mailobject.get_payload(decode=True), 'UTF-8')
msg.attach(body)
Response:
{
"errorMessage": "'bytes' object has no attribute 'encode'",
"errorType": "AttributeError",
"stackTrace": [
" File \"/var/task/lambda_function.py\", line 190, in lambda_handler\n message = create_message(file_dict)\n",
" File \"/var/task/lambda_function.py\", line 128, in create_message\n body = MIMEText(mailobject.get_payload(decode=True), 'UTF-8')\n",
" File \"/var/lang/lib/python3.8/email/mime/text.py\", line 34, in __init__\n _text.encode('us-ascii')\n"
]
}
To replicate this problem, get a new SES instance going, follow the tutorial for the original AWS SES Lambda forwarder Python sample code and verify it works. From there, replace the AWS sample code with my code in (A) or one of the other linked examples.
Also, I found this Lambda email test event code and was able to get it working enough to test this code from the AWS Lambda dashboard. To make it work I created two new test events (multipart/non-multipart) in the dashboard and pasted in that code changing (a) the email addresses as applicable, (b) the the lambda function ARN to my function's ARN, and (c) the message-id for an applicable email stored on my active s3 instance. This has saved me the trouble of manually creating and sending test emails for debugging and hopefully will do the same for somebody else that finds this post and is having similar problems. That test code will result in an actual email being forwarded.

For those getting this error:
'bytes' object has no attribute 'encode'
In this line:
body = MIMEText(mailobject.get_payload(decode=True), 'UTF-8')
I could make it work. I am not an expert on this so the code might need some improvement. Also the email body includes html tags. But at least it got delivered.
If decoding the email still fails the error message will appear in your CloudWatch log. Also you will receive an email with the error message.
payload = mailobject.get_payload(decode=True)
try:
decodedPayload = payload.decode()
body = MIMEText(decodedPayload, 'UTF-8')
msg.attach(body)
except Exception as error:
errorMsg = "An error occured when decoding the email payload:\n" + str(error)
print(errorMsg)
body = errorMsg + "\nPlease download it manually from the S3 bucket."
msg.attach(MIMEText(body, 'plain'))
It is up to you which information you want to add to the error email like the subject or the from address.

AWS EC2 Email Forwarding: Forward Incoming Email to an External Destination with Lambda

I'm following the AWS tutorial for forwarding emails to my Gmail account found here: https://aws.amazon.com/blogs/messaging-and-targeting/forward-incoming-email-to-an-external-destination/
I have all of that working, but the forwarded email comes in the form of an attachment that has to be downloaded to read.
What I wanted to know is how to get this forwarder to render the email to look like a normal email would. I searched SO but found very few threads on the topic. So, I worked on it myself and figured it out, and have provided the answer below for anyone else looking at this issue.

Use the following Python code instead of the code that is provided in the tutorial. Your forwarded emails will arrive looking the way you want them.
Keep in mind that there is a time limit on how long your lambda function can run. Decoding and encoding large attachments takes time. So, any attachment that is too large (like audio/video files) will probably cause the entire forwarding to fail. However, that failure won't stop AWS from retrying again and again, and it will continue to timeout again again with every attempt.
# Copyright 2010-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
# Altered from original by Adam Winter
#
# This file is licensed under the Apache License, Version 2.0 (the "License").
# You may not use this file except in compliance with the License. A copy of the
# License is located at
#
# http://aws.amazon.com/apache2.0/
#
# This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS
# OF ANY KIND, either express or implied. See the License for the specific
# language governing permissions and limitations under the License.
import os
import boto3
import email
import re
import html
from botocore.exceptions import ClientError
from email.mime.multipart import MIMEMultipart
from email.mime.text import MIMEText
from email.mime.application import MIMEApplication
from email.mime.image import MIMEImage
region = os.environ['Region']
def get_message_from_s3(message_id):
incoming_email_bucket = os.environ['MailS3Bucket']
incoming_email_prefix = os.environ['MailS3Prefix']
if incoming_email_prefix:
object_path = (incoming_email_prefix + "/" + message_id)
else:
object_path = message_id
object_http_path = (f"http://s3.console.aws.amazon.com/s3/object/{incoming_email_bucket}/{object_path}?region={region}")
# Create a new S3 client.
client_s3 = boto3.client("s3")
# Get the email object from the S3 bucket.
object_s3 = client_s3.get_object(Bucket=incoming_email_bucket,
Key=object_path)
# Read the content of the message.
file = object_s3['Body'].read()
file_dict = {
"file": file,
"path": object_http_path
}
return file_dict
def create_message(file_dict):
stringMsg = file_dict['file'].decode('utf-8')
# Create a MIME container.
msg = MIMEMultipart('alternative')
sender = os.environ['MailSender']
recipient = os.environ['MailRecipient']
# Parse the email body.
mailobject = email.message_from_string(file_dict['file'].decode('utf-8'))
#print(mailobject.as_string())
# Get original sender for reply-to
from_original = mailobject['Return-Path']
from_original = from_original.replace('<', '');
from_original = from_original.replace('>', '');
print(from_original)
# Create a new subject line.
subject = mailobject['Subject']
print(subject)
if mailobject.is_multipart():
#The quick and dirty way. If you don't like this, use the for loop below it.
index = stringMsg.find('Content-Type: multipart/')
stringBody = stringMsg[index:]
#print(stringBody)
stringData = 'Subject: ' + subject + '\nTo: ' + sender + '\nreply-to: ' + from_original + '\n' + stringBody
message = {
"Source": sender,
"Destinations": recipient,
"Data": stringData
}
return message
for part in mailobject.walk():
ctype = part.get_content_type()
cdispo = str(part.get('Content-Disposition'))
# case for each common content type
if ctype == 'text/plain' and 'attachment' not in cdispo:
bodyPart = MIMEText(part.get_payload(decode=True), 'plain', part.get_content_charset())
msg.attach(bodyPart)
if ctype == 'text/html' and 'attachment' not in cdispo:
mt = MIMEText(part.get_payload(decode=True), 'html', part.get_content_charset())
email.encoders.encode_quopri(mt)
del mt['Content-Transfer-Encoding']
mt.add_header('Content-Transfer-Encoding', 'quoted-printable')
msg.attach(mt)
if 'attachment' in cdispo and 'image' in ctype:
mi = MIMEImage(part.get_payload(decode=True), ctype.replace('image/', ''))
del mi['Content-Type']
del mi['Content-Disposition']
mi.add_header('Content-Type', ctype)
mi.add_header('Content-Disposition', cdispo)
msg.attach(mi)
if 'attachment' in cdispo and 'application' in ctype:
ma = MIMEApplication(part.get_payload(decode=True), ctype.replace('application/', ''))
del ma['Content-Type']
del ma['Content-Disposition']
ma.add_header('Content-Type', ctype)
ma.add_header('Content-Disposition', cdispo)
msg.attach(ma)
# not multipart - i.e. plain text, no attachments, keeping fingers crossed
else:
body = MIMEText(mailobject.get_payload(decode=True), 'UTF-8')
msg.attach(body)
# The file name to use for the attached message. Uses regex to remove all
# non-alphanumeric characters, and appends a file extension.
filename = re.sub('[^0-9a-zA-Z]+', '_', subject_original)
# Add subject, from and to lines.
msg['Subject'] = subject
msg['From'] = sender
msg['To'] = recipient
msg['reply-to'] = mailobject['Return-Path']
# Create a new MIME object.
att = MIMEApplication(file_dict["file"], filename)
att.add_header("Content-Disposition", 'attachment', filename=filename)
# Attach the file object to the message.
msg.attach(att)
message = {
"Source": sender,
"Destinations": recipient,
"Data": msg.as_string()
}
return message
def send_email(message):
aws_region = os.environ['Region']
# Create a new SES client.
client_ses = boto3.client('ses', region)
# Send the email.
try:
#Provide the contents of the email.
response = client_ses.send_raw_email(
Source=message['Source'],
Destinations=[
message['Destinations']
],
RawMessage={
'Data':message['Data']
}
)
# Display an error if something goes wrong.
except ClientError as e:
print('send email ClientError Exception')
output = e.response['Error']['Message']
else:
output = "Email sent! Message ID: " + response['MessageId']
return output
def lambda_handler(event, context):
# Get the unique ID of the message. This corresponds to the name of the file
# in S3.
message_id = event['Records'][0]['ses']['mail']['messageId']
print(f"Received message ID {message_id}")
# Retrieve the file from the S3 bucket.
file_dict = get_message_from_s3(message_id)
# Create the message.
message = create_message(file_dict)
# Send the email and print the result.
result = send_email(message)
print(result)

You can disregard this msg below, I see your comments about "quick and dirty", let me explore
Is the return message a typo, it doesn't make sense in this block of code?
if mailobject.is_multipart():
#The quick and dirty way. If you don't like this, use the for loop below it.
index = stringMsg.find('Content-Type: multipart/')
stringBody = stringMsg[index:]
#print(stringBody)
stringData = 'Subject: ' + subject + '\nTo: ' + sender + '\nreply-to: ' + from_original + '\n' + stringBody
message = {
"Source": sender,
"Destinations": recipient,
"Data": stringData
}
return message
btw: thank you for this code - I was looking for an alternative to the attachment example, and didn't want to wrap my head around what you are doing, so thanks!

Upload base64 image to couchdb

Im Trying to Upload a base64 image which I got from doing canvas.toDataUrl();
but nothing happens when I try and do a put request to couchdb. Im not sure how to retrieve the response to see if it worked.
Here is my request:
#app.route('/process', methods=['POST'])
def process():
email = request.form['email']
name = request.form['name'],
file = request.form['data'] #'data' is from html and was var srcdata = canvas.toDataURL()
if file:
print(file)
j = {"_id": "ab","_attachments":{"aboutme.jpg":{"content_type":"image/png","data":file}}}
requests.put('http://127.0.0.1:5984/test2/',data=None,json=j)
EDIT
the string that is in the data value in the jj json works and allows me to upload and attachment to couchdb but when I pass it the file that doesnt work. Im thinking it doesnt because its not a proper base64, I think it could actually be a domString.
#app.route('/process', methods=['POST'])
def process():
email = request.form['email']
name = request.form['name'],
file = request.form['data']
if file:
print(file)
jj = {
"_attachments":
{
"aboutme.png":
{
"content_type": "image/png",
"data":"VGhpcyBpcyBhIGJhc2U2NCBlbmNvZGVkIHRleHQ="
}
}
}
requests.post('http://127.0.0.1:5984/test2/', data=None, json=jj)
return jsonify({'name': "yes"})

unable to detect request content_type

On a django server, I process uploaded zip files sent from a python script. But I am getting "" (a blank string) for file.content_type. What am I doing wrong?
#csrf_exempt
def Import( request ):
if request.method != 'POST':
return HttpResponseNotAllowed('Only POST here')
if not request.FILES or not request.FILES.get( u'file' ):
return HttpResponse('Must upload a file')
file = request.FILES[u'file']
if file.content_type == 'application/zip':
unzipped_dir = unzip_file( file )
uid = create_project( unzipped_dir )
shutil.rmtree( unzipped_dir )
py_ob = { }
py_ob['success'] = uid is not None
if uid is not None:
py_ob['id'] = uid
json_ob = simplejson.dumps(py_ob)
return HttpResponse( json_ob, mimetype="application/json" )
else:
return HttpResponseNotAllowed( 'Only POST zip files here' )
This is the script which sends the zip file up:
import sys
import os
import requests
if len (sys.argv) < 5:
print "pass in url, username, password, file"
else:
url = sys.argv[1]
username = sys.argv[2]
password = sys.argv[3]
phile = sys.argv[4]
if os.path.exists(phile):
files = {'file': open( phile, 'rb' )}
r = requests.post( url, files=files, auth=( username, password ) )
if r.status_code == 200:
json_response = r.json()
if json_response['success']:
print "id: " + str( json_response['id'] )
else:
print "failure in processing bundle"
else:
print "server problem: " + str(r.status_code)
print r.text
else:
print "cannot find file to upload"

The Content-Type header is completely arbitrary (and optional) and not a good way to detect whether or not you're dealing with a valid ZIP file. Have you made sure your browser is supplying it?
Django's documentation tells us the same:
UploadedFile.content_type
The content-type header uploaded with the file (e.g. text/plain or application/pdf). Like any data supplied by the user, you shouldn’t
trust that the uploaded file is actually this type. You’ll still need
to validate that the file contains the content that the content-type
header claims – “trust but verify.”
You should be using zipfile.is_zipfile instead.