We Keep Coding

How to get specific details from email using python beautiful soup and extract to excel

i am trying to get specific data from my email such as check in & checkout date from my email, but i not sure what can i do to get the specific text from the email.
For the code below, it can search for specific hotel booking number and write in html file.
For now only not sure how can i read from those html file and get the specific details and extract to excel
my_mail = imaplib.IMAP4_SSL("imap.gmail.com","993")
my_mail.login(username, password)
my_mail.select('Inbox')
key = 'SUBJECT'
value = "721182693"
_, data = my_mail.search(None, key, value)
mail_id_list = data[0].split()
msgs = []
for num in mail_id_list:
_, data = my_mail.fetch(num, '(RFC822)') #RFC822 returns whole message (BODY fetches just body)
msgs.append(data)
def clean(text):
return "".join(c if c.isalnum() else "_" for c in text)
soup = BeautifulSoup(data)
for msg in msgs[::-1]:
for response in msg:
if isinstance(response, tuple):
msg = email.message_from_bytes(response[1])
subject, encoding = decode_header(msg["Subject"])[0]
if isinstance(subject, bytes):
subject = subject.decode(encoding="utf-8")
From, encoding = decode_header(msg.get("From"))[0]
if isinstance(From, bytes):
From = From.decode(encoding)
print("Subject:", subject)
print("From:", From)
if msg.is_multipart():
for part in msg.walk():
content_type = part.get_content_type()
content_disposition = str(part.get("Content-Disposition"))
try:
body = part.get_payload(decode=True).decode()
except:
pass
if content_type == "text/plain" and "attachment" not in content_disposition:
print(body)
elif "attachment" in content_disposition:
filename = part.get_filename()
if filename:
folder_name = clean(subject)
if not os.path.isdir(folder_name):
os.mkdir(folder_name)
filename = "index.html"
filepath = os.path.join(folder_name, filename)
open(filepath, "w",encoding="utf-8").write(body)
webbrowser.open(filepath)
else:
content_type = msg.get_content_type()
body = msg.get_payload(decode=True).decode()
if content_type == "text/plain":
print(body)
if content_type == "text/html":
folder_name = clean(subject)
if not os.path.isdir(folder_name):
os.mkdir(folder_name)
filename = "index.html"
filepath = os.path.join(folder_name, filename)
open(filepath, "w",encoding="utf-8").write(body)
webbrowser.open(filepath)
print("="*100)`

file upload from Flask template to external APIs

I have divided my flask app with two-part API (Flask API) and web app (Flask template).
From the web app, I am trying to upload a file but on the API side, I am not able to send.
File upload is working from APIs side, I have tested from the postman.
Template (UI with port: 3010) and API (port: 3009)
Below is running on API side with port 3009
#app.route('/sources', methods=['POST', 'GET'])
def upload_file():
if request.method == 'POST':
try:
print(request.form.get('file')) # Not able to print file here
print(request.form.get('Language1')) # I am able to print it
if 'file' not in request.files:
resp = jsonify({'message' : 'No file part in the request'})
resp.status_code = 400
return resp
file = request.files['file']
if file.filename == '':
resp = jsonify({'message' : 'No file selected for uploading'})
resp.status_code = 400
return resp
if file and allowed_file(file.filename,'sources'):
sourceFileName = secure_filename(file.filename)
Lang1 = request.form.get('Language1')
Lang2 = request.form.get('Language2')
except Exception as e:
print(e)
resp = jsonify({'message' : 'Server Error'})
resp.status_code = 500
return resp
Below is running on UI side with port 3010
#app.route('/sources', methods=['POST'])
def upload_sources():
if request.method == "POST":
Language1 = request.form["Language1"]
Language2 = request.form["Language2"]
file = request.files["file"]
# File object is printing here like: [<FileStorage: 'source_text.docx' ('application/vnd.openxmlformats-officedocument.wordprocessingml.document')>]
print(file)
params = {
"Language1":Language1,
"Language2":Language2,
"file":file
}
headers = {'content-type': 'application/json'}
req = requests.post('http://localhost:3009/sources', headers=headers, data=params)
r = req.json()
print(r['message']) # I am getting response message : ['No file part in the request']
return redirect(url_for('fetch_sources'))
Below is working fine, just fetching uploaded file
#app.route('/sources', methods=['GET'])
def fetch_sources():
sources = requests.get('http://localhost:3009/sources')
source_data = sources.json()
if source_data:
return render_template('sources.html', sources=source_data['sources'])

I was sending a file object and the way which I was following that was incorrect...
I can not send file directly to API from webapp.
Here I have changed my code and it is working fine.
if request.method == 'POST':
file = request.files["file"]
sourceFileName = secure_filename(file.filename)
cwd = os.getcwd()+'/'
if 'temp' not in os.listdir(cwd):
os.mkdir(cwd + 'temp')
file.save(os.path.join(cwd + 'temp', sourceFileName))
with open(cwd + 'temp/'+ sourceFileName, 'rb') as f:
data_file = ImmutableMultiDict([("file", f)])
resp = requests.post(api_base_url + "/sources",
files=data_file)

Django: How to catch a ValueError from URL?

I'm writing a single API function (as I don't need rest in this case).
Basically it's a URL with UUID4 as api-key.
The function goes as follow:
def jsonapi(request, apikey):
try:
test = get_object_or_404(ProfileDetails, apikey=apikey)
if ProfileDetails.objects.filter(apikey=apikey).exists():
details = get_object_or_404(ProfileDetails, apikey=apikey)
if Posts.objects.filter(published=True, user=details).exists():
data = Posts.objects.filter(published=True, user=details).order_by('-timestamp')
postData = []
for a in data:
user = a.user.username
title = a.title
post = a.post
date = a.timestamp
slug = a.slug
url = "https://app.domain.co/#" + a.user.username + "/" + a.slug
# print(date.date)
newjson = {
"username": user,
"title": title,
}
postData.append(newjson)
jsonStr = json.dumps(postData, cls=DjangoJSONEncoder)
qwe = json.loads(jsonStr)
returndata = JsonResponse(qwe, safe=False)
returndata['Access-Control-Allow-Origin'] = '*'
return returndata
else:
newjson = {
"error": "Please make sure you have at least 1 item live."
}
postData.append(newjson)
jsonStr = json.dumps(postData, cls=DjangoJSONEncoder)
qwe = json.loads(jsonStr)
returndata = JsonResponse(qwe, safe=False)
returndata['Access-Control-Allow-Origin'] = '*'
return returndata
else:
newjson = {
"error": "Make sure you have at least one item set as live."
}
postData.append(newjson)
jsonStr = json.dumps(postData, cls=DjangoJSONEncoder)
qwe = json.loads(jsonStr)
returndata = JsonResponse(qwe, safe=False)
returndata['Access-Control-Allow-Origin'] = '*'
return returndata
except ValueError:
return JsonResponse({"Error":"Invalid API Key"})
Now basically, what I need it to do is, when I enter something in the URL that's not a UUID, it returns ["'something' is not a valid UUID."]
I would've hoped the ValueError catches the error and let me write a response but no luck.
Any suggestions?
Thanks!

Flask and WTForms - How to determine if File Upload field is submitted for processing purposes

I've created a form with an optional WTForms File field to upload a file. The form works... but after submission, my view.py code is always trying to process the uploaded file, regardless of whether it was submitted or not.
How do I determine if a file has been uploaded or not? I'd like my code to only process the upload if something was uploaded.
Right now, I haven't found the correct way to validate, so my code is handling the uploaded file, even when no file has been uploaded.
I'm currently trying this in my views.py to distinguish, but it doesn't work (see below):
attachFile = False
if attachment:
attachFile = True
I've also tried the following to try to make something happen (these are commented out in the full code in views.py):
First attempt: if form.attachment.data is not str:
Second attempt: if not attachment.filename == "":
Third attempt: if (isinstance(attachment,str) == False):
(Fourth (and current) attempt is above)
I've also tried the following, but get the following error when a file is not uploaded:
if attachment.data:
attachFile = True
## AttributeError: 'str' object has no attribute 'data'
forms.py:
from flask_wtf import FlaskForm
from wtforms import StringField, PasswordField, BooleanField,
SubmitField, TextAreaField, FileField
from wtforms.fields.html5 import EmailField
from wtforms.validators import DataRequired, Email
class MailerForm(FlaskForm):
fromName = StringField('fromName', validators=[DataRequired()])
fromEmail = EmailField('fromEmail', validators=[DataRequired(), Email()])
subject = StringField('Subject', validators=[DataRequired()])
toAddress = TextAreaField('toAddress', validators=[DataRequired()])
message = TextAreaField('message', validators=[DataRequired()])
attachment = FileField('attachment')
submit = SubmitField('Send Email')
views.py
#app.route('/mailer/', methods=['GET','POST'])
def mailer():
# compiled regex to quick and dirty email validation
EMAIL_REGEX = re.compile(r"(^[a-zA-Z0-9_.+-]+#[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+$)")
form = MailerForm()
if form.validate_on_submit():
fromName = form.fromName.data
fromEmail = form.fromEmail.data
subject = form.subject.data
toAddress = form.toAddress.data
messageBody = form.message.data
attachment = form.attachment.data
newFileName = ""
attachFile = False
if attachment:
attachFile = True
basedir = os.path.abspath(os.path.dirname(__file__))
## lists to track successful and unsuccessful email addresses submitted
success = []
failure = []
##
## split email address
##
addresses = toAddress.split("\n")
##
## iterate through email addresses, validate, and send
##
for address in addresses:
address = address.strip()
if EMAIL_REGEX.match(address):
##if (isinstance(attachment,str) == False):
##if not attachment.filename == "":
if attachFile == True:
filename = os.path.join(basedir + "/static/" + app.config['UPLOAD_FOLDER'], attachment.filename)
attachment.save(filename)
msg = Message(subject)
msg.sender = (fromName,fromEmail)
msg.recipients = [address]
msg.body = messageBody
#if form.attachment.data is not str:
#if not attachment.filename == "":
#if (isinstance(attachment,str) == False):
if attachFile == True:
newFileName = attachment.filename
with app.open_resource(filename) as fp:
msg.attach(
newFileName,
"application/octet-stream",
fp.read())
mail.send(msg)
success.append(address)
else:
failure.append(address)
print("Failed:" + address)
else:
"""Renders index page."""
return render_template(
'mailer/mailer.html',
form = form
)
##
## Successfully emailed, time to nuke the temp attachment
##
os.system('rm ' + basedir + "/static/" + app.config['UPLOAD_FOLDER'] + "/'" + newFileName + "'")
##
##
##
return render_template(
'mailer/mailerCompleted.html',
form = form,
success = success,
failure = failure
)

You can also use not:
if not form.attachment.data:
print('no files has been uploaded')
not A is true when A is empty or None.
So, it triggers when no files are attached(form.attachment.data == None)

Add FileRequired() to validators:
from flask_wtf.file import FileField, FileRequired
...
attachment = FileField('attachment' , validators=[FileRequired()])
File is required and form will not validate if file is not submitted.
EDIT:
If you want file to be optional, remove validators=[FileRequired()] and check if file is passed:
if form.attachment.data is None:
print("File is empty")
# Code if file is empty
else:
# Code if file is passed

Using Flask-WTF==0.14.3, wtforms==2.3.3
The is None test does not work. The =='' test is works.
with a form definition using FileField
class ProfileForm(FlaskForm):
# other fields
photo = FileField('Profile photo')
when form is submitted
def profile():
form = ProfileForm()
if form.validate_on_submit():
print('form.photo.data: [{}]'.format(form.photo.data))
print('is None test: [{}]'.format(form.photo.data is None))
print('==\'\' test: [{}]'.format(form.photo.data == ''))
if form.photo.data is None:
# will NOT work
flash('you should upload a profile photo')
if form.photo.data == '':
# will work
flash('profile photo is highly recommended')
return render_template('profile.html', form=form)

unable to detect request content_type

On a django server, I process uploaded zip files sent from a python script. But I am getting "" (a blank string) for file.content_type. What am I doing wrong?
#csrf_exempt
def Import( request ):
if request.method != 'POST':
return HttpResponseNotAllowed('Only POST here')
if not request.FILES or not request.FILES.get( u'file' ):
return HttpResponse('Must upload a file')
file = request.FILES[u'file']
if file.content_type == 'application/zip':
unzipped_dir = unzip_file( file )
uid = create_project( unzipped_dir )
shutil.rmtree( unzipped_dir )
py_ob = { }
py_ob['success'] = uid is not None
if uid is not None:
py_ob['id'] = uid
json_ob = simplejson.dumps(py_ob)
return HttpResponse( json_ob, mimetype="application/json" )
else:
return HttpResponseNotAllowed( 'Only POST zip files here' )
This is the script which sends the zip file up:
import sys
import os
import requests
if len (sys.argv) < 5:
print "pass in url, username, password, file"
else:
url = sys.argv[1]
username = sys.argv[2]
password = sys.argv[3]
phile = sys.argv[4]
if os.path.exists(phile):
files = {'file': open( phile, 'rb' )}
r = requests.post( url, files=files, auth=( username, password ) )
if r.status_code == 200:
json_response = r.json()
if json_response['success']:
print "id: " + str( json_response['id'] )
else:
print "failure in processing bundle"
else:
print "server problem: " + str(r.status_code)
print r.text
else:
print "cannot find file to upload"

The Content-Type header is completely arbitrary (and optional) and not a good way to detect whether or not you're dealing with a valid ZIP file. Have you made sure your browser is supplying it?
Django's documentation tells us the same:
UploadedFile.content_type
The content-type header uploaded with the file (e.g. text/plain or application/pdf). Like any data supplied by the user, you shouldn’t
trust that the uploaded file is actually this type. You’ll still need
to validate that the file contains the content that the content-type
header claims – “trust but verify.”
You should be using zipfile.is_zipfile instead.