I am developing a Gmail extracting app and using Gmail API to fetch mail from server. the problem lies in the fact that fetch time for mails is too large even though I used threading in back end framework. now I am going to implement one feature which will suggest user opting for bulk download that "once your download is ready, we will mail you" but for that i want to run download.py mentioned below in app tree in background and once the fetch is over it will get terminated.
And in the very bottom of the code i want to mail user that their download is ready but its not working though i have defined the mail server in settings.py .
download.py
import httplib2, base64
from stripogram import html2text
from oauth2client.django_orm import Storage
from apiclient.discovery import build
from oauth2client import client
from django.contrib.auth.models import User
from .models import CredentialsModel
from django.conf import settings
from rest_framework.views import APIView
from rest_framework.response import Response
from rest_framework import authentication, permissions
from gextracto import models
from gextracto.models import UserData
from django.core.mail import EmailMessage
from django.core import mail
connection = mail.get_connection()
class ListMails(APIView):
"""
Gets a list of a specified number mail ids for a particular label
Extracts the email in the form of plain/text
The API returns all the extracted mails
"""
authentication_classes = (authentication.SessionAuthentication,)
permission_classes = (permissions.IsAuthenticated,)
def extract_headers(self, message):
"""
Extract the headers for a single mail and returns it
{To, From, Subject}
"""
needed_fields = ('From', 'To', 'Subject')
return {i['name']:i['value'] for i in message['payload']['headers'] if i['name'] in needed_fields}
def get_message_body(self, message):
"""
Get the body of an email
Recursively look for the body for different mimetypes
Returns the body as text/plain
"""
if 'payload' in message:
return self.get_message_body(message['payload'])
elif 'parts' in message:
return self.get_message_body(message['parts'][0])
else:
data = base64.urlsafe_b64decode(message['body']['data'].encode('ASCII'))
markdown_data = html2text(data)#.decode('utf-8', "replace")
data = data.replace("\n", "<br/>")
# return {markdown, html}
return {'markdown':unicode( markdown_data,"ISO-8859-1"), 'html':unicode(data,"ISO-8859-1")} if markdown_data else {'html':unicode(data,"ISO-8859-1")}
def message_content_html(self, userId, message_id, service):
"""
Make queries to get the content for a mail given its message id
Returns all the content
"""
content = {'id':message_id}
# try
message = service.users().messages().get(userId=userId, id=message_id).execute()
mimetype = message['payload']['mimeType']
if mimetype == 'text/html':
return {}
#
else:
body = self.get_message_body(message)
if body == "":
body = "<empty message>"
headers = self.extract_headers(message)
content['body'] = body
content.update(headers)
return content
def collect_mails(self, user, messages, service):
"""
Collect the content for all the mails currently downloaded
"""
all_messages = []
try:
for message in messages:
content = self.message_content_html(user.username, message['id'], service)
if content:
all_messages.append(content)
return all_messages
# return empty list if no messages were downloaded
except KeyError:
return []
def get(self, request, format=None):
"""
Handles the GET request to get all the mails for a label
Paginages through the GAPI content if required
API returns all the messages
{To, From, Subject, body}
"""
user = request.user
storage = Storage(CredentialsModel, 'id', user, 'credential')
credentials = storage.get()
http_auth = credentials.authorize(httplib2.Http())
service = build('gmail', 'v1', http=http_auth)
user_Id = user.username
label_id = request.GET['label']
# try
# call Google API with a request to get a list of all the labels
response = service.users().messages().list(userId=user_Id, labelIds=label_id, maxResults=100).execute()
all_messages = self.collect_mails(user, response['messages'], service)
if not all_messages:
return Response([])
else:
if 'nextPageToken' in response:
page_token_flag = True
# request more more mails if the download limit has not yet been satisfied
while(page_token_flag):
response = service.users().messages().list(userId=user_Id, pageToken=response['nextPageToken'], maxResults=100).execute()
all_messages.append(self.collect_mails(user, response['messages'], service))
print(all_messages)
#for x in range(0,len(all_messages)):
#b=all_messages[10]
#instance= UserData(user_id=user ,label=label_id, sender = b['From'] , subject=b['Subject'] , body=b['body'])
#instance.save()
page_token_flag = 'nextPageToken' in response
##
for x in range(0,len(all_messages)):
b=all_messages[10]
instance= UserData(user_id=user ,label=label_id, sender = b['From'] , subject=b['Subject'] , body=b['body'])
instance.save()
print ("Hi i am here!!!")
email = EmailMessage('Your Download Ready!', 'http://127.0.0.1:8000/admin/gextracto/userdata/', to=[user], connection=connection)
email.send()
connection.close()
return Response(all_messages)
Please tell me the way to run it in background. if need any other info please do ask. Thanks
Don't know the exact requirements but I'll think about Celery to run background tasks. This approach allows to manage all post-script activities in native Django manner.
Also you can think about running the Django script using cron (as manage.py command) - but it can lead to some limitations.
What about sending emails failure - believe, you don't need to close connection after sending email. Usually I use send_mail()/send_mass_mail() functions - please, check their code to get an idea.
Related
I'm trying to restrict incoming emails to my app so it only accepts mail from members of a google group. More specifically, I want to only add the contents of the email to my datastore if they are part of the group. I've found the hasMember/IsMember method here: https://developers.google.com/admin-sdk/directory/v1/reference/members/hasMember and think this may be what I am looking for, but I do not know how to use it as they haven't provided an example and I'm very new to this.
Would this be the correct API to use for this? Here is my incoming mail handler code, I have added the IF statement comment to show what I would like to do:
import webapp2
import logging
from google.appengine.ext.webapp import mail_handlers
from google.appengine.api import mail
import os
from main import WorkRequest
import re
class IncomingMailHandler(mail_handlers.InboundMailHandler):
def receive(self, message):
(encoding, payload) = list(message.bodies(content_type='text/plain'))[0]
body_text = payload.decode()
logging.info('Received email message from %s, subject "%s": %s' %
(message.sender, message.subject, body_text))
logging.info (message.sender)
logging.info(message.subject)
logging.info(body_text)
#IF MESSAGE_SENDER == MEMBER OF GOOGLE GROUP:
wr = WorkRequest()
wr.email = message.sender
wr.userId = None
wr.title = message.subject
wr.content = body_text
wr.status = "OPEN"
wr.submission_type = "EMAIL"
wr.assigned_to = "UNASSIGNED"
wr.put()
application = webapp2.WSGIApplication([('/_ah/mail/.+', IncomingMailHandler)],debug=True)
I want to have a continuous conversation with watson chatbot.
current situation :
The chatbox does not remember the status of the conversation you previously sent.
I installed the Django framework on the server and created a watson.py file to load the workspace and work with KakaoTalk, a Korean chat application.
The chatbot's conversation flow that I want is as follows.
User: I want to make a reservation
Chatbot: When is the meeting date?
User: tomorrow
Chatbot: How is your meeting time?
User: 14:00
We need your help very much.
watson.py
import json
from watson_developer_cloud import ConversationV1
from .models import Test
from . import views
import simplejson
conversation = ConversationV1(
username = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx",
password = "xxxxxxxxxxxxxxxxxxxxxx",
version = '2017-05-26' )
workspace_id = 'xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx' #workspace
def test(return_str):
result = ''
except Exception as err:
pass
response = conversation.message(
workspace_id = workspace_id,
message_input = {'text': return_str},
)
for i in range(len(response['output']['text'])):
result += response['output']['text'][i] +'\n'
return result
views.py
import json
from django.shortcuts import render
from django.http import JsonResponse
from django.views.decorators.csrf import csrf_exempt
from .models import Jidum, Test
from . import watson
# Create your views here.
def keyboard(request):
return JsonResponse({
'type':'text',
})
#csrf_exempt
def message(request):
message = ((request.body).decode('utf-8'))
return_json_str = json.loads(message)
return_str = return_json_str['content']
return JsonResponse({
'message': {
'text': watson.test(return_str),
},
'keyboard': {
'type':'text',
},
})
As you can see, has a lot examples inside Watson Developer Cloud for use each API from IBM Developers. Take a look in one example using Watson Conversation.
When you wanna use the same conversation for multiple requests (message), you need to include context object from the previous response.
But remember, you need to create the conversation flow inside your workspace.
For example:
import json
from watson_developer_cloud import ConversationV1
#########################
# message
#########################
conversation = ConversationV1(
username='YOUR SERVICE USERNAME',
password='YOUR SERVICE PASSWORD',
version='2017-04-21')
# replace with your own workspace_id
workspace_id = '0a0c06c1-8e31-4655-9067-58fcac5134fc'
# this example don't include
response = conversation.message(workspace_id=workspace_id, message_input={
'text': 'What\'s the weather like?'})
print(json.dumps(response, indent=2))
# This example include the context object from the previous response.
# response = conversation.message(workspace_id=workspace_id, message_input={
# 'text': 'turn the wipers on'},
# context=response['context']) //example
# print(json.dumps(response, indent=2))
See the Official API Reference using Python.
As #SayuriMizuguchi mentions the main issue is with not maintaining the context object.
Based on your sample above, you would do something like:
context = {}
def test(input_string):
except Exception as err:
pass
response = conversation.message(
workspace_id = workspace_id,
message_input = {'text': return_str},
context = context
)
result = '\n'.join(response['output']['text'])
context = response['context']
return result
Then to run that as an example:
response = test('I want to make a reservation')
print(response)
response = test('tomorrow')
print(response)
I have integrate the send-grid with my Django application and mails also sent successfully. But now I want to send email with designed template from my django application. I have read the docs also, but not get any idea how to use it programatically. This is my first time to use send-grid. Please can anyone help me to find out the way how can I send send-grid template from django application.
You can use SendGrid's Template Engine to store a template inside SendGrid. You then reference that template ID when sending an email via the SendGrid API, you can see an example of that code in the sendgrid-python library.
Here it is in a full example, it uses a SendGrid API key (you can find out how to get that set up by reading this guide):
import sendgrid
sg = sendgrid.SendGridClient('sendgrid_apikey')
message = sendgrid.Mail()
message.add_to('John Doe <john#email.com>')
message.set_subject('Example')
message.set_html('Body')
message.set_text('Body')
message.set_from('Doe John <doe#email.com>')
# This next section is all to do with Template Engine
# You pass substitutions to your template like this
message.add_substitution('-thing_to_sub-', 'Hello! I am in a template!')
# Turn on the template option
message.add_filter('templates', 'enable', '1')
# Tell SendGrid which template to use
message.add_filter('templates', 'template_id', 'TEMPLATE-ALPHA-NUMERIC-ID')
# Get back a response and status
status, msg = sg.send(message)
You need Sendgrid-Python interface first:
pip install sendgrid
after that try this:
import os
from sendgrid.helpers.mail import Mail
from sendgrid import SendGridAPIClient
FROM_EMAIL = 'Your_Name#SendGridTest.com'
# update to your dynamic template id from the UI
TEMPLATE_ID = 'd-d027f2806c894df38c59a9dec5460594'
# list of emails and preheader names, update with yours
TO_EMAILS = [('your_email#domain.com', 'Sourabh MbeforL'),]
def SendDynamic():
message = Mail(
from_email=FROM_EMAIL,
to_emails=TO_EMAILS)
# pass custom values for our HTML placeholders
message.dynamic_template_data = {
'subject': 'SendGrid Development',
'place': 'New York City',
'event': 'Twilio Signal'
}
message.template_id = TEMPLATE_ID
try:
sg = SendGridAPIClient(os.environ.get('YOUR_SENDGRID_API_KEY')) ## (^_^) This face is just to grab your attention for api key needed
response = sg.send(message)
code, body, headers = response.status_code, response.body, response.headers
print(f"Response code: {code}")
print(f"Response headers: {headers}")
print(f"Response body: {body}")
print("Dynamic Messages Sent!")
except Exception as e:
print("Error: {0}".format(e))
return str(response.status_code)
trying to list the google spreadsheet using oauth & GData in Google Appengine. But i'm getting following error. Anything wrong in my code?
Please advise.
File "/home/RKS/Appengine/test/main.py", line 99, in get
feed = client.get_spreadsheets()
File "/home/RKS/Appengine/test/gdata/spreadsheets/client.py", line 78, in get_spreadsheets
desired_class=desired_class, **kwargs)
File "/home/RKS/Appengine/test/gdata/client.py", line 640, in get_feed
**kwargs)
File "/home/RKS/Appengine/test/gdata/client.py", line 278, in request
version=get_xml_version(self.api_version))
File "/home/RKS/Appengine/test/atom/core.py", line 520, in parse
tree = ElementTree.fromstring(xml_string)
File "<string>", line 125, in XML
ParseError: no element found: line 1, column 0
Code:
#!/usr/bin/env python
import webapp2
import logging
from google.appengine.api import users
import gdata.auth
import gdata.gauth
import gdata.client
import gdata.service
import gdata.spreadsheets
import gdata.spreadsheets.client
import gdata.spreadsheets.data
SETTINGS = {
'APP_NAME': 'hidden', # intentionally hidden
'CONSUMER_KEY': 'hidden',
'CONSUMER_SECRET': 'hidden',
'SCOPES': ['https://spreadsheets.google.com/feeds'],
'CALLBACK_URL': '',
'SIG_METHOD': gdata.auth.OAuthSignatureMethod.HMAC_SHA1
}
class FetchToken(webapp2.RequestHandler):
def get(self):
current_uid = users.get_current_user().user_id()
if isinstance(gdata.gauth.AeLoad(current_uid), gdata.gauth.OAuthHmacToken):
# the user has gone through the process before
self.redirect('/')
else:
# the user has not gone through this process, or the authorization has expired or been revoked.
SETTINGS['CALLBACK_URL'] = 'http://%s/HandleOAuthCallback' % self.request.host
client = gdata.client.GDClient()
request_token = client.GetOAuthToken(
SETTINGS['SCOPES'],
SETTINGS['CALLBACK_URL'],
SETTINGS['CONSUMER_KEY'],
consumer_secret=SETTINGS['CONSUMER_SECRET'])
gdata.gauth.AeSave(request_token, current_uid)
self.redirect(str(request_token.generate_authorization_url()))
class HandleOAuthCallback(webapp2.RequestHandler):
def get(self):
current_uid = users.get_current_user().user_id()
client = gdata.client.GDClient()
saved_request_token = gdata.gauth.AeLoad(current_uid)
request_token = gdata.gauth.AuthorizeRequestToken(saved_request_token, self.request.uri)
access_token = client.GetAccessToken(request_token)
gdata.gauth.AeSave(access_token, current_uid)
self.redirect('/')
class GetDocsList(webapp2.RequestHandler):
def get(self):
if not users.get_current_user():
self.redirect(users.create_login_url('/'))
else:
current_uid = users.get_current_user().user_id()
if isinstance(gdata.gauth.AeLoad(current_uid), gdata.gauth.OAuthHmacToken):
# the user has gone through the process before
access_token = gdata.gauth.AeLoad(current_uid)
client = gdata.service.GDataService()
oauth_input_params = gdata.auth.OAuthInputParams(
gdata.auth.OAuthSignatureMethod.HMAC_SHA1,
SETTINGS['CONSUMER_KEY'],
SETTINGS['CONSUMER_SECRET']) # consumer_secret=
oauth_token = gdata.auth.OAuthToken(
key=access_token.token,
secret=access_token.token_secret,
scopes=SETTINGS['SCOPES'],
oauth_input_params=oauth_input_params)
client.SetOAuthToken(oauth_token)
client = gdata.spreadsheets.client.SpreadsheetsClient()
feed = client.get_spreadsheets()
else:
self.redirect('/FetchToken')
app = webapp2.WSGIApplication([('/HandleOAuthCallback', HandleOAuthCallback), ('/', GetDocsList), ('/FetchToken', FetchToken)], debug=False)
I recommend switching to OAuth2, and most of my answer will explain how to solve your problem by doing that. However, before trying to switch to OAuth2 I suggest you try this one line change, which may be a quick solution to your problem. The first thing I would try is adding a header to your client as follows.
client.additional_headers = {
'Authorization': 'Bearer %s' % access_token,
}
I have an application using the GData spreadsheets API with OAuth2 and adding that header was necessary for it to work. If I took that header out, I would get an error like the one you described.
Now, on to the answer:
Using OAuth2 in App Engine to do what you're trying to do is even simpler than what you've already done. Here's how it is done.
First create an OAuth2 decorator. (I describe where to create your client ID and client secret at the end of the answer.)
from oauth2client.appengine import OAuth2Decorator
GlobalOAuth2Decorator = OAuth2Decorator(
client_id=OAUTH2_CLIENT_ID,
client_secret=OAUTH2_CLIENT_SECRET,
scope='https://spreadsheets.google.com/feeds',
)
Then use this decorator when you create your request handler.
class SpreadsheetHandler(webapp2.RequestHandler):
#GlobalOAuth2Decorator.oauth_required
def get(self):
client = gdata.spreadsheet.service.SpreadsheetsService()
client.additional_headers = {
'Authorization': 'Bearer %s' % GlobalOAuth2Decorator.credentials.access_token,
}
Notice that I'm using SpreadsheetsService instead of SpreadsheetsClient. This has been working for me, though I haven't given SpreadsheetsClient a try (the client might actually be easier to use).
Once you've created this client object you can use it to read and write spreadsheets as you like. For example sheets = client.GetSpreadsheetsFeeds() will give you a list of spreadsheets that you can access.
Lastly, make sure to include the OAuth handler in your list of handlers when you create the application:
app = webapp2.WSGIApplication([..., (GlobalOAuth2Decorator.callback_path, GlobalOAuth2Decorator.callback_handler())]
In order for OAuth2 to work, you have to go to your developer console at https://console.developers.google.com and 1) select your app, 2) from the Credentials menu create a new Client ID (this will generate a client ID and secret for you), and 3) from the Consent Screen menu give your application an email address and name.
I'm using an open source web service python application to send email through GAE but if the name or email body contains Arabic or Hebrew characters the application throws some errors (e.g "The indicated parameters are not valid"). Therefore I need to know how to fix this issue. I have to note that I'm a Python beginner (one week since I started playing with Python).
#
import cgi
import os
import logging
import contextlib
from xml.dom import minidom
from xml.dom.minidom import Document
import exceptions
import warnings
import imghdr
from google.appengine.api import images
from google.appengine.api import users
from google.appengine.ext import db
from google.appengine.ext import webapp
from google.appengine.ext.webapp.util import run_wsgi_app
from google.appengine.ext.webapp import template
from google.appengine.api import mail
import wsgiref.handlers
# START Constants
CONTENT_TYPE_HEADER = "Content-Type"
CONTENT_TYPE_TEXT = "text/plain"
XML_CONTENT_TYPE = "application/xml"
XML_ENCODING = "utf-8"
"""
Allows you to specify IP addresses and associated "api_key"s to prevent others from using your app.
Storage and Manipulation methods will check for this "api_key" in the POST/GET params.
Retrieval methods don't use it (however you could enable them to use it, but maybe rewrite so you have a "read" key and a "write" key to prevent others from manipulating your data).
Set "AUTH = False" to disable (allowing anyone use your app and CRUD your data).
To generate a hash/api_key visit https://www.grc.com/passwords.htm
To find your ip visit http://www.whatsmyip.org/
"""
AUTH = False
# END Constants
# START Exception Handling
class Error(StandardError):
pass
class Forbidden(Error):
pass
logging.getLogger().setLevel(logging.DEBUG)
#contextlib.contextmanager
def mailExcpHandler(ctx):
try:
yield {}
except (ValueError), exc:
xml_error_response(ctx, 400 ,'app.invalid_parameters', 'The indicated parameters are not valid: ' + exc.message)
except (Forbidden), exc:
xml_error_response(ctx, 403 ,'app.forbidden', 'You don\'t have permission to perform this action: ' + exc.message)
except (Exception), exc:
xml_error_response(ctx, 500 ,'system.other', 'An unexpected error in the web service has happened: ' + exc.message)
def xml_error_response(ctx, status, error_id, error_msg):
ctx.error(status)
doc = Document()
errorcard = doc.createElement("error")
errorcard.setAttribute("id", error_id)
doc.appendChild(errorcard)
ptext = doc.createTextNode(error_msg)
errorcard.appendChild(ptext)
ctx.response.headers[CONTENT_TYPE_HEADER] = XML_CONTENT_TYPE
ctx.response.out.write(doc.toxml(XML_ENCODING))
# END Exception Handling
# START Helper Methods
def isAuth(ip = None, key = None):
if AUTH == False:
return True
elif AUTH.has_key(ip) and key == AUTH[ip]:
return True
else:
return False
# END Helper Methods
# START Request Handlers
class Send(webapp.RequestHandler):
def post(self):
"""
Sends an email based on POST params. It will queue if resources are unavailable at the time.
Returns "Success"
POST Args:
to: the receipent address
from: the sender address (must be a registered GAE email)
subject: email subject
body: email body content
"""
with mailExcpHandler(self):
# check authorised
if isAuth(self.request.remote_addr,self.request.POST.get('api_key')) == False:
raise Forbidden("Invalid Credentials")
# read data from request
mail_to = str(self.request.POST.get('to'))
mail_from = str(self.request.POST.get('from'))
mail_subject = str(self.request.POST.get('subject'))
mail_plain = str(self.request.POST.get('plain'))
mail_html = str(self.request.POST.get('html'))
message = mail.EmailMessage()
message.sender = mail_from
message.to = mail_to
message.subject = mail_subject
message.body = mail_plain
if mail_html != None and mail_html != "":
message.html = mail_html
message.send()
self.response.headers[CONTENT_TYPE_HEADER] = CONTENT_TYPE_TEXT
self.response.out.write("Success")
# END Request Handlers
# START Application
application = webapp.WSGIApplication([
('/send', Send)
],debug=True)
def main():
run_wsgi_app(application)
if __name__ == '__main__':
main()
# END Application
mail_to = str(self.request.POST.get('to'))
mail_from = str(self.request.POST.get('from'))
mail_subject = str(self.request.POST.get('subject'))
mail_plain = str(self.request.POST.get('plain'))
mail_html = str(self.request.POST.get('html'))
I doubt you need to convert them to strings. Try without str(), it could work.