We Keep Coding

FIX sessionNotFound at initiation

I'm trying to build my fix application with quickfix, but when starting it, it first sends a logout message before logging in, and raises Session not Found.
class Application(quickfix.Application):
def __init__(self, session, logger):
super(Application, self).__init__()
self.session = session
self.logger = logger
def onCreate(self, sessionID):
self.logger.info("Created session {}.".format(sessionID))
return
def onLogon(self, sessionID):
self.logger.info("Logon session {}.".format(sessionID))
return
def onLogout(self, sessionID):
self.logger.info("Logout session {}.".format(sessionID))
return
def toAdmin(self, message, sessionID):
msgType = quickfix.MsgType()
message.getHeader().getField(msgType)
if msgType.getValue() == quickfix.MsgType_Logon:
self.logger.info('LOGON SessionID {}'.format(sessionID))
elif msgType.getValue() == quickfix.MsgType_Logout:
self.logger.info('LOGOUT SessionID {}'.format(sessionID))
self.logger.info('to Admin session {} send {}'.format(sessionID, self.messageToString(message)))
self.session.sendToTarget(message)
return
def toApp(self, message, sessionID):
self.logger.info('to App: {}'.format(message))
self.session.sendToTarget(message)
return
def fromApp(self, message, sessionID):
self.logger.info('from App: {}'.format(message))
return
logger = create_logger(config)
settings = quickfix.SessionSettings(client_config)
application = Application(quickfix.Session, logger)
storeFactory = quickfix.FileStoreFactory(settings)
logFactory = quickfix.ScreenLogFactory(settings)
initiator = quickfix.SocketInitiator(application, storeFactory, settings, logFactory)
initiator.start()
I get the following:
LOGOUT SessionID FIX44:Client->Server
to Admin session FIX44:Client->Server send 8=FIX.4.4|9=62|35=5|34=26|49=Client|52=20200608-12:26:03|56=Server|10=168
File "FIx.py", line 42, in toAdmin self.session.sendToTarget(message)
SessionNotFound: Session Not Found
Any idea why it raises the message please?
Thank you folks!

The from/toApp or from/toAdmin methods are callbacks and you should NOT send the passed message by yourself via Session.sendToTarget.
Instead the message will be sent by quickfix when the callback returns.

How to send a message to a Channel from a Command

How do you send a message to a Django Consumer from a custom manage.py command
from django.core.management.base import BaseCommand, CommandError
from channels import Channel
class Command(BaseCommand):
help = 'Sends a message to a Django channel from the thing'
def add_arguments(self, parser):
parser.add_argument('json_object', nargs='+', type=str)
def handle(self, *args, **options):
self.stdout.write("TEST !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!")
print Channel("test").channel_layer
Channel("test").send({'op':options['json_object'][0]})
This is my consumer
class MyConsumer(WebsocketConsumer):
#classmethod
def channel_names(self):
return {"test"}
def connection_groups(self):
return ["test"]
def dispatch(self, message, **kwargs):
return self.get_handler(message, **kwargs)(message, **kwargs)
def get_handler(self, message, **kwargs):
channel_routing = [
consumers.MyConsumer.as_route(path=r"^/test/"),
route("test.receive", consumers.chat_join),
] for _filter, value in kwargs.items():
filter_mapping = getattr(self, _filter + '_mapping', None)
if not filter_mapping:
continue
consumer = getattr(self, filter_mapping.get(value), None)
if consumer:
return consumer
raise ValueError('Message')
def connect(self,message):
self.message.reply_channel.send({"accept": True})
def receive(self,text=None, bytes= None):
print text
def disconnect(self,message):
pass
When I try to run the command however, I get this message
2017-03-08 03:45:33,839 - ERROR - worker - Could not find match for message on test! Check your routing.
In case it is pertinent, here is my routing
channel_routing = [
consumers.MyConsumer.as_route(path=r"^/test/"),
]

In short, add the path to the content you'd like to send:
Channel("test").send({
'op':options['json_object'][0],
'path': '/test/',
})
And that's it!
I ran into the same problem and I found out that it is because I am using the as_route method of generic consumer to generate route_class, which always has path as its filter.
If we use route instead we do not necessarily provide the path argument and that's the reason why the code at the doc (https://channels.readthedocs.io/en/stable/getting-started.html#models) works

Run particular Django script in background

I am developing a Gmail extracting app and using Gmail API to fetch mail from server. the problem lies in the fact that fetch time for mails is too large even though I used threading in back end framework. now I am going to implement one feature which will suggest user opting for bulk download that "once your download is ready, we will mail you" but for that i want to run download.py mentioned below in app tree in background and once the fetch is over it will get terminated.
And in the very bottom of the code i want to mail user that their download is ready but its not working though i have defined the mail server in settings.py .
download.py
import httplib2, base64
from stripogram import html2text
from oauth2client.django_orm import Storage
from apiclient.discovery import build
from oauth2client import client
from django.contrib.auth.models import User
from .models import CredentialsModel
from django.conf import settings
from rest_framework.views import APIView
from rest_framework.response import Response
from rest_framework import authentication, permissions
from gextracto import models
from gextracto.models import UserData
from django.core.mail import EmailMessage
from django.core import mail
connection = mail.get_connection()
class ListMails(APIView):
"""
Gets a list of a specified number mail ids for a particular label
Extracts the email in the form of plain/text
The API returns all the extracted mails
"""
authentication_classes = (authentication.SessionAuthentication,)
permission_classes = (permissions.IsAuthenticated,)
def extract_headers(self, message):
"""
Extract the headers for a single mail and returns it
{To, From, Subject}
"""
needed_fields = ('From', 'To', 'Subject')
return {i['name']:i['value'] for i in message['payload']['headers'] if i['name'] in needed_fields}
def get_message_body(self, message):
"""
Get the body of an email
Recursively look for the body for different mimetypes
Returns the body as text/plain
"""
if 'payload' in message:
return self.get_message_body(message['payload'])
elif 'parts' in message:
return self.get_message_body(message['parts'][0])
else:
data = base64.urlsafe_b64decode(message['body']['data'].encode('ASCII'))
markdown_data = html2text(data)#.decode('utf-8', "replace")
data = data.replace("\n", "<br/>")
# return {markdown, html}
return {'markdown':unicode( markdown_data,"ISO-8859-1"), 'html':unicode(data,"ISO-8859-1")} if markdown_data else {'html':unicode(data,"ISO-8859-1")}
def message_content_html(self, userId, message_id, service):
"""
Make queries to get the content for a mail given its message id
Returns all the content
"""
content = {'id':message_id}
# try
message = service.users().messages().get(userId=userId, id=message_id).execute()
mimetype = message['payload']['mimeType']
if mimetype == 'text/html':
return {}
#
else:
body = self.get_message_body(message)
if body == "":
body = "<empty message>"
headers = self.extract_headers(message)
content['body'] = body
content.update(headers)
return content
def collect_mails(self, user, messages, service):
"""
Collect the content for all the mails currently downloaded
"""
all_messages = []
try:
for message in messages:
content = self.message_content_html(user.username, message['id'], service)
if content:
all_messages.append(content)
return all_messages
# return empty list if no messages were downloaded
except KeyError:
return []
def get(self, request, format=None):
"""
Handles the GET request to get all the mails for a label
Paginages through the GAPI content if required
API returns all the messages
{To, From, Subject, body}
"""
user = request.user
storage = Storage(CredentialsModel, 'id', user, 'credential')
credentials = storage.get()
http_auth = credentials.authorize(httplib2.Http())
service = build('gmail', 'v1', http=http_auth)
user_Id = user.username
label_id = request.GET['label']
# try
# call Google API with a request to get a list of all the labels
response = service.users().messages().list(userId=user_Id, labelIds=label_id, maxResults=100).execute()
all_messages = self.collect_mails(user, response['messages'], service)
if not all_messages:
return Response([])
else:
if 'nextPageToken' in response:
page_token_flag = True
# request more more mails if the download limit has not yet been satisfied
while(page_token_flag):
response = service.users().messages().list(userId=user_Id, pageToken=response['nextPageToken'], maxResults=100).execute()
all_messages.append(self.collect_mails(user, response['messages'], service))
print(all_messages)
#for x in range(0,len(all_messages)):
#b=all_messages[10]
#instance= UserData(user_id=user ,label=label_id, sender = b['From'] , subject=b['Subject'] , body=b['body'])
#instance.save()
page_token_flag = 'nextPageToken' in response
##
for x in range(0,len(all_messages)):
b=all_messages[10]
instance= UserData(user_id=user ,label=label_id, sender = b['From'] , subject=b['Subject'] , body=b['body'])
instance.save()
print ("Hi i am here!!!")
email = EmailMessage('Your Download Ready!', 'http://127.0.0.1:8000/admin/gextracto/userdata/', to=[user], connection=connection)
email.send()
connection.close()
return Response(all_messages)
Please tell me the way to run it in background. if need any other info please do ask. Thanks

Don't know the exact requirements but I'll think about Celery to run background tasks. This approach allows to manage all post-script activities in native Django manner.
Also you can think about running the Django script using cron (as manage.py command) - but it can lead to some limitations.
What about sending emails failure - believe, you don't need to close connection after sending email. Usually I use send_mail()/send_mass_mail() functions - please, check their code to get an idea.

google app engine, python , unicode, email, mail api

I'm using an open source web service python application to send email through GAE but if the name or email body contains Arabic or Hebrew characters the application throws some errors (e.g "The indicated parameters are not valid"). Therefore I need to know how to fix this issue. I have to note that I'm a Python beginner (one week since I started playing with Python).
#
import cgi
import os
import logging
import contextlib
from xml.dom import minidom
from xml.dom.minidom import Document
import exceptions
import warnings
import imghdr
from google.appengine.api import images
from google.appengine.api import users
from google.appengine.ext import db
from google.appengine.ext import webapp
from google.appengine.ext.webapp.util import run_wsgi_app
from google.appengine.ext.webapp import template
from google.appengine.api import mail
import wsgiref.handlers
# START Constants
CONTENT_TYPE_HEADER = "Content-Type"
CONTENT_TYPE_TEXT = "text/plain"
XML_CONTENT_TYPE = "application/xml"
XML_ENCODING = "utf-8"
"""
Allows you to specify IP addresses and associated "api_key"s to prevent others from using your app.
Storage and Manipulation methods will check for this "api_key" in the POST/GET params.
Retrieval methods don't use it (however you could enable them to use it, but maybe rewrite so you have a "read" key and a "write" key to prevent others from manipulating your data).
Set "AUTH = False" to disable (allowing anyone use your app and CRUD your data).
To generate a hash/api_key visit https://www.grc.com/passwords.htm
To find your ip visit http://www.whatsmyip.org/
"""
AUTH = False
# END Constants
# START Exception Handling
class Error(StandardError):
pass
class Forbidden(Error):
pass
logging.getLogger().setLevel(logging.DEBUG)
#contextlib.contextmanager
def mailExcpHandler(ctx):
try:
yield {}
except (ValueError), exc:
xml_error_response(ctx, 400 ,'app.invalid_parameters', 'The indicated parameters are not valid: ' + exc.message)
except (Forbidden), exc:
xml_error_response(ctx, 403 ,'app.forbidden', 'You don\'t have permission to perform this action: ' + exc.message)
except (Exception), exc:
xml_error_response(ctx, 500 ,'system.other', 'An unexpected error in the web service has happened: ' + exc.message)
def xml_error_response(ctx, status, error_id, error_msg):
ctx.error(status)
doc = Document()
errorcard = doc.createElement("error")
errorcard.setAttribute("id", error_id)
doc.appendChild(errorcard)
ptext = doc.createTextNode(error_msg)
errorcard.appendChild(ptext)
ctx.response.headers[CONTENT_TYPE_HEADER] = XML_CONTENT_TYPE
ctx.response.out.write(doc.toxml(XML_ENCODING))
# END Exception Handling
# START Helper Methods
def isAuth(ip = None, key = None):
if AUTH == False:
return True
elif AUTH.has_key(ip) and key == AUTH[ip]:
return True
else:
return False
# END Helper Methods
# START Request Handlers
class Send(webapp.RequestHandler):
def post(self):
"""
Sends an email based on POST params. It will queue if resources are unavailable at the time.
Returns "Success"
POST Args:
to: the receipent address
from: the sender address (must be a registered GAE email)
subject: email subject
body: email body content
"""
with mailExcpHandler(self):
# check authorised
if isAuth(self.request.remote_addr,self.request.POST.get('api_key')) == False:
raise Forbidden("Invalid Credentials")
# read data from request
mail_to = str(self.request.POST.get('to'))
mail_from = str(self.request.POST.get('from'))
mail_subject = str(self.request.POST.get('subject'))
mail_plain = str(self.request.POST.get('plain'))
mail_html = str(self.request.POST.get('html'))
message = mail.EmailMessage()
message.sender = mail_from
message.to = mail_to
message.subject = mail_subject
message.body = mail_plain
if mail_html != None and mail_html != "":
message.html = mail_html
message.send()
self.response.headers[CONTENT_TYPE_HEADER] = CONTENT_TYPE_TEXT
self.response.out.write("Success")
# END Request Handlers
# START Application
application = webapp.WSGIApplication([
('/send', Send)
],debug=True)
def main():
run_wsgi_app(application)
if __name__ == '__main__':
main()
# END Application

mail_to = str(self.request.POST.get('to'))
mail_from = str(self.request.POST.get('from'))
mail_subject = str(self.request.POST.get('subject'))
mail_plain = str(self.request.POST.get('plain'))
mail_html = str(self.request.POST.get('html'))
I doubt you need to convert them to strings. Try without str(), it could work.

Collate output in Python logging MemoryHandler with SMTPHandler

I have the logging module MemoryHandler set up to queue debug and error messages for the SMTPHandler target. What I want is for an email to be sent when the process errors that contains all debug statements up to that point (one per line). What I get instead is a separate email for every debug message.
This seems like it should be trivial, and part of the logging package, but I can't find anything about it, no examples, nothing on Google.
log = logging.getLogger()
log.setLevel(logging.DEBUG)
debug_format = logging.Formatter("%(levelname)s at %(asctime)s in %(filename)s (line %(lineno)d):: %(message)s")
# write errors to email
error_mail_subject = "ERROR: Script error in %s on %s" % (sys.argv[0], os.uname()[1])
error_mail_handler = logging.handlers.SMTPHandler(SMTP_HOST, 'errors#'+os.uname()[1], [LOG_EMAIL], error_mail_subject)
error_mail_handler.setLevel(logging.ERROR)
#error_mail_handler.setLevel(logging.DEBUG)
error_mail_handler.setFormatter(debug_format)
# buffer debug messages so they can be sent with error emails
memory_handler = logging.handlers.MemoryHandler(1024*10, logging.ERROR, error_mail_handler)
memory_handler.setLevel(logging.DEBUG)
# attach handlers
log.addHandler(memory_handler)
log.addHandler(error_mail_handler)
Related to this:
Do I need to add the error_mail_handler to the logger explicitly if it is a target of memory_handler anyway?
Should error_mail_handler be set to DEBUG or ERROR target? Does it even need a target when it is being fed from memory_handler?
Would love to see some working code from anyone who has solved this problem.

You might want to use or adapt the BufferingSMTPHandler which is in this test script.
In general, you don't need to add a handler to a logger if it's the target of a MemoryHandler handler which has been added to a logger. If you set the level of a handler, that will affect what the handler actually processes - it won't process anything which is less severe than its level setting.

Instead of buffering for email, consider posting unbuffered to a message stream on a messaging app, e.g. on Matrix, Discord, Slack, etc. Having said that, I wrote my own beastly thread-safe implementation of BufferingSMTPHandler (backup link) which sends emails from a separate thread. The primary goal is to not block the main thread.
As written, it uses two queues - this seemed necessary in order to implement some useful class-level parameters that are defined in the "Configurable parameters" section of the code. Although you can use the code as-is, it's probably better if you study and use it to write your own class.
Issues:
Some class-level parameters can perhaps be instance-level instead.
Either threading.Timer or the signal module could perhaps be used to avoid loops that run forever.

If you are using django - here is simple buffering handler, which will use standard django email methods:
import logging
from django.conf import settings
from django.core.mail import EmailMessage
class DjangoBufferingSMTPHandler(logging.handlers.BufferingHandler):
def __init__(self, capacity, toaddrs=None, subject=None):
logging.handlers.BufferingHandler.__init__(self, capacity)
if toaddrs:
self.toaddrs = toaddrs
else:
# Send messages to site administrators by default
self.toaddrs = zip(*settings.ADMINS)[-1]
if subject:
self.subject = subject
else:
self.subject = 'logging'
def flush(self):
if len(self.buffer) == 0:
return
try:
msg = "\r\n".join(map(self.format, self.buffer))
emsg = EmailMessage(self.subject, msg, to=self.toaddrs)
emsg.send()
except Exception:
# handleError() will print exception info to stderr if logging.raiseExceptions is True
self.handleError(record=None)
self.buffer = []
In django settings.py you will need to configure email and logging like this:
EMAIL_USE_TLS = True
EMAIL_PORT = 25
EMAIL_HOST = '' # example: 'smtp.yandex.ru'
EMAIL_HOST_USER = '' # example: 'user#yandex.ru'
EMAIL_HOST_PASSWORD = ''
DEFAULT_FROM_EMAIL = EMAIL_HOST_USER
SERVER_EMAIL = EMAIL_HOST_USER
LOGGING = {
'handlers': {
...
'mail_buffer': {
'level': 'WARN',
'capacity': 9999,
'class': 'utils.logging.DjangoBufferingSMTPHandler',
# optional:
# 'toaddrs': 'admin#host.com'
# 'subject': 'log messages'
}
},
...
}

For this purpose I use the BufferingSMTPHandler suggested by Vinay Sajip with one minor tweak: I set the buffer length to something really big (say 5000 log records) and manualy call the flush method of the handler every some seconds and after checking for internet conectivity.
# init
log_handler1 = BufferingSMTPHandler(
'smtp.host.lala', "from#test.com", ['to#test.com'], 'Log event(s)',5000)
...
logger.addHandler(log_handler1)
...
# main code
...
if internet_connection_ok and seconds_since_last_flush>60:
log_handler1.flush() # send buffered log records (if any)

Updated Vinay Sajip's answer for python3.
import logging
from logging.handlers import BufferingHandler
class BufferingSMTPHandler(BufferingHandler):
def __init__(self, mailhost, fromaddr, toaddrs, subject, capacity):
logging.handlers.BufferingHandler.__init__(self, capacity)
self.mailhost = mailhost
self.mailport = None
self.fromaddr = fromaddr
self.toaddrs = toaddrs
self.subject = subject
self.setFormatter(logging.Formatter("%(asctime)s %(levelname)-5s %(message)s"))
def flush(self):
if len(self.buffer) > 0:
try:
import smtplib
port = self.mailport
if not port:
port = smtplib.SMTP_PORT
smtp = smtplib.SMTP(self.mailhost, port)
msg = '''From: {}\r\nTo: {}\r\nSubject: {}\r\n\r\n'''.format(
self.fromaddr,
",".join(self.toaddrs),
self.subject
)
for record in self.buffer:
s = self.format(record)
print (s)
msg = msg + s + "\r\n"
smtp.sendmail(self.fromaddr, self.toaddrs, msg)
smtp.quit()
except:
self.handleError(None) # no particular record
self.buffer = []
#update for #Anant
if __name__ == '__main__'
buff_smtp_handler=BufferingSMTPHandler(...your args)
buff_smtp_handler.setLevel(logging.ERROR)
handlers=[buff_smtp_handler]
logging.basicConfig(handlers=handlers)

I think the point about the SMTP logger is that it is meant to send out a significant log message functioning as some kind of alert if sent to a human recipient or else to be further processed by an automated recipient.
If a collection of log messages is to be sent by email then that constitutes a report being sent at the end of execution of a task and writing that log to a file and then emailing the file would seem to be a reasonable solution.
I took a look at the basic FileHandler log handler and how to build a mechanism to write to a temp file then attach that temp file when the script exits.
I found the "atexit" module that allows for a method to be registered that will be executed against an object when the script is exiting.
import logging
import smtplib
from email.mime.multipart import MIMEMultipart
from email.mime.text import MIMEText
from email.mime.base import MIMEBase
import os
from email import encoders
import uuid
# atexit allows for a method to be set to handle an object when the script exits
import atexit
filename = uuid.uuid4().hex
class MailLogger:
def __init__(self, filePath, smtpDict):
self.filePath = filePath
self.smtpDict = smtpDict
# Generate random file name
filename = '%s.txt' % ( uuid.uuid4().hex )
# Create full filename
filename = '%s/%s' % (filePath,filename)
self.filename = filename
self.fileLogger = logging.getLogger('mailedLog')
self.fileLogger.setLevel(logging.INFO)
self.fileHandler = logging.FileHandler(filename)
self.fileHandler.setLevel(logging.INFO)
formatter = logging.Formatter('%(name)s - %(levelname)s - %(message)s')
self.fileHandler.setFormatter(formatter)
self.fileLogger.addHandler(self.fileHandler)
atexit.register(self.mailOut)
def mailOut(self):
'''
Script is exiting so time to mail out the log file
"emailSettings": {
"smtpServer" : "smtp.dom.com",
"smtpPort" : 25,
"sender" : "sender#dom.com>",
"recipients" : [
"recipient#dom.com"
],
"subject" : "Email Subject"
},
'''
# Close the file handler
smtpDict = self.smtpDict
self.fileHandler.close()
msg = MIMEMultipart('alternative')
s = smtplib.SMTP(smtpDict["smtpServer"], smtpDict["smtpPort"] )
msg['Subject'] = smtpDict["subject"]
msg['From'] = smtpDict["sender"]
msg['To'] = ','.join(smtpDict["recipients"])
body = 'See attached report file'
content = MIMEText(body, 'plain')
msg.attach(content)
attachment = MIMEBase('application', 'octet-stream')
attachment.set_payload(open(self.filename, 'rb').read())
encoders.encode_base64(attachment)
attachment.add_header('Content-Disposition', 'attachment; filename="%s"' % os.path.basename(self.filename))
msg.attach(attachment)
s.send_message(msg)
s.quit()
My basic test script is:
from EmailLogRpt import MailLogger
import time
smtpDict = {
"smtpServer" : "smtp.dom.com",
"smtpPort" : 25,
"sender" : "sender#dom.com",
"recipients" : [
"recpient#dom.com>"
],
"subject" : "Email Subject"
}
myMailLogger = MailLogger("/home/ed/tmp",smtpDict).fileLogger
myMailLogger.info("test msg 1")
time.sleep(5)
myMailLogger.info("test msg 2")
Hope this helps somebody.