Related
I have a python function that is supposed to send packet asynchronously in a loop.
import asyncio
import logging
import pickle
from time import sleep
from decoder import decode, encode
from socket import *
UDP_PORT_NUMBER = 44006
HOST_ADDRESS = "192.168.204.139"
server_address = (HOST_ADDRESS, UDP_PORT_NUMBER)
def get_data_packet():
with open('../packets/packet_15', mode='rb') as fpt:
packet = pickle.load(fpt, encoding='bytes')
return packet
class GcuUdpClient:
#staticmethod
async def send_packets():
data = get_data_packet()
client_socket = socket(AF_INET, SOCK_DGRAM)
client_socket.sendto(data, server_address)
print('sending data packets')
await asyncio.sleep(5)
I have another python program that calls this class, along with two other classes.
import asyncio
from http_server import PostServer
from grpc_server import GrpcServer
from gcu_udp_client import GcuUdpClient
async def main():
await asyncio.gather(GrpcServer.run(), PostServer.run_server(), GcuUdpClient.send_packets())
if __name__ == '__main__':
asyncio.run(main())
This code starts up the first two meaning
GrpcServer.run()
and
PostServer.run_server()
However I can't get the last one started, to send out the packets. What am I missing.
Adding the code for http_server.py
import threading
import zlib
from http import HTTPStatus
from http.server import HTTPServer, BaseHTTPRequestHandler
from io import BytesIO
from socket import *
from time import sleep
#from GCUSimulator.generated.GDataPayload_pb2 import GDataPayload
from generated.GDataPayload_pb2 import GDataPayload
HTTP_PORT_NUMBER = 8085
HTTP_SERVER_IP="192.168.204.1"
class SimpleHTTPRequestHandler(BaseHTTPRequestHandler):
def __init__(self, *args, **kwargs):
self.subscribed = False
super(SimpleHTTPRequestHandler, self).__init__(*args, *kwargs)
def do_GET(self):
self.send_response(200)
self.end_headers()
def do_POST(self):
content_length = int(self.headers['Content-Length']) # <--- Gets the size of data
post_data = self.rfile.read(content_length) # <--- Gets the data itself
print("POST Path: {0} Headers: {1} Body: {2}".
format(str(self.path), str(self.headers), post_data.decode('utf-8')))
if str(self.path) == '/setSubscriber.php':
message = "Subscription Result:Subscribed,APIVersion:1.0.0"
self.send_response(HTTPStatus.OK)
self.end_headers()
self.wfile.write(message.encode('utf-8'))
self.subscribed = True
elif str(self.path) == '/removeSubscriber.php':
self.send_response(HTTPStatus.OK)
self.end_headers()
self.subscribed = False
def get_subscription_status(self):
return self.subscribed
class PostServer:
#staticmethod
async def run_server(server_class=HTTPServer, handler_class=SimpleHTTPRequestHandler, addr=HTTP_SERVER_IP, port=HTTP_PORT_NUMBER):
server_address = (addr, port)
print('Http Post server : {0}:{1}'.format(addr, port))
httpd = server_class(server_address, handler_class)
await httpd.serve_forever()
Adding the code for grpc_server.py
from concurrent import futures
import grpc
from generated import guidance_grpc_service_definitions_pb2_grpc
from gcu.ops_station_services import OPSStationServices
GRPC_PORT_NUMBER = 12074
class GrpcServer:
#staticmethod
async def run():
server = grpc.server(futures.ThreadPoolExecutor(max_workers=10))
guidance_grpc_service_definitions_pb2_grpc.add_OpRPCServicesServicer_to_server(OServices(), server)
server.add_insecure_port('[::]:{0}'.format(GRPC_PORT_NUMBER))
print('Grpc Server : {0}:{1}'.format('localhost', GRPC_PORT_NUMBER))
server.start()
# server.wait_for_termination()
Earlier I wrote multi threaded web server, which at times would simply stop processing requests and also getting terminated at peak times.
I've implemented same opencv based processing in Python Websocket based server too which is working fine.
For very old browsers, I also need POST based processing using web server. I converted from multithreading to single but that also is stopping different times and not printing any log etc.
I checked syslog but not clue. More than a week has gone by without finding a solution. I suspect something related to Digital Ocean VPS or network.
I've this code and can't figure why it should stop responding:
from http.server import HTTPServer, BaseHTTPRequestHandler
import threading
import cgi
import tempfile
import resource
import base64
from common import *
from datetime import datetime
print( datetime.now());
gg_hashmap = getHash()
USE_HTTPS = True
def dump(obj):
for attr in dir(obj):
print("obj.%s = %r" % (attr, getattr(obj, attr)))
class PostHandler(BaseHTTPRequestHandler):
def handle(self):
try:
BaseHTTPRequestHandler.handle(self)
except :
pass
def do_POST(self):
try:
print("new req="+str( datetime.now()),flush=True);
form = cgi.FieldStorage(
fp=self.rfile,
headers=self.headers,
environ={'REQUEST_METHOD': 'POST',
'CONTENT_TYPE': self.headers['Content-Type'],
})
self.send_response(200)
self.send_header("Content-type", "text/html")
self.send_header("Access-Control-Allow-Origin", "*")
self.end_headers()
for field in form.keys():
field_item = form[field]
if field_item.filename:
file_data = field_item.file.read()
file_len = len(file_data)
del file_data
self.wfile.write('\tUploaded %s as "%s" (%d bytes)\n' % \
(field, field_item.filename, file_len))
else:
pass
if ('base64' in form and 'license' in form):
print("license=",form['license'].value);
global gg_hashmap
file_content = form['base64'].value
try:
f, temp_file_path = tempfile.mkstemp(prefix='sand', suffix='jpg')
os.close(f)
with open(temp_file_path, 'wb') as w:
w.write(base64.b64decode (file_content))
input_hashes = get_input_img(temp_file_path)
all_letters = ""
if input_hashes != None:
for inp_hash in input_hashes:
lowest = 1000
lowest_letter = ''
for letter, arr in gg_hashmap.items():
for hashval in arr:
if int(inp_hash - hashval) < lowest:
lowest = int(inp_hash - hashval)
lowest_letter = letter
all_letters += lowest_letter
self.wfile.write(bytes(all_letters, "utf8"))
except Exception as e:
print("exception3 caught")
print(e)
print(str(e))
return
except Exception as e:
print("Caught unknown exception",e)
def do_GET(self):
self.send_response(200)
self.end_headers()
message = threading.currentThread().getName()
self.wfile.write(bytes(message,'utf-8'))
self.wfile.write('\n')
return
form = cgi.FieldStorage(
fp=self.rfile,
headers=self.headers,
environ={'REQUEST_METHOD': 'POST',
'CONTENT_TYPE': self.headers['Content-Type'],
})
self.send_response(200)
self.end_headers()
for field in form.keys():
field_item = form[field]
if field_item.filename:
file_data = field_item.file.read()
file_len = len(file_data)
del file_data
self.wfile.write('\tUploaded %s as "%s" (%d bytes)\n' % \
(field, field_item.filename, file_len))
else:
pass
return
def run():
# resource.setrlimit(resource.RLIMIT_STACK, (2**29,-1))
# threading.stack_size(24*1048576)
server = HTTPServer(('0.0.0.0', 443), PostHandler)
if USE_HTTPS:
import ssl
server.socket = ssl.wrap_socket(server.socket, keyfile='./ssl/key.pem', certfile='./ssl/public.pem'
, ca_certs="./ssl/cap1_transactionfailed_com.ca-bundle" , server_side=True)
server.serve_forever()
if __name__ == '__main__':
run()
I don't think much anyone will want to read through all 157 lines of convoluted HTTP request handling code (of which some isn't even posted, from common import *) to try and decipher why it might stop at some given time.
It's likely not the answer you want to hear, but HTTPServer really isn't what anyone uses in production for Python.
You should look into rewriting your code with either (my recommendations at the time of writing)
FastAPI (or its underlying Starlette framework) on Uvicorn (Uvicorn will let you do the websocket stuff in the same process), or
Flask on Gunicorn or uWSGI
For instance, here's a rough estimation of what your code would look like with Starlette. (There may be bugs since it's dry-coded, and it's certainly not fully async, but that doesn't matter here.)
import tempfile
import base64
from starlette.applications import Starlette
from starlette.requests import Request
from starlette.responses import PlainTextResponse
app = Starlette()
def get_all_letters(input_hashes):
all_letters = ""
if input_hashes:
for inp_hash in input_hashes:
lowest = 1000
lowest_letter = ""
for letter, arr in gg_hashmap.items():
for hashval in arr:
if int(inp_hash - hashval) < lowest:
lowest = int(inp_hash - hashval)
lowest_letter = letter
all_letters += lowest_letter
return all_letters
#app.route("/", methods=["GET", "POST"])
async def handle(request: Request):
if request.method == "GET":
return PlainTextResponse("Hello!")
form = await request.form()
if not ("base64" in form and "license" in form):
return PlainTextResponse("Missing data!", status_code=400)
with tempfile.NamedTemporaryFile(prefix="sand", suffix="jpg") as f:
content = await form["base64"].read()
f.write(base64.b64decode(content))
f.flush()
input_hashes = get_input_img(f)
if not input_hashes:
return PlainTextResponse("No input hashes!", status_code=400)
all_letters = get_all_letters(input_hashes)
return PlainTextResponse(all_letters)
You could then run this using Uvicorn (which will also handle all of that HTTPS stuff for you).
With mkstemp you must delete the tempfile. You probably run out of disk space or max out files in temp directory. As AKX mentioned though you should look into using a more robust http server. if the file thing isn't your problem there are numerous other issues that could come up when using a non production HTTP server.
I'm new in cherrypy, and I try to develop a small application by using this framework. My problem is, this application can serves well and browser can access when cherrypy works in undaemonizer mode. But when I write code by using cherrypy.process.plugins.Daemonizer(), cherrypy started in background, and it listen on specific port, but browser will get a connection refused( iptables or ufw already shutted down but still inaccessible ). The incredible thing is, when I start it with daemoned mode, I can still start a undaemoned process , and they listen on the same port. I wonder why would this happend, and how to solve it?
Simply saids: With Daemonizer starts, cherrypy listened on specified port, but browser connection refused; without Daemonizer, cherrypy works very well.
Thanks alot
with my code
from optparse import OptionParser
from cherrypy.process.plugins import Daemonizer
from cherrypy.process.plugins import PIDFile
import cherrypy
import json
import urllib
import datetime
try:
import cPickle as pickle
except:
import pickle
import time
import base64
import os
import sys
'''
cherrypy class
'''
class Index(object):
#cherrypy.expose
def index(self):
return "Say hello to the yellow elephant"
class System(object):
#cherrypy.expose
def env(self, token):
local_token = Token()
if local_token.AuthToken(token) is True:
env = get_env()
return json.dumps(env)
return '{"errcode", "Invalid token"}'
class Jmx(object):
#cherrypy.expose
def get(self, token, host, port, qry):
local_token = Token()
if local_token.AuthToken(token) is True:
url = 'http://' + host + ':' + port + '/jmx?qry=' + qry
jmx = urllib.urlopen(url)
jmx_data = jmx.read().replace('\n', '')
jmx.close()
return jmx_data
return '{"errcode", "Invalid token"}'
"""
command uses base64 encode by using http post method
"""
class Command(object):
def __init__(self):
self.fname = datetime.datetime.now().strftime('%Y-%m-%d_%M-%M-%S') + '.log'
#cherrypy.expose
def run(self, token, command):
local_token = Token()
command = base64.b64decode(command)
if local_token.AuthToken(token) is True:
os.popen(command + ' 2>&1 > /usr/lib/agent/output/' + self.fname)
return '{"errcode", "Invalid token"}'
#cherrypy.expose
def readlog(self, token):
local_token = Token()
if local_token.AuthToken(token) is True:
log = open('/usr/lib/agent/output/' + self.fname)
lines = log.readlines()
log.close()
return json.dumps(lines, ensure_ascii=False)
return '{"errcode", "Invalid token"}'
"""
First time access from central, it will create a new token on slave node, the token is pickle.dump(cacl_mysql_passwd(conf['agent']['secret']))
By token created , if central makes change to secret, the slave node will be inaccessible!!!
"""
class Token(object):
def AuthToken(self, token):
if(os.path.isfile('/usr/lib/agent/key/authenticate.key')) is False:
return self.CreateToken(token)
else:
try:
k = open('/usr/lib/agent/key/authenticate.key', 'rb')
tokenizer = pickle.load(k)
k.close()
if token == tokenizer:
return True
else:
return False
except IOError, e:
return '{"errcode":"' + str(e).replace('\n', '<br/>') + '"}'
#cherrypy.expose
def CreateToken(self, token):
if(os.path.isfile('/usr/lib/agent/key/authenticate.key')) is False:
try:
k = open('/usr/lib/agent/key/authenticate.key', 'wb')
pickle.dump(token, k)
k.close()
return True
except IOError, e:
return '{"Exception":"' + str(e).replace('\n', '<br/>') + '"}'
else:
return '{"errcode":"token exists"}'
class Controller:
def __init__(self, pidfile='/var/run/agent/agent.pid', host='0.0.0.0', port=30050):
self.port = port
self.host = host
self.pidfile = pidfile
self.settings = {
'global': {
'server.socket_port': port,
'server.socket_host': host,
'server.socket_file': '',
'server.socket_queue_size': 5,
'server.protocol_version': 'HTTP/1.1',
'server.log_to_screen': True,
'server.log_file': '',
'server.reverse_dns': False,
'server.thread_pool': 10,
'server.environment': 'production',
'engine.timeout_monitor.on': False
}
}
def start(self):
if os.path.exists(self.pidfile):
sys.stderr.write('PID file exists, server running?\n')
sys.exit(1)
else:
Daemonizer(cherrypy.engine, stdin='/dev/stdin', stdout='/dev/stdout', stderr='/dev/stderr').subscribe()
PIDFile(cherrypy.engine, self.pidfile).subscribe()
cherrypy.tree.mount(Index(), '/')
cherrypy.tree.mount(System(), '/system')
cherrypy.tree.mount(Command(), '/command')
cherrypy.tree.mount(Jmx(), '/jmx')
cherrypy.config.update(self.settings)
cherrypy.engine.start()
cherrypy.engine.block()
def stop(self):
cherrypy.config.update(self.settings)
if os.path.exists(self.pidfile):
cherrypy.engine.stop()
cherrypy.engine.exit()
try:
process = open(self.pidfile).read().strip()
if process != 0:
os.popen('kill -9 %s' % process)
os.remove(self.pidfile)
except IOError, e:
sys.stderr.write(str(e))
else:
sys.stderr.write('PID file does not exist, server gone?\n')
sys.exit(1)
if '__main__' == __name__:
cherrypy.engine.autoreload.stop()
cherrypy.engine.autoreload.unsubscribe()
syntax = 'Syntax: %prog -b 192.168.1.1 -s start'
parser = OptionParser(usage=syntax)
ip = os.popen('hostname -i').read().strip()
hostname = os.popen('hostname --fqdn').read().strip()
parser.add_option('-b', '--bind', action='store', type='string', dest='bind', default=ip, help='Inner network IP address, default value is hostname -i')
parser.add_option('-s', '--signal', action='store', type='string', dest='signal', help='Valid signal is {start|stop|restart}')
options, args = parser.parse_args()
if len(sys.argv) == 1:
print 'Use %s -h or --help for help.' % sys.argv[0]
else:
if options.signal == '':
print 'Must give -s option\'s value'
else:
daemon = Controller(pidfile='/var/run/agent/agent.pid', host=options.bind)
if 'start' == options.signal:
daemon.start()
elif 'stop' == options.signal:
daemon.stop()
else:
print 'Invalid signal'
sys.exit(1)
I have the following code
# logging
from twisted.python import log
import sys
# MIME Multipart handling
import email
import email.mime.application
import uuid
# IMAP Connection
from twisted.mail import imap4
from twisted.internet import protocol
#SMTP Sending
import os.path
from OpenSSL.SSL import SSLv3_METHOD
from twisted.internet import ssl
from twisted.mail.smtp import ESMTPSenderFactory
from twisted.internet.ssl import ClientContextFactory
from twisted.internet.defer import Deferred
from twisted.internet import reactor
#class AccountsManager(object):
def connectToIMAPServer(imap_server, username, password):
factory = IMAP4ClientFactory(username, password, login_insecure = True)
host, port = imap_server.split(":")
# connect to reactor
if port == '993':
reactor.connectSSL(host, int(port), factory, ssl.ClientContextFactory())
else:
if not port:
port = 143
reactor.connectTCP(host, int(port), factory)
d = factory.deferred
d.addCallback(lambda r: factory.proto)
return d
class IMAP4Client(imap4.IMAP4Client):
"""
A client with callbacks for greeting messages from an IMAP server.
"""
greetDeferred = None
def serverGreeting(self, caps):
self.serverCapabilities = caps
if self.greetDeferred is not None:
d, self.greetDeferred = self.greetDeferred, None
d.callback(self)
class IMAP4ClientFactory(protocol.ClientFactory):
usedUp = False
protocol = IMAP4Client
def __init__(self, username, password, mailbox = "INBOX", login_insecure = False):
self.ctx = ssl.ClientContextFactory()
self.username = username
self.password = password
self.mailbox = mailbox
self.login_insecure = login_insecure
self.deferred = Deferred()
def buildProtocol(self, addr):
"""
Initiate the protocol instance. Since we are building a simple IMAP
client, we don't bother checking what capabilities the server has. We
just add all the authenticators twisted.mail has. Note: Gmail no
longer uses any of the methods below, it's been using XOAUTH since
2010.
"""
assert not self.usedUp
self.usedUp = True
p = self.protocol(self.ctx)
p.factory = self
p.greetDeferred = self.deferred
p.registerAuthenticator(imap4.PLAINAuthenticator(self.username))
p.registerAuthenticator(imap4.LOGINAuthenticator(self.username))
p.registerAuthenticator(imap4.CramMD5ClientAuthenticator(self.username))
self.deferred.addCallback(self.GreetingCallback)
self.deferred.addErrback(self.GreetingErrback)
self.proto = p
return p
def GreetingCallback(self, result):
print "Secure Login"
auth_d = self.proto.authenticate(self.password)
auth_d.addCallback(self.AuthenticationCallback)
auth_d.addErrback(self.AuthenticationErrback)
return auth_d # attach it to the main deferred
def GreetingErrback(self, error):
log.err(error)
self.CloseConnection()
return error
def AuthenticationCallback(self, result):
print "Selecting Mailbox"
d = self.proto.examine(self.mailbox)
return d
def AuthenticationErrback(self, failure):
if self.login_insecure:
failure.trap(imap4.NoSupportedAuthentication)
return self.InsecureLogin()
else:
return error
def InsecureLogin(self):
print "Insecure Login"
d = self.proto.login(self.username, self.password)
d.addCallback(self.AuthenticationCallback)
return d
def CloseConnection(self):
self.proto.transport.loseConnection()
def clientConnectionFailed(self, connector, reason):
d, self.deferred = self.deferred, None
d.errback(reason)
class MailServer(object):
"Manages a server"
size = 0
used_space = 0
def __init__(self, smtp_server, imap_server, username, password):
self.smtp_server, self.smtp_port = smtp_server.split(":")
self.imap_server, self.imap_port = imap_server.split(":")
self.username = username
self.password = password
self.imap_connection = IMAP4ClientFactory(username, password)
def upload_data(self, data):
"""
Uploads data to email server returns deferred that will return with the imap uid
"""
# Create a text/plain message
id = str(uuid.uuid4()).upper()
msg = email.mime.Multipart.MIMEMultipart()
msg['Subject'] = 'GMA ID: %s' % id
msg['From'] = self.email_address
msg['To'] = self.email_address
# The main body is just another attachment
body = email.mime.Text.MIMEText("GMA ID: %s" % (self.uuid_id))
msg.attach(body)
att = email.mime.application.MIMEApplication(data,_subtype="raw")
att.add_header('Content-Disposition','attachment',filename = os.path.basename(self.filename))
msg.attach(att)
# Create a context factory which only allows SSLv3 and does not verify
# the peer's certificate.
contextFactory = ClientContextFactory()
contextFactory.method = SSLv3_METHOD
d = Deferred()
mime_obj = StringIO(str(msg))
senderFactory = ESMTPSenderFactory(
self.username,
self.password,
self.email_address,
self.email_address,
mime_obj,
d,
contextFactory=contextFactory)
d.addCallback(lambda r: self.email_sent(id, int(self.parts)) )
d.addErrback(self.email_error)
reactor.connectTCP(self.smtp_server, self.smtp_port, senderFactory)
d.addCallback(self.upload_success, *args, **kw)
d.addErrback(self.upload_error, 1)
return d
def upload_success(self, result):
print "upload was succesful!"
def upload_error(self, result):
print "upload error"
def download_data(self, uid):
"""
Downloads data from the email server returns a deferred that will return with the data
"""
print "uid"
if __name__ == "__main__":
log.startLogging(sys.stdout)
d = connectToIMAPServer("imap.gmail.com:993", "username", "password")
def f(s):
print s
d.addCallback(lambda r: f("These are fired before the auth and examine callbacks, why?"))
d.addCallback(lambda r: f("These are fired before the auth and examine callbacks, why?"))
reactor.run()
The class is suppose to handle logging in and selecting a mailbox and nicely return a IMAP proto ready to use however the two callbacks at the bottom are fired before the other ones, I get why, the callbacks are added before the other ones because buildProtocol hasn't been called yet so what is the best way to handle this, just have a dummy callback added in init that "holds" the first spot?
from twisted.internet.endpoints import TCP4ClientEndpoint
d = TCP4ClientEndpoint(reactor, host, int(port)).connect(factory)
and
d.addCallback(lambda r: factory.deferred)
instead of
d = factory.deferred
in connectToIMAPServer should do it - your factory.deferred will be returned only after protocol is ready. (Twisted Documentation on writing clients)
I eventually edited the code around and just managed the deferred's callback or errback internally
update code
# logging
from twisted.python import log
import sys
# MIME Multipart handling
import email
import email.mime.application
import uuid
# IMAP Connection
from twisted.mail import imap4
from twisted.internet import protocol
#SMTP Sending
import os.path
from OpenSSL.SSL import SSLv3_METHOD
from twisted.internet import ssl
from twisted.mail.smtp import ESMTPSenderFactory
from twisted.internet.ssl import ClientContextFactory
from twisted.internet.defer import Deferred
from twisted.internet import reactor
#class AccountsManager(object):
def connectToIMAPServer(imap_server, username, password):
factory = IMAP4ClientFactory(username, password, login_insecure = True)
host, port = imap_server.split(":")
# connect to reactor
if port == '993':
reactor.connectSSL(host, int(port), factory, ssl.ClientContextFactory())
else:
if not port:
port = 143
reactor.connectTCP(host, int(port), factory)
return factory.deferred
class IMAP4Client(imap4.IMAP4Client):
"""
A client with callbacks for greeting messages from an IMAP server.
"""
greetDeferred = None
def serverGreeting(self, caps):
self.serverCapabilities = caps
if self.greetDeferred is not None:
d, self.greetDeferred = self.greetDeferred, None
d.callback(self)
class IMAP4ClientFactory(protocol.ClientFactory):
usedUp = False
protocol = IMAP4Client
def __init__(self, username, password, mailbox = "INBOX", login_insecure = False):
self.ctx = ssl.ClientContextFactory()
self.username = username
self.password = password
self.mailbox = mailbox
self.login_insecure = login_insecure
# called when the protocol is all set up or there is an error setting it up
self.deferred = Deferred()
def buildProtocol(self, addr):
"""
Initiate the protocol instance. Since we are building a simple IMAP
client, we don't bother checking what capabilities the server has. We
just add all the authenticators twisted.mail has. Note: Gmail no
longer uses any of the methods below, it's been using XOAUTH since
2010.
"""
assert not self.usedUp
self.usedUp = True
p = self.protocol(self.ctx)
p.factory = self
# deferred for when the IMAP Greeting is done
p.greetDeferred = Deferred()
p.registerAuthenticator(imap4.PLAINAuthenticator(self.username))
p.registerAuthenticator(imap4.LOGINAuthenticator(self.username))
p.registerAuthenticator(imap4.CramMD5ClientAuthenticator(self.username))
p.greetDeferred.addCallback(self.GreetingCallback)
p.greetDeferred.addErrback(self.GreetingErrback)
self.proto = p
return p
def GreetingCallback(self, result):
log.msg("Succesfully sent IMAP Greeting.")
auth_d = self.proto.authenticate(self.password)
auth_d.addCallback(self.AuthenticationCallback)
auth_d.addErrback(self.AuthenticationErrback)
return auth_d
def GreetingErrback(self, error):
log.msg("Problem sending greeting")
log.err(error)
self.CloseConnection()
self.deferred.errback(error)
def AuthenticationCallback(self, result):
log.msg("Authenticated")
log.msg("Selecting Mailbox")
d = self.proto.examine(self.mailbox)
d.addCallback(self.MailboxSelectCallback)
d.addErrback(self.MailboxSelectErrback)
return d
def AuthenticationErrback(self, failure):
log.msg("Unable to authenticate securly")
if self.login_insecure:
log.msg("Trying to login insecurly")
failure.trap(imap4.NoSupportedAuthentication)
return self.InsecureLogin()
else:
log.err(failure)
self.deferred.errback(failure)
def InsecureLogin(self):
log.msg("Logging in insecurly")
d = self.proto.login(self.username, self.password)
d.addCallback(self.AuthenticationCallback)
return d
def MailboxSelectCallback(self, result):
# connected and protocol set up
log.msg("IMAP4 protocol setup")
self.deferred.callback(self.proto)
def MailboxSelectErrback(self, error):
log.msg("Cannot select mailbox %s" % self.mailbox)
log.err(error)
self.deferred.errback(error)
def CloseConnection(self):
self.proto.transport.loseConnection()
def clientConnectionFailed(self, connector, reason):
log.msg("Connecting was lost")
log.err(reason)
d, self.deferred = self.deferred, None
d.errback(reason)
class MailServer(object):
"Manages a server"
size = 0
used_space = 0
def __init__(self, smtp_server, imap_server, username, password):
self.smtp_server, self.smtp_port = smtp_server.split(":")
self.imap_server, self.imap_port = imap_server.split(":")
self.username = username
self.password = password
self.imap_connection = IMAP4ClientFactory(username, password)
def upload_data(self, data):
"""
Uploads data to email server returns deferred that will return with the imap uid
"""
# Create a text/plain message
id = str(uuid.uuid4()).upper()
msg = email.mime.Multipart.MIMEMultipart()
msg['Subject'] = 'GMA ID: %s' % id
msg['From'] = self.email_address
msg['To'] = self.email_address
# The main body is just another attachment
body = email.mime.Text.MIMEText("GMA ID: %s" % (self.uuid_id))
msg.attach(body)
att = email.mime.application.MIMEApplication(data,_subtype="raw")
att.add_header('Content-Disposition','attachment',filename = os.path.basename(self.filename))
msg.attach(att)
# Create a context factory which only allows SSLv3 and does not verify
# the peer's certificate.
contextFactory = ClientContextFactory()
contextFactory.method = SSLv3_METHOD
d = Deferred()
mime_obj = StringIO(str(msg))
senderFactory = ESMTPSenderFactory(
self.username,
self.password,
self.email_address,
self.email_address,
mime_obj,
d,
contextFactory=contextFactory)
d.addCallback(lambda r: self.email_sent(id, int(self.parts)) )
d.addErrback(self.email_error)
reactor.connectTCP(self.smtp_server, self.smtp_port, senderFactory)
d.addCallback(self.upload_success, *args, **kw)
d.addErrback(self.upload_error, 1)
return d
def upload_success(self, result):
print "upload was succesful!"
def upload_error(self, result):
print "upload error"
def download_data(self, uid):
"""
Downloads data from the email server returns a deferred that will return with the data
"""
print "uid"
if __name__ == "__main__":
log.startLogging(sys.stdout)
d = connectToIMAPServer("imap.gmail.com:993", "email", "password")
def f(s):
print s
d.addCallback(lambda r: f("These are fired before the auth and examine callbacks, why?"))
d.addCallback(lambda r: f("These are fired before the auth and examine callbacks, why?"))
reactor.run()
Ok,
This should be simple, since people do it all the time. I want to get the body of a POST request sent a twisted Agent. This is created with a twisted FileBodyProducer. On the server side, I get a request object for my render_POST method.
How do I retrieve the body?
server:
from twisted.web import server, resource
from twisted.internet import reactor
class Simple(resource.Resource):
isLeaf = True
def render_GET(self, request):
return "{0}".format(request.args.keys())
def render_POST(self, request):
return "{0}".format(request.data)
with open(request.args['filename'][0], 'rb') as fd:
fd.write(request.write())
site = server.Site(Simple())
reactor.listenTCP(8080, site)
reactor.run()
client:
from StringIO import StringIO
from twisted.internet import reactor
from twisted.web.client import Agent
from twisted.web.http_headers import Headers
from twisted.web.client import FileBodyProducer
from twisted.internet.defer import Deferred
from twisted.internet.protocol import Protocol
from pprint import pformat
class BeginningPrinter(Protocol):
def __init__(self, finished):
self.finished = finished
self.remaining = 1024 * 10
def dataReceived(self, bytes):
if self.remaining:
display = bytes[:self.remaining]
print 'Some data received:'
print display
self.remaining -= len(display)
def connectionLost(self, reason):
print 'Finished receiving body:', reason.getErrorMessage()
self.finished.callback(None)
agent = Agent(reactor)
body = FileBodyProducer(StringIO("hello, world"))
d = agent.request(
'POST',
'http://127.0.0.1:8080/',
Headers({'User-Agent': ['Twisted Web Client Example'],
'Content-Type': ['text/x-greeting']}),
body)
def cbRequest(response):
print 'Response version:', response.version
print 'Response code:', response.code
print 'Response phrase:', response.phrase
print 'Response headers:'
print pformat(list(response.headers.getAllRawHeaders()))
finished = Deferred()
response.deliverBody(BeginningPrinter(finished))
return finished
d.addCallback(cbRequest)
def cbShutdown(ignored):
reactor.stop()
d.addBoth(cbShutdown)
reactor.run()
The only docs I can find for setting up the consumer side leave something to be desired. Primarily, how can a consumer use the write(data) method to receive results?
Which bit am I missing to plug these two components together?
All right, so it's as simple as calling request.content.read(). This, as far as I can tell, is undocumented in the API.
Here's the updated code for the client:
from twisted.internet import reactor
from twisted.web.client import Agent
from twisted.web.http_headers import Headers
from twisted.web.client import FileBodyProducer
from twisted.internet.defer import Deferred
from twisted.internet.protocol import Protocol
from pprint import pformat
class BeginningPrinter(Protocol):
def __init__(self, finished):
self.finished = finished
self.remaining = 1024 * 10
def dataReceived(self, bytes):
if self.remaining:
display = bytes[:self.remaining]
print 'Some data received:'
print display
self.remaining -= len(display)
def connectionLost(self, reason):
print 'Finished receiving body:', reason.getErrorMessage()
self.finished.callback(None)
class SaveContents(Protocol):
def __init__(self, finished, filesize, filename):
self.finished = finished
self.remaining = filesize
self.outfile = open(filename, 'wb')
def dataReceived(self, bytes):
if self.remaining:
display = bytes[:self.remaining]
self.outfile.write(display)
self.remaining -= len(display)
else:
self.outfile.close()
def connectionLost(self, reason):
print 'Finished receiving body:', reason.getErrorMessage()
self.outfile.close()
self.finished.callback(None)
agent = Agent(reactor)
f = open('70935-new_barcode.pdf', 'rb')
body = FileBodyProducer(f)
d = agent.request(
'POST',
'http://127.0.0.1:8080?filename=test.pdf',
Headers({'User-Agent': ['Twisted Web Client Example'],
'Content-Type': ['multipart/form-data; boundary=1024'.format()]}),
body)
def cbRequest(response):
print 'Response version:', response.version
print 'Response code:', response.code
print 'Response phrase:', response.phrase
print 'Response headers:'
print 'Response length:', response.length
print pformat(list(response.headers.getAllRawHeaders()))
finished = Deferred()
response.deliverBody(SaveContents(finished, response.length, 'test2.pdf'))
return finished
d.addCallback(cbRequest)
def cbShutdown(ignored):
reactor.stop()
d.addBoth(cbShutdown)
reactor.run()
And here's the server:
from twisted.web import server, resource
from twisted.internet import reactor
import os
# multi part encoding example: http://marianoiglesias.com.ar/python/file-uploading-with-multi-part-encoding-using-twisted/
class Simple(resource.Resource):
isLeaf = True
def render_GET(self, request):
return "{0}".format(request.args.keys())
def render_POST(self, request):
with open(request.args['filename'][0], 'wb') as fd:
fd.write(request.content.read())
request.setHeader('Content-Length', os.stat(request.args['filename'][0]).st_size)
with open(request.args['filename'][0], 'rb') as fd:
request.write(fd.read())
request.finish()
return server.NOT_DONE_YET
site = server.Site(Simple())
reactor.listenTCP(8080, site)
reactor.run()
I can now write the file contents I receive, and read back the results.
If the content type is application/x-www-form-urlencoded or multipart/form-data,
the body will be parsed and put in the request.args dict.
If the body is too big, it is written in temp file, otherwise in StringIO.
After the body is read, the method finish() is called. You can subclass Request and
pares the body in this method or do sth else.
if you want to make a simple POST with body (not a file) you can do as follows
import urllib
from twisted.internet import protocol
from twisted.internet import defer
from twisted.web.http_headers import Headers
from twisted.internet import reactor
from twisted.web.client import Agent
from twisted.web.iweb import IBodyProducer
from zope.interface import implements
from twisted.internet.defer import succeed
class StringProducer(object):
implements(IBodyProducer)
def __init__(self, body):
self.body = body
self.length = len(body)
def startProducing(self, consumer):
consumer.write(self.body)
return succeed(None)
def pauseProducing(self):
pass
def stopProducing(self):
pass
class SimpleReceiver(protocol.Protocol):
def __init__(self, d):
self.buf = ''; self.d = d
def dataReceived(self, data):
self.buf += data
def connectionLost(self, reason):
self.d.callback(self.buf)
def httpRequest(url, values=None, headers=None, method='POST'):
agent = Agent(reactor)
data = urllib.urlencode(values) if values else None
d = agent.request(method, url, Headers(headers) if headers else {},
StringProducer(data) if data else None
)
def handle_response(response):
if response.code == 204:
d = defer.succeed('')
else:
d = defer.Deferred()
response.deliverBody(SimpleReceiver(d))
return d
d.addCallback(handle_response)
return d
Now to use above in real code you can i.e.
d = httpRequest('htpp://...', post_data_as_dictionary, some_headers, 'POST')
d.addCallback(your_ok_callback_function)
d.addErrback(your_errorback_function)
Example headers should look like
headers = {'Accept' : ['application/json',],
'Content-Type': ['application/x-www-form-urlencoded',]
}
I hope that helps