We Keep Coding

Why does the paho python client publish short messages but not long ones

This works :
while True:
print('')
command_input = input()
if command_input == 'q':
break
mcp = Mqtt_command_publisher
mcp.publish_command(device_ids, command_input)
But this does not:
class Mqtt_command_bl:
def update_minutes_to_run_at(json):
if not json['device_ids']:
return 'Request must contain device ids'
device_ids = json['device_ids']
minutes_to_run_at = json['minutes_to_run_at']
minutes_to_run_at_command_section = ''
for i in minutes_to_run_at:
m = '"{}",'.format(i)
if i == minutes_to_run_at[len(minutes_to_run_at) - 1]:
m = '"{}"'.format(i)
minutes_to_run_at_command_section += m
#command_input = 'jq \'.+{{minutes_to_run_at:[{}]}}\' /home/pi/hallmonitor_lite/config.json > /home/pi/hallmonitor_lite/tmp.json && mv /home/pi/hallmonitor_lite/tmp.json /home/pi/hallmonitor_lite/new_config.json'.format(minutes_to_run_at_command_section)
command_input = 'mkdir /home/pi/hallmonitor_lite/hello_world'
mcp = Mqtt_command_publisher
mcp.publish_command(device_ids, command_input)
return 'Success'
The class they both call:
class Mqtt_command_publisher:
def publish_command(device_ids, command_input):
mqtt_msg = json.dumps({'device_ids':device_ids,'command':command_input})
print('\n{}'.format(mqtt_msg))
client = mqtt.Client()
client.connect('********', ****, 30)
client.publish('topic/commands', mqtt_msg)
client.disconnect()
Looking at the print statements output from the Mqtt_command_publisher, the output can be the exact same, however, only one of them will execute, and I don't see why one works and the other does not.
I tried this command for testing: mkdir /home/pi/hallmonitor_lite/hello_world
This is the receiving part:
device_id = 0
with open('/home/pi/hallmonitor_lite/config.json') as json_data_file:
data = json.load(json_data_file)
device_id = data['device_id']
def on_connect(client, userdata, flags, rc):
print("Connected with result code: " + str(rc))
client.subscribe("topic/commands")
def on_message(client, userdata, msg):
mqtt_message = msg.payload.decode()
print(mqtt_message)
ids_and_command = json.loads(mqtt_message)
if str(device_id) in ids_and_command['device_ids'] or not ids_and_command['device_ids']:
print(('Executing: {}').format(ids_and_command['command']))
os.system(ids_and_command['command'])
client = mqtt.Client()
client.connect("********", ****, 30)
client.on_connect = on_connect
client.on_message = on_message
client.loop_forever()
Any ideas?

The problem is most likely because the second set of code is creating a message bigger than will fit in a single TCP packet.
This is a problem because you are not running the client network loop so the client.publish command can only send a single packet, the rest of the message would have been sent by the network loop, but even if it was running you are calling disconnect immediately after the publish call.
The client is not meant to be spun up for a single message like that, it is meant to be started and then left running with you just calling the publish method when you want to send a message. If you don't want to do that or can't for some reason there is a specific helper class in the paho python package that will do all the heavy lifting of starting the client, sending the message and then tearing everything down nicely. The docs for the single publish are here.
import paho.mqtt.publish as publish
publish.single("paho/test/single", "payload", hostname="mqtt.eclipse.org")

How to Send New Messages from Azure IoT Edge Module Python

It seems there is not very much support for what I am trying to do, but it is supposed to be possible since it is demonstrated in temperature sensor and sensor filter tutorial. However, there are no examples for the actual message creation from an edge module in python. That tutorial only shows forwarding messages. There are examples of sending from a device, but devices use a different class than edge modules. From the filter example and from a couple of device examples I have pieced together the following:
# Copyright (c) Microsoft. All rights reserved.
# Licensed under the MIT license. See LICENSE file in the project root for
# full license information.
import random
import time
import sys
import iothub_client
from iothub_client import IoTHubModuleClient, IoTHubClientError, IoTHubTransportProvider
from iothub_client import IoTHubMessage, IoTHubMessageDispositionResult, IoTHubError
# messageTimeout - the maximum time in milliseconds until a message times out.
# The timeout period starts at IoTHubModuleClient.send_event_async.
# By default, messages do not expire.
MESSAGE_TIMEOUT = 10000
# global counters
RECEIVE_CALLBACKS = 0
SEND_CALLBACKS = 0
# Choose HTTP, AMQP or MQTT as transport protocol. Currently only MQTT is supported.
PROTOCOL = IoTHubTransportProvider.MQTT
# Callback received when the message that we're forwarding is processed.
def send_confirmation_callback(message, result, user_context):
global SEND_CALLBACKS
print ( "Confirmation[%d] received for message with result = %s" % (user_context, result) )
map_properties = message.properties()
key_value_pair = map_properties.get_internals()
print ( " Properties: %s" % key_value_pair )
SEND_CALLBACKS += 1
print ( " Total calls confirmed: %d" % SEND_CALLBACKS )
# receive_message_callback is invoked when an incoming message arrives on the specified
# input queue (in the case of this sample, "input1"). Because this is a filter module,
# we will forward this message onto the "output1" queue.
def receive_message_callback(message, hubManager):
global RECEIVE_CALLBACKS
message_buffer = message.get_bytearray()
size = len(message_buffer)
print ( " Data: <<<%s>>> & Size=%d" % (message_buffer[:size].decode('utf-8'), size) )
map_properties = message.properties()
key_value_pair = map_properties.get_internals()
print ( " Properties: %s" % key_value_pair )
RECEIVE_CALLBACKS += 1
print ( " Total calls received: %d" % RECEIVE_CALLBACKS )
hubManager.forward_event_to_output("output1", message, 0)
return IoTHubMessageDispositionResult.ACCEPTED
def construct_message(message_body, topic):
try:
msg_txt_formatted = message_body
message = IoTHubMessage(msg_txt_formatted)
# Add a custom application property to the message.
# An IoT hub can filter on these properties without access to the message body.
prop_map = message.properties()
prop_map.add("topic", topic)
# TODO Use logging
# Send the message.
print( "Sending message: %s" % message.get_string() )
except IoTHubError as iothub_error:
print ( "Unexpected error %s from IoTHub" % iothub_error )
return
return message
class HubManager(object):
def __init__(
self,
protocol=IoTHubTransportProvider.MQTT):
self.client_protocol = protocol
self.client = IoTHubModuleClient()
self.client.create_from_environment(protocol)
# set the time until a message times out
self.client.set_option("messageTimeout", MESSAGE_TIMEOUT)
# sets the callback when a message arrives on "input1" queue. Messages sent to
# other inputs or to the default will be silently discarded.
self.client.set_message_callback("input1", receive_message_callback, self)
# Forwards the message received onto the next stage in the process.
def forward_event_to_output(self, outputQueueName, event, send_context):
self.client.send_event_async(
outputQueueName, event, send_confirmation_callback, send_context)
def send_message(self, message):
# No callback
# TODO what is the third arg?
self.client.send_event_async(
"output1", message, send_confirmation_callback, 0)
self.client.send_message()
def mypublish(self, topic, msg):
message = construct_message(msg, topic)
self.send_message(message)
print('publishing %s', msg)
def main(protocol):
try:
print ( "\nPython %s\n" % sys.version )
print ( "IoT Hub Client for Python" )
hub_manager = HubManager(protocol)
print ( "Starting the IoT Hub Python sample using protocol %s..." % hub_manager.client_protocol )
print ( "The sample is now waiting for messages and will indefinitely. Press Ctrl-C to exit. ")
while True:
hub_manager.mypublish('testtopic', 'hello world this is a module')
time.sleep(1)
except IoTHubError as iothub_error:
print ( "Unexpected error %s from IoTHub" % iothub_error )
return
except KeyboardInterrupt:
print ( "IoTHubModuleClient sample stopped" )
if __name__ == '__main__':
main(PROTOCOL)
When I build and deploy this it executes on the edge device without errors and in the log, the callback reports that the messages are sent ok. However, no messages come through when I attempt to monitor D2C messages.

I used this to create and send a message from a JSON dict.
new_message = json.dumps(json_obj)
new_message = IoTHubMessage(new_message)
hubManager.forward_event_to_output("output1", new_message, 0)
You can send anything you need, even strings or whatever.

To narrow down the issue, you can install the azureiotedge-simulated-temperature-sensor module published by Microsoft to see whether the issue relative to the Edge environment issue or coding.
I also wrote a sample Python code module based on the Python Module templates which works well for me, you can refer the code below:
# Copyright (c) Microsoft. All rights reserved.
# Licensed under the MIT license. See LICENSE file in the project root for
# full license information.
import random
import time
import sys
import iothub_client
from iothub_client import IoTHubModuleClient, IoTHubClientError, IoTHubTransportProvider
from iothub_client import IoTHubMessage, IoTHubMessageDispositionResult, IoTHubError
# messageTimeout - the maximum time in milliseconds until a message times out.
# The timeout period starts at IoTHubModuleClient.send_event_async.
# By default, messages do not expire.
MESSAGE_TIMEOUT = 10000
# global counters
RECEIVE_CALLBACKS = 0
SEND_CALLBACKS = 0
# Choose HTTP, AMQP or MQTT as transport protocol. Currently only MQTT is supported.
PROTOCOL = IoTHubTransportProvider.MQTT
# Callback received when the message that we're forwarding is processed.
def send_confirmation_callback(message, result, user_context):
global SEND_CALLBACKS
print ( "Confirmation[%d] received for message with result = %s" % (user_context, result) )
map_properties = message.properties()
key_value_pair = map_properties.get_internals()
print ( " Properties: %s" % key_value_pair )
SEND_CALLBACKS += 1
print ( " Total calls confirmed: %d" % SEND_CALLBACKS )
# receive_message_callback is invoked when an incoming message arrives on the specified
# input queue (in the case of this sample, "input1"). Because this is a filter module,
# we will forward this message onto the "output1" queue.
def receive_message_callback(message, hubManager):
global RECEIVE_CALLBACKS
message_buffer = message.get_bytearray()
size = len(message_buffer)
print ( " Data: <<<%s>>> & Size=%d" % (message_buffer[:size].decode('utf-8'), size) )
map_properties = message.properties()
key_value_pair = map_properties.get_internals()
print ( " Properties: %s" % key_value_pair )
RECEIVE_CALLBACKS += 1
print ( " Total calls received: %d" % RECEIVE_CALLBACKS )
hubManager.forward_event_to_output("output1", message, 0)
return IoTHubMessageDispositionResult.ACCEPTED
class HubManager(object):
def __init__(
self,
protocol=IoTHubTransportProvider.MQTT):
self.client_protocol = protocol
self.client = IoTHubModuleClient()
self.client.create_from_environment(protocol)
# set the time until a message times out
self.client.set_option("messageTimeout", MESSAGE_TIMEOUT)
# sets the callback when a message arrives on "input1" queue. Messages sent to
# other inputs or to the default will be silently discarded.
self.client.set_message_callback("input1", receive_message_callback, self)
# Forwards the message received onto the next stage in the process.
def forward_event_to_output(self, outputQueueName, event, send_context):
self.client.send_event_async(
outputQueueName, event, send_confirmation_callback, send_context)
def SendSimulationData(self, msg):
print"sending message..."
message=IoTHubMessage(msg)
self.client.send_event_async(
"output1", message, send_confirmation_callback, 0)
print"finished sending message..."
def main(protocol):
try:
print ( "\nPython %s\n" % sys.version )
print ( "IoT Hub Client for Python" )
hub_manager = HubManager(protocol)
print ( "Starting the IoT Hub Python sample using protocol %s..." % hub_manager.client_protocol )
print ( "The sample is now waiting for messages and will indefinitely. Press Ctrl-C to exit. ")
while True:
hub_manager.SendSimulationData("test msg")
time.sleep(1)
except IoTHubError as iothub_error:
print ( "Unexpected error %s from IoTHub" % iothub_error )
return
except KeyboardInterrupt:
print ( "IoTHubModuleClient sample stopped" )
if __name__ == '__main__':
main(PROTOCOL)

If it can help someone, I think you miss await send_message.
Seems the same problem I answered here

Python/Matlab comm over TCP

Edit
Worked around this problem by switching to UDP. See that updated code snippets after the break.
I'm having trouble with my threaded tcp server in python. The goal is to establish a communication link between Matlab and a Python server to allow for querying a database. This example only works properly once, then I have to restart Matlab to get the message echoed back again.
The Python side:
import SocketServer
from threading import Thread
class service(SocketServer.BaseRequestHandler):
def handle(self):
self.data = 'dummy'
print "Client connected with ", self.client_address
while len(self.data):
self.data = self.request.recv(1024).strip()
print self.data
self.request.sendall(self.data.upper())
print "Client exited"
self.request.close()
class ThreadedTCPServer(SocketServer.ThreadingMixIn, SocketServer.TCPServer):
pass
t = ThreadedTCPServer(('',1522), service)
t.serve_forever()
And the Matlab Code:
t = ThreadedTCPServer(('',1522), service)
t.serve_forever()
t = tcpip('localhost', 1522);
fopen(t);
% write a message
fwrite(t, 'This is a test message.');
% read the echo
bytes = fread(t, [1, t.BytesAvailable]);
char(bytes)
% close the connection
fclose(t);
So, while i'm expecting the data to get echoed back, this only occurs on the first run, i.e. right after I've started Matlab. If I run it again, I get the message on the python side, but it is not echoed again. Ideas?
The Updated Python Code:
x = np.array([[55, 1000, 45], [20, 3, 10]])
class UDP_Interrupt(SocketServer.BaseRequestHandler):
def setup(self):
pass
def handle(self):
data = self.request[0].strip()
print data
socket = self.request[1]
print "{}{} wrote:".format(self.client_address[0], self.client_address)
print data
print x
socket.sendto(x.tostring('C'), self.client_address)
#scipy.io.savemat('/Users/empire/Documents/MATLAB/hybridQuadSim/quaternionController/models/mapData.mat', mdict={'mapData': x})
def finish(self):
pass
class ThreadedUDPServer(SocketServer.ThreadingMixIn, SocketServer.UDPServer):
pass
if __name__ == "__main__":
map_server = ThreadedUDPServer((HOST, PORT), UDP_Interrupt)
# terminate with Ctrl-C
try:
server_thread = Thread(target=map_server.serve_forever)
server_thread.daemon = False
server_thread.start()
print "Server loop running in thread:", server_thread.name
except KeyboardInterrupt:
sys.exit(0)
And for Matlab:
% connect to the server
t = udp('localhost', 2002);
fopen(t);
% write a message
fwrite(t, 'This is a test message.');
% read the echo
bytes = fread(t, [t.BytesAvailable, 1], 'char');
temp = reshape(bytes, [8 6]);
z = [0,0,0,0,0,0]
for col = 1:6
bytepack=uint64(0);
for row = 1:8
temp(9-row, col)
bytepack = bitshift(temp(9 - row, col),8*(8-row));
z(col) = bitor(bytepack,z(col));
temp(row, col);
end;
end;
% close the connection
fclose(t);
This setup seems to work well enough, but would appreciate any feedback or corrections offered. I would like a more robust solution to the problem of reconstructing the numpy array into a matrix in Matlab.

python chat using poll

I need to built a python chat and I'm stacked in the very final step. I've built the server and the client and I have the following problem while running the code:
server.py 127.0.0.1
-in a separate window client.py 127.0.0.1
-another client
-type the nicknames to chat for both clients and get the correct answer 'yuppie' meaning you are connected
a client try to speak
message is not read by the other client until it doesn't print something, after printing it get the message printed on its screen correctly.
I'd like to get the message without being obliged to print something, it's pretty unrealistic!!! Code of client and server are below in 2 different classes. Thank you!
#! /usr/bin/env python
import socket,sys,select,re
PORT=1060
class Server():
def __init__(self,host):
#building listen_sock
self.listen_sock=socket.socket(socket.AF_INET,socket.SOCK_STREAM)
self.listen_sock.setsockopt(socket.SOL_SOCKET,socket.SO_REUSEADDR,1)
self.listen_sock.bind((host,PORT))
self.listen_sock.listen(20)
#building dict for socket and socket state
self.sockets={self.listen_sock.fileno(): self.listen_sock}
self.socket_state={self.listen_sock.fileno():''}
#building poll object
self.poll=select.poll()
self.poll.register(self.listen_sock,select.POLLIN)
#users' list
self.users_list={}
#DON'T LOOK HERE
#initialize the sender
#self.sender=0
# self.users=re.compile("\s*\$(get users connected)$\s*",re.IGNORECASE)
# self.nick=re.compile("\s*\$\$(\w*\d*)\$\$\s*",re.IGNORECASE)
# self.quit=re.compile("\s*\$(quit)\$\s*",re.IGNORECASE)
#self.commands=[self.users,self.nick,self.quit]
#funcion to receive message from client (work well)
def recv_until(self,fd,suffix):
self.message=''
#checking the end of the message
while not self.message.endswith(suffix):
data=self.sockets[fd].recv(16)
if not data:
raise EOFError('socket closed before we saw %r' % suffix)
self.message+=data
self.message=self.message[:-1]
#delete client (work well)
def del_client(self,fd):
del self.users_list[fd]
del self.socket_state[fd]
self.poll.unregister(fd)
#print the remaining active connections
if not len(self.users_list):
print 'Anyone is connected, waiting for new connection'
else:
print self.users_list
#add new client and change the of the file descriptor for that client (work well)
def new_client(self,fd):
newsock, sockname = self.listen_sock.accept()
print 'new connection from ', newsock.getpeername()
newsock.setblocking(False)
#recording the new connection
fd=newsock.fileno()
self.sockets[fd]=newsock
self.poll.register(fd,select.POLLOUT)
self.socket_state[fd]='ask nick'
#DON'T LOOK HERE
# def handle_query(self,fd):
# for n,command in enumerate(self.commands):
# match=command.search(self.message)
# if n==1 and match:
# self.users_list[self.sockets[fd].getpeername()]=match.group(1)
# print self.users_list
# for value in self.users_list.values():
# self.sockets[fd].sendall(value+'\n')
#starting the main function of the class
def chat(self):
while True:
#here il where the code hangs up waitng and waiting (WORKS BAD)
#return a tuple, identify where (fd) the event (event) is happening
for fd,event in self.poll.poll():
#print the state of each socket and the poll object
print self.socket_state
print self.poll.poll()
#starting the state machine
#remove closed sockets
if event & (select.POLLHUP | select.POLLERR |
select.POLLNVAL):
#deleting the socket closed at fd
self.del_client(fd)
#if the socket referred to is our listen_sock and we have a new connection request
elif self.sockets[fd] is self.listen_sock:
#recording the new entry!
self.new_client(fd)
#managing all the situation where it is necessary to answer to a client
#and changing the state of the socket and that of the sockets[fd]
elif event & select.POLLOUT:
if self.socket_state[fd]=='ask nick':
self.sockets[fd].sendall('identify\n')
self.poll.modify(self.sockets[fd],select.POLLIN)
self.socket_state[fd]='get user'
if self.socket_state[fd]=='invalid nick':
self.sockets[fd].sendall('invalid nick\n')
for value in self.users_list.values():
self.sockets[fd].sendall('\n'+value+'\n')
self.socket_state[fd]='ask nick'
if self.socket_state[fd]=='connected':
print '3'
self.sockets[fd].sendall('yuppie\n')
self.poll.modify(self.sockets[fd],select.POLLIN)
self.socket_state[fd]='ready to communicate'
if self.socket_state[fd]=='ready to receive':
self.sockets[fd].sendall(self.message)
print '4'
self.poll.modify(self.sockets[fd],select.POLLIN)
self.socket_state[fd]='ready to communicate'
#managing all the situation where it is necessary to get values from clients
elif event & select.POLLIN:
if self.socket_state[fd]=='get user':
self.recv_until(fd,'\n')
if self.message not in self.users_list.values():
self.users_list[fd]=self.message
self.poll.modify(self.sockets[fd],select.POLLOUT)
self.socket_state[fd]='connected'
else:
self.poll.modify(self.sockets[fd],select.POLLOUT)
self.socket_state[fd]='invalid nick'
if self.socket_state[fd]=='ready to communicate':
self.recv_until(fd,'\n')
print '5'
for i in self.users_list.keys():
if i!=fd:
self.poll.modify(self.sockets[i],select.POLLOUT)
self.socket_state[i]='ready to receive'
if __name__ == '__main__':
se=Server(sys.argv[1])
se.chat()
#! /usr/bin/env python
import sys,socket,select,threading,time
s=socket.socket(socket.AF_INET,socket.SOCK_STREAM)
HOST=sys.argv.pop()
PORT=1060
class Client():
def setup(self):
server_address=(HOST,PORT)
self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.sock.connect(server_address)
def chat(self):
while True:
time.sleep(1)
text=raw_input('>>> ')
self.sock.sendall(text+'\n')
def rec(self):
while True:
mess=self.sock.recv(16)
if mess:
print '$$$ ', mess,
def start(self):
l=threading.Thread(target=self.rec)
t=threading.Thread(target=self.chat)
t.start()
l.start()
if __name__=='__main__':
cl=Client()
cl.setup()
cl.start()

Next time take a look at http://www.zeromq.org/, it has a nice python binding http://zeromq.github.com/pyzmq/. It's perfect for this kind of stuff.

Non blocking python sockets

I'd like to write a small Bluetooth server application to my Nokia phone in PyS60. It needs to be able to send response to the client's request and be able to push data to the client as well.
option 1:
if I use socket.recv(1024), the program waits until something is received, therefore the server can't push data to the client. The Python for S60 implementation is missing the socket.settimeout() method, so I couldn't write a proper non-blocking code.
oprion 2:
The socket.makefile() approach was looking good, but couldn't make it work. When I replaced the conn.recv(1024) to fd = socket.makefile() fd.readline(), it didn't read a thing.
option 3:
Looked into the select() function, but had no luck with it. When I changed the conn.recv() to the r,w,e = select.select([conn],[],[]) like it's been suggested the client doesn't even connect. It hangs at "Waiting for the client...". Strange...
I know that there are pretty nice server implementations and asynchronous API-s as well, but I only need a really basic stuff here. Thanks in advance!
here's what I have:
sock = btsocket.socket(btsocket.AF_BT, btsocket.SOCK_STREAM)
channel = btsocket.bt_rfcomm_get_available_server_channel(sock)
sock.bind(("", channel))
sock.listen(1)
btsocket.bt_advertise_service(u"name", sock, True, btsocket.RFCOMM)
print "Waiting for the client..."
conn, client_mac = sock.accept()
print "connected: " + client_mac
while True:
try:
data = conn.recv(1024)
if len(data) != 0:
print "received [%s]" % data
if data.startswith("something"): conn.send("something\r\n")
else:
conn.send("some other data \r\n")
except:
pass
It's obviously blocking, so the "some other data" is never sent, but it's the best I've got so far. At least I can send something in reply to the client.

Found the solution finally!
The select function wasn't working with the btsocket module of the newer PyS60 ports.
Someone wrote a new_btsocket (available here) with a working select function.

Here is a simple example based on an echo server
#!/usr/bin/python
import socket
import select
server = socket.socket( socket.AF_INET, socket.SOCK_STREAM )
server.bind( ('localhost', 12556) )
server.listen( 5 )
toread = [server]
running = 1
# we will shut down when all clients disconenct
while running:
rready,wready,err = select.select( toread, [], [] )
for s in rready:
if s == server:
# accepting the socket, which the OS passes off to another
# socket so we can go back to selecting. We'll append this
# new socket to the read list we select on next pass
client, address = server.accept()
toread.append( client ) # select on this socket next time
else:
# Not the server's socket, so we'll read
data = s.recv( 1024 )
if data:
print "Received %s" % ( data )
else:
print "Client disconnected"
s.close()
# remove socket so we don't watch an invalid
# descriptor, decrement client count
toread.remove( s )
running = len(toread) - 1
# clean up
server.close()
That said, I still find socketserver cleaner and easier. Implement handle_request and call serve_forever

Here's an Epoll Server Implementation (non-blocking)
http://pastebin.com/vP6KPTwH (same thing as below, felt this might be easier to copy)
use python epollserver.py to start the server.
Test it using wget localhost:8888
import sys
import socket, select
import fcntl
import email.parser
import StringIO
import datetime
"""
See:
http://docs.python.org/library/socket.html
"""
__author__ = ['Caleb Burns', 'Ben DeMott']
def main(argv=None):
EOL1 = '\n\n'
EOL2 = '\n\r\n'
response = 'HTTP/1.0 200 OK\r\nDate: Mon, 1 Jan 1996 01:01:01 GMT\r\n'
response += 'Content-Type: text/plain\r\nContent-Length: 13\r\n\r\n'
response += 'Hello, world!'
serversocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
# Tell the server socket file descriptor to destroy itself when this program ends.
socketFlags = fcntl.fcntl(serversocket.fileno(), fcntl.F_GETFD)
socketFlags |= fcntl.FD_CLOEXEC
fcntl.fcntl(serversocket.fileno(), fcntl.F_SETFD, socketFlags)
serversocket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
serversocket.bind(('0.0.0.0', 8888))
serversocket.listen(1)
# Use asynchronous sockets.
serversocket.setblocking(0)
# Allow a queue of up to 128 requests (connections).
serversocket.listen(128)
# Listen to socket events on the server socket defined by the above bind() call.
epoll = select.epoll()
epoll.register(serversocket.fileno(), select.EPOLLIN)
print "Epoll Server Started..."
try:
#The connection dictionary maps file descriptors (integers) to their corresponding network connection objects.
connections = {}
requests = {}
responses = {}
while True:
# Ask epoll if any sockets have events and wait up to 1 second if no events are present.
events = epoll.poll(1)
# fileno is a file desctiptor.
# event is the event code (type).
for fileno, event in events:
# Check for a read event on the socket because a new connection may be present.
if fileno == serversocket.fileno():
# connection is a new socket object.
# address is client IP address. The format of address depends on the address family of the socket (i.e., AF_INET).
connection, address = serversocket.accept()
# Set new socket-connection to non-blocking mode.
connection.setblocking(0)
# Listen for read events on the new socket-connection.
epoll.register(connection.fileno(), select.EPOLLIN)
connections[connection.fileno()] = connection
requests[connection.fileno()] = b''
responses[connection.fileno()] = response
# If a read event occured, then read the new data sent from the client.
elif event & select.EPOLLIN:
requests[fileno] += connections[fileno].recv(1024)
# Once we're done reading, stop listening for read events and start listening for EPOLLOUT events (this will tell us when we can start sending data back to the client).
if EOL1 in requests[fileno] or EOL2 in requests[fileno]:
epoll.modify(fileno, select.EPOLLOUT)
# Print request data to the console.
epoll.modify(fileno, select.EPOLLOUT)
data = requests[fileno]
eol = data.find("\r\n") #this is the end of the FIRST line
start_line = data[:eol] #get the contents of the first line (which is the protocol information)
# method is POST|GET, etc
method, uri, http_version = start_line.split(" ")
# re-used facebooks httputil library (works well to normalize and parse headers)
headers = HTTPHeaders.parse(data[eol:])
print "\nCLIENT: FD:%s %s: '%s' %s" % (fileno, method, uri, datetime.datetime.now())
# If the client is ready to receive data, sent it out response.
elif event & select.EPOLLOUT:
# Send response a single bit at a time until the complete response is sent.
# NOTE: This is where we are going to use sendfile().
byteswritten = connections[fileno].send(responses[fileno])
responses[fileno] = responses[fileno][byteswritten:]
if len(responses[fileno]) == 0:
# Tell the socket we are no longer interested in read/write events.
epoll.modify(fileno, 0)
# Tell the client we are done sending data and it can close the connection. (good form)
connections[fileno].shutdown(socket.SHUT_RDWR)
# EPOLLHUP (hang-up) events mean the client has disconnected so clean-up/close the socket.
elif event & select.EPOLLHUP:
epoll.unregister(fileno)
connections[fileno].close()
del connections[fileno]
finally:
# Close remaining open socket upon program completion.
epoll.unregister(serversocket.fileno())
epoll.close()
serversocket.close()
#!/usr/bin/env python
#
# Copyright 2009 Facebook
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""HTTP utility code shared by clients and servers."""
class HTTPHeaders(dict):
"""A dictionary that maintains Http-Header-Case for all keys.
Supports multiple values per key via a pair of new methods,
add() and get_list(). The regular dictionary interface returns a single
value per key, with multiple values joined by a comma.
>>> h = HTTPHeaders({"content-type": "text/html"})
>>> h.keys()
['Content-Type']
>>> h["Content-Type"]
'text/html'
>>> h.add("Set-Cookie", "A=B")
>>> h.add("Set-Cookie", "C=D")
>>> h["set-cookie"]
'A=B,C=D'
>>> h.get_list("set-cookie")
['A=B', 'C=D']
>>> for (k,v) in sorted(h.get_all()):
... print '%s: %s' % (k,v)
...
Content-Type: text/html
Set-Cookie: A=B
Set-Cookie: C=D
"""
def __init__(self, *args, **kwargs):
# Don't pass args or kwargs to dict.__init__, as it will bypass
# our __setitem__
dict.__init__(self)
self._as_list = {}
self.update(*args, **kwargs)
# new public methods
def add(self, name, value):
"""Adds a new value for the given key."""
norm_name = HTTPHeaders._normalize_name(name)
if norm_name in self:
# bypass our override of __setitem__ since it modifies _as_list
dict.__setitem__(self, norm_name, self[norm_name] + ',' + value)
self._as_list[norm_name].append(value)
else:
self[norm_name] = value
def get_list(self, name):
"""Returns all values for the given header as a list."""
norm_name = HTTPHeaders._normalize_name(name)
return self._as_list.get(norm_name, [])
def get_all(self):
"""Returns an iterable of all (name, value) pairs.
If a header has multiple values, multiple pairs will be
returned with the same name.
"""
for name, list in self._as_list.iteritems():
for value in list:
yield (name, value)
def items(self):
return [{key: value[0]} for key, value in self._as_list.iteritems()]
def get_content_type(self):
return dict.get(self, HTTPHeaders._normalize_name('content-type'), None)
def parse_line(self, line):
"""Updates the dictionary with a single header line.
>>> h = HTTPHeaders()
>>> h.parse_line("Content-Type: text/html")
>>> h.get('content-type')
'text/html'
"""
name, value = line.split(":", 1)
self.add(name, value.strip())
#classmethod
def parse(cls, headers):
"""Returns a dictionary from HTTP header text.
>>> h = HTTPHeaders.parse("Content-Type: text/html\\r\\nContent-Length: 42\\r\\n")
>>> sorted(h.iteritems())
[('Content-Length', '42'), ('Content-Type', 'text/html')]
"""
h = cls()
for line in headers.splitlines():
if line:
h.parse_line(line)
return h
# dict implementation overrides
def __setitem__(self, name, value):
norm_name = HTTPHeaders._normalize_name(name)
dict.__setitem__(self, norm_name, value)
self._as_list[norm_name] = [value]
def __getitem__(self, name):
return dict.__getitem__(self, HTTPHeaders._normalize_name(name))
def __delitem__(self, name):
norm_name = HTTPHeaders._normalize_name(name)
dict.__delitem__(self, norm_name)
del self._as_list[norm_name]
def get(self, name, default=None):
return dict.get(self, HTTPHeaders._normalize_name(name), default)
def update(self, *args, **kwargs):
# dict.update bypasses our __setitem__
for k, v in dict(*args, **kwargs).iteritems():
self[k] = v
#staticmethod
def _normalize_name(name):
"""Converts a name to Http-Header-Case.
>>> HTTPHeaders._normalize_name("coNtent-TYPE")
'Content-Type'
"""
return "-".join([w.capitalize() for w in name.split("-")])
if(__name__ == '__main__'):
sys.exit(main(sys.argv))