We Keep Coding

Receiving and transmitting data with socket

I'm having trouble with receiving and sending data with Python's socket. In my script I need to listen to incoming data in socket and reading a FIFO file for a response and send it with socket when I find \n. I created separate thread for reading FIFO and it works but sometimes it is really slow. Is it possible to do both things in a main thread? My code:
#!/usr/bin/python
from __future__ import absolute_import, print_function, unicode_literals
from optparse import OptionParser, make_option
import os
import errno
import sys
import socket
import uuid
import dbus
import dbus.service
import dbus.mainloop.glib
import time
from threading import Thread
try:
from gi.repository import GObject
except ImportError:
import gobject as GObject
class ArduinoFifo:
fifofile = -1
OUT_PIPE_FILE = '/tmp/ble_pipe_out'
def removeFile(self, filename):
try:
os.remove(filename)
except OSError as e: # this would be "except OSError, e:" before Python 2.6
if e.errno != errno.ENOENT: # errno.ENOENT = no such file or directory
print(e)
raise # re-raise exception if a different error occured
def createFifo(self):
print('removing pipe file\n')
self.removeFile(self.OUT_PIPE_FILE)
print('making pipe\n')
try:
os.mkfifo(self.OUT_PIPE_FILE, 0777)
except OSError as err:
print (err)
raise
def openFifo(self):
print('waiting to open pipe\n')
try:
self.fifofile = os.open(self.OUT_PIPE_FILE, os.O_WRONLY) # | os.O_NONBLOCK)
except OSError as err:
print (err)
def writeFifo(self, data):
try:
if (self.fifofile == -1):
openFifo(self)
os.write(self.fifofile, data)
except OSError as err:
print (err)
class FIFOReader(Thread):
def __init__(self, server_sock):
super(FIFOReader, self).__init__()
self.server_sock = server_sock
self.daemon = True
self.received_msg = ""
self.cancelled = False
print('remove in fifo')
try:
os.remove("/tmp/ble_pipe_in")
except OSError as e: # this would be "except OSError, e:" before Python 2.6
if e.errno != errno.ENOENT: # errno.ENOENT = no such file or directory
print(e)
raise
print('create in fifo')
try:
os.mkfifo("/tmp/ble_pipe_in", 0777)
except OSError as err:
print (err)
raise
print('open in fifo')
try:
self.fifofile = os.open("/tmp/ble_pipe_in", os.O_RDWR)
except OSError as err:
print (err)
print('fifo in opened')
def run(self):
while not self.cancelled:
print("READING")
self.received_msg += os.read(self.fifofile, 1)
print("read: %s\n" % self.received_msg)
if "\n" in self.received_msg :
print("Sending Message...")
self.server_sock.send(self.received_msg)
self.received_msg = ""
def cancel(self):
self.cancelled = True
myfifo = ArduinoFifo()
class Profile(dbus.service.Object):
fd = -1
#dbus.service.method("org.bluez.Profile1",
in_signature="", out_signature="")
def Release(self):
print("Release")
mainloop.quit()
#dbus.service.method("org.bluez.Profile1",
in_signature="", out_signature="")
def Cancel(self):
print("Cancel")
#dbus.service.method("org.bluez.Profile1",
in_signature="oha{sv}", out_signature="")
def NewConnection(self, path, fd, properties):
global received_msg
self.fd = fd.take()
print("NewConnection(%s, %d)" % (path, self.fd))
server_sock = socket.fromfd(self.fd, socket.AF_UNIX, socket.SOCK_STREAM)
server_sock.setblocking(1)
myfifo.openFifo()
infifo = FIFOReader(server_sock)
infifo.start()
print('enter recv loop\n')
try:
while True:
data = server_sock.recv(1024)
#print("received: %s" % data)
if data:
myfifo.writeFifo(data)
#if data == "h":
#server_sock.send("Hello!\n")
except IOError as err:
print (err)
pass
server_sock.close()
print("all done")
os.kill(os.getpid(), 9)
#dbus.service.method("org.bluez.Profile1",
in_signature="o", out_signature="")
def RequestDisconnection(self, path):
print("RequestDisconnection(%s)" % (path))
if (self.fd > 0):
os.close(self.fd)
self.fd = -1
if __name__ == '__main__':
dbus.mainloop.glib.DBusGMainLoop(set_as_default=True)
bus = dbus.SystemBus()
manager = dbus.Interface(bus.get_object("org.bluez",
"/org/bluez"), "org.bluez.ProfileManager1")
option_list = [
make_option("-C", "--channel", action="store",
type="int", dest="channel",
default=None),
]
parser = OptionParser(option_list=option_list)
(options, args) = parser.parse_args()
options.uuid = "1101"
options.psm = "3"
options.role = "server"
options.name = "Edison SPP Loopback"
options.service = "spp char loopback"
options.path = "/foo/bar/profile"
options.auto_connect = False
options.record = ""
profile = Profile(bus, options.path)
mainloop = GObject.MainLoop()
opts = {
"AutoConnect" : options.auto_connect,
}
if (options.name):
opts["Name"] = options.name
if (options.role):
opts["Role"] = options.role
if (options.psm is not None):
opts["PSM"] = dbus.UInt16(options.psm)
if (options.channel is not None):
opts["Channel"] = dbus.UInt16(options.channel)
if (options.record):
opts["ServiceRecord"] = options.record
if (options.service):
opts["Service"] = options.service
if not options.uuid:
options.uuid = str(uuid.uuid4())
manager.RegisterProfile(options.path, options.uuid, opts)
myfifo.createFifo()
mainloop.run()
EDIT: I think the problem is in writing data to FIFO or receiving incoming data from socket because in my code in C I've got this delay when I want to read a data from the input FIFO using fgets function.
EIDT2: I use this to instantly receive a message and sends a response one after another

I doubt the issue has to do with the separate thread. "Threads" in Python aren't necessarily OS-level threads but could just be operations the main OS-level thread processes asynchronously. In cPython, which most people use, this is how they work. But I do see a couple of possible issues:
I'm not familiar with some of these libs, but os.read(self.fifofile, 1) stands out. If you use the builtin open() (not the one in os) or BufferedReader, this would be buffered and therefore ok. But os.open is a low-level call that doesn't buffer reads or writes, so you're actually reading 1 byte at a time from the file handle this way, which isn't a good idea as it can cause slowdowns for a variety of hard-to-trace reasons. You should either use a higher level library for this or do the buffering yourself.
Secondly, your +='ing of the read input to the message string repeatedly is going to be slow if your Python interpreter is creating a new string internally each time. So you could be looking at O(N^2) where N is message size time complexity for something that should be O(N). It depends on your interpreter, so to make things portable, you should be appending to a list instead.
Unrelated, but if you don't know whether your FIFO file is text, you shouldn't open it in text mode or else you'll run into errors. Strings only allow valid text bytes, UTF-8 if it's Py3 and I think ASCII if it's Py2, and you'll get an error if you receive, say, 0x00.
Hope this helps.

python socket tchat issue

I started to code in python with sockets and I have a little problem for my chat script.
Server script
import pickle, socket, struct, sys, threading
SERVERADDRESS = ("localhost", 6030)
class helloChatServer(threading.Thread):
def __init__(self):
threading.Thread.__init__(self)
self.__server = socket.socket()
self.users = []
try:
self.__server.bind(SERVERADDRESS)
except socket.error:
print('Bind failed {}'.format(socket.error))
self.__server.listen(10)
def exit(self):
self.__server.close()
def run(self):
print( "Listening... {}".format(SERVERADDRESS))
while True:
client, addr = self.__server.accept()
try:
threading.Thread(target=self._handle, args=(client, addr)).start()
except OSError:
print('Error during processing the message')
def _handle(self, client, addr):
print('Client connected with {}:{}'.format(addr[0], str(addr[1])))
self.users.append(addr)
while True:
data = client.recv(1024)
print(data)
client.send(data)
client.close()
if __name__ == '__main__':
helloChatServer().run()
Client script
import pickle, socket, struct, sys, threading
SERVERADDRESS = (socket.gethostname(), 6050)
class helloChatClient():
def __init__(self, host='localhost', port=5000, pseudo="Visitor"):
self.__socket = socket.socket()
self.__socket.bind((host, port))
self.__pseudo = pseudo
print('Listening on {}:{}'.format(host, port))
def run(self):
handlers = {
'/exit': self._exit,
'/quit': self._quit,
'/join': self._join,
'/send': self._send
}
self.__running = True
self.__address = None
threading.Thread(target=self._receive).start()
while self.__running:
line = sys.stdin.readline().rstrip() + ' '
# Extract the command and the param
command = line[:line.index(' ')]
param = line[line.index(' ')+1:].rstrip()
# Call the command handler
if command in handlers:
try:
handlers[command]() if param == '' else handlers[command](param)
except:
print("Error during the execution of the message")
else:
print('Command inconnue:', command)
def _exit(self):
self.__running = False
self.__address = None
self.__socket.close()
def _quit(self):
self.__address = None
def _join(self, param):
if self.__pseudo == "Visitor":
self.__pseudo = input("Choose a username: ")
tokens = param.split(' ')
if len(tokens) == 2:
try:
self.__address = (tokens[0], int(tokens[1]))
self.__socket.connect(self.__address)
print('~~~~~~~~~~~~~~~~~~~~~~~~~~')
print('Connected at {}:{}'.format(*self.__address))
print('~~~~~~~~~~~~~~~~~~~~~~~~~~')
except OSError:
print("Error during the sending of the message")
self.__socket.send(self.__pseudo.encode())
def _send(self, param):
if self.__address is not None:
try:
message = param.encode()
totalsent = 0
while totalsent < len(message):
sent = self.__socket.send(message[totalsent:])
totalsent += sent
print(self.__socket.recv(1024).decode())
except OSError:
print('Error during the reception of the message')
def _receive(self):
while self.__running:
try:
data = self.__socket.recv(1024).decode()
print(data)
except socket.timeout:
pass
except OSError:
return
if __name__ == '__main__':
if len(sys.argv) == 4:
helloChatClient(sys.argv[1], int(sys.argv[2]), sys.argv[3]).run()
else:
helloChatClient().run()
Well when I run the script on the terminal, I see this.
Server
MacBook-Pro-de-Saikou:labo2 saikouah$ python3.4 helloChatServer.py
En écoute sur... ('MacBook-Pro-de-Saikou.local', 6030)
Client connected with 127.0.0.1:5004
Il y a actuellement 1 connecté
b'bluebeel'
b'hello'
Client
MacBook-Pro-de-Saikou:labo2 saikouah$ python3.4 helloChatClient.py localhost 5004 bluebeel
Écoute sur localhost:5004
/join MacBook-Pro-de-Saikou.local 6030
~~~~~~~~~~~~~~~~~~~~~~~~~~
Connecté à MacBook-Pro-de-Saikou.local:6030
~~~~~~~~~~~~~~~~~~~~~~~~~~
/send hello
bluebeel
On the client terminal he doesn't print me hello but bluebeel. I made several test and he took me every time the previous one message. Looks like he is late.
Someone can help me? :)

PROBLEM ANALYSIS
Your code fails in _receive function:
data = self.__socket.recv(1024).decode()
This line throws OSError because you try to call .recv before connecting to the server. Thus the exception handler is fired and the function exits. So what happens is that after calling
threading.Thread(target=self._receive).start()
function _receive exits before you call /join. So watch what happens
You call /join.
bluebeel is send to the server
Server receives it and sends it back to the client
But _receive function is no longer there. So the message is "stacked" on the socket (it will wait for next .recv() call)
You call /send hello
Server receives hello and sends it back
Client calls print(self.__socket.recv(1024).decode()) in _send method
But .recv retrieves the first message that is stacked on the socket. In that case it is not hello, it is bluebeel.
Now this schema continues to work. You send message, server pings it back but there's always 1 message in front of the received one. The "late" message.
SOLUTION
One way of solving this issue is to call
threading.Thread(target=self._receive).start()
in ._join method after .connect. Remember to remove print(self.__socket.recv(1024).decode()) from _send method, otherwise it will block stdin.
Of course you will have problems when issuing multiple /join commands. To properly address that you would have to keep track of _receive thread and kill it at the begining of ._join method. This however is beyond the scope of this question IMHO.
SIDE NOTE
Don't ever handle exceptions like you did. This is wrong:
try:
data = self.__socket.recv(1024).decode()
print(data)
except socket.timeout:
pass
except OSError:
return
At least do this:
import traceback
try:
data = self.__socket.recv(1024).decode()
print(data)
except socket.timeout:
traceback.print_exc()
except OSError:
traceback.print_exc()
return

python Time a try except

My problem is very simple.
I have a try/except code. In the try I have some http requests attempts and in the except I have several ways to deal with the exceptions I'm getting.
Now I want to add a time parameter to my code. Which means the try will only last for 'n' seconds. otherwise catch it with except.
In free language it would appear as:
try for n seconds:
doSomthing()
except (after n seconds):
handleException()
this is mid-code. Not a function. and I have to catch the timeout and handle it. I cannot just continue the code.
while (recoveryTimes > 0):
try (for 10 seconds):
urllib2.urlopen(req)
response = urllib2.urlopen(req)
the_page = response.read()
recoveryTimes = 0
except (urllib2.URLError, httplib.BadStatusLine) as e:
print str(e.__unicode__())
print sys.exc_info()[0]
recoveryTimes -= 1
if (recoveryTimes > 0):
print "Retrying request. Requests left %s" %recoveryTimes
continue
else:
print "Giving up request, changing proxy."
setUrllib2Proxy()
break
except (timedout, 10 seconds has passed)
setUrllib2Proxy()
break
The solution I need is for the try (for 10 seconds)
and the except (timeout, after 10 seconds)

Check the documentation
import urllib2
request = urllib2.Request('http://www.yoursite.com')
try:
response = urllib2.urlopen(request, timeout=4)
content = response.read()
except urllib2.URLError, e:
print e
If you want to catch more specific errors check this post
or alternatively for requests
import requests
try:
r = requests.get(url,timeout=4)
except requests.exceptions.Timeout as e:
# Maybe set up for a retry
print e
except requests.exceptions.RequestException as e:
print e
More about exceptions while using requests can be found in docs or in this post

A generic solution if you are using UNIX:
import time as time
import signal
#Close session
def handler(signum, frame):
print 1
raise Exception('Action took too much time')
signal.signal(signal.SIGALRM, handler)
signal.alarm(3) #Set the parameter to the amount of seconds you want to wait
try:
#RUN CODE HERE
for i in range(0,5):
time.sleep(1)
except:
print 2
signal.alarm(10) #Resets the alarm to 10 new seconds
signal.alarm(0) #Disables the alarm

Ignoring exceptions for a specific amount of time

Trying to make the try to run in a loop, since I am booting a machine containing the webserver and I want to make it run and not just go direct to the except and stop the script. I have made a while for the http-status code, but that does only work if the machine is up.
So my question is how can I make the try loop for like 5 minutes before it goes to the except? Sorry for my poor explanation.
try:
r = requests.head("http://www.testing.co.uk")
while r.status_code != 200:
print "Response not == to 200."
time.sleep(30)
r = requests.head("http://www.testing.co.uk")
else:
print "Response is 200 - OK"
except requests.ConnectionError:
print "NOT FOUND - ERROR"

You could do something like:
import requests, time, datetime
# Determine "end" time -- in this case, 5 minutes from now
t_end = datetime.datetime.now() + datetime.timedelta(minutes=5)
while True:
try:
r = requests.head("http://www.testing.co.uk")
if r.status_code != 200:
# Do something
print "Response not == to 200."
else:
# Do something else
print "Response is 200 - OK"
break # Per comments
time.sleep(30) # Wait 30 seconds between requests
except requests.ConnectionError as e:
print "NOT FOUND - ERROR"
# If the time is past the end time, re-raise the exception
if datetime.datetime.now() > t_end: raise e
time.sleep(30) # Wait 30 seconds between requests
The important line is:
if datetime.datetime.now() > t_end: raise e
If the condition isn't met (less that 5 minutes have elapsed), the exception is silently ignored and the while loop continues.
If the condition is met, the exception is re-raised to be handled by some other, outer code or not handled at all -- in which case you'll see the exception "break" (in your words) the program.
The benefit of using this approach over something like (instead of while True:):
while datetime.datetime.now() > t_end:
is that if you find yourself outside of the while loop, you know you got there from break and not from 5 minutes elapsing. You also preserve the exception in case you want to do something special in that case.

detect disconnect persistant curl connection

Where should I check for a disconnect in a pycurl persistant connection?
Somewhere in my script the connection is dying/timing out/throwing an error but the script stays open. I need to detect the problem so I can restart the script.
We are connecting to gnip (a social media data provider)
My code is here: https://gist.github.com/3353033
I've read over the options for libcurl and I read through the php curl_setopts docs because they also leverage libcurl.
class Client:
time_start = time.time()
content = ""
def __init__(self,options):
self.options = options
self.buffer = ""
self.conn = pycurl.Curl()
self.conn.setopt(pycurl.USERPWD, "%s:%s" % (USER, PASS))
self.conn.setopt(pycurl.ENCODING,'gzip')
self.conn.setopt(pycurl.URL, STREAM_URL)
self.conn.setopt(pycurl.WRITEFUNCTION, self.on_receive)
self.conn.setopt(pycurl.FOLLOWLOCATION,1)
self.conn.setopt(pycurl.MAXREDIRS, 5)
self.conn.setopt(pycurl.COOKIEFILE,"cookie.txt")
try:
self.conn.perform()
except Exception,e:
print e.message
def on_receive(self, data):
self.buffer += data
if data.endswith("\r\n") and self.buffer.strip():
if(self.triggered()):
if(len(self.buffer) != 0 ):
try:
SaveThread(self.buffer).start()
except Exception, e:
print "something i commented would have told you there was an error"
system.exit(1)
self.buffer = ""
def triggered(self):
# First trigger based on size then based on time..
if (len(self.buffer) > SAVE_FILE_LENGTH):
return True
time_end = time.time()
if (((time_end - self.time_start) > ROLL_DURATION)): #for the time frame
self.time_start=time.time()
return True
return False
edit: i've fixed the gist

In the above code system.exit(1) should be sys.exit(1) right?
Other than that do you have any more bare except clauses that might be catching the SystemExit exception raised by sys.exit(1)?