Where should I check for a disconnect in a pycurl persistant connection?
Somewhere in my script the connection is dying/timing out/throwing an error but the script stays open. I need to detect the problem so I can restart the script.
We are connecting to gnip (a social media data provider)
My code is here: https://gist.github.com/3353033
I've read over the options for libcurl and I read through the php curl_setopts docs because they also leverage libcurl.
class Client:
time_start = time.time()
content = ""
def __init__(self,options):
self.options = options
self.buffer = ""
self.conn = pycurl.Curl()
self.conn.setopt(pycurl.USERPWD, "%s:%s" % (USER, PASS))
self.conn.setopt(pycurl.ENCODING,'gzip')
self.conn.setopt(pycurl.URL, STREAM_URL)
self.conn.setopt(pycurl.WRITEFUNCTION, self.on_receive)
self.conn.setopt(pycurl.FOLLOWLOCATION,1)
self.conn.setopt(pycurl.MAXREDIRS, 5)
self.conn.setopt(pycurl.COOKIEFILE,"cookie.txt")
try:
self.conn.perform()
except Exception,e:
print e.message
def on_receive(self, data):
self.buffer += data
if data.endswith("\r\n") and self.buffer.strip():
if(self.triggered()):
if(len(self.buffer) != 0 ):
try:
SaveThread(self.buffer).start()
except Exception, e:
print "something i commented would have told you there was an error"
system.exit(1)
self.buffer = ""
def triggered(self):
# First trigger based on size then based on time..
if (len(self.buffer) > SAVE_FILE_LENGTH):
return True
time_end = time.time()
if (((time_end - self.time_start) > ROLL_DURATION)): #for the time frame
self.time_start=time.time()
return True
return False
edit: i've fixed the gist
In the above code system.exit(1) should be sys.exit(1) right?
Other than that do you have any more bare except clauses that might be catching the SystemExit exception raised by sys.exit(1)?
Related
i'm getting some occasional Unpickling errors, but more times than not, it works fine. Essentially I'm generating images on the server side, and using pickle to transmit them to the client side.
Essentially, I am using my send() function to let the client know how many bytes the pickled data is, so it can use rscSock.recv() with the amount of bytes+1 when I use conn.send(graphs) to prevent this exact thing. And it works, most of the time. Occasionally I'm getting pickle truncated, and I can't seem to find out why. I tried using a while loop to receive it in blocks of 4096 from code I found on here (python 3.6 socket pickle data was truncated), but it hangs on the recv. Not sure what to do.
Server Code:
elif cmd['cmd'] == 'RSC_VIEW_GRAPHS':
graphs = pickle.dumps(genGraphs(userSession['uid'], cmd['arg0'], cmd['arg1']))
send(conn, 'RSC_IMG_DATA', len(graphs))
conn.send(graphs)
del graphs
Client Code
send(rscSock, 'RSC_VIEW_GRAPHS', radioVar.get(), str(dateObj.date()))
resp = receive(rscSock)
if resp['resp'] == 'RSC_IMG_DATA':
graphs = pickle.loads(rscSock.recv(int(resp['arg0'])+1))
The graphs variable being fulfilled by the genGraphs() function is returning an array of BytesIO objects, as shown here by the end of the genGraphs() function:
imgs = []
for x in statDict:
# Filler Code removed, irrelevant to post
imgs.append(io.BytesIO())
plt.savefig(imgs[-1], format='png')
plt.close()
return imgs
And lastly, here are the send() and receive() functions for both the client and server:
Client
def send(conn, cmd, *argv):
try:
cmdObj = {'cmd': cmd}
y = 0
for x in argv:
cmdObj['arg'+str(y)] = x
y+=1
cmdObj['key'] = sessionKey
obj = str.encode(json.dumps(cmdObj))
objLen = str(len(obj)).encode()
if conn.send(objLen):
if conn.recv(12).decode() == "RSC_LEN_OK":
if conn.send(obj):
if debug == 1: print("Sending '", obj, "' with length '", objLen ,"'")
return True
return False
except (ConnectionResetError, ConnectionAbortedError):
if cmdObj['arg1'] == 0:
return True
else:
messagebox.showerror("Real Estate Stat Counter", "Lost server connection. Please log back in.")
return False
def receive(conn):
try:
dataSize = int(conn.recv(8))
if dataSize < 16384:
conn.send(str.encode("RSC_LEN_OK"))
data = json.loads(conn.recv(dataSize).decode())
if debug == 1: print("Received '", data, "' with length '", dataSize ,"'")
return data
else:
conn.send(str.encode("RSC_LEN_NO"))
return False
except (OSError, UnicodeDecodeError, json.decoder.JSONDecodeError) as e:
return False
Server
def send(conn, resp, *argv):
try:
respObj = {'resp': resp}
y = 0
for x in argv:
respObj["arg"+str(y)] = x
y+=1
obj = str.encode(json.dumps(respObj))
objLen = str(len(obj)).encode()
if conn.send(objLen):
if conn.recv(12).decode() == "RSC_LEN_OK":
if conn.send(obj):
if debug == 1: print("Sending '", obj, "' with length '", objLen ,"'")
return True
logging.log("WARN: send() failure")
return False
except ConnectionResetError:
logging.log("INFO: Client connection lost, terming socket")
conn.close()
return False
def receive(conn):
try:
dataSize = int(conn.recv(8))
if dataSize < 16384:
conn.send(str.encode("RSC_LEN_OK"))
data = json.loads(conn.recv(dataSize).decode())
if debug == 1: print("Received '", data, "' with length '", dataSize ,"'")
return data
else:
conn.send(str.encode("RSC_LEN_NO"))
return False
except (OSError, UnicodeDecodeError, json.decoder.JSONDecodeError) as e:
logging.log("WARN: receive() received raw data:", conn.recv(16384).decode())
logging.log("WARN: receive() exception:", e)
return False
except ValueError:
logging.log("WARN: receive() did not get a valid byte length first")
return False
So based on jasonharper's comment, I ended up revising only the client code:
data = []
while len(b"".join(data)) < int(resp['arg0']):
data.append(rscSock.recv(4096))
graphs = pickle.loads(b"".join(data))
Now it will check in a loop if it's received the amount of bytes that it was originally told before sending the BytesIO object array.
Working good so far!!
I've recently been trying to create a torrent client in python, and have just got the UDP announce protocol to work.
The tracker accepts my connect request just fine but only returns my IP and port as the peer list when I announce to it...
I've tried to look at the same torrents in other torrent clients and they have multiple working peers while my request only shows me my computer (I've tried this on many torrents, all return just my IP and port)
Here's the code for the sending function itself:
async def announce_udp(self, try_num = 1):
self.sock.settimeout(15)
answer = {}
inner_while = False
while try_num < 4:
while try_num < 4:
try:
print("trying to send")
sended = self.send(1, self.announce_payload())
print("sending the following packet: {0}".format(sended))
print(self.url)
inner_while = True
break
except Exception:
print("problem in sending")
try_num += 1
if not inner_while:
break
try:
answer = self.interpret(15)
break
except Exception:
print("problem in receiving")
try_num += 1
print("announce answer is: {0}".format(answer))
return answer
here's the code for the make payload function:
def announce_payload(self, downloaded = 0, left = 0, uploaded = 0, event = 0, key = get_transaction_id()):
payload = [self.torrent.get_torrent_info_hash_decoded(), get_peer_id().encode(), downloaded,
self.torrent.get_torrent_size(), uploaded, event, 0, key, -1, 6988]
p_tosend = None
try:
p_tosend = struct.pack('!20s20sqqqiIIiH', *payload)
except Exception as e:
print("there was an error: {0}".format(e))
return p_tosend
here's the code for the interpret + process function:
def interpret(self, timeout=10):
self.sock.settimeout(timeout)
print("got to interpret")
try:
response = self.sock.recv(10240)
print("answer recieved")
except socket.timeout:
print("no answer, try again")
raise TrackerResponseException("no answer", 0)
headers = response[:8]
payload = response[8:]
action, trans_id = struct.unpack('!ll', headers)
try:
trans = self.transactions[trans_id]
except KeyError:
raise TrackerResponseException("InvalidTransaction: id not found", trans_id)
try:
trans['response'] = self.process(action, payload, trans)
except Exception as e:
trans['response'] = None
print("error occured: {0}".format(e))
trans['completed'] = True
del self.transactions[trans_id]
#print(trans)
return trans
def process_announce(self, payload, trans):
response = {}
info = payload[:struct.calcsize("!lll")]
interval, leechers, seeders = struct.unpack("!lll", info)
print(interval, leechers, seeders, "noamsssssss")
peer_data = payload[struct.calcsize("!lll"):]
peer_size = struct.calcsize("!lH")
num_of_peers = int(len(peer_data) / peer_size)
print("the number of peers is: {0} and the peer data is: {1}".format(num_of_peers, peer_data))
print()
peers = []
for peer_offset in range(num_of_peers):
off = peer_size * peer_offset
peer = peer_data[off:off + peer_size]
addr, port = struct.unpack("!lH", peer)
peers.append({
'addr': socket.inet_ntoa(struct.pack('!L', addr)),
'port': port,
})
print(payload)
return dict(interval=interval, leechers=leechers, seeders=seeders, peers=peers)
I'm sorry if any of this is irrelevant, but I want to give you all of the code incase it tells you something.
(get_peer_id() returns a random peer id per the tracker protocol specification, and the get_transaction_id() returns random.randint(0, 1 << 32 - 1))
EDIT:
Alright, I've found the problem and now I'm feeling pretty dumb...
turns out even in the udp tracker whenever you send the info hash it has to be SHA1 encoded.
Hopefully this can help someone if they are stuck in the same problem :)
I'm having trouble with receiving and sending data with Python's socket. In my script I need to listen to incoming data in socket and reading a FIFO file for a response and send it with socket when I find \n. I created separate thread for reading FIFO and it works but sometimes it is really slow. Is it possible to do both things in a main thread? My code:
#!/usr/bin/python
from __future__ import absolute_import, print_function, unicode_literals
from optparse import OptionParser, make_option
import os
import errno
import sys
import socket
import uuid
import dbus
import dbus.service
import dbus.mainloop.glib
import time
from threading import Thread
try:
from gi.repository import GObject
except ImportError:
import gobject as GObject
class ArduinoFifo:
fifofile = -1
OUT_PIPE_FILE = '/tmp/ble_pipe_out'
def removeFile(self, filename):
try:
os.remove(filename)
except OSError as e: # this would be "except OSError, e:" before Python 2.6
if e.errno != errno.ENOENT: # errno.ENOENT = no such file or directory
print(e)
raise # re-raise exception if a different error occured
def createFifo(self):
print('removing pipe file\n')
self.removeFile(self.OUT_PIPE_FILE)
print('making pipe\n')
try:
os.mkfifo(self.OUT_PIPE_FILE, 0777)
except OSError as err:
print (err)
raise
def openFifo(self):
print('waiting to open pipe\n')
try:
self.fifofile = os.open(self.OUT_PIPE_FILE, os.O_WRONLY) # | os.O_NONBLOCK)
except OSError as err:
print (err)
def writeFifo(self, data):
try:
if (self.fifofile == -1):
openFifo(self)
os.write(self.fifofile, data)
except OSError as err:
print (err)
class FIFOReader(Thread):
def __init__(self, server_sock):
super(FIFOReader, self).__init__()
self.server_sock = server_sock
self.daemon = True
self.received_msg = ""
self.cancelled = False
print('remove in fifo')
try:
os.remove("/tmp/ble_pipe_in")
except OSError as e: # this would be "except OSError, e:" before Python 2.6
if e.errno != errno.ENOENT: # errno.ENOENT = no such file or directory
print(e)
raise
print('create in fifo')
try:
os.mkfifo("/tmp/ble_pipe_in", 0777)
except OSError as err:
print (err)
raise
print('open in fifo')
try:
self.fifofile = os.open("/tmp/ble_pipe_in", os.O_RDWR)
except OSError as err:
print (err)
print('fifo in opened')
def run(self):
while not self.cancelled:
print("READING")
self.received_msg += os.read(self.fifofile, 1)
print("read: %s\n" % self.received_msg)
if "\n" in self.received_msg :
print("Sending Message...")
self.server_sock.send(self.received_msg)
self.received_msg = ""
def cancel(self):
self.cancelled = True
myfifo = ArduinoFifo()
class Profile(dbus.service.Object):
fd = -1
#dbus.service.method("org.bluez.Profile1",
in_signature="", out_signature="")
def Release(self):
print("Release")
mainloop.quit()
#dbus.service.method("org.bluez.Profile1",
in_signature="", out_signature="")
def Cancel(self):
print("Cancel")
#dbus.service.method("org.bluez.Profile1",
in_signature="oha{sv}", out_signature="")
def NewConnection(self, path, fd, properties):
global received_msg
self.fd = fd.take()
print("NewConnection(%s, %d)" % (path, self.fd))
server_sock = socket.fromfd(self.fd, socket.AF_UNIX, socket.SOCK_STREAM)
server_sock.setblocking(1)
myfifo.openFifo()
infifo = FIFOReader(server_sock)
infifo.start()
print('enter recv loop\n')
try:
while True:
data = server_sock.recv(1024)
#print("received: %s" % data)
if data:
myfifo.writeFifo(data)
#if data == "h":
#server_sock.send("Hello!\n")
except IOError as err:
print (err)
pass
server_sock.close()
print("all done")
os.kill(os.getpid(), 9)
#dbus.service.method("org.bluez.Profile1",
in_signature="o", out_signature="")
def RequestDisconnection(self, path):
print("RequestDisconnection(%s)" % (path))
if (self.fd > 0):
os.close(self.fd)
self.fd = -1
if __name__ == '__main__':
dbus.mainloop.glib.DBusGMainLoop(set_as_default=True)
bus = dbus.SystemBus()
manager = dbus.Interface(bus.get_object("org.bluez",
"/org/bluez"), "org.bluez.ProfileManager1")
option_list = [
make_option("-C", "--channel", action="store",
type="int", dest="channel",
default=None),
]
parser = OptionParser(option_list=option_list)
(options, args) = parser.parse_args()
options.uuid = "1101"
options.psm = "3"
options.role = "server"
options.name = "Edison SPP Loopback"
options.service = "spp char loopback"
options.path = "/foo/bar/profile"
options.auto_connect = False
options.record = ""
profile = Profile(bus, options.path)
mainloop = GObject.MainLoop()
opts = {
"AutoConnect" : options.auto_connect,
}
if (options.name):
opts["Name"] = options.name
if (options.role):
opts["Role"] = options.role
if (options.psm is not None):
opts["PSM"] = dbus.UInt16(options.psm)
if (options.channel is not None):
opts["Channel"] = dbus.UInt16(options.channel)
if (options.record):
opts["ServiceRecord"] = options.record
if (options.service):
opts["Service"] = options.service
if not options.uuid:
options.uuid = str(uuid.uuid4())
manager.RegisterProfile(options.path, options.uuid, opts)
myfifo.createFifo()
mainloop.run()
EDIT: I think the problem is in writing data to FIFO or receiving incoming data from socket because in my code in C I've got this delay when I want to read a data from the input FIFO using fgets function.
EIDT2: I use this to instantly receive a message and sends a response one after another
I doubt the issue has to do with the separate thread. "Threads" in Python aren't necessarily OS-level threads but could just be operations the main OS-level thread processes asynchronously. In cPython, which most people use, this is how they work. But I do see a couple of possible issues:
I'm not familiar with some of these libs, but os.read(self.fifofile, 1) stands out. If you use the builtin open() (not the one in os) or BufferedReader, this would be buffered and therefore ok. But os.open is a low-level call that doesn't buffer reads or writes, so you're actually reading 1 byte at a time from the file handle this way, which isn't a good idea as it can cause slowdowns for a variety of hard-to-trace reasons. You should either use a higher level library for this or do the buffering yourself.
Secondly, your +='ing of the read input to the message string repeatedly is going to be slow if your Python interpreter is creating a new string internally each time. So you could be looking at O(N^2) where N is message size time complexity for something that should be O(N). It depends on your interpreter, so to make things portable, you should be appending to a list instead.
Unrelated, but if you don't know whether your FIFO file is text, you shouldn't open it in text mode or else you'll run into errors. Strings only allow valid text bytes, UTF-8 if it's Py3 and I think ASCII if it's Py2, and you'll get an error if you receive, say, 0x00.
Hope this helps.
I started to code in python with sockets and I have a little problem for my chat script.
Server script
import pickle, socket, struct, sys, threading
SERVERADDRESS = ("localhost", 6030)
class helloChatServer(threading.Thread):
def __init__(self):
threading.Thread.__init__(self)
self.__server = socket.socket()
self.users = []
try:
self.__server.bind(SERVERADDRESS)
except socket.error:
print('Bind failed {}'.format(socket.error))
self.__server.listen(10)
def exit(self):
self.__server.close()
def run(self):
print( "Listening... {}".format(SERVERADDRESS))
while True:
client, addr = self.__server.accept()
try:
threading.Thread(target=self._handle, args=(client, addr)).start()
except OSError:
print('Error during processing the message')
def _handle(self, client, addr):
print('Client connected with {}:{}'.format(addr[0], str(addr[1])))
self.users.append(addr)
while True:
data = client.recv(1024)
print(data)
client.send(data)
client.close()
if __name__ == '__main__':
helloChatServer().run()
Client script
import pickle, socket, struct, sys, threading
SERVERADDRESS = (socket.gethostname(), 6050)
class helloChatClient():
def __init__(self, host='localhost', port=5000, pseudo="Visitor"):
self.__socket = socket.socket()
self.__socket.bind((host, port))
self.__pseudo = pseudo
print('Listening on {}:{}'.format(host, port))
def run(self):
handlers = {
'/exit': self._exit,
'/quit': self._quit,
'/join': self._join,
'/send': self._send
}
self.__running = True
self.__address = None
threading.Thread(target=self._receive).start()
while self.__running:
line = sys.stdin.readline().rstrip() + ' '
# Extract the command and the param
command = line[:line.index(' ')]
param = line[line.index(' ')+1:].rstrip()
# Call the command handler
if command in handlers:
try:
handlers[command]() if param == '' else handlers[command](param)
except:
print("Error during the execution of the message")
else:
print('Command inconnue:', command)
def _exit(self):
self.__running = False
self.__address = None
self.__socket.close()
def _quit(self):
self.__address = None
def _join(self, param):
if self.__pseudo == "Visitor":
self.__pseudo = input("Choose a username: ")
tokens = param.split(' ')
if len(tokens) == 2:
try:
self.__address = (tokens[0], int(tokens[1]))
self.__socket.connect(self.__address)
print('~~~~~~~~~~~~~~~~~~~~~~~~~~')
print('Connected at {}:{}'.format(*self.__address))
print('~~~~~~~~~~~~~~~~~~~~~~~~~~')
except OSError:
print("Error during the sending of the message")
self.__socket.send(self.__pseudo.encode())
def _send(self, param):
if self.__address is not None:
try:
message = param.encode()
totalsent = 0
while totalsent < len(message):
sent = self.__socket.send(message[totalsent:])
totalsent += sent
print(self.__socket.recv(1024).decode())
except OSError:
print('Error during the reception of the message')
def _receive(self):
while self.__running:
try:
data = self.__socket.recv(1024).decode()
print(data)
except socket.timeout:
pass
except OSError:
return
if __name__ == '__main__':
if len(sys.argv) == 4:
helloChatClient(sys.argv[1], int(sys.argv[2]), sys.argv[3]).run()
else:
helloChatClient().run()
Well when I run the script on the terminal, I see this.
Server
MacBook-Pro-de-Saikou:labo2 saikouah$ python3.4 helloChatServer.py
En écoute sur... ('MacBook-Pro-de-Saikou.local', 6030)
Client connected with 127.0.0.1:5004
Il y a actuellement 1 connecté
b'bluebeel'
b'hello'
Client
MacBook-Pro-de-Saikou:labo2 saikouah$ python3.4 helloChatClient.py localhost 5004 bluebeel
Écoute sur localhost:5004
/join MacBook-Pro-de-Saikou.local 6030
~~~~~~~~~~~~~~~~~~~~~~~~~~
Connecté à MacBook-Pro-de-Saikou.local:6030
~~~~~~~~~~~~~~~~~~~~~~~~~~
/send hello
bluebeel
On the client terminal he doesn't print me hello but bluebeel. I made several test and he took me every time the previous one message. Looks like he is late.
Someone can help me? :)
PROBLEM ANALYSIS
Your code fails in _receive function:
data = self.__socket.recv(1024).decode()
This line throws OSError because you try to call .recv before connecting to the server. Thus the exception handler is fired and the function exits. So what happens is that after calling
threading.Thread(target=self._receive).start()
function _receive exits before you call /join. So watch what happens
You call /join.
bluebeel is send to the server
Server receives it and sends it back to the client
But _receive function is no longer there. So the message is "stacked" on the socket (it will wait for next .recv() call)
You call /send hello
Server receives hello and sends it back
Client calls print(self.__socket.recv(1024).decode()) in _send method
But .recv retrieves the first message that is stacked on the socket. In that case it is not hello, it is bluebeel.
Now this schema continues to work. You send message, server pings it back but there's always 1 message in front of the received one. The "late" message.
SOLUTION
One way of solving this issue is to call
threading.Thread(target=self._receive).start()
in ._join method after .connect. Remember to remove print(self.__socket.recv(1024).decode()) from _send method, otherwise it will block stdin.
Of course you will have problems when issuing multiple /join commands. To properly address that you would have to keep track of _receive thread and kill it at the begining of ._join method. This however is beyond the scope of this question IMHO.
SIDE NOTE
Don't ever handle exceptions like you did. This is wrong:
try:
data = self.__socket.recv(1024).decode()
print(data)
except socket.timeout:
pass
except OSError:
return
At least do this:
import traceback
try:
data = self.__socket.recv(1024).decode()
print(data)
except socket.timeout:
traceback.print_exc()
except OSError:
traceback.print_exc()
return
Im coding a python script that connects to a remote server, and parses the returned response. For some odd reason, 9 out of 10 times, Once the header is read, the script continues and returns before getting the body of the response. Im no expert at python, but im certain that my code is correct on the python side of things. Here is my code:
class miniclient:
"Client support class for simple Internet protocols."
def __init__(self, host, port):
"Connect to an Internet server."
self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.sock.settimeout(30)
try:
self.sock.connect((host, port))
self.file = self.sock.makefile("rb")
except socket.error, e:
#if e[0] == 111:
# print "Connection refused by server %s on port %d" % (host,port)
raise
def writeline(self, line):
"Send a line to the server."
try:
# Updated to sendall to resolve partial data transfer errors
self.sock.sendall(line + CRLF) # unbuffered write
except socket.error, e:
if e[0] == 32 : #broken pipe
self.sock.close() # mutual close
self.sock = None
raise e
except socket.timeout:
self.sock.close() # mutual close
self.sock = None
raise
def readline(self):
"Read a line from the server. Strip trailing CR and/or LF."
s = self.file.readline()
if not s:
raise EOFError
if s[-2:] == CRLF:
s = s[:-2]
elif s[-1:] in CRLF:
s = s[:-1]
return s
def read(self, maxbytes = None):
"Read data from server."
if maxbytes is None:
return self.file.read()
else:
return self.file.read(maxbytes)
def shutdown(self):
if self.sock:
self.sock.shutdown(1)
def close(self):
if self.sock:
self.sock.close()
self.sock = None
I use the ReadLine() method to read through the headers until i reach the empty line (Delimiter between headers and body). From there, my objects just call the "Read()" method to read the body. As stated before, 9 of 10 times, read returns nothing, or just partial data.
Example use:
try:
http = miniclient(host, port)
except Exception, e:
if e[0] == 111:
print "Connection refused by server %s on port %d" % (host,port)
raise
http.writeline("GET %s HTTP/1.1" % str(document))
http.writeline("Host: %s" % host)
http.writeline("Connection: close") # do not keep-alive
http.writeline("")
http.shutdown() # be nice, tell the http server we're done sending the request
# Determine Status
statusCode = 0
status = string.split(http.readline())
if status[0] != "HTTP/1.1":
print "MiniClient: Unknown status response (%s)" % str(status[0])
try:
statusCode = string.atoi(status[1])
except ValueError:
print "MiniClient: Non-numeric status code (%s)" % str(status[1])
#Extract Headers
headers = []
while 1:
line = http.readline()
if not line:
break
headers.append(line)
http.close() # all done
#Check we got a valid HTTP response
if statusCode == 200:
return http.read()
else:
return "E\nH\terr\nD\tHTTP Error %s \"%s\"\n$\tERR\t$" % (str(statusCode), str(status[2]))
You call http.close() before you call http.read(). Delay the call to http.close() until after you have read all of the data.