We Keep Coding

How to pause a thread until data received in Python

I do not have much experience with threads (and networking in general). I am creating a script which receives data from a client (1, 2 or 3). All these have a meaning:
1 = NEW Apache Benchmark ITERATION - we must run new top command and append every second
2 = END Apache Benchmark ITERATION - we must end top command
3 = STOP ENTIRE PROGRAM
The top command on Linux just records the CPU and memory usage.
I have created an initial thread which listens for data from the client and targets the get_data() function.
The run() function waits for data to be returned from get_data() but if it isn't getting any data then both the run() and get_data() function will halt.
Is there a way to pause the thread targeting the get_data function until data is sent from the client side so that the run() function doesn't halt?
My Current Code:
import socket
import sys
import threading
import subprocess
import sched
import time
import os
import signal
sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
server_socket = ('xxx.xxx.x.xx', 5000)
sock.bind(server_socket)
process = None
def run_command(i):
global process
print("Running top")
process = subprocess.Popen('top -b -n 1 | head -n 5 >> htop-' + str(i) + '.txt', shell=True)
print("Finished running top")
return process
def get_data():
while True:
# data = ''
data, address = sock.recvfrom(1024)
print("waiting?")
data = data.split(",")
iteration = data[1]
data = data[0]
print("Data: " + data + " and iteration: " + iteration)
time.sleep(1.0)
# run(data, iteration)
return data, iteration
'''
1 = NEW AB ITERATION - we must run new top command and append every second
2 = END OF AB - we must end top command
3 = STOP ENTIRE PROGRAM
'''
def run():
while True:
print("New")
data = get_data()
# data = data.split(",")
iteration = data[1]
data = data[0]
print("Data: " + data + " and iteration: " + iteration)
if data == '1' or data == '':
run_command(iteration)
print("We ran the command")
time.sleep(1.0)
print("Terminating")
process.kill()
print("We terminated the process")
if data == '2':
print("We got 2")
process.kill()
if data == '3':
print("Closing program")
exit()
runThread = threading.Thread(target = run)
runThread.start()
run()
print("Starting the listening thread...")

You need to make your socket non-blocking (see setblocking()) and rewrite your run() procedure to handle cases when data in socket is not received yet.

run code every X seconds

I Have infinite loop that read bytes from serial port, I want to save the read data to firebase database every X seconds.
I used this code snippet but it's not helping:
import threading
def printit():
threading.Timer(5.0, printit).start()
print "Hello, World!"
printit()
This is my code
import serial
ser = serial.Serial()
ser.baudrate = 115200
ser.port = "/dev/ttyUSB0"
ser.timeout = 30
try:
try:
while 1:
line = ser.readline().rstrip().decode('utf-8')
# print("save data here every X seconds)
except KeyboardInterrupt:
ser.close() # Close port
pass
except serial.serialutil.SerialException as e:
print(str(e))
I can't use sleep because it blocking the main thread,So how to let the code read continuously and print "data saved" every X seconds (I'll save to database in my case)

Thanks to Lutz Horn for the suggestion in the comment, I resolve the problem like that :
import schedule
import time
import serial
ser = serial.Serial()
ser.baudrate = 115200
ser.port = "/dev/ttyUSB0"
ser.timeout = 30
schedule.every(10).seconds.do(save_db)
try:
try:
while 1:
schedule.run_pending()
line = ser.readline().rstrip().decode('utf-8')
# here every 10 seconds the function save_db will be called
except KeyboardInterrupt:
ser.close() # Close port
pass
except serial.serialutil.SerialException as e:
print(str(e))

I hope i have understood you correctly. Use time.time() to set timer.
import time
def doEvery_X_Seconds():
print("I do it every X seconds")
time.sleep(1)
TIMER_LIMIT = 5
setTimer = time.time()
while(1):
print("hello world")
if(time.time() - setTimer >= TIMER_LIMIT):
doEvery_X_Seconds()
setTimer = time.time()
There is time.sleep(1) only to demonstrate, that it works.

How to finish a socket file transfer in Python?

I have a Client and a Server and I need to transfer some files using sockets. I can send small messages, but when I try to send a File, the problems begins...
client.py:
from socket import *
from threading import Thread
import sys
import hashlib
class Client(object):
ASK_LIST_FILES = "#001" # 001 is the requisition code to list
# all the files
ASK_SPECIFIC_FILE = "#002" # 002 is the requisition code to a
# specific file
SEND_FILE = "#003" # 003 is the requisition code to send one
# file
AUTHENTICATION = "#004" # 004 is the requisition code to user
# authentication
listOfFiles = []
def __init__(self):
try:
self.clientSocket = socket(AF_INET, SOCK_STREAM)
except (error):
print("Failed to create a Socket.")
sys.exit()
def connect(self, addr):
try:
self.clientSocket.connect(addr)
except (error):
print("Failed to connect.")
sys.exit()
print(self.clientSocket.recv(1024).decode())
def closeConnection(self):
self.clientSocket.close()
def _askFileList(self):
try:
data = Client.ASK_LIST_FILES
self.clientSocket.sendall(data.encode())
# self._recvFileList()
except (error):
print("Failed asking for the list of files.")
self.closeConnection()
sys.exit()
thread = Thread(target = self._recvFileList)
thread.start()
def _recvFileList(self):
print("Waiting for the list...")
self.listOfFiles = []
while len(self.listOfFiles) == 0:
data = self.clientSocket.recv(1024).decode()
if (data):
self.listOfFiles = data.split(',')
if(len(self.listOfFiles) > 0):
print (self.listOfFiles)
def _askForFile(self, fileIndex):
fileIndex = fileIndex - 1
try:
data = Client.ASK_SPECIFIC_FILE + "#" + str(fileIndex)
self.clientSocket.sendall(data.encode())
except(error):
print("Failed to ask for an specific file.")
self.closeConnection()
sys.exit()
self._downloadFile(fileIndex)
def _downloadFile(self, fileIndex):
print("Starting receiving file")
f = open("_" + self.listOfFiles[fileIndex], "wb+")
read = self.clientSocket.recv(1024)
# print(read)
# f.close
while len(read) > 0:
print(read)
f.write(read)
f.flush()
read = self.clientSocket.recv(1024)
f.flush()
f.close()
self.closeConnection()
server.py
from socket import *
from threading import Thread
import sys
import glob
class Server(object):
def __init__(self):
try:
self.serverSocket = socket(AF_INET, SOCK_STREAM)
except (error):
print("Failed to create a Socket.")
sys.exit()
def connect(self, addr):
try:
self.serverSocket.bind(addr)
except (error):
print ("Failed on binding.")
sys.exit()
def closeConnection(self):
self.serverSocket.close()
def waitClients(self, num):
while True:
print("Waiting for clients...")
self.serverSocket.listen(num)
conn, addr = self.serverSocket.accept()
print("New client found...")
thread = Thread(target = self.clientThread, args = (conn,))
thread.start()
def clientThread(self, conn):
WELCOME_MSG = "Welcome to the server"
conn.send(WELCOME_MSG.encode())
while True:
data = conn.recv(2024).decode()
if(data):
# print(data)
# reply = 'OK: ' + data
# conn.sendall(reply.encode())
if(data == "#001"):
listOfFiles = self.getFileList()
strListOfFiles = ','.join(listOfFiles)
self._sendFileList(strListOfFiles, conn)
else:
dataCode = data.split('#')
print(dataCode)
if(dataCode[1] == "002"):
print("Asking for file")
self._sendFile(int(dataCode[2]), conn)
if(dataCode[1] == "003"):
print("Pedido de login")
if self._authentication(dataCode[2]):
conn.send("OK".encode())
# self._recvFile(conn)
else:
conn.send("FAILED".encode())
def _sendFile(self, fileIndex, conn):
listOfFiles = self.getFileList()
print(fileIndex)
print(listOfFiles[fileIndex])
f = open(listOfFiles[fileIndex], "rb")
read = f.read(1024)
while len(read) > 0:
conn.send(read)
read = f.read(1024)
f.close()
def _sendFileList(self, strList, conn):
try:
conn.sendall(strList.encode())
except (error):
print("Failed to send list of files.")
def getFileList(self):
return glob.glob("files/*")
When I try to get a file from my server, I can transfer everything but the connection never ends. What is going on with my code?

First, you are doing here the most common error using TCP: assume all data sent in a single send() will be got identically in a single recv(). This is untrue for TCP, because it is an octet stream, not a message stream. Your code will work only under ideal (lab) conditions and could mysteriously fail in a real world usage. You should either explicitly invent message boundaries in TCP streams, or switch e.g. to SCTP. The latter is available now almost everywhere and keeps message boundaries across a network connection.
The second your error is directly connected to the first one. When sending file, you don't provide any explicit mark that file has been finished. So, clients waits forever. You might try to close server connection to show that file is finished, but in that case client won't be able to distinguish real file end and connection loss; moreover, the connection won't be reusable for further commands. You would select one of the following ways:
Prefix a file contents with its length. In this case, client will know how many bytes shall be received for the file.
Send file contents as a chunk sequence, prefixing each chunk with its length (only for TCP) and with mark whether this chunk is last (for both transports). Alternatively, a special mark "EOF" can be sent without data.
Similarly, control messages and their responses shall be provided with either length prefix or a terminator which can't appear inside such message.
When you finish developing this, you would look at FTP and HTTP; both addresses all issues I described here but in principally different ways.

Using pyserial for sending keystrokes

I have a task where I have to send keystrokes to a machine and to achieve the same I have two machine out of which one is Host Machine (running python scripts) and other is the target machine (where I have to send keystrokes). I am using "L3 Systems Inc's KeyAT device" on COM1.
Now the problem is I am unable to send keystrokes and following is the code I am running.
import serial, time
#initialization and open the port
#possible timeout values:
# 1. None: wait forever, block call
# 2. 0: non-blocking mode, return immediately
# 3. x, x is bigger than 0, float allowed, timeout block call
ser = serial.Serial()
#ser.port = "/dev/ttyUSB0"
ser.port = "COM1"
#ser.port = "/dev/ttyS2"
ser.baudrate = 9600
ser.bytesize = serial.EIGHTBITS #number of bits per bytes
ser.parity = serial.PARITY_NONE #set parity check: no parity
ser.stopbits = serial.STOPBITS_ONE #number of stop bits
#ser.timeout = None #block read
ser.timeout = 1 #non-block read
#ser.timeout = 2 #timeout block read
ser.xonxoff = False #disable software flow control
ser.rtscts = False #disable hardware (RTS/CTS) flow control
ser.dsrdtr = False #disable hardware (DSR/DTR) flow control
ser.writeTimeout = 2 #timeout for write
try:
ser.open()
except Exception, e:
print "error open serial port: " + str(e)
exit()
if ser.isOpen():
try:
ser.flushInput() #flush input buffer, discarding all its contents
ser.flushOutput()#flush output buffer, aborting current output
#and discard all that is in buffer
#write data
# ser.write("AT+CSQ")
ser.write("~~~~~~~~~~\r")
ser.write("~:04\r")
# ser.write('\x03')
#print("write data: AT+CSQ")
time.sleep(0.5) #give the serial port sometime to receive the data
numOfLines = 0
while True:
response = ser.readline()
print("read data: " + response)
numOfLines = numOfLines + 1
if (numOfLines >= 5):
break
ser.close()
except Exception, e1:
print "error communicating...: " + str(e1)
else:
print "cannot open serial port "
Can anyone please help
Thanks,
Vipul

The only problem is the ser.readline() returns binary , you need transform it into string: str(response), so it will work perfectly!

pyserial readline() : SerialException

I'm writing a code used to send order to an avr. I send several information but between each write, I have to wait for an answer (I have to wait for the robot to reach a point on the coordinate system). As I read in the documentation, readline() should at least read until the timeout but as soon as I send the first coordinate, the readline() automatically return :
SerialException: device reports readiness to read but returned no data (device disconnected?)
When I put a sleep() between each write() in the for loop, everything works fine. I tried to use inWaiting() but it still does not work. Here is an example of how I used it:
for i in chemin_python:
self.serieInstance.ecrire("goto\n" + str(float(i.x)) + '\n' + str(float(-i.y)) + '\n')
while self.serieInstance.inWaiting():
pass
lu = self.serieInstance.readline()
lu = lu.split("\r\n")[0]
reponse = self.serieInstance.file_attente.get(lu)
if reponse != "FIN_GOTO":
log.logger.debug("Erreur asservissement (goto) : " + reponse)

Here an snipet how to use serial in python
s.write(command);
st = ''
initTime = time.time()
while True:
st += s.readline()
if timeout and (time.time() - initTime > t) : return TIMEOUT
if st != ERROR: return OK
else: return ERROR

This method allows you to separately control the timeout for gathering all the data for each line, and a different timeout for waiting on additional lines.
def serial_com(self, cmd):
'''Serial communications: send a command; get a response'''
# open serial port
try:
serial_port = serial.Serial(com_port, baudrate=115200, timeout=1)
except serial.SerialException as e:
print("could not open serial port '{}': {}".format(com_port, e))
# write to serial port
cmd += '\r'
serial_port.write(cmd.encode('utf-8'))
# read response from serial port
lines = []
while True:
line = serial_port.readline()
lines.append(line.decode('utf-8').rstrip())
# wait for new data after each line
timeout = time.time() + 0.1
while not serial_port.inWaiting() and timeout > time.time():
pass
if not serial_port.inWaiting():
break
#close the serial port
serial_port.close()
return lines