We Keep Coding

Python Process won't stop by calling kill method

So I'm trying having a self-made led-controller (raspberry pi).
The Controller should be able to play different scenes which were pre-defined by myself.
Now to the main problem...
The controller runs as TCP server and gets his scene-changes by tcp messages.
i coded a lot of scenes which need to run in an endless while loop.
So I decided to use multiprocessing to be able killing the running "scene process" if a new tcp message arrives.
So the awkward result of my two scripts are that if I'm running the server script on windows its perfectly working, but if im changing to run the server script on raspberry pi the running process isn't getting killed like it should.
so as my server test script I used the following:
import multiprocessing
import time
from time import sleep
try:
from analyse import *
from handler import *
from history import writeState
from led import *
except: pass
import socket
from subprocess import check_output #um die IP-Adresse unter Raspian auszulesen
from platform import system
class SYSINFO():
os=system() #Fragt das System nach dem laufenden OS-Typ ab
if os=="Linux":
IP_ADDRESS=check_output(["hostname",'-I']).decode().split(" ")[0]
elif os=="Windows" or os=="Darwin":
IP_ADDRESS= socket.gethostbyname(socket.gethostname())#"192.168.168.97"
class MyProcess(multiprocessing.Process):
def __init__(self, ):
multiprocessing.Process.__init__(self)
self.exit = multiprocessing.Event()
def run(self):
while not self.exit.is_set():
print(round(time.perf_counter()), self.pid)
time.sleep(1)
print("You exited!")
def shutdown(self):
print("Shutdown initiated")
self.exit.set()
class TCPController(multiprocessing.Process):
def __init__(self, ):
multiprocessing.Process.__init__(self)
self.exit = multiprocessing.Event()
def run(self):
counter=0
def shutdown(self):
print("Shutdown initiated")
self.exit.set()
if __name__ == "__main__":
HOST = SYSINFO.IP_ADDRESS # Standard loopback interface address (localhost
PORT = 6060 # Port to listen on (non-privileged ports are > 1023)
while True:
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
s.bind((HOST, PORT))
print(f"server listens under {HOST!r} , {PORT!r} now")
s.listen()
while True:
try:
conn, addr = s.accept()
print("waiting for connection")
with conn:
print(f"Connected by {addr}")
data = conn.recv(1024).decode()
print(data)
if data=="on":
process = MyProcess()
process.daemon=True
process.start()
time.sleep(3)
elif data=="off":
#process.shutdown()
process.kill()
time.sleep(3)
print("Child process state: %d" % process.is_alive())
except: pass
sleep(.5)
my client test cycle script looks like that
# echo-client.py
import socket
from time import sleep
class heimkinoSteuereinheit:
HOST = "192.168.168.97" #"192.168.168.97" # The server's hostname or IP address
PORT = 6060 # The port used by the server
def cinemaclient(msg):
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
s.connect((heimkinoSteuereinheit.HOST, heimkinoSteuereinheit.PORT))
s.sendall(msg.encode())
data = s.recv(1024).decode()
print(f"Received {data!r}")
return data
while True:
sleep(1)
cinemaclient("on")
sleep(5)
cinemaclient("off")
Hope you guys could help.
Thanks for your help,
Luca

The variable process is only defined in the
if data=="on"
While you use the variable process in the
if data=="off
It has not been defined. Is that done intentionally?
Furthermore what do you mean by the code isn't working. Do you get any errors?

How to run an instance of a python script with arguments in parallel?

I am trying to get the Script(flowcheck.py) to run as an instance, so that if the client sends more than one request in quick succession the scripts don't hang and wait for the previous request to finish.
Is there a way to make the server just run an instance of the script + arguments without monitoring it, then go back into listening for the next request?
Server Code
#!/usr/bin/env python
import SocketServer
import flowcheck
import os
class MyTCPHandler(SocketServer.BaseRequestHandler):
def handle(self):
# self.request is the TCP socket connected to the client
self.data = self.request.recv(1024).strip()
print "{} wrote:".format(self.client_address[0])
print self.data
flowcheck.flow(self.data)
if __name__ == "__main__":
HOST, PORT = "", 5007
# Create the server, binding to localhost on port 5007
server = SocketServer.TCPServer((HOST, PORT), MyTCPHandler)
# Activate the server; this will keep running until you
# interrupt the program with Ctrl-C
server.serve_forever()
Client Code
#!/usr/bin/env python
import socket
import sys
HOST, PORT = "192.168.8.201", 5007
data = " ".join(sys.argv[1:])
# Create a socket (SOCK_STREAM means a TCP socket)
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try:
# Connect to server and send data
sock.connect((HOST, PORT))
sock.sendall(data + "\n")
finally:
sock.close()
print "Sent: {}".format(data)
Script
#!/usr/bin/env python
import sys
import time
#data = " ".join(sys.argv[1:])
def flow(data):
var=0
while var<3:
print "FlowCheck",data,var
var = var +1
time.sleep(5)

Instead of importing the script, import subprocess using Popen to run an instance of the script in the background and continue taking requests.
#!/usr/bin/env python
import SocketServer
import os
import subprocess
class MyTCPHandler(SocketServer.BaseRequestHandler):
def handle(self):
# self.request is the TCP socket connected to the client
self.data = self.request.recv(1024).strip()
print "{} wrote:".format(self.client_address[0])
print self.data
var1 =self.data
subprocess.Popen([ "python", "flowcheck.py",var1])
Change to script to take system arguments with import sys
#!/usr/bin/env python
import sys
import time
print "start"
data = sys.argv[1]
def flow(data):
var=0
while var<3:
print "FlowCheck",data,var
var = var +1
time.sleep(5)
flow(data)

Python server daemon not working

I need to make a basic python server run as a daemon but when I set the thread to run as a daemon, the socket doesnt listen anymore. I try to connect to it with a client to send a file to it but I get a connection error. When I leave out the part that sets it to a daemon, it works just fine. Suggestions? Thanks
Server code:
import socket
import sys
import threading
HOST = "localhost"
PORT = 3000
exit = False
def handle_client(client):
global exit
filename = client.recv(50).decode()
client.send(b"/ok")
if filename == '':
client.close()
return
elif filename == '/kill':
exit = True
client.send(b'killing server...\n')
killserver = socket.socket()
killserver.connect((HOST, PORT))
killserver.close()
client.send(b'success.\n')
client.close()
return
f = open(filename, "wb")
data = client.recv(1024)
while(data):
f.write(data)
data = client.recv(1024)
f.close()
print("Wrote file " + filename)
client.close()
def server():
servsock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
servsock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
servsock.bind((HOST,PORT))
print("[INFO]: listening on " + str(PORT))
servsock.listen(10)
while True:
client, addr = servsock.accept()
if exit is True:
print("[INFO]: server received kill request")
break
print("[INFO]: connection from " + addr[0])
hc = threading.Thread(target=handle_client, args=(client,))
hc.start()
servsock.close()
s = threading.Thread(target=server)
s.setDaemon(True)
s.start()

When there are only daemon threads running, the process exits. After you start the thread, you exit the main thread. You don't want the server thread to be daemonic.

Connections outlive SocketServer.TCPServer?

How can I make a TCP server properly close sockets?
I wrote a simple TCP server to echo some information back to client. Now I would like to use it on localhost:8088 and stop it using killall and restart at any time as I'm working on the code.
However, I'm having trouble making it close all sockets and "free" the address, so when I quickly make a few tests, fix something in the code and then stop (Ctrl+C) and start the server again, I get socket.error: [Errno 98] Address already in use.
When I sudo netstat -ntap, I can still see few 127.0.0.1:8088 sockets in TIME_WAIT state. So I have to wait until they "die out".
My test case:
#!/usr/bin/python
import SocketServer
class MyEchoHandler(SocketServer.BaseRequestHandler):
def handle(self):
self.data = self.request.recv(1024).strip()
# do something smart with the data, but for now, just say hello.
self.reply = "Content-Type: text/plain\r\n\r\nhello."
self.request.send(self.reply)
self.request.close()
def main():
server = SocketServer.TCPServer((HOST, PORT), MyEchoHandler)
server.serve_forever()
if __name__ == '__main__':
HOST, PORT = "localhost", 8088
main()
What am I doing wrong? Shouldn't self.request.close() be enough?
I'm trying this on Debian with Python 2.7.3, although I need to support Squeeze with Python 2.6 as well.

The TCP stack puts the port into timed-wait for awhile (something like 30 seconds to multiple minutes depending on your system). You can change that behavior with the SO_REUSEADDR socket option. The trick is that the option must be set before the port is bound.
If you have the raw socket, you can:
s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
SocketServer lets you set the option globally with the allow_reuse_address attribute:
SocketServer.TCPServer.allow_reuse_address = True
Or you can do it when you create the server:
def main():
server = SocketServer.TCPServer((HOST, PORT), MyEchoHandler, False)
server.allow_reuse_address = True
server.server_bind()
server.serve_forever()
Or even override the bind method:
class MyTCPServer(SocketServer.TCPServer):
def server_bind(self):
self.allow_reuse_address = True
super(MyTCPServer, self).server_bind()
Here are 3 solutions that worked for me on my Windows machine.
(1) set globally
#!/usr/bin/python
import SocketServer
import time
SocketServer.allow_reuse_address = True
class MyEchoHandler(SocketServer.BaseRequestHandler):
def handle(self):
self.data = self.request.recv(1024).strip()
# do something smart with the data, but for now, just say hello.
self.reply = "Content-Type: text/plain\r\n\r\n" + time.asctime()
self.request.send(self.reply)
self.request.close()
def main():
server = SocketServer.TCPServer((HOST, PORT), MyEchoHandler)
server.serve_forever()
if __name__ == '__main__':
HOST, PORT = "localhost", 8088
main()
(2) Set locally
#!/usr/bin/python
import SocketServer
import time
class MyEchoHandler(SocketServer.BaseRequestHandler):
def handle(self):
self.data = self.request.recv(1024).strip()
# do something smart with the data, but for now, just say hello.
self.reply = "Content-Type: text/plain\r\n\r\n" + time.asctime()
self.request.send(self.reply)
self.request.close()
def main():
server = SocketServer.TCPServer((HOST, PORT), MyEchoHandler, False)
server.allow_reuse_address = True
server.server_bind()
server.server_activate()
server.serve_forever()
if __name__ == '__main__':
HOST, PORT = "localhost", 8088
main()
(3) Inherit
#!/usr/bin/python
import SocketServer
import time
class MyEchoHandler(SocketServer.BaseRequestHandler):
def handle(self):
self.data = self.request.recv(1024).strip()
# do something smart with the data, but for now, just say hello.
self.reply = "Content-Type: text/plain\r\n\r\n" + time.asctime()
self.request.send(self.reply)
self.request.close()
class MyTCPServer(SocketServer.TCPServer):
def server_bind(self):
self.allow_reuse_address = True
SocketServer.TCPServer.server_bind(self)
def main():
server = MyTCPServer((HOST, PORT), MyEchoHandler)
server.serve_forever()
if __name__ == '__main__':
HOST, PORT = "localhost", 8088
main()

You can call server.shutdown() but you have to do it from another thread 8-( IMO this is weak area in Python's TCPServer.

Multiprocessing and sockets in Python

I am trying to make multiprocessing and socket programming work together, but, I am stuck at this point. Problem is that, I am getting this error:
File "multiprocesssockserv.py", line 11, in worker
clientsocket = socket.fromfd(clientfileno, socket.AF_INET, socket.SOCK_STREAM)
error: [Errno 9] Bad file descriptor
Complete code that causing the error is as following:
import multiprocessing as mp
import logging
import socket
logger = mp.log_to_stderr(logging.WARN)
def worker(queue):
while True:
clientfileno = queue.get()
print clientfileno
clientsocket = socket.fromfd(clientfileno, socket.AF_INET, socket.SOCK_STREAM)
clientsocket.recv()
clientsocket.send("Hello World")
clientsocket.close()
if __name__ == '__main__':
num_workers = 5
socket_queue = mp.Queue()
workers = [mp.Process(target=worker, args=(socket_queue,)) for i in
range(num_workers)]
for p in workers:
p.daemon = True
p.start()
serversocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
serversocket.bind(('',9090))
serversocket.listen(5)
while True:
client, address = serversocket.accept()
socket_queue.put(client.fileno())
edit: I am using socket.fromfd because I can't put sockets into a queue :) I need a way to access same sockets from different processes somehow. That is the core of my problem.

After working on this for a while, I decided to approach this problem from a different angle, and following method seems to be working for me.
import multiprocessing as mp
import logging
import socket
import time
logger = mp.log_to_stderr(logging.DEBUG)
def worker(socket):
while True:
client, address = socket.accept()
logger.debug("{u} connected".format(u=address))
client.send("OK")
client.close()
if __name__ == '__main__':
num_workers = 5
serversocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
serversocket.bind(('',9090))
serversocket.listen(5)
workers = [mp.Process(target=worker, args=(serversocket,)) for i in
range(num_workers)]
for p in workers:
p.daemon = True
p.start()
while True:
try:
time.sleep(10)
except:
break

I'm not an expert so I can't give the real explanation but if you want to use queues, you need to reduce the handle and then recreate it:
in your main :
client, address = serversocket.accept()
client_handle = multiprocessing.reduction.reduce_handle(client.fileno())
socket_queue.put(client_handle)
and in your worker:
clientHandle = queue.get()
file_descriptor = multiprocessing.reduction.rebuild_handle(client_handle)
clientsocket = socket.fromfd(file_descriptor, socket.AF_INET, socket.SOCK_STREAM)
also
import multiprocessing.reduction
That will work with your original code.
However, I am currently having problems with closing sockets in worker processes after they were created as I described.

Here is some working code on what's mentioned above - https://gist.github.com/sunilmallya/4662837 multiprocessing.reduction socket server with parent processing passing connections to client after accepting connections