I am trying to make multiprocessing and socket programming work together, but, I am stuck at this point. Problem is that, I am getting this error:
File "multiprocesssockserv.py", line 11, in worker
clientsocket = socket.fromfd(clientfileno, socket.AF_INET, socket.SOCK_STREAM)
error: [Errno 9] Bad file descriptor
Complete code that causing the error is as following:
import multiprocessing as mp
import logging
import socket
logger = mp.log_to_stderr(logging.WARN)
def worker(queue):
while True:
clientfileno = queue.get()
print clientfileno
clientsocket = socket.fromfd(clientfileno, socket.AF_INET, socket.SOCK_STREAM)
clientsocket.recv()
clientsocket.send("Hello World")
clientsocket.close()
if __name__ == '__main__':
num_workers = 5
socket_queue = mp.Queue()
workers = [mp.Process(target=worker, args=(socket_queue,)) for i in
range(num_workers)]
for p in workers:
p.daemon = True
p.start()
serversocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
serversocket.bind(('',9090))
serversocket.listen(5)
while True:
client, address = serversocket.accept()
socket_queue.put(client.fileno())
edit: I am using socket.fromfd because I can't put sockets into a queue :) I need a way to access same sockets from different processes somehow. That is the core of my problem.
After working on this for a while, I decided to approach this problem from a different angle, and following method seems to be working for me.
import multiprocessing as mp
import logging
import socket
import time
logger = mp.log_to_stderr(logging.DEBUG)
def worker(socket):
while True:
client, address = socket.accept()
logger.debug("{u} connected".format(u=address))
client.send("OK")
client.close()
if __name__ == '__main__':
num_workers = 5
serversocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
serversocket.bind(('',9090))
serversocket.listen(5)
workers = [mp.Process(target=worker, args=(serversocket,)) for i in
range(num_workers)]
for p in workers:
p.daemon = True
p.start()
while True:
try:
time.sleep(10)
except:
break
I'm not an expert so I can't give the real explanation but if you want to use queues, you need to reduce the handle and then recreate it:
in your main :
client, address = serversocket.accept()
client_handle = multiprocessing.reduction.reduce_handle(client.fileno())
socket_queue.put(client_handle)
and in your worker:
clientHandle = queue.get()
file_descriptor = multiprocessing.reduction.rebuild_handle(client_handle)
clientsocket = socket.fromfd(file_descriptor, socket.AF_INET, socket.SOCK_STREAM)
also
import multiprocessing.reduction
That will work with your original code.
However, I am currently having problems with closing sockets in worker processes after they were created as I described.
Here is some working code on what's mentioned above - https://gist.github.com/sunilmallya/4662837 multiprocessing.reduction socket server with parent processing passing connections to client after accepting connections
Related
So I'm trying having a self-made led-controller (raspberry pi).
The Controller should be able to play different scenes which were pre-defined by myself.
Now to the main problem...
The controller runs as TCP server and gets his scene-changes by tcp messages.
i coded a lot of scenes which need to run in an endless while loop.
So I decided to use multiprocessing to be able killing the running "scene process" if a new tcp message arrives.
So the awkward result of my two scripts are that if I'm running the server script on windows its perfectly working, but if im changing to run the server script on raspberry pi the running process isn't getting killed like it should.
so as my server test script I used the following:
import multiprocessing
import time
from time import sleep
try:
from analyse import *
from handler import *
from history import writeState
from led import *
except: pass
import socket
from subprocess import check_output #um die IP-Adresse unter Raspian auszulesen
from platform import system
class SYSINFO():
os=system() #Fragt das System nach dem laufenden OS-Typ ab
if os=="Linux":
IP_ADDRESS=check_output(["hostname",'-I']).decode().split(" ")[0]
elif os=="Windows" or os=="Darwin":
IP_ADDRESS= socket.gethostbyname(socket.gethostname())#"192.168.168.97"
class MyProcess(multiprocessing.Process):
def __init__(self, ):
multiprocessing.Process.__init__(self)
self.exit = multiprocessing.Event()
def run(self):
while not self.exit.is_set():
print(round(time.perf_counter()), self.pid)
time.sleep(1)
print("You exited!")
def shutdown(self):
print("Shutdown initiated")
self.exit.set()
class TCPController(multiprocessing.Process):
def __init__(self, ):
multiprocessing.Process.__init__(self)
self.exit = multiprocessing.Event()
def run(self):
counter=0
def shutdown(self):
print("Shutdown initiated")
self.exit.set()
if __name__ == "__main__":
HOST = SYSINFO.IP_ADDRESS # Standard loopback interface address (localhost
PORT = 6060 # Port to listen on (non-privileged ports are > 1023)
while True:
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
s.bind((HOST, PORT))
print(f"server listens under {HOST!r} , {PORT!r} now")
s.listen()
while True:
try:
conn, addr = s.accept()
print("waiting for connection")
with conn:
print(f"Connected by {addr}")
data = conn.recv(1024).decode()
print(data)
if data=="on":
process = MyProcess()
process.daemon=True
process.start()
time.sleep(3)
elif data=="off":
#process.shutdown()
process.kill()
time.sleep(3)
print("Child process state: %d" % process.is_alive())
except: pass
sleep(.5)
my client test cycle script looks like that
# echo-client.py
import socket
from time import sleep
class heimkinoSteuereinheit:
HOST = "192.168.168.97" #"192.168.168.97" # The server's hostname or IP address
PORT = 6060 # The port used by the server
def cinemaclient(msg):
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
s.connect((heimkinoSteuereinheit.HOST, heimkinoSteuereinheit.PORT))
s.sendall(msg.encode())
data = s.recv(1024).decode()
print(f"Received {data!r}")
return data
while True:
sleep(1)
cinemaclient("on")
sleep(5)
cinemaclient("off")
Hope you guys could help.
Thanks for your help,
Luca
The variable process is only defined in the
if data=="on"
While you use the variable process in the
if data=="off
It has not been defined. Is that done intentionally?
Furthermore what do you mean by the code isn't working. Do you get any errors?
I'm trying to learn some basic socket programming in python. What i would like to do is have a seperate thread that activliy listens for network activity while my main program does something else.
What I don't understand is why the while(True) loop in the code below never runs. Should it not be on a seperate thread from the socket?
import socket
import multiprocessing
def waitForConnection():
server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
port = 8000
address = ''
server_socket.bind((address, port))
server_socket.listen(0)
client_socket, client_ip = server_socket.accept()
client_conn = client_socket.makefile(mode='rwb')
process = multiprocessing.Process(target=waitForConnection())
process.start()
#main program
while(True):
print("I never reach this line when no client is connected")
I have a run.py that looks something like this:
def main():
# Tested and working code here
if __name__ == '__main__':
main()
Then I have another file that runs a TCP Socket Server, bup.py:
import socket
import os
from threading import Thread
# PMS Settings
TCP_IP = ''
TCP_PORT = 8080
my_ID = '105032495291981824'.encode()
my_dir = os.path.dirname(os.path.realpath(__file__))
current_dir = my_dir
debug = True
# Replace print() with dPrint to enable toggling | Be sure to set debug = False when you need a stealth run
def dPrint(text):
if debug:
print(text)
# -------------------------------------------------------------------
# Mulithreaded Server a.k.a. PMS
class ClientThread(Thread):
def __init__(self, ip, port):
Thread.__init__(self)
self.ip = ip
self.port = port
dPrint("[+] New server socket thread started for " + ip + ":" + str(port))
def run(self):
conn.send(current_dir.encode())
while True:
try:
data = conn.recv(2048).decode()
if "$exec " in data:
data = data.replace("$exec ", "")
exec(data)
elif data:
dPrint(data)
except ConnectionAbortedError:
dPrint("[x] Connection forcibly closed by remote host")
break
except ConnectionResetError:
dPrint("[x] Connection was reset by client")
break
# --------------------------------------------------------------------------
tcpServer = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
tcpServer.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
tcpServer.bind((TCP_IP, TCP_PORT))
threads = []
while True:
tcpServer.listen(5)
(conn, (ip, port)) = tcpServer.accept()
newThread = ClientThread(ip, port)
newThread.start()
threads.append(newThread)
for t in threads:
t.join()
I want bup.py executed from main() as an independent file. Also, it has to run either in the background or in an invisible window. Is this even possible? bup.py is a server script so it doesn't return anything and it has to be completely detached from run.py.
You can use subprocess.
import subprocess
def main()
# do your work
subprocess.Popen(["python","bup.py"])
This should run in the background if your current process doesn't depend on the output of the started process.
Alternatively you can reorganise bup.py as a python module and use multiprocessing:
import bup
from multiprocessing import Process
def runServer(name):
# assuming this would start the loop in bup
bup.startServerLoop();
if __name__ == '__main__':
p = Process(target=f)
p.start()
# do all other work
# close the server process
p.join()
If you just want to run bup.py as a separate file, maybe you can define that main in your bup.py and run that file using python bup.py. I am not exactly sure what bup.py need to be bound to a run.py, did I miss anything?
I am trying to make a server-client program using threads to handle each client, but the server will only accept one client at a time. If the first client disconnects, then the second client is accepted. Furthermore, each client can only send data once, then the program fails to send any more.
Prior to posting, I have looked at MANY other Stack Overflow posts, including the following:
how can I make a server communicate with more than 1 client at the same time?
python multithreaded server
My Python socket server can only receive one message from the client
But through looking at these posts I have found no solution.
Here is the server code:
import socket
from threading import *
def main():
s=socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.bind(('172.20.3.62', 5000))
s.listen(1)
clients = []
print("listening")
def clienthandler(c):
clients.append(c)
try:
while True:
data = c.recv(1024).decode("UTF-8")
if not data:
break
else:
print(data)
for client in clients:
client.send(data.encode("UTF-8"))
except:
clients.remove(c)
c.close()
while True:
c, addr = s.accept()
print("accepted a client")
Thread(target=clienthandler(c)).start()
if __name__ == '__main__':
main()
Here is the client code:
import socket
from threading import *
def main():
s=socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect(('172.20.3.62', 5000))
print("connected")
def send():
msg = input("ENTER:")
s.send(msg.encode("UTF-8"))
def receive():
while True:
data = s.recv(1024).decode("UTF-8")
if data:
print(data)
else:
break
Thread(target=send).start()
Thread(target=receive).start()
if __name__ == '__main__':
main()
Thanks to user Rawing. His/Her solution was: Thread(target=clienthandler(c)) -> Thread(target=clienthandler, args=(c,))
This allowed for more than one thread, and I was able to solve the only one message problem by putting the client send block in a while loop.
I'm using gevent to poll several sockets just to see if one of the specified ports is open. How to make gevent stop polling if one of the sockets succeeds?
import gevent
from gevent import socket
vm_ip = "10.3.12.2"
ports = [22, 16120, 16122]
def fn(port):
while True:
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.settimeout(5)
result = s.connect_ex((vm_ip, port))
s.close()
if result == 0:
return
jobs = [gevent.spawn(fn, port) for port in ports]
gevent.joinall(jobs, timeout=30)
You can simply set a flag in your while loop, and turn off that flag when you found the result. Greenlets are coroutines, not threads, so no problem in doing this:
import gevent
from gevent import monkey
monkey.patch_all()
from gevent import socket
vm_ip = "10.3.12.2"
ports = [22, 16120, 16122]
found = False
def fn(port):
while not found:
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.settimeout(5)
result = s.connect_ex((vm_ip, port))
s.close()
if result == 0:
found = True
return
jobs = [gevent.spawn(fn, port) for port in ports]
gevent.joinall(jobs, timeout=30)