Few days ago I coded a mini game to learn socket and threading. When I run my game server and client on windows it just works perfectly but when I moved my server file to my test server it gives me this pickle error:
Exception in thread Thread-2:
Traceback (most recent call last):
File "/usr/lib/python3.8/threading.py", line 932, in _bootstrap_inner
self.run()
File "/usr/lib/python3.8/threading.py", line 870, in run
self._target(*self._args, **self._kwargs)
File "server.py", line 46, in handle_client
obj = pickle.loads(conn.recv(obj_length))
_pickle.UnpicklingError: invalid load key, ' '.
What could be the problem?
Whole game files: ---------------------------------------------------------------------------------
Codes that might help:
server.py:
import socket
import threading
import pickle
import time
import random
import ast
#-------------------------------------------------------------------------
class Server():
def __init__(self):
self.HEADER = 2048
self.PORT = 6000
self.SERVER = "ip"
self.ADDR = (self.SERVER, self.PORT)
self.FORMAT = 'utf-8'
self.DISCONNECT_MESSAGE = "!DISCONNECT"
self.ROLES = ["Mafya", "Mafya", "Köylü", "Doktor","Gözcü"]
self.server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.server.bind(self.ADDR)
#---------------------------------------------------------------------------------------------
self.names = list()
self.addresses = list()
self.lobby_dict = dict()
self.game_dict = dict()
self.ready_list = list()
self.alive_list = list()
self.vote = list()
self.kill_vote = list()
self.who_voted = list()
self.ready_for_day = list()
self.protected = None
def handle_client(self, conn, addr):
try:
if addr[0] not in self.addresses:
print(f"[NEW CONNECTION] {addr[0]} connected.")
self.addresses.append(addr[0])
connected = True
while connected:
obj_length = conn.recv(self.HEADER).decode(self.FORMAT)
if obj_length:
obj_length = int(obj_length)
obj = pickle.loads(conn.recv(obj_length))
if obj == self.DISCONNECT_MESSAGE:
connected = False
print(f"[DISCONNECTED] {addr[0]} disconnected.")
elif "?ONLINE" in obj:
lobby_id = obj.split(":")[1]
conn.send(pickle.dumps(self.lobby_dict[lobby_id]["Players"]))
elif "!NEWLOBBY" in obj:
splitted_obj = obj.split(":")
lobby_id = splitted_obj[1]
admin = splitted_obj[2]
capacity = splitted_obj[3]
self.lobby_dict[lobby_id] = {"Players":[admin],"Capacity":capacity}
elif "!JOINLOBBY" in obj:
splitted_obj = obj.split(":")
lobby_id = splitted_obj[1]
name = splitted_obj[2]
if lobby_id in self.lobby_dict.keys():
self.lobby_dict[lobby_id]["Players"].append(name)
conn.send(pickle.dumps(f"True:{self.lobby_dict[lobby_id]['Capacity']}"))
else:
conn.send(pickle.dumps("False"))
elif "?ALIVE" in obj:
conn.send(pickle.dumps(self.alive_list))
#-----------------------------------------------------------------------------------------------
#Game commands:
elif "!NAME" in obj:
name = obj.split(":")[1]
self.names.append(name)
elif "!ALIVE" in obj:
conn.send(pickle.dumps(self.names))
elif "!READY" in obj:
ready_player = obj.split(":")[1]
self.ready_list.append(ready_player)
elif "!SHUFFLE" in obj:
if len(self.ready_list) == len(self.names):
temp = self.ROLES
if len(self.names) > len(self.ROLES):
for i in range(0,len(self.names) - len(self.ROLES)):
temp.append("Köylü")
random.shuffle(temp)
for i in range(len(self.names)):
self.game_dict[self.names[i]] = temp[i]
conn.send(pickle.dumps(f"True/{self.game_dict}"))
with open("shuffled_roles.txt", "w", encoding="utf-8") as file:
file.write(str(self.game_dict))
print(f"[SHUFFLED LIST] {self.game_dict}")
else:
conn.send(pickle.dumps("False"))
elif "!ROLES" in obj:
if len(self.ready_list) == len(self.names):
with open("shuffled_roles.txt", "r", encoding="utf-8") as file:
line = file.readline()
self.game_dict = ast.literal_eval(line)
conn.send(pickle.dumps(f"True/{self.game_dict}"))
else:
conn.send(pickle.dumps("False"))
elif "!VOTE" in obj:
voted_player = obj.split(":")[1]
who = obj.split(":")[2] + ": " + voted_player
self.who_voted.append(who)
self.vote.append(voted_player)
elif "!VRESULTS" in obj:
conn.send(pickle.dumps(self.vote))
conn.send(pickle.dumps(self.who_voted))
elif "!VCLEAN" in obj:
self.vote = []
self.who_voted = []
elif "!PROTECTED" in obj:
protected = obj.split(":")[1]
self.protected = obj
elif "!NIGHT_KILL" in obj:
kill = obj.split(":")[1]
self.kill_vote.append(kill)
elif "!NKRESULTS" in obj:
nk_results = self.kill_vote
nk_protected = self.protected
if len(nk_results) == 1:
if nk_results[0] != nk_protected:
conn.send(pickle.dumps(nk_results[0]))
elif len(nk_results) == 2:
if nk_results[0] == nk_results[1]:
if nk_results[0] != nk_protected and nk_results[0] != "None":
conn.send(pickle.dumps(nk_results[0]))
elif nk_results[0] == "None" and nk_results[1] != "None":
conn.send(pickle.dumps(nk_results[1]))
elif nk_results[1] == "None" and nk_results[0] != "None":
conn.send(pickle.dumps(nk_results[0]))
else:
conn.send(pickle.dumps(None))
else:
conn.send(pickle.dumps(None))
elif "!NKCLEAN" in obj:
self.protected = "None"
self.kill_vote = []
elif "!RFORDAY" in obj:
rplayer = obj.split(":")[1]
self.ready_for_day.append(rplayer)
elif "!RFDLIST" in obj:
conn.send(pickle.dumps(self.ready_for_day))
elif "!RFDCLEAN" in obj:
self.ready_for_day = list()
else:
print(f"[{addr}] {obj}") #İsimler Buradan -> addr
except ConnectionResetError:
print(f"[CONNECTION] {addr} Connection reset exception has been handled.")
finally:
conn.close()
def start(self):
print("[STARTING] Server is starting...")
self.server.listen()
print("[LISTENING] Server is listening on {}".format(self.SERVER))
while True:
conn, addr = self.server.accept()
thread = threading.Thread(target=self.handle_client, args=(conn, addr))
thread.start()
if __name__ == "__main__":
server = Server()
server.start()
app.py:
import sys
import os
import time
import random
import socket
import threading
import pickle
class Config():
def __init__(self):
self.HEADER = 2048
self.PORT = 6000
self.SERVER = "ip" #socket.gethostbyname(socket.gethostname())
self.ADDR = (self.SERVER, self.PORT)
self.FORMAT = 'utf-8'
self.DISCONNECT_MESSAGE = "!DISCONNECT"
self.client = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.menuText = """
░██╗░░░░░░░██╗██╗░██████╗███████╗ ████████╗░█████╗░░██╗░░░░░░░██╗███╗░░██╗
░██║░░██╗░░██║██║██╔════╝██╔════╝ ╚══██╔══╝██╔══██╗░██║░░██╗░░██║████╗░██║
░╚██╗████╗██╔╝██║╚█████╗░█████╗░░ ░░░██║░░░██║░░██║░╚██╗████╗██╔╝██╔██╗██║
░░████╔═████║░██║░╚═══██╗██╔══╝░░ ░░░██║░░░██║░░██║░░████╔═████║░██║╚████║
░░╚██╔╝░╚██╔╝░██║██████╔╝███████╗ ░░░██║░░░╚█████╔╝░░╚██╔╝░╚██╔╝░██║░╚███║
░░░╚═╝░░░╚═╝░░╚═╝╚═════╝░╚══════╝ ░░░╚═╝░░░░╚════╝░░░░╚═╝░░░╚═╝░░╚═╝░░╚══╝
"""
class Client(Config):
def __init__(self):
super().__init__()
self.client.connect(self.ADDR)
self.admin = None
def send(self, obj):
obj = pickle.dumps(obj)
obj_length = len(obj)
send_length = str(obj_length).encode(self.FORMAT)
send_length += b' ' * (self.HEADER - len(send_length))
self.client.send(send_length)
self.client.send(obj)
def messenger(self):
while True:
msg = input("Mesaj: ")
self.send(msg)
class Menu(Client):
def __init__(self):
super().__init__()
class MainMenu(Menu):
def __init__(self, player_name):
super().__init__()
self.player_name = player_name
def printMainMenu(self, game_is_on = False):
print("\n" * 7 + self.menuText)
time.sleep(2)
os.system("cls")
print(self.menuText)
print("""
1. Yeni Oyun Oluştur
2. Oyuna Katıl
""")
class Lobby(Menu):
def __init__(self):
super().__init__()
def lobbyMenu(self):
name_send = f"!NAME:{self.player_name}"
self.send(name_send)
os.system("cls")
print(self.menuText)
print(f"Lobby ID: {self.id}")
self.send(f"?ONLINE:{self.id}")
self.online_list = pickle.loads(self.client.recv(2048))
temp_list = self.online_list
sys.stdout.write("Aktif Oyuncular:| ")
for i in self.online_list:
sys.stdout.write(f"{i} | ")
sys.stdout.flush()
while not self.game_started:
time.sleep(1)
self.send("?ONLINE:" + str(self.id))
self.online_list = pickle.loads(self.client.recv(2048))
if temp_list != self.online_list:
sys.stdout.write("\rAktif Oyuncular:| ")
for i in self.online_list:
sys.stdout.write(f"{i} | ")
sys.stdout.flush()
temp_list = self.online_list
if len(self.online_list) == self.capacity:
self.game_started = True
class CreateLobby(Lobby):
def __init__(self, capacity, player_name, admin):
super().__init__()
self.player_name = player_name
self.admin = admin
self.id = random.randint(100000,999999)
self.capacity = int(capacity)
self.game_started = False
self.send(f"!NEWLOBBY:{self.id}:{self.player_name}:{self.capacity}")
self.lobbyMenu()
class JoinLobby(Lobby):
def __init__(self, id, player_name):
super().__init__()
self.id = id
self.player_name = player_name
self.game_started = False
self.lobby_joiner()
def lobby_joiner(self):
self.send(f"!JOINLOBBY:{self.id}:{self.player_name}")
bool_obj = pickle.loads(self.client.recv(2048))
while bool_obj == "False":
print("Bu ID'ye ait lobby bulunmamaktadır.")
self.id = input(" Lobby ID: ")
self.send(f"!JOINLOBBY:{self.id}:{self.player_name}")
bool_obj = pickle.loads(self.client.recv(2048))
self.capacity = int(bool_obj.split(":")[1])
self.lobbyMenu()
#-------------------------------------------------------------------------------------------------------------
Your code includes
obj = pickle.loads(conn.recv(obj_length))
The problem is that TCP is a streaming protocol and the entire obj_length of data may not have been received when the call is made. When you run client and server on the same machine, you don't have the real network with real segmentation and delays, so you don't see the problem.
The solution is your own receiver that knows to keep asking for data until it sees it all
def recvall(conn, count):
recvlist = []
recvcount = 0
while recvcount < count:
buf = conn.recv(count-recvcount)
if not buf:
# replace with your error handling here
raise OSError("Connection terminated")
recvlist.append(buf)
recvcount += len(buf)
return b"".join(recvlist)
Replace your original line with
obj = pickle.loads(recvall(conn, obj_length))
and it should work
Related
From the server side, I have prepared a python script where the following tasks are performed.
Task 1 => Acquiring data from the sensors
Task 2 => Setting a certain register in the robot control unit to 1 or 0 depending upon the acquired data from the sensors
To do the above-mentioned tasks, i implemented two individual threads. To perform task 1, i have defined the class "SensorEvaluationBoard(Thread)" and for performing task 2 i have defined the class "UR_RTDE(Thread)". For more details, please see the following python script from the server side.
Server side:
#!/usr/bin/env python
import time
from time import sleep
import sys
import string
import traceback
import logging
import socket
import struct
import copy
import xmlrpclib
# xmlrpclib.Marshaller.dispatch[type(0L)] = lambda _, v, w: w("<value><i8>%d</i8></value>" % v)
# xmlrpclib.dumps((2**63-1,))
xmlrpclib.Binary
from SimpleXMLRPCServer import SimpleXMLRPCServer, SimpleXMLRPCRequestHandler
import threading
from threading import Thread, Lock
# "https://www.universal-robots.com/articles/ur-articles/real-time-data-exchange-rtde-guide/"
import rtde.rtde as rtde
import rtde.rtde_config as rtde_config
# urHost = "127.0.0.1" # UR robot's IP address
# "https://www.universal-robots.com/articles/ur-articles/remote-control-via-tcpip/"
urPort = 30004 # Port number dedicated for UR robot's secondary interface
config_filename = 'control_loop_configuration.xml'
mutexFreedriveMode = Lock()
loadCycleOfSensor1, loadCycleOfSensor2, loadCycleOfSensor3 = "", "", ""
c1, c2, c3 = 0, 0, 0
cnt = 0
data = bytearray()
sensEvalBoard_ipAddr = "" # IP address of sensor evaluation board
sensEvalBoard_port = 0 # Port number of of sensor evaluation board
DEFAULT_TIMEOUT = 1
mutexSensVal = Lock()
sebSocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) # initializing the socket object
mutexSEBSocket = Lock()
def set_ipaddr(new_daemon_host):
global sensEvalBoard_ipAddr
sensEvalBoard_ipAddr = new_daemon_host
return sensEvalBoard_ipAddr
def get_ipaddr():
tmp = ""
if str(sensEvalBoard_ipAddr):
tmp = sensEvalBoard_ipAddr
else:
tmp = "No ip address set"
return tmp
def set_portnum(new_daemon_port):
global sensEvalBoard_port
sensEvalBoard_port = int (new_daemon_port)
return sensEvalBoard_port
def get_portnum():
tmp = 0
if sensEvalBoard_port != 0:
tmp = sensEvalBoard_port
else:
tmp = 0
return tmp
class ConnectionState:
DISCONNECTED = 0
CONNECTED = 1
STARTED = 2
PAUSED = 3
class SensorEvaluationBoardException(Exception):
def __init__(self, msg):
self.msg = msg
def __str__(self):
return repr(self.msg)
class SensorEvaluationBoard(Thread):
def __init__(self, hostname, port):
# Call the Thread class's init function
Thread.__init__(self)
self.__hostname = hostname
self.__port = port
self.__conn_state = ConnectionState.DISCONNECTED
self.__sock = None
logging.basicConfig(level=logging.DEBUG)
self.__logger = logging.getLogger(self.__class__.__name__)
# self.__streamHandler = logging.StreamHandler()
# self.__streamHandler.setLevel(logging.DEBUG)
# self.__formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
# self.__streamHandler.setFormatter(self.__formatter)
# self.__logger.addHandler(self.__streamHandler)
def connect(self):
global sebSocket
if self.__sock:
return
self.__buf = b''
try:
self.__sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.__sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
self.__sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
self.__sock.settimeout(DEFAULT_TIMEOUT)
self.__sock.connect((self.__hostname, self.__port))
self.__conn_state = ConnectionState.CONNECTED
sebSocket = copy.deepcopy(self.__sock)
except (socket.timeout, socket.error):
self.__sock = None
raise
def disconnect(self):
if self.__sock:
self.__sock.close()
self.__sock = None
self.__conn_state = ConnectionState.DISCONNECTED
def is_connected(self):
return self.__conn_state is not ConnectionState.DISCONNECTED
def get_connection_state(self):
return self.__conn_state
def send_input_data(self, data):
self.__sock.send(data)
def receive_output_data(self, input_data):
self.send_input_data(input_data)
self.__rcvData = self.__sock.recv(1024)
return self.__rcvData
def run(self):
global loadCycleOfSensor1, loadCycleOfSensor2, loadCycleOfSensor3
self.connect()
while True:
if self.is_connected:
# print("Socket => connection state: " + str(self.get_connection_state()) + " means CONNECTED")
self.__input_data = bytearray()
self.__input_data.append(0x1)
self.__rcvData = self.receive_output_data(self.__input_data)
self.__unpacker_string = 3*'I I I I I B B B B'
self.__unpacker = struct.Struct('<B '+ self.__unpacker_string)
self.__unpacked = self.__unpacker.unpack(self.__rcvData)
# print("Slave 1: "+ repr(self.__unpacked[1:10]))
# print("Slave 2: "+ repr(self.__unpacked[10:19]))
# print("Slave 3: "+ repr(self.__unpacked[19:28]))
mutexSensVal.acquire()
loadCycleOfSensor1 = str(self.__unpacked[1])
loadCycleOfSensor2 = str(self.__unpacked[12])
loadCycleOfSensor3 = str(self.__unpacked[20])
# print("Load cycle of sensors 1, 2 and 3: ["+ loadCycleOfSensor1 + ", " + loadCycleOfSensor2 + ", " + loadCycleOfSensor3 + "]")
mutexSensVal.release()
sleep(0.1)
else:
print("Socket => connection state: " + str(self.get_connection_state()) + " means DISCONNECTED")
class UR_RTDE(Thread):
def __init__(self, host, port, config_filename):
# Call the Thread class's init function
Thread.__init__(self)
self.__host = host
self.__port = port
self.__config_filename = config_filename
logging.basicConfig(level=logging.DEBUG)
self.__logger = logging.getLogger(self.__class__.__name__)
# self.__streamHandler = logging.StreamHandler()
# self.__streamHandler.setLevel(logging.INFO)
# self.__formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
# self.__streamHandler.setFormatter(self.__formatter)
# self.__logger.addHandler(self.__streamHandler)
self.__con = None
self._rtde_ok = False
self._recept_ok = False
self._dataAccessConfig = Lock()
self.__keep_running = True
def init_rtde(self):
self.__logger.debug("init_rtde")
if not self._rtde_ok :
if self.__con is None:
self.connect_rtde()
if not self._recept_ok:
self.init_conf()
self.send_conf_to_robot()
self.start_communication()
self._rtde_ok = True
def connect_rtde(self):
self.__logger.debug("connect_rtde")
try:
self.__con = rtde.RTDE(self.__host, self.__port)
self.__con.connect()
except socket.timeout as e:
self.__logger.error("failed to connect with robot", exc_info=True)
self.__controller_info = None
self.__con = None
self.__controller_info = self.__con.get_controller_version()
self.__logger.info("connected with UR established")
self.__logger.info(self.__controller_info)
return True
def disconnect_rtde(self):
self.__con.send_pause()
self.__con.disconnect()
self.__logger.debug("disconnect_rtde")
def init_conf(self):
self.__logger.debug("init_conf")
with self._dataAccessConfig:
self._conf = rtde_config.ConfigFile(self.__config_filename)
self._state_names, self._state_types = self._conf.get_recipe('state')
self._set_freedrive_name, self._set_freedrive_type = self._conf.get_recipe('set_freedrive')
def send_conf_to_robot(self):
self.__logger.debug("send_conf_to_robot")
print("in send conf")
try:
self.__con.send_output_setup(self._state_names, self._state_types)
self.__set_freedrive = self.__con.send_input_setup(self._set_freedrive_name, self._set_freedrive_type)
self.__set_freedrive.__dict__[b"input_bit_register_64"] = 0
self._recept_ok = True
except Exception as e:
self.__set_freedrive = None
self.__logger.error("rtde recepts error", exc_info=True)
self._recept_ok = False
def start_communication(self):
self.__logger.debug("start_communication")
if not self.__con.is_connected():
self.__logger.warning("no connection established")
if not self.__con.send_start():
self.__logger.warning("starting data_sync failed")
sys.exit()
self.__logger.info("started communication")
return True
def run(self):
global loadCycleOfSensor1, loadCycleOfSensor2, loadCycleOfSensor3
global c1, c2, c3
lowerThr_loadCycleOfSensor1, upperThr_loadCycleOfSensor1 = 400, 700
lowerThr_loadCycleOfSensor2, upperThr_loadCycleOfSensor2 = 100, 400
lowerThr_loadCycleOfSensor3, upperThr_loadCycleOfSensor3 = 200, 425
lTh_c1 = lowerThr_loadCycleOfSensor1
uTh_c1 = upperThr_loadCycleOfSensor1
lTh_c2 = lowerThr_loadCycleOfSensor2
uTh_c2 = upperThr_loadCycleOfSensor2
lTh_c3 = lowerThr_loadCycleOfSensor3
uTh_c3 = upperThr_loadCycleOfSensor3
self.init_rtde()
# Set the 'input_bit_register_64' to 0 by default
self.__set_freedrive.__dict__[b"input_bit_register_64"] = 0
if self.__con.is_connected():
while self.__keep_running:
# receive the current state
self.__state = self.__con.receive()
if self.__state is None:
break
mutexSensVal.acquire()
c1 = int (loadCycleOfSensor1)
self.__logger.info("Loading cycle of CapSen1 is " + str(c1))
c2 = int (loadCycleOfSensor2)
self.__logger.info("Loading cycle of CapSen2 is " + str(c2))
c3 = int (loadCycleOfSensor3)
self.__logger.info("Loading cycle of CapSen3 is " + str(c3))
mutexSensVal.release()
mutexFreedriveMode.acquire()
# input_bit_register_64 refers to "general purpose input register 64"
# if "general purpose input register 64" variable is set to 1 => Freedrive mode is activated
if ((lTh_c1 < c1 < uTh_c1) and (lTh_c2 < c2 < uTh_c2)):
self.__set_freedrive.__dict__[b"input_bit_register_64"] = 1
self.__logger.info("Capacitive sensors 1 and 2 are touched by the human operator, Freedrive mode activated")
elif ((lTh_c2 < c2 < uTh_c2) and (lTh_c3 < c3 < uTh_c3)):
self.__set_freedrive.__dict__[b"input_bit_register_64"] = 1
self.__logger.info("Capacitive sensors 2 and 3 are touched by the human operator, Freedrive mode activated")
elif ((lTh_c3 < c3 < uTh_c3) and (lTh_c1 < c1 < uTh_c1)):
self.__set_freedrive.__dict__[b"input_bit_register_64"] = 1
self.__logger.info("Capacitive sensors 1 and 3 are touched by the human operator, Freedrive mode activated")
else:
self.__set_freedrive.__dict__[b"input_bit_register_64"] = 0
self.__logger.info("No two capacitive sensors are touched by the human operator, Freedrive mode deactivated")
self.__con.send(self.__set_freedrive)
mutexFreedriveMode.release()
sleep(0.1)
self.disconnect_rtde()
def main():
try:
# threadevent = threading.Event()
# Create an object of Thread
th_sensevalboard = SensorEvaluationBoard(sensEvalBoard_ipAddr, sensEvalBoard_port)
# start the SensorEvaluationBoard thread
th_sensevalboard.start()
sleep(1)
# Create an object of Thread
th_rtde = UR_RTDE(urHost, urPort, config_filename)
# start the RTDE thread
th_rtde.start()
return "SensorEvaluationBoard and RTDE threads has started..."
except KeyboardInterrupt:
print("Terminating communication with the sensor evaluation board... ")
th_sensevalboard.disconnect()
print("Socket => connection state: " + str(th_sensevalboard.get_connection_state()) + " means DISCONNECTED")
th_sensevalboard.join()
print ("SensorEvaluationBoard thread successfully closed")
print("Terminating communication with the RTDE server... ")
th_rtde.disconnect()
th_rtde.join()
print ("RTDE thread successfully closed")
return "SensorEvaluationBoard and RTDE threads has stopped..."
sys.stdout.write("CapSens daemon started")
sys.stderr.write("CapSens daemon started with caught exception")
# server_addr = ("127.0.0.1", 40405)
# server = SimpleXMLRPCServer(server_addr, SimpleXMLRPCRequestHandler, allow_none=True)
server = SimpleXMLRPCServer(("127.0.0.1", 40405), allow_none=True)
server.register_function(set_ipaddr, "set_ipaddr")
server.register_function(get_ipaddr, "get_ipaddr")
server.register_function(set_portnum, "set_portnum")
server.register_function(get_portnum, "get_portnum")
server.register_instance(UR_RTDE(urHost, urPort, config_filename), allow_dotted_names=True)
server.register_instance(SensorEvaluationBoard(sensEvalBoard_ipAddr, sensEvalBoard_port), allow_dotted_names=True)
server.register_function(main, "main")
server.serve_forever()
In the main() function of the above-mentioned python script, i initialize the two threads 'th_sensevalboard' and 'th_rtde' and start them one after the other. After the main() function, i tried to register the class objects which are individual threads as instances to an XMLRPC server. I am not sure, if this is the right way to do it.
Can anyone please have a look at the above-mentioned python script and tell me if the way in which i register multiple class objects which which are individual threads as instances to an XMLRPC server is proper or not? If it is not the proper way, can you please give me your suggestions.
Best Regards
I'm trying to find a way to forward stdin input from my main process to a child process, and what I've found that works is basically to open a socket on the main process and then send text via the socket to the children processes. But what I'm finding is that half of the time my socket gets refused, and I have no idea what's going on.
I've followed the instructions on this question 16130786 but to no avail, I can connect via telnet, but the software still fails.
Here is the minimally reproducable example I've made
from multiprocessing import Process, Queue
from queue import Full, Empty
from io import TextIOBase
import socket
import selectors
class SocketConsoleClient(TextIOBase):
def __init__(self, port: int):
self.port = port
self.conn = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.conn.connect(('', self.port))
self.selector = selectors.DefaultSelector()
self.conn.setblocking(False)
self.selector.register(self.conn, selectors.EVENT_WRITE, data='hello')
def readline(self, size: int = ...) -> str:
while True:
for k, _ in self.selector.select(timeout=None):
if k.data == 'hello':
try:
return str(self.conn.recv(1024).decode('latin1'))
except Exception as e:
# print(e)
continue
class SocketConsoleWriter(Process):
def __init__(self):
super().__init__()
self.writes = Queue()
self.connections = []
self.listener = None
self.selector = None
self.port = 10000
def run(self) -> None:
while True:
try:
self.listener = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.listener.bind(('', self.port))
self.listener.listen()
print('listening on', ('', self.port))
self.listener.setblocking(False)
break
except Exception as _:
self.port += 1 # if errno is 98, then port is not available.
self.selector = selectors.DefaultSelector()
self.selector.register(self.listener, selectors.EVENT_READ, data='test')
while True:
try:
w = self.writes.get_nowait()
if w == '$$$EXIT!!!':
break
else:
for c in self.connections:
c.send(w.encode('latin1'))
except Empty:
pass
try:
d = self.selector.select(1)
for k, _ in d:
if k.data == 'test':
conn, addr = self.listener.accept()
print('{} connected'.format(addr))
self.connections.append(conn)
except Exception as e:
# print(e)
pass
class SocketConsoleServer:
server = None
def __init__(self):
if SocketConsoleServer.server is None:
SocketConsoleServer.server = SocketConsoleWriter()
SocketConsoleServer.server.start()
#staticmethod
def port() -> int:
if SocketConsoleServer.server is None:
SocketConsoleServer.server = SocketConsoleWriter()
SocketConsoleServer.server.start()
return SocketConsoleServer.server.port
#staticmethod
def write(msg: str):
if SocketConsoleServer.server is None:
SocketConsoleServer.server = SocketConsoleWriter()
SocketConsoleServer.server.start()
SocketConsoleServer.server.writes.put(msg)
if __name__ == '__main__':
import sys, time
serv = SocketConsoleServer()
time.sleep(1)
class TestProcessSocket(Process):
def run(self):
sys.stdin = SocketConsoleClient(serv.port())
time.sleep(1)
print(input())
client = TestProcessSocket()
client.start()
serv.write(input('Type something: '))
client.join()
Why is my socket connection getting refused, I'm using ubuntu?
I want to edit all client QTextBrowser logs when any client sends some message to server.
my intended procedure is this:
[client]enter -> [client]chatUI.handleEnter -> (RFC)[server]exposed_send -> [server]broadcast -> (RFC)[clients]update.emit() -> [clients]listen -> log changed
When I run this code, other clients logs are not changed and only the client that give input to server has an updated log.
How I can solve this to update all clients properly?
chat_server.py
import rpyc
import random
import string
from threading import RLock
users = dict()
callbacks = dict()
user_num = 0
lock = RLock()
buf = dict()
class chatService(rpyc.Service):
def on_connect(self):
global user_num
with lock:
user_num = user_num+1
print ("connect user: %d" % user_num)
def on_disconnect(self):
global user_num
with lock:
user_num = user_num-1
print ("disconnect user: %d" % user_num)
def exposed_accept(self, idt, callback):
with lock:
global users
global callbacks
if not isinstance(idt, str) or len(idt) != 6:
return False
elif idt in users:
return -1
else:
pw = ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(32))
users[idt] = pw
callbacks[idt] = rpyc.async(callback)
return pw
def exposed_send(self, target, msg, idt, pw):
print ('here')
name = self.identify(idt, pw)
if name == False:
print ('here2')
return False
else:
print ('here3')
global callbacks
if target == None:
self.broadcast("[%s] %s" % (name, msg))
elif msg.target in callbacks:
self.send("[%s] %s" %(name, msg), target)
else:
return False
def exposed_order(self, msg, idt, pw):
pass
def identify(self, idt, pw):
global users
if users[idt] == pw:
return idt
else:
return False
def broadcast(self, msg):
with lock:
print("bloadcast calls")
global callbacks
global buf
for user, callback in callbacks.items():
if user not in buf or buf[user] == None:
buf[user] = (msg,)
else:
buf[user] = buf[user] + (msg,)
callback()
def send(self, msg, target):
global callbacks
global buf
if user not in buf or buf[user] == None:
buf[target] = (msg,)
else:
buf[target] = buf[target] + (msg,)
callbacks[target]()
def exposed_get_buf(self, user):
global buf
temp = buf[user]
buf[user] = None
return temp
if __name__ == '__main__':
from rpyc.utils.server import ThreadedServer
t = ThreadedServer(chatService, port = 3743)
t.start()
chat_client.py
from chatUI import *
import rpyc
import random
import string
if __name__ == '__main__':
service = rpyc.connect('floating.tk', 3743)
app, chat = UIReady(service)
while True:
idt = ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(6))
pw = service.root.accept(idt, chat.update.update.emit)
if pw != False and pw != -1:
break
chat.idt = idt
chat.pw = pw
sys.exit(app.exec_())
chatUI.py
import sys
from PyQt5.QtWidgets import *
from PyQt5.QtCore import *
class Updater(QObject):
update = pyqtSignal()
class Chat(QWidget):
log = None
enter = None
def __init__(self, service) :
super().__init__()
self.service = service
self.idt = None
self.pw = None
self.initUI()
self.update = Updater()
self.update.update.connect(self.listen)
def initUI(self):
logLabel = QLabel('chat log')
enterLabel = QLabel('enter')
self.log = QTextBrowser()
self.enter = QLineEdit()
self.enter.returnPressed.connect(self.handleEnter)
layout = QGridLayout()
layout.addWidget(logLabel, 0, 0)
layout.addWidget(self.log, 0, 1, 5, 1)
layout.addWidget(enterLabel, 6, 0)
layout.addWidget(self.enter, 6, 1)
self.setLayout(layout)
self.setWindowTitle('chat')
self.resize(600, 600)
self.show()
def handleEnter(self):
msg = self.enter.text()
self.enter.setText("")
self.service.root.send(None, msg, self.idt, self.pw)
print('get enter')
def listen(self):
msg = self.service.root.get_buf(self.idt)
for m in msg:
self.log.append(m)
def UIReady(service):
app = QApplication(sys.argv)
chat = Chat(service)
return app, chat
Please help me to improve this code:
import base64
import hashlib
import threading
import socket
class WebSocketServer:
def __init__(self, host, port, limit, **kwargs):
"""
Initialize websocket server.
:param host: Host name as IP address or text definition.
:param port: Port number, which server will listen.
:param limit: Limit of connections in queue.
:param kwargs: A dict of key/value pairs. It MAY contains:<br>
<b>onconnect</b> - function, called after client connected.
<b>handshake</b> - string, containing the handshake pattern.
<b>magic</b> - string, containing "magic" key, required for "handshake".
:type host: str
:type port: int
:type limit: int
:type kwargs: dict
"""
self.host = host
self.port = port
self.limit = limit
self.running = False
self.clients = []
self.args = kwargs
def start(self):
"""
Start websocket server.
"""
self.root = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.root.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
self.root.bind((self.host, self.port))
self.root.listen(self.limit)
self.running = True
while self.running:
client, address = self.root.accept()
if not self.running: break
self.handshake(client)
self.clients.append((client, address))
onconnect = self.args.get("onconnect")
if callable(onconnect): onconnect(self, client, address)
threading.Thread(target=self.loop, args=(client, address)).start()
self.root.close()
def stop(self):
"""
Stop websocket server.
"""
self.running = False
def handshake(self, client):
handshake = 'HTTP/1.1 101 Switching Protocols\r\nConnection: Upgrade\r\nUpgrade: websocket\r\nSec-WebSocket-Accept: %s\r\n\r\n'
handshake = self.args.get('handshake', handshake)
magic = "258EAFA5-E914-47DA-95CA-C5AB0DC85B11"
magic = self.args.get('magic', magic)
header = str(client.recv(1000))
try:
res = header.index("Sec-WebSocket-Key")
except ValueError:
return False
key = header[res + 19: res + 19 + 24]
key += magic
key = hashlib.sha1(key.encode())
key = base64.b64encode(key.digest())
client.send(bytes((handshake % str(key,'utf-8')), 'utf-8'))
return True
def loop(self, client, address):
"""
:type client: socket
"""
while True:
message = ''
m = client.recv(1)
while m != '':
message += m
m = client.recv(1)
fin, text = self.decodeFrame(message)
if not fin:
onmessage = self.args.get('onmessage')
if callable(onmessage): onmessage(self, client, text)
else:
self.clients.remove((client, address))
ondisconnect = self.args.get('ondisconnect')
if callable(ondisconnect): ondisconnect(self, client, address)
client.close()
break
def decodeFrame(self, data):
if (len(data) == 0) or (data is None):
return True, None
fin = not(data[0] & 1)
if fin:
return fin, None
masked = not(data[1] & 1)
plen = data[1] - (128 if masked else 0)
mask_start = 2
if plen == 126:
mask_start = 4
plen = int.from_bytes(data[2:4], byteorder='sys.byteorder')
elif plen == 127:
mask_start = 10
plen = int.from_bytes(data[2:10], byteorder='sys.byteorder')
mask = data[mask_start:mask_start+4]
data = data[mask_start+4:mask_start+4+plen]
decoded = []
i = 0
while i < len(data):
decoded.append(data[i] ^ mask[i%4])
i+=1
text = str(bytearray(decoded), "utf-8")
return fin, text
def sendto(self, client, data, **kwargs):
"""
Send <b>data</b> to <b>client</b>. <b>data</b> can be of type <i>str</i>, <i>bytes</i>, <i>bytearray</i>, <i>int</i>.
:param client: Client socket for data exchange.
:param data: Data, which will be sent to the client via <i>socket</i>.
:type client: socket
:type data: str|bytes|bytearray|int|float
"""
if type(data) == bytes or type(data) == bytearray:
frame = data
elif type(data) == str:
frame = bytes(data, kwargs.get('encoding', 'utf-8'))
elif type(data) == int or type(data) == float:
frame = bytes(str(data), kwargs.get('encoding', 'utf-8'))
else:
return None
framelen = len(frame)
head = bytes([0x81])
if framelen < 126:
head += bytes(int.to_bytes(framelen, 1, 'big'))
elif 126 <= framelen < 0x10000:
head += bytes(126)
head += bytes(int.to_bytes(framelen, 2, 'big'))
else:
head += bytes(127)
head += bytes(int.to_bytes(framelen, 8, 'big'))
client.send(head + frame)
It works fine.
I want the server to use all the processor cores for improved performance. And this code is not effective in high quantities connections. How to implement a multi-threaded solution for this case?
sorry for my bad english.
In CPython, the global interpreter lock, or GIL, is a mutex that
prevents multiple native threads from executing Python bytecodes at
once.
So your code won't work. You can use processeses instead of threads (not on Windows*), twisted or asyncore if you want to support more than one client at the same time.
If your choice is multiprocessing, try this:
client.py:
import socket
def main():
s = socket.socket()
s.connect(("localhost", 5555))
while True:
data = raw_input("> ")
s.send(data)
if data == "quit":
break
s.close()
if __name__ == "__main__":
main()
server.py:
from multiprocessing import Process
from os import getpid
import socket
def receive(conn):
print "(%d) connected." % getpid()
while True:
data = conn.recv(1024)
if data:
if data == "quit":
break
else:
print "(%s) data" % getpid()
def main():
s = socket.socket()
s.bind(("localhost", 5555))
s.listen(1)
while True:
conn, address = s.accept()
print "%s:%d connected." % address
Process(target=receive, args=(conn,)).start()
s.close()
if __name__ == "__main__":
main()
*On Windows this code will throw an error when pickling the socket:
File "C:\Python27\lib\pickle.py", line 880, in load_eof
raise EOFError
(never mind...solved it)
I've written a milter in Python using pymilter that extracts attachments and saves them to a folder (thanks SO for the recommendation). I've gotten to the point in writing the milter where I need to put the original body of the message back in with the original formatting, but I'm having difficulty seeing exactly how to do this.
FYI - I am a NOVICE python programmer, but have learned much in the past few days.
Here is my current code for examination:
#! /usr/bin/env python
import Milter
import StringIO
import email
import email.Message
import hashlib
import mime
import os
import sys
import tempfile
import time
import rfc822
from Milter.utils import parse_addr
from email import Errors
from email.Message import Message
## == IP Information
from socket import AF_INET, AF_INET6
## ==
## === Define multiprocessing == ##
if True:
from multiprocessing import Process as Thread, Queue
else:
from threading import Thread
from Queue import Queue
logq = Queue(maxsize=4)
def background():
while True:
t = logq.get()
if not t: break
msg,id,ts = t
print "%s [%d]" % (time.strftime('%Y%b%d %H:%M:%S',time.localtime(ts)),id),
# 2005Oct13 02:34:11 [1] msg1 msg2 msg3 ...
for i in msg: print i,
print
## === End Define Multiprocesing === ##
class mltr_SaveAttachments(Milter.Base):
def __init__(self):
self.id = Milter.uniqueID()
def close(self):
# always called, even when abort is called. Clean up
# any external resources here.
return Milter.CONTINUE
def abort(self):
# client disconnected prematurely
return Milter.CONTINUE
def log(self,*msg):
logq.put((msg,self.id,time.time()))
#Milter.noreply
def connect(self, IPname, family, hostaddr):
self.IP = hostaddr[0]
self.port = hostaddr[1]
if family == AF_INET6:
self.flow = hostaddr[2]
self.scope = hostaddr[3]
else:
self.flow = None
self.scope = None
self.IPname = IPname # Name from a reverse IP lookup
self.H = None
self.fp = None
self.receiver = self.getsymval('j')
self.log("connect from %s at %s" % (IPname, hostaddr) )
return Milter.CONTINUE
#Milter.noreply
def header(self, name, hval):
self.fp.write("%s: %s\n" % (name,hval)) # add header to buffer
return Milter.CONTINUE
#Milter.noreply
def body(self, chunk):
self.fp.write(chunk)
return Milter.CONTINUE
#Milter.noreply
def eoh(self):
self.fp.write("\n") # terminate headers
return Milter.CONTINUE
def envfrom(self,mailfrom,*str):
# self.log("envfrom")
self.F = mailfrom
self.R = []
self.fromparms = Milter.dictfromlist(str)
self.user = self.getsymval('{auth_authen}')
self.log("mail from:", mailfrom, *str)
self.fp = StringIO.StringIO()
self.canon_from = '#'.join(parse_addr(mailfrom))
self.fp.write('From %s %s\n' % (self.canon_from,time.ctime()))
return Milter.CONTINUE
## def envrcpt(self, to, *str):
#Milter.noreply
def envrcpt(self, recipient, *str):
rcptinfo = recipient,Milter.dictfromlist(str)
self.R.append(rcptinfo)
return Milter.CONTINUE
def attachment(self):
msg = self._msg
attachDir = attach_dir(msg)
removedParts = []
payload = []
for part in msg.walk():
fname = ""
self.log(part.getparams())
if part.is_multipart():
continue
dtypes = part.get_params(None, 'Content-Disposition')
if not dtypes:
if part.get_content_type() == 'text/plain':
payload.append(part)
continue
ctypes = part.getparams()
if not ctypes:
continue
for key,val in ctypes:
if key.lower() == 'name':
fname = val
else:
for key,val in dtypes:
if key.lower() == 'filename':
fname = val
if fname:
removedParts.append(fname)
data = part.get_payload(decode=1)
extract_attachment(data, attachDir, fname)
part = self.delete_attachments(part, fname)
payload.append(part)
# del msg["content-type"]
# del msg["content-disposition"]
# del msg["content-transfer-encoding"]
# msg.set_payload(payload)
self._msg = msg
out = tempfile.TemporaryFile()
try:
self.log("dumping")
msg.dump(out)
out.seek(0)
msg = rfc822.Message(out)
msg.rewindbody()
while 1:
buf = out.read(8192)
if len(buf) == 0: break
self.replacebody(buf)
finally:
out.close()
self._msg.attach(payload)
return Milter.CONTINUE
def delete_attachments(self, part,fname):
for key,value in part.get_params():
part.del_param(key)
part.set_payload('[DELETED]\n')
del part["content-type"]
del part["content-disposition"]
del part["content-transfer-encoding"]
part["Content-Type"] = "text/html, name="+fname+".html"
return part
def eom(self):
self.fp.seek(0)
msg = mime.message_from_file(self.fp)
self._msg = msg
self.attachment()
# self.log("### MESSAGE ###")
# self.log(self._msg)
# return Milter.ACCEPT
return Milter.TEMPFAIL
## ===
def attach_dir(msg):
tempname = fname = tempfile.mktemp(".tmp")
out = tempfile.TemporaryFile()
msg.dump(out)
out.seek(0)
buf = out.read()
hashDir = hashit(buf)
attachDir = dropDir + hashDir
if not os.path.isdir(hashDir):
os.mkdir(attachDir)
return attachDir
def extract_attachment(data, attachDir, fname):
exdir_file = attachDir + "/" + fname
extracted = open(exdir_file, "wb")
extracted.write(data)
extracted.close()
def hashit(data):
sha1 = hashlib.sha1()
sha1.update(data)
return sha1.hexdigest()
dropDir = "/dropdir/"
def main():
bt = Thread(target=background)
bt.start()
socketname = "/tmp/py_testmilter.sock"
timeout = 600
Milter.factory = mltr_SaveAttachments
flags = Milter.CHGBODY + Milter.CHGHDRS + Milter.ADDHDRS
flags += Milter.ADDRCPT
flags += Milter.DELRCPT
Milter.set_flags(flags) # tell Sendmail/Postfix which features we use
print "%s milter startup" % time.strftime('%Y%b%d %H:%M:%S')
sys.stdout.flush()
Milter.runmilter("py_testmilter",socketname,timeout)
logq.put(None)
bt.join()
print "%s milter shutdown" % time.strftime('%Y%b%d %H:%M:%S')
if __name__ == "__main__":
main()