We Keep Coding

Adapt thread to select in Python

While studying about networks, i found a very common exercise that i thought was pretty insteresting, that is an application to manage simple chatrooms using sockets in Python
The thing is that i found a solution that uses thread, and was wondering how to adapt the solution i found from thread use to select.
The server.py :
from http import client
import os
import socket
import threading
import time
class Server:
def __init__(self, host, port):
self.HOST = host
self.PORT = port
self.rooms_list = []
def get_network(self):
return (self.HOST,self.PORT)
def run(self):
try:
self.create_connection_TCP()
self.accept_connection_rooms()
except:
print("Ocorreu um erro com o servidor principal")
os._exit(1)
def getList(self):
...
def create_connection_TCP(self):
server = (self.HOST, self.PORT)
self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
try:
self.socket.bind(server)
except:
print("Bind failed")
os._exit(1)
self.socket.listen(100)
def accept_connection_rooms(self):
while True:
try:
client, client_address = self.socket.accept()
thread = threading.Thread(target = self.control_connection, args = (client, ))
thread.start()
except:
print("Failing while creating conection")
os._exit(1)
def check_comand(self, client_socket):
message = client_socket.recv(1024).decode('utf-8')
command = message.split(':')
if command[0] == '/shutdown':
self.socket.close()
if command[0] == '/add_room':
room = ':'.join(command[1:4])
print(room)
if not room in self.rooms_list:
qtd_clients = len(self.rooms_list)
print(f"servidor: {room} | max clients: {command[4]}")
room = ':'.join(command[1:5])
self.rooms_list.append(room)
if command[0] == '/get_room':
index = int(command[1])
try:
room = self.rooms_list[index].split(':')
room = ':'.join(room[1:3])
client_socket.send(f"{room}".encode('utf-8'))
except IndexError:
client_socket.send("error: invald option".encode('utf-8'))
if command[0] == '/get_room_id':
message = len(self.rooms_list)
client_socket.send(message.encode('utf-8'))
if command[0] == '/list_rooms':
rooms = []
for index in range(len(self.rooms_list)):
room_name = self.rooms_list[index].split(':')[0]
rooms.append(f"{index} - {room_name}")
rooms = '\n'.join(rooms)
client_socket.send(f"{rooms}".encode('utf-8'))
# print(f"{rooms}")
if command[0] == '/close_room':
room = ':'.join(command[1:4])
self.rooms_list.remove(room)
print(f"closed_room: {room}")
def control_connection(self, client):
self.check_comand(client)
def close_server(self):
self.socket.close()
server = Server('127.0.0.1', 5000)
server.run()

Probably a good solution is to use select to listen all sockets connections and implement on accept_connection_rooms to manage new sockets

Why is my connection being refused half of the time

I'm trying to find a way to forward stdin input from my main process to a child process, and what I've found that works is basically to open a socket on the main process and then send text via the socket to the children processes. But what I'm finding is that half of the time my socket gets refused, and I have no idea what's going on.
I've followed the instructions on this question 16130786 but to no avail, I can connect via telnet, but the software still fails.
Here is the minimally reproducable example I've made
from multiprocessing import Process, Queue
from queue import Full, Empty
from io import TextIOBase
import socket
import selectors
class SocketConsoleClient(TextIOBase):
def __init__(self, port: int):
self.port = port
self.conn = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.conn.connect(('', self.port))
self.selector = selectors.DefaultSelector()
self.conn.setblocking(False)
self.selector.register(self.conn, selectors.EVENT_WRITE, data='hello')
def readline(self, size: int = ...) -> str:
while True:
for k, _ in self.selector.select(timeout=None):
if k.data == 'hello':
try:
return str(self.conn.recv(1024).decode('latin1'))
except Exception as e:
# print(e)
continue
class SocketConsoleWriter(Process):
def __init__(self):
super().__init__()
self.writes = Queue()
self.connections = []
self.listener = None
self.selector = None
self.port = 10000
def run(self) -> None:
while True:
try:
self.listener = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.listener.bind(('', self.port))
self.listener.listen()
print('listening on', ('', self.port))
self.listener.setblocking(False)
break
except Exception as _:
self.port += 1 # if errno is 98, then port is not available.
self.selector = selectors.DefaultSelector()
self.selector.register(self.listener, selectors.EVENT_READ, data='test')
while True:
try:
w = self.writes.get_nowait()
if w == '$$$EXIT!!!':
break
else:
for c in self.connections:
c.send(w.encode('latin1'))
except Empty:
pass
try:
d = self.selector.select(1)
for k, _ in d:
if k.data == 'test':
conn, addr = self.listener.accept()
print('{} connected'.format(addr))
self.connections.append(conn)
except Exception as e:
# print(e)
pass
class SocketConsoleServer:
server = None
def __init__(self):
if SocketConsoleServer.server is None:
SocketConsoleServer.server = SocketConsoleWriter()
SocketConsoleServer.server.start()
#staticmethod
def port() -> int:
if SocketConsoleServer.server is None:
SocketConsoleServer.server = SocketConsoleWriter()
SocketConsoleServer.server.start()
return SocketConsoleServer.server.port
#staticmethod
def write(msg: str):
if SocketConsoleServer.server is None:
SocketConsoleServer.server = SocketConsoleWriter()
SocketConsoleServer.server.start()
SocketConsoleServer.server.writes.put(msg)
if __name__ == '__main__':
import sys, time
serv = SocketConsoleServer()
time.sleep(1)
class TestProcessSocket(Process):
def run(self):
sys.stdin = SocketConsoleClient(serv.port())
time.sleep(1)
print(input())
client = TestProcessSocket()
client.start()
serv.write(input('Type something: '))
client.join()
Why is my socket connection getting refused, I'm using ubuntu?

Python: How do I lock an object until all threads have done a thing

I am writing a simple threaded server that will send a message to all clients. I have an object that is reset after posting the change message, however I am having a hard time figuring out how to reset that object only after all threads have posted the change message.
To add some context to the problem. I am building a multi user Tkinter python app which connects to a remote database to retrieve information and the application needs to know when data changes so that when a user updates data, all other running instances of the app will get the update. From what I understand, MySQL does not support asynchronous application updates. Instead of running a query every 5 seconds on the database to see if there is a change, I am putting this code server side so that it will send a message to a socket on the client that a change has occurred on the database.
The main loop is just a dummy that will simulate a change
Here is my code:
import socket, threading, time, select, os
class dbMonitor:
isDBAltered = False
def postChange(self):
self.isDBAltered = True
def __str__(self):
return str(self.isDBAltered)
class ThreadedServer(object):
def __init__(self, port,dbMon):
self.port = port
self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
self.sock.setblocking(0)
self.sock.bind((socket.gethostname(), self.port))
self.dbMon = dbMon
def listen(self):
self.sock.listen(100)
read_list = [self.sock]
while True:
read,write,error = select.select(read_list,[],[],1)
for s in read:
if s is self.sock:
client, address = self.sock.accept()
client.settimeout(60)
threading.Thread(target = self.listenToClient, args = (client,address)).start()
def listenToClient(self, client, address):
read_list = [client]
size = 1024
while True:
response = b'Ack'
if self.dbMon.isDBAltered:
response = b'CHANGE'
try:
client.send(response)
except:
client.close()
return False
self.dbMon.isDBAltered = False
read,write,error = select.select(read_list,[],[],1)
for s in read:
if s is client:
try:
data = client.recv(size)
print(data)
if data:
client.send(response)
else:
raise error('Client disconnected')
except:
client.close()
return False
def mainLoop():
while True:
time.sleep(15)
print(dbMon)
dbMon.postChange()
dbMon = dbMonitor()
server = ThreadedServer(5005,dbMon)
threading.Thread(target = mainLoop, args=()).start()
threading.Thread(target = server.listen(), args=()).start()
How do I get self.dbMon.isDBAltered = False to execute only after all threads have executed:
response = b'CHANGE'
try:
client.send(response)

You're trying to synchronize something that's asynchronous... This is massively more complicated than it should be. Your dbmon is only storing a boolean flag... why not just asynchronously modify the "database" instead? For example, if the "database" was a thread-safe buffer, you could just append to that buffer or modify that buffer without synchronizing each thread individually, pull the information written to that buffer and write it to the client socket they belong to in another event loop (this is pretty much what asyncore does)
That said, I have some (probably nonworking, but I hope you get the idea) reference modified code for you to go off of if you want to continue pursing this avenue.
Basically, dbmon will keep a mapping of thread ids to [creation time, modified flag]
Our predicate returns true iff all threads created before a certain threshold have ALL set the modified flag. We set the modified flag when we send the response in the data = client.recv(size) portion of your code. And then we wait on that condition in the server send. We keep notifying all waiting threads on each client receive so that when the condition is finally met, our waiting server threads will all unblock and send the subsequent response.
import socket, threading, time, select, os
import collections
class dbMonitor:
def __init__(self):
self.isDBAltered = {}
self.lock = threading.Lock()
def newThread(self, tid):
self.lock.acquire()
# time of creation, boolean whether that thread has sent response
self.isDBAltered[tid] = [time.time(), False]
self.lock.release()
def threadDone(self, tid):
self.lock.acquire()
self.isDBAltered.pop(tid, None)
self.lock.release()
def altered(self, tid):
self.lock.acquire()
self.isDBAltered[tid][1] = True
self.lock.release()
def reset(self, tid):
self.lock.acquire()
self.isDBAltered(tid)[1] = False
self.lock.release()
def __str__(self):
return str(self.isDBAltered)
class ThreadedServer(object):
def __init__(self, port,dbMon):
self.port = port
self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
self.sock.setblocking(0)
self.sock.bind((socket.gethostname(), self.port))
self.dbMon = dbMon
self.lock = threading.lock()
self.cv = threading.Condition()
self.thresh = 2000
def condition_pred(self):
# unblock if threads currently running for longer than self.thresh have all set their flags
return all([timecreate[1] if time.time() - timecreate[0] > self.thresh else True for tid,timecreate in self.dbMon.isDBAltered])
def listen(self):
self.sock.listen(100)
read_list = [self.sock]
while True:
read,write,error = select.select(read_list,[],[],1)
for s in read:
if s is self.sock:
self.lock.acquire()
client, address = self.sock.accept()
client.settimeout(60)
T = threading.Thread(target = self.listenToClient, args = (client,address)).start()
self.dbmon.newThread(T.ident)
self.lock.release()
def listenToClient(self, client, address):
read_list = [client]
size = 1024
while True:
response = b'Ack'
with self.cv:
self.cv.wait_for(self.condition_pred)
self.dbMon.reset(threading.get_ident())
response = b'CHANGE'
try:
client.send(response)
except:
client.close()
self.dbmon.threadDone(threading.get_ident())
return False
read,write,error = select.select(read_list,[],[],1)
for s in read:
if s is client:
with self.cv:
try:
data = client.recv(size)
print(data)
if data:
client.send(response)
self.dbMon.altered(threading.get_ident())
self.cv.notifyAll()
else:
raise error('Client disconnected')
except:
client.close()
self.dbmon.threadDone(threading.get_ident())
return False

Modify server's variable from client's thread (threading, python)

I implemented a simple network 'game' in Python - server draws a random number, and then the client tries to guess it. My application works great, when the client guesses the number, it disconnects from server (it is handled on client's side).
However, after the proper guess, the number is still the same. I would like to modify the application, such that when the client guesses the number, the server should then rand a new number, so other clients should guess the new one. How can I do this?
Some template, just to draw an attention to the problem:
#!/usr/bin/env python
from random import randint
import socket, select
from time import gmtime, strftime
import threading
import sys
class Handler(threading.Thread):
def __init__(self, connection, randomnumber):
threading.Thread.__init__(self)
self.connection = connection
self.randomnumber = randomnumber
def run(self):
while True:
try:
data = self.connection.recv(1024)
if data:
print data
try:
num = int(data)
if Server.guess(num) :
msg = "You won! This is the right number!"
self.connection.send(msg)
break
else :
msg = "Try again!"
self.connection.send(msg)
except ValueError, e:
msg = "%s" % e
self.connection.send(msg)
else:
msg = "error"
self.connection.send(msg)
except socket.error:
self.connection.close()
break
self.connection.close()
class Server:
def __init__(self, ip, port):
self.ip = ip
self.port = port
self.address = (self.ip, self.port)
self.server_socket = None
self.randnum = randint(1, 100)
#classmethod
def guess(cls, no):
if cls.randnum == no:
cls.randnum = randint(1, 1000)
result = True
else:
result = False
return reslut
def run(self):
try:
self.server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.server_socket.bind((self.ip, self.port))
self.server_socket.listen(10)
print 'Num is %s' % self.randnum
while True:
connection, (ip, port) = self.server_socket.accept()
c = Handler(connection, self.randnum)
c.start()
except socket.error, e:
if self.server_socket:
self.server_socket.close()
sys.exit(1)
if __name__ == '__main__':
s = Server('127.0.0.1', 1234)
s.run()

Generate the random number that is shared between both server and all the client, there should be only instance of this, hence this should be class attribute.
Add a class function guess which return False upon incorrect guess and upon correct guess changes the randnum and returns True
class Server:
randnum = randint(1, 1000) # class attribute created
#classmethod
def guess(cls, no): # To be used "guess" if `no` attribute if the same as `cls.randnum`
if cls.randnum == no:
cls.randnum = randint(1, 1000)
result = True
else:
result = False
return result
def __init__(self, ip, port):
# ...
The client should call this Server.guess function each time.

Actually your issue comes from the fact that you create randnum as an instance method (see your self.randnum) as #shanmuga explained, if you simply declare it as being a class attribute, and remove the instance method it solves your issue (i.e. declaring it in the class directly).
As a side issue (not being an expert on socket), when you send message to the client, you might want to encode them as a byte object (in the run method of Handler, I changed self.connection.send(msg) to self.connection.send(msg.encode())). Also note that I used Python 3.6 (which mainly change the style of print statements)
See the code below:
#!/usr/bin/env python
from random import randint
import socket, select
from time import gmtime, strftime
import threading
import sys
class Handler(threading.Thread):
def __init__(self, connection, randomnumber):
threading.Thread.__init__(self)
self.connection = connection
self.randomnumber = randomnumber
def run(self):
while True:
try:
data = self.connection.recv(1024)
if data:
print(data)
try:
num = int(data)
if Server.guess(num) :
msg = "You won! This is the right number!"
self.connection.send(msg.encode())
break
else :
msg = "Try again!"
self.connection.send(msg.encode())
except ValueError as e:
msg = "%s" % e
self.connection.send(msg.encode())
else:
msg = "error"
self.connection.send(msg.encode())
except socket.error:
self.connection.close()
break
self.connection.close()
class Server:
randnum = randint(1,100)
def __init__(self, ip, port):
self.ip = ip
self.port = port
self.address = (self.ip, self.port)
self.server_socket = None
#classmethod
def guess(cls, no):
if cls.randnum == no:
cls.randnum = randint(1, 1000)
print("New number is ", cls.randnum )
result = True
else:
result = False
return result
def run(self):
try:
self.server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.server_socket.bind((self.ip, self.port))
self.server_socket.listen(10)
print('Num is %s' % self.randnum)
while True:
connection, (ip, port) = self.server_socket.accept()
c = Handler(connection, self.randnum)
c.start()
except socket.error as e:
if self.server_socket:
self.server_socket.close()
sys.exit(1)
if __name__ == '__main__':
s = Server('127.0.0.1', 1234)
s.run()

Python socket programming with multiple threads

I'm trying to use sockets in python. For now, I'm trying to get it such that if any client sends any message it is received at all clients. However I'm getting very weird results. I think it's because I'm using multiple threads. The output of the program changes every time I run it. Is this a threading problem or is it something else?
import socket
import sys
from thread import *
from server import Server
from client import Client
s = Server()
start_new_thread(s.acceptConnection,())
m = Client("m")
k = Client("k")
start_new_thread(m.recieveData,())
start_new_thread(k.recieveData,())
k.sendData("Hey!")
print "*"*100
print repr(k.data()), repr(m.data())
print "*"*100
m.sendData("okay okay")
print "*"*100
print repr(k.data()), repr(m.data())
print "*"*100
m.client.close()
k.client.close()
s.s.close()
Server Class:
import socket
import sys
from thread import *
class Server(object):
def __init__(self,port = 5555):
self.host = 'localhost' # '' means connect to all hosts
self.port = port
self.text = ""
self.s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
try:
self.s.bind((self.host, self.port))
except socket.error as e:
print(str(e))
self.s.listen(2)
print "Waiting for a connection.\n"
self.connections = []
def threaded_client(self,conn):
# conn.send("Connected to server\n")
while True:
try:
data = conn.recv(2048)
except:
data = ""
if(not data):
break
# conn.sendall(reply)
for c,a in self.connections:
try:
c.sendall(data + "\n")
except:
print "connection lost\n"
self.connections.remove((c,a))
conn.close()
def acceptConnection(self):
while True:
conn, addr = self.s.accept()
self.connections += [(conn,addr)]
start_new_thread(self.threaded_client,(conn,))
Client class:
import socket
import sys
from thread import *
class Client(object):
def __init__(self,name):
self.host = 'localhost'
self.port = 5555
self.name = name
self.client= socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.client.connect((self.host,self.port))
self.text = ""
def sendData(self,data):
self.client.send(data)
def recieveData(self):
while True:
try:
data = self.client.recv(2048)
except:
break
if data:
self.text = data
self.client.close()
def data(self):
return self.text
def closeClient(self):
self.client.close()

Anyway you have no warrants about the data was already come back to the clients when you try to print it. You should introduce some Conditions and use wait() and notifyAll() to make sure the data was arrive.... To check if my guess is correct put some sleep() in your test:
import time
k.sendData("Hey!")
print "*"*100
time.sleep(200)
print repr(k.data()), repr(m.data())
print "*"*100
m.sendData("okay okay")
print "*"*100
time.sleep(200)
print repr(k.data()), repr(m.data())
print "*"*100
If it works you should use conditions and notify to do your tests.
Moreover you must protect data access by a Lock().
def recieveData(self):
while True:
try:
data = self.client.recv(2048)
except:
break
if data:
self.l.acquire()
self.text = data
self.l.release()
self.client.close()
def data(self):
self.l.acquire()
ret = self.text
self.l.release()
return ret
Where attribute l of clients are defined in __init__ by
self.l=threading.Lock()