For some reason, self.connected of the asyncore.dispatcher class doesn't consider my socket to be connected on the client side. The server side sees the socket as connected and treats it as such, but the client doesn't know if it's connected or not and handle_connect doesn't "proc", so i can't use a overridden version of it to check if the socket is connected either.
Any thoughts on this code why it ain't working:
#!/usr/bin/python
# -*- coding: iso-8859-15 -*-
import asyncore
from threading import *
from socket import *
from time import sleep
from os import _exit
from logger import *
from config import *
class logDispatcher(asyncore.dispatcher):
def __init__(self, config=None):
self.inbuffer = ''
self.buffer = ''
self.lockedbuffer = False
self.is_writable = False
asyncore.dispatcher.__init__(self)
#Thread.__init__(self)
self.create_socket(AF_INET, SOCK_STREAM)
#self.is_writable = True
#self.start()
def compare(self, obj, otherobj):
return (str(obj).lower() == str(otherobj).lower()[:len(str(obj))])
def _in(self, obj, otherobj):
return (str(obj).lower() in str(otherobj).lower())
def parse(self):
if self.inbuffer[-2:] != '\r\n':
return False
self.lockedbuffer = True
self.inbuffer = ''
self.lockedbuffer = False
def readable(self):
return True
def handle_connect(self):
log('Connected to ' + str(server), 'SOCK_CORE')
def handle_close(self):
self.close()
def handle_read(self):
data = self.recv(8192)
while self.lockedbuffer:
sleep(0.01)
self.inbuffer += data
def writable(self):
return (len(self.buffer) > 0)
def handle_write(self):
while self.is_writable:
sent = self.send(self.buffer)
sleep(1)
self.buffer = self.buffer[sent:]
if len(self.buffer) <= 0:
self.is_writable = False
sleep(0.01)
def _send(self, what):
self.buffer += what + '\r\n'
self.is_writable = True
def handle_error(self):
log('Error, closing socket!', 'SOCK_CORE')
self.close()
def run(self):
log('Log socket engine initating', 'SOCK_CORE')
self.connect((server, server_port))
print self.connected
sleep(3)
print self.connected
class start(Thread):
def __init__(self):
Thread.__init__(self)
self.start()
def run(self):
asyncore.loop(0.1)
start()
logDisp = logDispatcher()
logDisp.run()
def handle_connect_event(self):
self.is_connected = True
Adding that to your dispatcher will give you a way to check if the socket is connected or not, thanks to some stack trace (python -m trace -t script.py) in Python I managed to figure out that the asyncore class automaticly created that function for whatever reason, and it was called continiously as long as the socket was connected or in a connected state.
After that, i also replaced the threaded asyncore.loop() and replaced it with a "static" placement locking your main thread, one of these two combinations (or both) solved the issue for now.. the logic isn't the same as in my problem which i don't like but i assume that i'll be needing to create my own dispach_event just like if i were to do a OpenGL GUI class where i would call dispatch_event() manually every loop some how in the thread to "keep things alive".. it's just a thought..
Anyway, here's a working example:
#!/usr/bin/python
# -*- coding: iso-8859-15 -*-
import asyncore, socket
from threading import *
from time import sleep
from os import _exit
from logger import *
from config import *
def _map():
return {}
def _array():
return []
class logDispatcher(Thread, asyncore.dispatcher):
def __init__(self, config=None):
self.inbuffer = ''
self.buffer = ''
self.lockedbuffer = False
self.is_writable = False
self.is_connected = False
self.exit = False
self.initated = False
asyncore.dispatcher.__init__(self)
Thread.__init__(self)
self.create_socket(socket.AF_INET, socket.SOCK_STREAM)
try:
self.connect((server, server_port))
except:
log('Could not connect to ' + server, 'LOG_SOCK')
return None
self.start()
def handle_connect_event(self):
self.is_connected = True
def handle_connect(self):
self.is_connected = True
log('Connected to ' + str(server), 'LOG_SOCK')
def handle_close(self):
self.is_connected = False
self.close()
def handle_read(self):
data = self.recv(8192)
while self.lockedbuffer:
sleep(0.01)
self.inbuffer += data
def handle_write(self):
while self.is_writable:
sent = self.send(self.buffer)
sleep(1)
self.buffer = self.buffer[sent:]
if len(self.buffer) <= 0:
self.is_writable = False
sleep(0.01)
def _send(self, what):
self.buffer += what + '\r\n'
self.is_writable = True
def run(self):
sleep(1)
log('Log engine initating (hooking on to main)', 'LOG_CORE')
main = None
for t in enumerate():
if t.getName() == 'MainThread':
main = t
log('Log engine attached to main', 'LOG_CORE')
while (main and main.isAlive()) and (self.connected or self.is_connected):
print 'WHAM', self.connected, self.is_connected
sleep(1)
while 1:
logDisp = logDispatcher()
asyncore.loop(0.1)
log('Logserver disconnected, trying to reconnect!', 'CORE')
sleep(10)
Related
I'm trying to find a way to forward stdin input from my main process to a child process, and what I've found that works is basically to open a socket on the main process and then send text via the socket to the children processes. But what I'm finding is that half of the time my socket gets refused, and I have no idea what's going on.
I've followed the instructions on this question 16130786 but to no avail, I can connect via telnet, but the software still fails.
Here is the minimally reproducable example I've made
from multiprocessing import Process, Queue
from queue import Full, Empty
from io import TextIOBase
import socket
import selectors
class SocketConsoleClient(TextIOBase):
def __init__(self, port: int):
self.port = port
self.conn = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.conn.connect(('', self.port))
self.selector = selectors.DefaultSelector()
self.conn.setblocking(False)
self.selector.register(self.conn, selectors.EVENT_WRITE, data='hello')
def readline(self, size: int = ...) -> str:
while True:
for k, _ in self.selector.select(timeout=None):
if k.data == 'hello':
try:
return str(self.conn.recv(1024).decode('latin1'))
except Exception as e:
# print(e)
continue
class SocketConsoleWriter(Process):
def __init__(self):
super().__init__()
self.writes = Queue()
self.connections = []
self.listener = None
self.selector = None
self.port = 10000
def run(self) -> None:
while True:
try:
self.listener = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.listener.bind(('', self.port))
self.listener.listen()
print('listening on', ('', self.port))
self.listener.setblocking(False)
break
except Exception as _:
self.port += 1 # if errno is 98, then port is not available.
self.selector = selectors.DefaultSelector()
self.selector.register(self.listener, selectors.EVENT_READ, data='test')
while True:
try:
w = self.writes.get_nowait()
if w == '$$$EXIT!!!':
break
else:
for c in self.connections:
c.send(w.encode('latin1'))
except Empty:
pass
try:
d = self.selector.select(1)
for k, _ in d:
if k.data == 'test':
conn, addr = self.listener.accept()
print('{} connected'.format(addr))
self.connections.append(conn)
except Exception as e:
# print(e)
pass
class SocketConsoleServer:
server = None
def __init__(self):
if SocketConsoleServer.server is None:
SocketConsoleServer.server = SocketConsoleWriter()
SocketConsoleServer.server.start()
#staticmethod
def port() -> int:
if SocketConsoleServer.server is None:
SocketConsoleServer.server = SocketConsoleWriter()
SocketConsoleServer.server.start()
return SocketConsoleServer.server.port
#staticmethod
def write(msg: str):
if SocketConsoleServer.server is None:
SocketConsoleServer.server = SocketConsoleWriter()
SocketConsoleServer.server.start()
SocketConsoleServer.server.writes.put(msg)
if __name__ == '__main__':
import sys, time
serv = SocketConsoleServer()
time.sleep(1)
class TestProcessSocket(Process):
def run(self):
sys.stdin = SocketConsoleClient(serv.port())
time.sleep(1)
print(input())
client = TestProcessSocket()
client.start()
serv.write(input('Type something: '))
client.join()
Why is my socket connection getting refused, I'm using ubuntu?
I have set up a socket server with a client and a host.
It works fine until the client has disconnected, with both .shutdown() and .close().
When I then launch the client again, it can't connect.
I presume this is not because of how I've written my code but rather what I haven't written.
How do I make the server truly disconnect the client's connection so that it can connect again?
Server:
import socket, threading, time, json
ONLINE_USERS = []
SESSION = None
class User():
def __init__(user, connection, address):
print('for', address, '{Connection established}')
user.connection = connection
user.address = address
user.character = None
threading.Thread(target=user.process, args=(), daemon=True).start()
def process(user):
time.sleep(1)
user.send("&pLogin\n^^^^^\n")
username = user.send("&iUser>")
password = user.send("&iPass>")
print(user.ping())
print(user.logout())
def send(user, *x):
user.connection.sendall(str.encode(str(x)))
data = user.connection.recv(1024)
return data if data!=b'\x01' else True
def recv(user, x):
user.connection.recv(x)
def ping(user):
start = time.time()
user.connection.sendall(b'\x02')
end = float(user.connection.recv(1024))
return round((end - start) * 1000)
def logout(user):
user.connection.sendall(b'\x04')
return user.connection.recv(4)
class Session():
def __init__(session, host='', port=12345):
session.host = host
session.port = port
session.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
session.socket.bind((host, port))
session.socket.listen(10)
def accept():
conn = User(*session.socket.accept())
session.thread_accept = threading.Thread(target=accept, args=(), daemon=True).start()
def shutdown():
for user in ONLINE_USERS.keys():
ONLINE_USERS[user].connection.sendall(bytes([0xF]))
if __name__ == '__main__':
SESSION = Session()
input('Press heart to continue!\n')
Client:
import socket, sys, threading, time, os
def clear(t=0.5):
time.sleep(t)
os.system('cls')
def tryeval(x, default):
try:
return eval(x)
except:
return default
class Client():
def __init__(client):
try:
server_info = input('IP_ADDRESS:PORT>').split(':')
client.host = server_info[0]
client.port = int(server_info[1])
except:
client.host = 'localhost'
client.port = 12345
client.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
client.socket.settimeout(10)
try:
client.socket.connect((client.host, client.port))
clear()
client.data_exchange()
finally:
client.shutdown()
def data_exchange(client):
while True:
data = client.socket.recv(1024)
if data:
if data==b'\x02':
client.socket.sendall(str.encode(str(time.time())))
elif data==b'\x04':
client.shutdown()
else:
data = tryeval(data, ())
response = []
for item in data:
try:
prefix, content = item[:2], item[2:]
if prefix=='&p':
print(content, end='')
elif prefix=='&i':
response.append(input(content))
if prefix=='&c':
time.sleep(float(content))
clear()
except:
pass
if len(response)>0:
client.socket.sendall(str.encode(str(tuple(response))))
else:
client.socket.sendall(b'\x01')
time.sleep(0.001)
def shutdown(client):
try:
client.socket.sendall(b'\x04')
except:
pass
print('Shutting down program.')
client.socket.shutdown(socket.SHUT_RDWR)
print('Socket has been shutdown.')
client.socket.close()
print('Socket has been closed.')
print('Exiting program')
time.sleep(1)
sys.exit()
if __name__ == '__main__':
client = Client()
"The server repeatedly calls accept waiting for new incoming connections." No it doesn't. It calls accept once in a thread...which exits. – Mark Tolonen
I tried to create multithreaded echo server:
echomain.py:
#!/usr/bin/python
from echoserver import echoserver
server = echoserver()
print server.isRunning()
print server.port()
server.start()
print "Main program continues..."\\This part is not displayed(((
echoserver.py:
#!/usr/bin/python
import threading
import socket
class connection(threading.Thread):
def __init__(self, sock, addr):
self.sock = sock
self.addr = addr
threading.Thread.__init__(self)
def run (self):
while True:
buffer = self.sock.recv(1024)
if buffer == "disconnect\r\n":
self.sock.send("bye")
break
elif buffer:
self.sock.send(buffer)
self.sock.close()
class echoserver(object):
def __init__(self, port=12119):
self.running = False
self._port = port
self._socket = None
def isRunning(self):
return self.running
def port(self):
return self._port
def start(self):
self.running = True
self._socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self._socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
self._socket.bind(("0.0.0.0", self.port()))
self._socket.listen(5)
while True:
conn, addr = self._socket.accept()
connection(conn, addr).start()
def stop(self):
self._socket.close()
print "Server is closed..."
Could somebody help me in how I can launch echoserver class as a thread so it run simultaneously with main program so I could stop it with stop() method in echomain.py part?
Change your runner program to run the server as a thread:
echomain.py:
#!/usr/bin/python
from echoserver import echoserver
from threading import Thread
import time
server = echoserver()
print server.isRunning()
print server.port()
# server.start()
# run server in a different thread
serverThread = Thread(target=server.start)
serverThread.start()
print "main - server started"
# wait ten seconds before stopping
time.sleep(10)
server.stop()
print "main - server stopped"
print "Main program continues..."
This example simply stops the server after 10 seconds.
The simplest way is to have your echoserver itself be a Thread as proposed by Reut Sharabani, but IMHO, you should also implement a correct stop() method, ensuring that all children have ended.
Here is my implementation of your script :
#!/usr/bin/python
import threading
import socket
class connection(threading.Thread):
def __init__(self, sock, addr, parent):
self.sock = sock
self.addr = addr
self.parent = parent
threading.Thread.__init__(self)
self.sock.settimeout(None)
self.closed = False # will be set to True on thread end
def run (self):
while not self.parent._stopped:
buffer = self.sock.recv(1024)
if buffer == "disconnected\r\n":
self.sock.send("bye")
break
elif buffer:
self.sock.send(buffer)
self.sock.close()
self.closed = True
class echoserver(threading.Thread):
def __init__(self, port=12119):
threading.Thread.__init__(self)
self.running = False
self._port = port
self._socket = None
self._stopped = False
self._conns = [] # list of active connections
def isRunning(self):
return self.running
def port(self):
return self._port
def run(self):
self.running = True
self._socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self._socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
self._socket.bind(("0.0.0.0", self.port()))
self._socket.listen(5)
self._socket.settimeout(5) # use a timeout to respond to stop()
while not self._stopped:
try:
conn, addr = self._socket.accept()
c = connection(conn, addr, self)
self._conns.append(c) # add child the the list
c.start()
except Exception as e:
# print e # in debug
pass
self._conns = self.child_list() # remove closed child from list
self._socket.close()
print "Server is closing..."
for connect in self._conns: # join active children
connect.join()
print "Server is closed"
def stop(self):
self._stopped = True
def child_list(self):
l = []
for conn in self._conns:
if conn.closed:
conn.join()
else:
l.append(conn)
return l
Remarks :
you simply use it that way :
serv=echoserver()
serv.start()
... # sleep of do anything you want
serv.stop()
if no connection is active when you call stop() all stops at the end of the accept timeout and you get :
Server is closing...
Server is closed
if at least one connection is active when you call stop(), you get only Server is closing... at the end of the accept timeout. Then for each connection, it will end as soon as it receives a packet, and will be joined by echoserver. Then when all connection are over, you will get Server is closed and echoserver thread will terminate
that means that in you main thread you have only to do
serv.stop()
serv.join()
to be sure that all other threads are correctly terminated, and that all sockets are closed
I'm trying to use sockets in python. For now, I'm trying to get it such that if any client sends any message it is received at all clients. However I'm getting very weird results. I think it's because I'm using multiple threads. The output of the program changes every time I run it. Is this a threading problem or is it something else?
import socket
import sys
from thread import *
from server import Server
from client import Client
s = Server()
start_new_thread(s.acceptConnection,())
m = Client("m")
k = Client("k")
start_new_thread(m.recieveData,())
start_new_thread(k.recieveData,())
k.sendData("Hey!")
print "*"*100
print repr(k.data()), repr(m.data())
print "*"*100
m.sendData("okay okay")
print "*"*100
print repr(k.data()), repr(m.data())
print "*"*100
m.client.close()
k.client.close()
s.s.close()
Server Class:
import socket
import sys
from thread import *
class Server(object):
def __init__(self,port = 5555):
self.host = 'localhost' # '' means connect to all hosts
self.port = port
self.text = ""
self.s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
try:
self.s.bind((self.host, self.port))
except socket.error as e:
print(str(e))
self.s.listen(2)
print "Waiting for a connection.\n"
self.connections = []
def threaded_client(self,conn):
# conn.send("Connected to server\n")
while True:
try:
data = conn.recv(2048)
except:
data = ""
if(not data):
break
# conn.sendall(reply)
for c,a in self.connections:
try:
c.sendall(data + "\n")
except:
print "connection lost\n"
self.connections.remove((c,a))
conn.close()
def acceptConnection(self):
while True:
conn, addr = self.s.accept()
self.connections += [(conn,addr)]
start_new_thread(self.threaded_client,(conn,))
Client class:
import socket
import sys
from thread import *
class Client(object):
def __init__(self,name):
self.host = 'localhost'
self.port = 5555
self.name = name
self.client= socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.client.connect((self.host,self.port))
self.text = ""
def sendData(self,data):
self.client.send(data)
def recieveData(self):
while True:
try:
data = self.client.recv(2048)
except:
break
if data:
self.text = data
self.client.close()
def data(self):
return self.text
def closeClient(self):
self.client.close()
Anyway you have no warrants about the data was already come back to the clients when you try to print it. You should introduce some Conditions and use wait() and notifyAll() to make sure the data was arrive.... To check if my guess is correct put some sleep() in your test:
import time
k.sendData("Hey!")
print "*"*100
time.sleep(200)
print repr(k.data()), repr(m.data())
print "*"*100
m.sendData("okay okay")
print "*"*100
time.sleep(200)
print repr(k.data()), repr(m.data())
print "*"*100
If it works you should use conditions and notify to do your tests.
Moreover you must protect data access by a Lock().
def recieveData(self):
while True:
try:
data = self.client.recv(2048)
except:
break
if data:
self.l.acquire()
self.text = data
self.l.release()
self.client.close()
def data(self):
self.l.acquire()
ret = self.text
self.l.release()
return ret
Where attribute l of clients are defined in __init__ by
self.l=threading.Lock()
I want to stop the thread with simple CTR+C, but data is keep coming and coming, so I have to close the window and start it again. Help me please to fix the issue. In order to see what I'm saying, just type your twitter username and password in user and pass
import threading
import random
import time
import Queue
import urllib2
import sys
import simplejson, pycurl
import sys, signal
queue = Queue.Queue()
keep_running = True
user = "" # username
pswd = "" # password
headers = [ "Content-Type: text/xml; charset: UTF-8; "]
class MyThread(threading.Thread):
def __init__(self, queue):
threading.Thread.__init__(self)
self.queue = queue
self.buffer = ""
self.streaming_url = "https://stream.twitter.com/1/statuses/filter.json?track=justinbieber"
def start_crawl(self, user, pswd):
self.conn = pycurl.Curl()
self.conn.setopt(pycurl.USERPWD, "%s:%s" % (user, pswd))
self.conn.setopt(pycurl.URL, self.streaming_url)
#self.conn.setopt(pycurl.POST, 1)
self.conn.setopt(pycurl.HTTPHEADER, headers)
self.conn.setopt(pycurl.READFUNCTION, self.storage)
self.conn.setopt(pycurl.VERBOSE, 1)
self.conn.setopt(pycurl.HTTPAUTH, pycurl.HTTPAUTH_BASIC)
self.conn.perform()
def storage(self, data):
self.buffer += str(data)
if isinstance(self.buffer, str):
if self.buffer.endswith("\r\n") and self.buffer.strip():
content = json.loads(self.buffer)
self.buffer = ""
self.queue.put(content)
self.queue.task_done()
#if "text" in content:
#print u"{0[user][name]}: {0[text]}".format(content)
else:
return data
def run(self):
while keep_running:
self.start_crawl(user,pswd)
if signal.signal(signal.SIGINT, signal.SIG_DFL):
sys.exit()
#line = self.names[random.randint(0,len(self.names)-1)]
#queue.put(line)
class Starter():
def __init__(self):
self.queue = queue
t = MyThread(self.queue)
t.start()
self.next()
def next(self):
while True:
time.sleep(0.1)
if not self.queue.empty():
line = self.queue.get(timeout=0.2)
print line, self.queue.qsize()
else:
print 'waiting for queue'
def main():
try:
Starter()
queue.join()
except KeyboardInterrupt, e:
print 'Stopping'
global keep_running
keep_running = False
#Join all existing threads to main thread.
for thread in threading.enumerate():
if thread is not threading.currentThread():
thread.join()
sys.exit(1)
main()
Set the thread as daemon and it will be killed with your program
import threading
import random
import time
import Queue
import urllib2
import sys
import simplejson, pycurl
import sys, signal
queue = Queue.Queue()
keep_running = True
user = "" # username
pswd = "" # password
headers = [ "Content-Type: text/xml; charset: UTF-8; "]
class MyThread(threading.Thread):
def __init__(self, queue):
threading.Thread.__init__(self)
self.queue = queue
self.buffer = ""
self.streaming_url = "https://stream.twitter.com/1/statuses/filter.json?track=justinbieber"
def start_crawl(self, user, pswd):
self.conn = pycurl.Curl()
self.conn.setopt(pycurl.USERPWD, "%s:%s" % (user, pswd))
self.conn.setopt(pycurl.URL, self.streaming_url)
#self.conn.setopt(pycurl.POST, 1)
self.conn.setopt(pycurl.HTTPHEADER, headers)
self.conn.setopt(pycurl.READFUNCTION, self.storage)
self.conn.setopt(pycurl.VERBOSE, 1)
self.conn.setopt(pycurl.HTTPAUTH, pycurl.HTTPAUTH_BASIC)
self.conn.perform()
def storage(self, data):
self.buffer += str(data)
if isinstance(self.buffer, str):
if self.buffer.endswith("\r\n") and self.buffer.strip():
content = json.loads(self.buffer)
self.buffer = ""
self.queue.put(content)
self.queue.task_done()
#if "text" in content:
#print u"{0[user][name]}: {0[text]}".format(content)
else:
return data
def run(self):
while keep_running:
self.start_crawl(user,pswd)
if signal.signal(signal.SIGINT, signal.SIG_DFL):
sys.exit()
#line = self.names[random.randint(0,len(self.names)-1)]
#queue.put(line)
class Starter():
def __init__(self):
self.queue = queue
t = MyThread(self.queue)
t.daemon = True
t.start()
self.next()
def next(self):
while True:
time.sleep(0.1)
if not self.queue.empty():
line = self.queue.get(timeout=0.2)
print line, self.queue.qsize()
else:
print 'waiting for queue'
def main():
try:
Starter()
queue.join()
except KeyboardInterrupt, e:
print 'Stopping'
raise
main()