I have Python 2.7 code that makes 1,000 requests to check my current IP address. I have written it using 200 threads and sockets. The code does the same task with two methods. I couldn't find any differences between the two below, besides that one subclasses threading.Thread
#!/usr/bin/env python2.7
import sys, ssl, time, socket, threading
class Counter:
def __init__(self):
self._value = 0
self._LOCK = threading.Lock()
def increment(self):
with self._LOCK:
self._value += 1
return self._value
def pr(out):
sys.stdout.write('{}\n'.format(out))
sys.stdout.flush()
def recvAll(sock):
data = ''
BUF_SIZE = 1024
while True:
part = sock.recv(BUF_SIZE)
data += part
length = len(part)
if length < BUF_SIZE: break
return data
class Speed(threading.Thread):
_COUNTER = Counter()
def __init__(self):
super(Speed, self).__init__()
self.daemon = True
self._sock = ssl.wrap_socket(socket.socket(socket.AF_INET, socket.SOCK_STREAM))
self._sock.settimeout(5)
self._sock.connect(('checkip.amazonaws.com', 443))
self._request = 'GET / HTTP/1.1\r\n'
self._request += 'Host: checkip.amazonaws.com\r\n\r\n'
def run(self):
i = 0
while i < 5:
self._sock.write(self._request)
response = recvAll(self._sock)
if '200 OK' not in response: continue
count = Speed._COUNTER.increment()
pr('#{} - {}'.format(count, response))
i += 1
self._sock.close()
def speed():
sock = ssl.wrap_socket(socket.socket(socket.AF_INET, socket.SOCK_STREAM))
sock.settimeout(5)
sock.connect(('checkip.amazonaws.com', 443))
request = 'GET / HTTP/1.1\r\n'
request += 'Host: checkip.amazonaws.com\r\n\r\n'
i = 0
while i < 5:
sock.write(request)
response = recvAll(sock)
if '200 OK' not in response: continue
count = counter.increment()
pr('#{} - {}'.format(count, response))
i += 1
sock.close()
slow = False
if slow:
for _ in xrange(200):
thread = Speed()
thread.start()
else:
counter = Counter()
for _ in xrange(200):
thread = threading.Thread(target = speed)
thread.daemon = True
thread.start()
while threading.active_count() > 1: time.sleep(1)
I expected both to have similar speeds. However, the variation that subclasses threading.Thread is much, much, much slower. Any ideas as to why?
Your Thread subclass is doing far too much of its work in __init__, which doesn't execute in the new thread. The version that uses the subclass ends up executing largely sequentially as a result.
Related
I am writing a program which requires multiple threads (attackers):
MESSAGE = b"flood"
IP = "127.0.0.1"
BREAKER_PORT = 5005
CONTROLLER_PORT = 49153
ATTACKER_PORT1 = 49154
ATTACKER_PORT2 = 49155
def send_flood_messages(port_number):
print("port number: " + str(port_number))
attacker_socket = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
attacker_socket.bind((IP, port_number))
x = 0 #used to interchange between attackers to mitigate detection
counter = 0 #handle replays
while True:
frame, meta = receiving_socket.recvfrom(65565)
if len(frame) >= 34:
UDP_dictionary = promisc.unpack_UDP(frame[34:])
if UDP_dictionary.get("udp_dest") == CONTROLLER_PORT:
counter += 1
if counter == 2:
time.sleep(float(wait_time))
print("Sending flood message at time: " + str(time.time()))
attacker_socket.sendto(MESSAGE, (IP, BREAKER_PORT))
counter = 0
class attackerThread(threading.Thread):
def __init__(self, threadID, name):
threading.Thread.__init__(self)
self.threadID = threadID
self.name = name
def run(self, port_number):
send_flood_messages(port_number)
attacker1 = attackerThread(1, "attacker1")
attacker2 = attackerThread(2, "attacker2")
attacker1.run(ATTACKER_PORT1)
attacker2.run(ATTACKER_PORT2)
When I run both threads, only one thread executes and prints (the first one). The output is as follows:
port number: 49154
I cannot figure out why the second thread is not executing and running the function? I was expecting to see the following output:
port number: 49154
port number: 49155
The run() method is representing the thread's activity AFAIK.
To run a thread you must use threadobject.start().
(and don't forget the GIL... :( )
I am using python twisted library and am making a server to receive data to do some processing on the received data and then close the connection. I observe that the program hangs in dataReceived without a print statement. With print statement it goes through. Wondering is print is somehow slowing down the execution to avoid race condition or if I have coded a bug?
My code is as follows:
class Stack(Protocol):
def __init__(self, factory):
self.factory = factory
self.bytesremaining = None
self.payload = ""
self.headerseen = False
def dataReceived(self, data):
if self.headerseen == False:
header = unpack('B',data[0])[0]
if header == 128:
self.pop()
return
self.bytesremaining = self.datalength = unpack('B',data[0])[0]
print self.datalength #without this print the execution hangs in the middle.
if len(data) > 1 and (len(self.factory.pushstack) < 100):
self.payload += data[1:]
self.bytesremaining -= len(data) - 1
self.headerseen = True
elif len(self.factory.pushstack) < 100:
self.payload += data
self.bytesremaining -= len(data) - 1
if self.bytesremaining == 0:
self.factory.pushstack.appendleft(self.payload)
retval = pack('B',0)
self.transport.write(retval)
self.transport.loseConnection()
class StackFactory(ServerFactory):
def __init__(self):
self.clients = []
self.pushstack = collections.deque()
self.popstack = collections.deque()
self.clientsmap = {}
def buildProtocol(self, addr):
return Stack(self)
It appears to me that the default twisted reactor for OS X (selectreactor) is not as stable as kqueue.
I am not seeing the issue anymore after installing the kqueue reactor.
from twisted.internet import kqreactor
kqreactor.install()
from twisted.internet import reactor
i am trying to receive about 1000 connections to my server but it cannot receive more than 512. What can i do to increase the amount of open connections? I am running windows 8.1
Not: I am very new to this stuff so, thanks for help
Here is my code;
import asyncore
import socket
import uuid
import time
import threading
class statistics(threading.Thread):
def __init__(self):
threading.Thread.__init__(self)
def run(self):
while True:
entry = raw_input("")
zaman = int(time.time())
cmd = receivedCmd
print "calculating.."
time.sleep(1)
if entry == 'istatistik':
print str(receivedCmd-cmd) + " command/second"
print "total received commands: " + str(receivedCmd)
entry = ""
class tcpClient:
def __init__(self):
self.clientid = uuid.uuid1(int(time.time()))
self.buffer = ""
self.buffer_size = 0
self.conn_time = time.time()
self.overflow = 0
#print str(self.clientid) + " assingned"
def recv_msg(self, msg):
global receivedCmd
self.buffer = msg
self.buffer_size = len(self.buffer)
receivedCmd = receivedCmd + 1
if self.buffer_size >= 1024:
self.overflow = 1
def __del__(self):
print str(self.clientid) + " has left."
class TCPHandler(asyncore.dispatcher_with_send):
global clist
def handle_read(self):
data = self.recv(1024)
if data:
if clist[self].overflow:
self.send("overflow")
self.handle_close()
else:
self.send(data)
clist[self].recv_msg(data)
def handle_close(self):
del clist[self]
self.close()
def handle_error(self):
del clist[self]
self.close()
class TCPServer(asyncore.dispatcher):
global clist
def __init__(self, host, port):
asyncore.dispatcher.__init__(self)
self.create_socket(socket.AF_INET, socket.SOCK_STREAM)
self.set_reuse_addr()
self.bind((host, port))
self.listen(5)
def handle_accept(self):
self.clist = clist
pair = self.accept()
if pair is None:
pass
else:
sock, addr = pair
#print 'Connection : %s' % repr(addr)
clist[TCPHandler(sock)] = tcpClient()
if __name__ == '__main__':
clist = {}
receivedCmd = 0
server = TCPServer('', 5000)
server2 = TCPServer('',5001)
StaticsThread = statistics()
StaticsThread.start()
asyncore.loop()
Note: I still cannot receive more than 512 connections with the Twisted Framework, i don't know what to do. There have to be thousands of connected clients. Please help.
The asyncore module relies in the select OS function, which only supports a limited number of file descriptors.
As an alternative use a multi-threading server (I won't recommend this) or, better, the Twisted framework which is event-driven (highly recommended!).
Hope this helps!
Since Twisted's default reactor under Windows is also select-based then you should consider using the IOCP reactor instead.
from twisted.internet import iocpreactor
iocpreactor.install()
from twisted.internet import reactor
But also take into account that Twisted prefers Linux systems (where the default reactor is epoll-based) rather than Windows. Maybe switching to Linux is a better choice.
I'm working on server written in python. When the client sends a cmd the server will call a function with unknown running time. So to avoid blocking I used threading. But when looking at the child process it seems that they're not terminating, causing a lot of memory usage.
EDIT : Here is the tree of the directory : http://pastebin.com/WZDxLquC
Following answers I found on stackoverflow I implemented a custom Thread class:
sThreads.py :
import threading
class Thread(threading.Thread):
def __init__(self, aFun, args = ()):
super(Thread, self).__init__(None, aFun, None, args)
self.stopped = threading.Event()
def stop(self):
self.stopped.set()
def isStopped(self):
return self.stopped.isSet()
Then here is the server's loop:
some where in mainServer.py:
def serve_forever(self, aCustomClass, aSize = 1024):
while True:
self.conn, self.addr = self.sock.accept()
msg = self.recvMSG(4096)
if(msg):
self.handShake(msg)
print 'Accepted !'
while True:
msg = self.recvMSG(aSize)
if(msg):
t = sThreads.Thread(self.handle, (aCustomClass,))
t.start()
self.currentThreads.append(t)
if(self.workers > 0):
tt = sThreads.Thread(self.respond)
tt.start()
if(self.workers == 0 and len(self.currentThreads) > 0):
for th in self.currentThreads:
th.stop()
Using a custom Thread class will not solve the issue and it still does not stop the terminated threads!
EDIT : added the handle() and respond() methods :
def handle(self, aClass):
self.workers += 1
self.queue.put(aClass._onRecieve(self.decodeStream()))
def respond(self):
while self.workers > 0:
msgToSend, wantToSend = self.queue.get()
self.workers -= 1
if(wantToSend):
print 'I want to send :', msgToSend
continue #Send is not yet implemented !
It seems that self.queue.get() was causing all the issue ...
I must be missing something in the code. I've rewritten an 'echo server' example to do a bit more when it receives something.
This is how it currently looks:
#!/usr/bin/env python
import select
import socket
import sys
import threading
import time
import Queue
globuser = {}
queue = Queue.Queue()
class Server:
def __init__(self):
self.host = ''
self.port = 2000
self.backlog = 5
self.size = 1024
self.server = None
self.threads = []
def open_socket(self):
try:
self.server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.server.bind((self.host,self.port))
self.server.listen(5)
except socket.error, (value,message):
if self.server:
self.server.close()
print "Could not open socket: " + message
sys.exit(1)
def run(self):
self.open_socket()
input = [self.server,sys.stdin]
running = 1
while running:
inputready,outputready,exceptready = select.select(input,[],[])
for s in inputready:
if s == self.server:
# handle the server socket
c = Client(self.server.accept(), queue)
c.start()
self.threads.append(c)
elif s == sys.stdin:
# handle standard input
junk = sys.stdin.readline()
running = 0
# close all threads
self.server.close()
for c in self.threads:
c.join()
class Client(threading.Thread):
initialized=0
def __init__(self,(client,address), queue):
threading.Thread.__init__(self)
self.client = client
self.address = address
self.size = 1024
self.queue = queue
self.threads = []
global globuser
print 'Client thread created!'
def run(self):
running = 1
while running:
print 'While running client'
data = self.client.recv(self.size)
print 'Dit we receive data?'
if data:
print 'Data received!'
print 'Fetching data from socket: ',
if data[0] == 'I':
print 'Initializing user: ' + data
user = {'uid': data[1:6] ,'x': data[6:9], 'y': data[9:12]}
globuser[user['uid']] = user
print globuser
initialized=1
m=updateClient(user['uid'], queue)
m.start()
self.threads.append(m)
self.client.send('Beginning - Initialized'+';')
elif data[0] == 'A':
print 'Client has received previous data: ' + data
#On deactivation, nothing works.
self.client.send(data+';')
#print 'Client Data sent: ' + data
else:
print 'Closing'
self.client.close()
running = 0
if self.queue.empty():
print 'Queue is empty'
else:
print 'Queue has information: ',
data2 = self.queue.get(1, 1)
isdata2 = 1
if data2 == 'Exit':
running = 0
print 'Client is being closed'
self.client.close()
if isdata2 == 1:
print 'Sending data to client: ' + data2,
self.client.send(data2)
self.queue.task_done()
isdata2 = 0
time.sleep(1)
class updateClient(threading.Thread):
def __init__(self,uid, queue):
threading.Thread.__init__(self)
self.uid = uid
self.queue = queue
global globuser
print 'updateClient thread started!'
def run(self):
running = 20
test=0
while running > 0:
test = test + 1
self.queue.put('Test Queue Data #' + str(test))
running = running - 1
time.sleep(1)
print 'Updateclient has stopped'
if __name__ == "__main__":
s = Server()
s.run()
This runs fine, although it's kind of silly to keep sending the same data back again along with other data.
In the middle of the code you'll see
#On deactivation, nothing works.
self.client.send(data+';')
#print 'Client Data sent: ' + data
When I DO deactive the self.client.send(data+';') or change it into self.client.send('something else;') it does not work! And the client receives nothing.
Is there something special about the "data" variable? Do I need to format the string in some way?
Here's a cleaned-up, functional version of your code! I tested it myself, though I didn't write unit tests.
There were some syntax errors and other miscellaneous problems with the
original code, so I took some liberties. I'm assuming that the protocol is
framed by using ; as a delimiter, since a ; is sent at the end of every
message to the client, though no framing was being done in the original code.
from twisted.internet import reactor, protocol, task
from twisted.protocols import basic
from twisted.python import log
import sys
class ServerProtocol(basic.LineOnlyReceiver):
delimiter = ';'
def lineReceived(self, line):
if line.startswith('I'):
user = dict(uid=line[1:6], x=line[6:9], y=line[9:12])
self.factory.users[user['uid']] = user
log.msg(repr(self.factory.users))
self.startUpdateClient()
self.sendLine('Beginning - Initialized')
elif line.startswith('A'):
self.sendLine(line)
else:
self.transport.loseConnection()
def _updateClient(self):
if self._running == 0:
self._looper.stop()
return
self._running -= 1
self._test += 1
self.sendLine('Test Queue Data #%d' % (self._test,))
def startUpdateClient(self):
self._running, self._test = 20, 0
self._looper = task.LoopingCall(self._updateClient)
self._looper.start(1, now=False)
class Server(protocol.ServerFactory):
protocol = ServerProtocol
def __init__(self):
self.users = {}
if __name__ == '__main__':
log.startLogging(sys.stderr)
reactor.listenTCP(2000, Server())
reactor.run()