We Keep Coding

Python 3.6 Object/Class Threading

I'm looking to create a "self contained threaded class" using Python 3.
At a high level, want I would liked to do is to spawn up 50 asynchronous device objects from my "main" class and then just use their methods as needed. This is not difficult when just dealing with objects in synchronous situation but gets cloudy quite quickly as we move to asynchronous processing. The primary idea to keep the threading self contained in the device class so my base (main.py) code stays streamlined/clean and without any of the thread management.
I don't plan on any resource sharing in this case so I think I'm clear of any thread lock issues.
Here is some sample code that I hope someone can provide some hints or samples into making it a self threaded class (meaning I don't want to manage threads at the main.py level):
Sample main.py
from deviceworker import Device
availableworkers = {'USA':'services.groupkt.com', 'IND':'services.groupkt.com'}
Activeworkers = []
for name, ip in availableworkers.items():
Activeworkers.append(Device(name, ip))
for worker in Activeworkers:
worker.checkcountry() # asynchronous call - (we don't want to wait for a response)
# The idea is to keep this code as clean as possible.
Sample Object: deviceworker.py
import urllib.request
import urllib.parse
import json
class Device:
def __init__(self, name, endpoint, preamble = 'state', port = 80 ):
self.name = name
self.connected =False
self.connection = HTTPConnection(endpoint, preamble, port)
self.getStatus()
def getStatus(self, check_for = None):
self.urlresponse = json.loads(self.connection.GET('get/USA/all')) #Use USA just to verify connection
if check_for:
pass
self.connected = True
def checkcountry(self):
print(self.connection.GET('get/%s/all' % self.name))
class HTTPConnection:
def __init__(self, endpoint, preamble = None, port = 80):
if preamble: # specificing a version after the port and before method
self.url = 'http://%s:%s/%s/' % (endpoint, port, preamble)
else:
self.url = 'http://%s:%s/' % (endpoint, port)
print('_init_ url=%s' % self.url)
def GET(self, operation):
#try:
#print('%s%s' % (self.url, operation))
with urllib.request.urlopen('%s%s' % (self.url, operation)) as f:
return f.read().decode('utf-8')
#except Exception as e:
#raise Exception("GET Request Failed")
I've stripped most of the exception handling for simplicity. The sample above should work.
--- UPDATE ---
So I think I've sort of figured it out. Still not getting the parrellism I would expect from the documentation.
import threading
import urllib.request
import urllib.parse
import json
import time
class Device(threading.Thread):
def __init__(self, name, endpoint, preamble = 'state', port = 80 ):
threading.Thread.__init__(self)
self.name = name
self.connected = False
self.connection = HTTPConnection(endpoint, preamble, port)
print('%s: __init__' % self.name)
def run(self):
self.getStatus()
print('%s: hit run()' % self.name)
def getStatus(self):
self.urlresponse = json.loads(self.connection.GET('get/USA/all')) #Use USA just to verify connection
self.connected = True
def checkcountry(self):
if (self.name == 'USA'): self.waittime = 10
else: self.waittime = 0
print('%s: Getting Codes - wait time: %s' % (self.name, self.waittime))
start_time=time.time()
time.sleep(self.waittime)
result =self.connection.GET('get/%s/all' % self.name)
elapsed_time=time.time() - start_time
print('%s: Got Codes - second: %s' % (self.name, elapsed_time))
class HTTPConnection:
def __init__(self, endpoint, preamble = None, port = 80):
if preamble: # specificing a version after the port and before method
self.url = 'http://%s:%s/%s/' % (endpoint, port, preamble)
else:
self.url = 'http://%s:%s/' % (endpoint, port)
def GET(self, operation):
with urllib.request.urlopen('%s%s' % (self.url, operation)) as f:
return f.read().decode('utf-8')
DeviceList = {'USA':'services.groupkt.com', 'IND':'services.groupkt.com'}
ActiveDevices = []
DeviceList = {'USA':'services.groupkt.com', 'IND':'services.groupkt.com'}
ActiveDevices = []
for name, ip in DeviceList.items():
print('main: creating object for: %s' % name)
newDevice = Device(name, ip)
ActiveDevices.append(newDevice)
newDevice.start()
for device in ActiveDevices:
print('main: calling checkcountry() for: %s' % device.name)
device.checkcountry()
Here are the results:
main: creating object for: USA
USA: __init__
main: creating object for: IND
IND: __init__
main: calling checkcountry() for: USA
USA: Getting Codes - wait time: 10
USA: Got Codes - second: 10.167016744613647
main: calling checkcountry() for: IND
IND: Getting Codes - wait time: 0
IND: Got Codes - second: 0.11001110076904297
I by adding in the delay to the USA search I would have expected the IND to finish first but it appears that it serialized.
I'm running this on:
Python 3.6.0 (v3.6.0:41df79263a11, Dec 23 2016, 07:18:10) [MSC v.1900 32 bit (Intel)] on win32

Here is a custom thread sample with locking worked great for me, better than using the event.
Try it in Colab.
import threading,time
i=0
luk=threading.Lock()
global i
global y
global t_num
class special_thread(threading.Thread):
"""This function starts a Thread class"""
def __init__(self, execute,threadID , name, daemon,args=(), repetitive=False,kwargs=None, interval_sec=60 ):
threading.Thread.__init__(self)
self.daemon = daemon
self.stopped = threading.Event()
self.interval_sec = interval_sec
self.execute = execute
self.name = name
if kwargs is None:
kwargs = {}
self.args = args
self.kwargs=kwargs
self.repetitive=repetitive
self.threadID = threadID
print(args)
def stop(self):
self.stopped.set()
self.join()
def run(self):
if self.repetitive:
while not self.stopped.wait(self.interval_sec):
self.execute(*self.args,**self.kwargs)
else:
self.execute(*self.args,**self.kwargs)
def center(t_num):
y=0
luk.acquire()
caller = inspect.getouterframes(inspect.currentframe())[1][3]
print(' {} is aquiring by {} '.format( caller, str(time.ctime())))
y+=t_num
print( "Inside %s()" % caller)
print('thread number is ',t_num,y)
time.sleep(2*t_num)
luk.release()
print(' {} is releasing by {} '.format( caller, str(time.ctime())))
def target_uno():
t_num=1
center(t_num)
def target_dos():
t_num=2
center(t_num)
target_uno=special_thread(execute=target_uno, args=(),repetitive=True, interval_sec=1,threadID=10004,
name='target_uno',
daemon=False )
target_dos=special_thread(execute=target_dos, args=(),repetitive=True, interval_sec=1,threadID=10004,
name='target_dos',
daemon=False )
if __name__ == "__main__":
target_uno.start()
target_dos.start()
time.sleep(20)
target_uno.stop()
target_dos.stop()

Python cant find class attribute

Why is not Factory.rdyq visible to inherited classes?
bar.py
import Queue
import threading
class Factory:
name = ""
rdyQ = Queue.Queue()
dispQ = Queue.Queue()
def __init__(self):
self.worker_count = 1
self.worker_thread = None
def worker(self): pass
def initialize(self):
for i in range(self.worker_count):
t = threading.Thread(target=self.worker)
t.daemon = True
t.start()
self.worker_thread = t
#staticmethod
def joinQ():
Factory.rdyQ.join()
Factory.dispQ.join()
return
#staticmethod
def getFactory(factory_name):
if factory_name == "setup":
return SetupFactory()
elif factory_name == "dispatch":
return DispatchFactory()
elif factory_name == "complete":
return CompleteFactory()
else:
return None
class SetupFactory(Factory):
name = "setup"
def worker(self):
while True:
try:
item = Factory.rdyq.get(timeout=1)
print "setup: %s" % item
Factory.rdyq.task_done()
Factory.dispQ.put(item)
except Queue.Empty, msg:
continue
class DispatchFactory(Factory):
name = "dispatch"
def worker(self):
while True:
try:
item = Factory.dispQ.get(timeout=1)
print "dispQ: %s" % item
Factory.dispQ.task_done()
except Queue.Empty, msg:
continue
class CompleteFactory(Factory):
name = "complete"
def worker(self):
for i in range(10):
Factory.rdyq.put(i)
foo.py
import bar as df
setup = df.Factory.getFactory("setup")
dispatch = df.Factory.getFactory("dispatch")
complete = df.Factory.getFactory("complete")
setup.initialize()
dispatch.initialize()
complete.initialize()
df.Factory.joinQ()
setup.worker_thread.join()
dispatch.worker_thread.join()
complete.worker_thread.join()
python foo.py
File "/u/bar.py", line 73, in worker
Factory.rdyq.put(i)
AttributeError: class Factory has no attribute 'rdyq'

You should use a classmethod instead of a staticmethod for joinQ.
You also had a rdyq (bar.py last line) instead of the expected rdyQ and Python is case-sensitive.
bar.py
import Queue
import threading
class Factory:
name = ""
rdyQ = Queue.Queue()
dispQ = Queue.Queue()
def __init__(self):
self.worker_count = 1
self.worker_thread = None
def worker(self): pass
def initialize(self):
for i in range(self.worker_count):
t = threading.Thread(target=self.worker)
t.daemon = True
t.start()
self.worker_thread = t
#classmethod
def joinQ(cls):
cls.rdyQ.join()
cls.dispQ.join()
return
#staticmethod
def getFactory(factory_name):
if factory_name == "setup":
return SetupFactory()
elif factory_name == "dispatch":
return DispatchFactory()
elif factory_name == "complete":
return CompleteFactory()
else:
return None
class SetupFactory(Factory):
name = "setup"
def worker(self):
while True:
try:
item = Factory.rdyq.get(timeout=1)
print "setup: %s" % item
Factory.rdyq.task_done()
Factory.dispQ.put(item)
except Queue.Empty, msg:
continue
class DispatchFactory(Factory):
name = "dispatch"
def worker(self):
while True:
try:
item = Factory.dispQ.get(timeout=1)
print "dispQ: %s" % item
Factory.dispQ.task_done()
except Queue.Empty, msg:
continue
class CompleteFactory(Factory):
name = "complete"
def worker(self):
for i in range(10):
Factory.rdyQ.put(i)

Python variables and attributes are case sensitive, so "rdyq" is not the same as "rdyQ".

You set the name with a capital q, so maybe this will fix it?
Factory.rdyQ.put(i)
I recommend using_underscores_for_variables as you avoid issues camelCase can cause.

Why doesn’t this thread terminate?

I am trying to use the python threading module. As I am sysadmin, I struggle a little bit when developing; and this concept is kind of new for me. I launch two threads and I want to stop them, when the main thread sets a flag to False:
class My_Thread( threading.Thread):
def __init__(self, thread_id, thread_name, count):
threading.Thread.__init__(self)
self.thread_id = thread_id
self.thread_name = thread_name
self.count = count
def run(self):
do_job(self.thread_name, self.thread_id, self.count)
def do_job(t_name, t_id, count):
while not get_kill():
print "It is "+str(time.time())+" and I am "+str(t_name)
print get_kill()
time.sleep(count)
kill = False
def get_kill():
return kill
def set_kill(state):
kill = state
if __name__ == '__main__':
a = My_Thread(1, "Thread-1", 2)
b = My_Thread(2, "Thread-2", 1)
a.start()
b.start()
while(True):
try:
pass
except KeyboardInterrupt,ki:
set_kill(True)
sys.exit(0)
But the value is never read as changed in both threads and they don't exit. Why is this value not properly read from threads?

The problem
In set_kill(), you are creating a new local variable kill setting it to state, and returning from the function. You are not actually updating the value of kill in the global scope.
To do that, you would need to have:
def set_kill(state):
global kill
kill = state
A better way
Using globals like that is generally considered bad practice, you probably want to convert your kill variable and functions into an object, to encapsulate that data and behaviour together:
class Kill(object):
kill = False
def get(self):
return self.kill
def set(self, value):
self.kill = value
Which you would use like this:
class MyThread(Thread):
def __init__(self, thread_id, thread_name, count, kill):
self.kill = kill
...
def do_job(self, ...):
while not self.kill.get():
...
if __name__ == '__main__':
kill = Kill()
a = My_Thread(1, "Thread-1", 2, kill)
b = My_Thread(2, "Thread-2", 1, kill)
...
kill.set(True)

Limiting Threads within Python Threading, Queue

Im using the following code to multithread urlib2. However what is the best way to limit the number of threads that it consumes ??
class ApiMultiThreadHelper:
def __init__(self,api_calls):
self.q = Queue.Queue()
self.api_datastore = {}
self.api_calls = api_calls
self.userpass = '#####'
def query_api(self,q,api_query):
self.q.put(self.issue_request(api_query))
def issue_request(self,api_query):
self.api_datastore.update({api_query:{}})
for lookup in ["call1","call2"]:
query = api_query+lookup
request = urllib2.Request(query)
request.add_header("Authorization", "Basic %s" % self.userpass)
f = urllib2.urlopen(request)
response = f.read()
f.close()
self.api_datastore[api_query].update({lookup:response})
return True
def go(self):
threads = []
for i in self.api_calls:
t = threading.Thread(target=self.query_api, args = (self.q,i))
t.start()
threads.append(t)
for t in threads:
t.join()

You should use a thread pool. Here's my implementation I've made years ago (Python 3.x friendly):
import traceback
from threading import Thread
try:
import queue as Queue # Python3.x
except ImportError:
import Queue
class ThreadPool(object):
def __init__(self, no=10):
self.alive = True
self.tasks = Queue.Queue()
self.threads = []
for _ in range(no):
t = Thread(target=self.worker)
t.start()
self.threads.append(t)
def worker(self):
while self.alive:
try:
fn, args, kwargs = self.tasks.get(timeout=0.5)
except Queue.Empty:
continue
except ValueError:
self.tasks.task_done()
continue
try:
fn(*args, **kwargs)
except Exception:
# might wanna add some better error handling
traceback.print_exc()
self.tasks.task_done()
def add_job(self, fn, args=[], kwargs={}):
self.tasks.put((fn, args, kwargs))
def join(self):
self.tasks.join()
def deactivate(self):
self.alive = False
for t in self.threads:
t.join()
You can also find a similar class in multiprocessing.pool module (don't ask me why it is there). You can then refactor your code like this:
def go(self):
tp = ThreadPool(20) # <-- 20 thread workers
for i in self.api_calls:
tp.add_job(self.query_api, args=(self.q, i))
tp.join()
tp.deactivate()
Number of threads is now defined a priori.

Using threads in the right way

I'm working on server written in python. When the client sends a cmd the server will call a function with unknown running time. So to avoid blocking I used threading. But when looking at the child process it seems that they're not terminating, causing a lot of memory usage.
EDIT : Here is the tree of the directory : http://pastebin.com/WZDxLquC
Following answers I found on stackoverflow I implemented a custom Thread class:
sThreads.py :
import threading
class Thread(threading.Thread):
def __init__(self, aFun, args = ()):
super(Thread, self).__init__(None, aFun, None, args)
self.stopped = threading.Event()
def stop(self):
self.stopped.set()
def isStopped(self):
return self.stopped.isSet()
Then here is the server's loop:
some where in mainServer.py:
def serve_forever(self, aCustomClass, aSize = 1024):
while True:
self.conn, self.addr = self.sock.accept()
msg = self.recvMSG(4096)
if(msg):
self.handShake(msg)
print 'Accepted !'
while True:
msg = self.recvMSG(aSize)
if(msg):
t = sThreads.Thread(self.handle, (aCustomClass,))
t.start()
self.currentThreads.append(t)
if(self.workers > 0):
tt = sThreads.Thread(self.respond)
tt.start()
if(self.workers == 0 and len(self.currentThreads) > 0):
for th in self.currentThreads:
th.stop()
Using a custom Thread class will not solve the issue and it still does not stop the terminated threads!
EDIT : added the handle() and respond() methods :
def handle(self, aClass):
self.workers += 1
self.queue.put(aClass._onRecieve(self.decodeStream()))
def respond(self):
while self.workers > 0:
msgToSend, wantToSend = self.queue.get()
self.workers -= 1
if(wantToSend):
print 'I want to send :', msgToSend
continue #Send is not yet implemented !

It seems that self.queue.get() was causing all the issue ...