We Keep Coding

Non-blocking methods, without parallelism

I'm trying to create a class with methods that must not block the execution (I guess it is called "non blocking"), but only one of those could be executing at a given time.
Suppose this:
import time
import random
class Foo():
def do_stuff_a(self):
print("Start a")
time.sleep(0.5)
print("Finished a")
def do_stuff_b(self):
print("Start b")
time.sleep(0.5)
print("Finished b")
def do_other_stuff():
print("doing other stsuff")
foo = Foo()
for x in range(15):
time.sleep(0.1)
# These calls should be non-blocking
if random.randint(0,1) == 1:
foo.do_stuff_a()
else:
foo.do_stuff_b()
do_other_stuff()
do_stuff_a and do_stuff_b should not block the flow, so do_other_stuff() should be executing every 0.1 seconds. Additionally, do_stuff_a and b should be skipped if there is any of them still running (i.e. methods of class Foo cannot run in parallel, if there is one running, the call to any of them should be just skipped (rather than put in a queue for later).
Any pointer friends on how to get started?

This is what I have come up with; I need two decorators, one to manage the threading of the methods (works fine) the other one to make sure that only one method is executed, and skip the calls if it is running (rather than leaving it in a queue)
import time
import random
import threading
def thread(func):
def wrapper( self, *args, **kwargs):
th = threading.Thread(target=func,args=(self, *args))
th.start()
return wrapper
class Foo():
def __init__(self):
self.lock = threading.Lock()
def non_parallel(fun):
def wrapper( self, *args, **kwargs):
if self.lock.locked(): return
self.lock.acquire()
try:
fun( self , *args)
except ValueError as e:
print (e)
self.lock.release()
return wrapper
#thread
#non_parallel
def do_stuff_a(self):
print("Start a\n")
print("Finished a\n")
#thread
#non_parallel
def do_stuff_b(self):
print("Start b\n")
time.sleep(1)
print("Finished b\n")
def do_other_stuff():
print("doing other stsuff\n")
foo = Foo()
for x in range(5):
time.sleep(0.1)
foo.do_stuff_a()
foo.do_stuff_b()
do_other_stuff()

Python - Starting two threads - Thread (t2) name error

This simple example seems to work fine but when I implement this same code in a only slightly more complicated codebase I get a error NameError: name 't2' is not defined when add is called from class QuoteReport using t2.add(key[0], key[1]). Is the way I am starting the two threads in main okay? Though this code seems to work is there anything here that is not good form? The only differnce I see from my simplified example and the one that fails is QueueReport and CallReport are in a seperate file from the main file. But I know the import is working because 1)if I change the code to t2 = CallReport_badname() it fails calls is cant find that class 2)I run some logging from the run method of CallReport and it does print to log. I would assume once those classes are imported I would call it as t2? is there anthing speical I should be doing when the instance of one class calls an instance of another class?
from threading import *
import time
class QueueReport(Thread):
def __init__(self):
Thread.__init__(self)
self.reports_to_call = {}
def add(self, user, report):
self.reports_to_call.update({(user, report): None})
print("Added Value - Queue Report")
print(len(self.reports_to_call))
def run(self):
print("Running - que\n")
time.sleep(10)
while True:
for key in self.reports_to_call:
print(key)
print("about to add to t2")
t2.add(key[0], key[1])
time.sleep(120)
class CallReport(Thread):
def __init__(self):
Thread.__init__(self)
self.reports_called = {}
def add(self, user, report):
self.reports_called.update({(user, report): None})
print("Added Value to reports called")
print("len2", len(self.reports_called))
def run(self):
print("Running - Call\n")
time.sleep(10)
while True:
for key in self.reports_called:
print("key from running", key)
#do something with this key and delete if succcessful
print("Call Report done- waiting to loop")
time.sleep(20)
if __name__ == '__main__':
t1 = QueueReport()
thread = Thread(target=t1.run)
thread.start()
t2 = CallReport()
thread2 = Thread(target=t2.run)
thread2.start()
t1.add("bob", "report2r")
time.sleep(1)
t1.add("tom", "report2")
time.sleep(1)
t1.add("harry", "report3")
time.sleep(15) # add new after first loop
t1.add("john", "report4")

Yes, there will be a difference if you move the logic into different files. t1 and t2 are declared in main method which makes them global variables and thus can be accessible by both the classes. If they are in different files you have to pass them around to access.
from threading import *
import time
class QueueReport(Thread):
def __init__(self, call_report_thread):
Thread.__init__(self)
self.reports_to_call = {}
self.call_report_thread = call_report_thread
def run(self):
print("Running - que\n")
time.sleep(10)
while True:
for key in self.reports_to_call:
print(key)
print("about to add to t2")
self.call_report_thread.add(key[0], key[1])
time.sleep(120)
class CallReport(Thread):
def __init__(self):
Thread.__init__(self)
self.reports_called = {}
def add(self, user, report):
self.reports_called.update({(user, report): None})
print("Added Value to reports called")
print("len2", len(self.reports_called))
def run(self):
print("Running - Call\n")
time.sleep(10)
while True:
for key in self.reports_called:
print("key from running", key)
#do something with this key and delete if succcessful
print("Call Report done- waiting to loop")
time.sleep(20)
if __name__ == '__main__':
t2 = CallReport()
t2.start()
t1 = QueueReport(t2)
t1.start()
I am assuming here that which thread starts first won't matter.

Python send variables to thread

I'm trying to create my own threading class in Python2.7. I want it to be able to stop that thread with my own class function. Currently I have something like this:
class loop(threading.Thread):
def __init__(self, myvar):
super(loop, self).__init__()
self.terminate = False
self.myvar = myvar
def run(self):
while not self.terminate:
do.smthng.useful(self.myvar)
def change(self, newvar):
self.myvar = newvar #Doesnt work, in run() my old var is still being used
def stoploop(self):
self.terminate = True #Also not working
l = loop(1)
l.start()
time.sleep(1)
l.change(2) #thread still using "1"
time.sleep(1)
l.stoploop() #doesnt stop
I've read some posts here about this, but it wasnt what I needed.
Any help would be appreciated.
Thank you.
EDIT:
As some of the commenters already stated, this part of code looks like to be really working! Problem is in another place of my project. I've found it, but can't solve it. Maybe some of you could help.
So, my project uses Apache Thrift library and the server is in python.
Server.py:
loo = loop(0)
handler = ServHandler(loo)
processor = serv.Processor(handler)
transport = TSocket.TServerSocket('0.0.0.0', port=9090)
tfactory = TTransport.TBufferedTransportFactory()
pfactory = TBinaryProtocol.TBinaryProtocolFactory()
server = TProcessPoolServer.TProcessPoolServer(processor, transport, tfactory, pfactory)
print 'Starting the server...'
server.serve()
ServHandler.py:
class ServHandler:
def __init__(self, loo):
self.loo = loo
def terminate(self): #Function that can be called remotely
self.loo.stoploop() #Doesn't work
In above case thread isn't terminated and I don't why. There's no error, object exists, but it sets self.terminate value somewhere else. The object id seems to be the same as well as memory address, but it just looks like object is different although loop init function is called only once...
Below is the example, when the loop is terminated successfully.
ServHandler.py:
class ServHandler:
def __init__(self, loo):
self.loo = None
def terminate(self): #Function that can be called remotely
self.loo.stoploop() #Does work!!!!!!
def create(self):
self.loo = loop(0) #Function that can be called remotely
When I create loop object remotely, I can terminate it remotely. But it doesn't fit me. There should be a thread created before thrift server is served and multiple users have to be able to change vars/terminate/etc of that thread. How can I achieve this?
Thank you!

Not a answer per sae, but a useful debug code for the OP
from time import sleep
from threading import Thread
class loop(Thread):
def __init__(self, myvar):
Thread.__init__(self)
self.terminate = False
self.myvar = myvar
def run(self):
while self.terminate is False:
print('Run says myvar is:',self.myvar)
sleep(0.5)
def change(self, newvar):
self.myvar = newvar
def stoploop(self):
self.terminate = True
l = loop(1)
l.start()
sleep(1)
l.change(2)
sleep(1)
l.stoploop()
print('Final product:',l.myvar)
sleep(2)
print('Is the thread alive:',l.isAlive())
Tried your code with some debugging prints, and it's working?
Following code produced:
[torxed#archie ~]$ python test.py
Run says myvar is: 1
Run says myvar is: 1
Run says myvar is: 2 <-- Proves that change() does change `myvar`
Run says myvar is: 2
Final product: 2 <-- Also the global scope knows about the change
Is the thread alive: False <-- And the thread got terminated as intended
However, these are not bulletproof ideas when fetching data or dealing with thread-returns for a number of reasons (even tho i use this method myself from time to time), you should consider using thread.join which should be used in combination with l.toplooop() like so:
l = loop(1)
l.start()
l.change(2)
l.stoploop()
ret = l.join()
Also when updating data you should aquire locks on your data so collisions don't occur, have a look at semaphore objects.

Is it what you need?
import threading
import time
class Worker(threading.Thread):
def __init__(self):
threading.Thread.__init__(self)
self.state = threading.Condition()
self.variable = 10
self.paused = False
def run(self):
while True:
with self.state:
if self.paused:
self.state.wait()
self.do_stuff()
def do_stuff(self):
time.sleep(.1)
print self.variable
def resume(self):
with self.state:
self.paused = False
self.state.notify()
def pause(self):
with self.state:
self.paused = True
loop = Worker()
loop.start()
time.sleep(1)
loop.pause()
loop.variable = 11
print 'CHANGED!'
loop.resume()
time.sleep(1)

How to communicate with worker thread

I'm using a library which heaviliy uses I/O. For that reason calls to that library can last very long (more than 5 seconds) possible.
Using that directly inside an UI is not a good idea because it will freeze.
For that reason I outsourced the library calls to a thread queue like shown in this example: Python threads: communication and stopping
Nevertheless I'm not very happy with that solution since this has a major drawback:
I cannot really communicate with the UI.
Every lib command returns a return message, which can either be an error message or some computational result.
How would I get this?
Consider a library call do_test(foo):
def do_test(foo):
time.sleep(10)
return random.random() * foo
def ui_btn_click():
threaded_queue.put((do_test, 42))
# Now how to display the result without freezing the UI?
Can someone give me advice how to realize such a pattern?
Edit:
This here is a minimal example:
import os, time, random
import threading, queue
CMD_FOO = 1
CMD_BAR = 2
class ThreadedQueue(threading.Thread):
def __init__(self):
super().__init__()
self.in_queue = queue.Queue()
self.out_queue = queue.Queue()
self.__stoprequest = threading.Event()
def run(self):
while not self.__stoprequest.isSet():
(cmd, arg) = self.in_queue.get(True)
if cmd == CMD_FOO:
ret = self.handle_foo(arg)
elif cmd == CMD_BAR:
ret = self.handle_bar(arg)
else:
print("Unsupported cmd {0}".format(cmd))
self.out_queue.put(ret)
self.in_queue.task_done()
def handle_foo(self, arg):
print("start handle foo")
time.sleep(10)
return random.random() * arg
def handle_bar(self, arg):
print("start handle bar")
time.sleep(2)
return (random.random() * arg, 2 * arg)
if __name__ == "__main__":
print("START")
t = ThreadedQueue()
t.start()
t.in_queue.put((CMD_FOO, 10))
t.in_queue.put((CMD_BAR, 10))
print("Waiting")
while True:
x = t.out_queue.get(True)
t.out_queue.task_done()
print(x)
I personally use PySide but I don't want to depend this library on PySide or any other ui-related library.

I thought a bit about my implementations. THe conclusion is that I start another thread for picking the results of the queue:
class ReceiveThread(threading.Thread):
"""
Processes the output queue and calls a callback for each message
"""
def __init__(self, queue, callback):
super().__init__()
self.__queue = queue
self.__callback = callback
self.__stoprequest = threading.Event()
self.start()
def run(self):
while not self.__stoprequest.isSet():
ret = self.__queue.get(True)
self.__callback(ret)
self.__queue.task_done()
The given callback from an UI or elsewhere is called with every result from the queue.

Cancellable threading.Timer in Python

I am trying to write a method that counts down to a given time and unless a restart command is given, it will execute the task. But I don't think Python threading.Timer class allows for timer to be cancelable.
import threading
def countdown(action):
def printText():
print 'hello!'
t = threading.Timer(5.0, printText)
if (action == 'reset'):
t.cancel()
t.start()
I know the above code is wrong somehow. Would appreciate some kind guidance over here.

You would call the cancel method after you start the timer:
import time
import threading
def hello():
print "hello, world"
time.sleep(2)
t = threading.Timer(3.0, hello)
t.start()
var = 'something'
if var == 'something':
t.cancel()
You might consider using a while-loop on a Thread, instead of using a Timer.
Here is an example appropriated from Nikolaus Gradwohl's answer to another question:
import threading
import time
class TimerClass(threading.Thread):
def __init__(self):
threading.Thread.__init__(self)
self.event = threading.Event()
self.count = 10
def run(self):
while self.count > 0 and not self.event.is_set():
print self.count
self.count -= 1
self.event.wait(1)
def stop(self):
self.event.set()
tmr = TimerClass()
tmr.start()
time.sleep(3)
tmr.stop()

I'm not sure if I understand correctly. Do you want to write something like in this example?
>>> import threading
>>> t = None
>>>
>>> def sayHello():
... global t
... print "Hello!"
... t = threading.Timer(0.5, sayHello)
... t.start()
...
>>> sayHello()
Hello!
Hello!
Hello!
Hello!
Hello!
>>> t.cancel()
>>>

The threading.Timer class does have a cancel method, and although it won't cancel the thread, it will stop the timer from actually firing. What actually happens is that the cancel method sets a threading.Event, and the thread actually executing the threading.Timer will check that event after it's done waiting and before it actually executes the callback.
That said, timers are usually implemented without using a separate thread for each one. The best way to do it depends on what your program is actually doing (while waiting for this timer), but anything with an event loop, like GUI and network frameworks, all have ways to request a timer that is hooked into the eventloop.

Im not sure if best option but for me is woking like this:
t = timer_mgr(.....) append to list "timers.append(t)" and then after all created you can call:
for tm in timers:#threading.enumerate():
print "********", tm.cancel()
my timer_mgr() class is this:
class timer_mgr():
def __init__(self, st, t, hFunction, id, name):
self.is_list = (type(st) is list)
self.st = st
self.t = t
self.id = id
self.hFunction = hFunction
self.thread = threading.Timer(t, self.handle_function, [id])
self.thread.name = name
def handle_function(self, id):
if self.is_list:
print "run_at_time:", datetime.now()
self.hFunction(id)
dt = schedule_fixed_times(datetime.now(), self.st)
print "next:", dt
self.t = (dt-datetime.now()).total_seconds()
else:
self.t = self.st
print "run_every", self.t, datetime.now()
self.hFunction(id)
self.thread = threading.Timer(self.t, self.handle_function, [id])
self.thread.start()
def start(self):
self.thread.start()
def cancel(self):
self.thread.cancel()

Inspired by above post.
Cancelable and Resetting Timer in Python. It uses thread.
Features: Start, Stop, Restart, callback function.
Input: Timeout, sleep_chunk values, and callback_function.
Can use or inherit this class in any other program. Can also pass arguments to the callback function.
Timer should respond in middle also. Not just after completion of full sleep time. So instead of using one full sleep, using small chunks of sleep and kept checking event object in loop.
import threading
import time
class TimerThread(threading.Thread):
def __init__(self, timeout=3, sleep_chunk=0.25, callback=None, *args):
threading.Thread.__init__(self)
self.timeout = timeout
self.sleep_chunk = sleep_chunk
if callback == None:
self.callback = None
else:
self.callback = callback
self.callback_args = args
self.terminate_event = threading.Event()
self.start_event = threading.Event()
self.reset_event = threading.Event()
self.count = self.timeout/self.sleep_chunk
def run(self):
while not self.terminate_event.is_set():
while self.count > 0 and self.start_event.is_set():
# print self.count
# time.sleep(self.sleep_chunk)
# if self.reset_event.is_set():
if self.reset_event.wait(self.sleep_chunk): # wait for a small chunk of timeout
self.reset_event.clear()
self.count = self.timeout/self.sleep_chunk # reset
self.count -= 1
if self.count <= 0:
self.start_event.clear()
#print 'timeout. calling function...'
self.callback(*self.callback_args)
self.count = self.timeout/self.sleep_chunk #reset
def start_timer(self):
self.start_event.set()
def stop_timer(self):
self.start_event.clear()
self.count = self.timeout / self.sleep_chunk # reset
def restart_timer(self):
# reset only if timer is running. otherwise start timer afresh
if self.start_event.is_set():
self.reset_event.set()
else:
self.start_event.set()
def terminate(self):
self.terminate_event.set()
#=================================================================
def my_callback_function():
print 'timeout, do this...'
timeout = 6 # sec
sleep_chunk = .25 # sec
tmr = TimerThread(timeout, sleep_chunk, my_callback_function)
tmr.start()
quit = '0'
while True:
quit = raw_input("Proceed or quit: ")
if quit == 'q':
tmr.terminate()
tmr.join()
break
tmr.start_timer()
if raw_input("Stop ? : ") == 's':
tmr.stop_timer()
if raw_input("Restart ? : ") == 'r':
tmr.restart_timer()