I have this
#threads
import thread
import threading
import time
class ThreadTask(threading.Thread):
def __init__(self,name,delay,callback):
threading.Thread.__init__(self)
self.name = name
self.counter = 0
self.delay = delay
self.callback = callback
self.lock = threading.Lock()
def run(self):
while True:
self.counter += 1
print 'running ', self.name , self.counter
time.sleep(self.delay)
if self.counter % 5 == 0:
self.callback(self)
class Barrier(object):
def __init__(self):
self.locks = []
def wait_task(self,task):
print 'lock acquire'
self.locks.append(task.lock)
task.lock.acquire(True)
task.lock.acquire(True)
def notity_task(self,task):
print 'release lock'
for i in self.locks:
try:
i.release()
except Exception, e:
print 'Error', e.message
print 'Lock released'
self.locks = []
try:
barrier = Barrier()
task1 = ThreadTask('Task_1',1,barrier.wait_task)
task4 = ThreadTask('Task_4',1,barrier.wait_task)
task3 = ThreadTask('Task_3',2,barrier.wait_task)
task2 = ThreadTask('Task_2',3,barrier.notity_task)
task2.start()
task1.start()
task3.start()
task4.start()
except Exception as e:
raise e
while 1:
pass
These Thread run ok but if I put 2 task.lock.acquire(True) consecutive otherwise do not work, they stop each 10 when need to be each 5. Any one know what happening?
Thanks
You create a regular, non-reentrant lock here:
self.lock = threading.Lock()
And then try to acquire it twice here
task.lock.acquire(True)
task.lock.acquire(True)
This is illegal, regular locks cannot be acquired twice by same thread.
Perhaps you means to use a threading.RLock()?
Related
I am trying to end a thread execution without directly referencing the thread. because it is not possible to do that in the full program.
for reference the main program is for the Raspberry Pi and I need it to stop executing a function/thread immediately once a button is pressed.
I have tried raising an exception from main but the other do not catch it for some reason.
Here is the scrap program that I have been testing on:
import threading
import time
class Thread_Exception(Exception):
def __init__(self, msg):
return super().__init__(msg)
def thread_function(index):
bool = True
try:
while bool:
print("Print from thread #", index)
time.sleep(4)
except Thread_Exception:
print('Exception thrown, thread #', index)
bool = False
if __name__ == "__main__":
try:
for index in range(3):
x = threading.Thread(target=thread_function, args=(index,))
x.start()
time.sleep(20)
raise Thread_Exception("intr")
while True:
continue
except KeyboardInterrupt:
print('Interrupted main')
an example of how it can be done:
import threading
import ctypes
import time
class thread_with_exception(threading.Thread):
def __init__(self, name):
threading.Thread.__init__(self)
self.name = name
def run(self):
# target function of the thread class
try:
while True:
print('running ' + self.name)
finally:
print('ended')
def get_id(self):
# returns id of the respective thread
if hasattr(self, '_thread_id'):
return self._thread_id
for id, thread in threading._active.items():
if thread is self:
return id
def raise_exception(self):
thread_id = self.get_id()
res = ctypes.pythonapi.PyThreadState_SetAsyncExc(thread_id,
ctypes.py_object(SystemExit))
if res > 1:
ctypes.pythonapi.PyThreadState_SetAsyncExc(thread_id, 0)
print('Exception raise failure')
t1 = thread_with_exception('Thread 1')
t1.start()
time.sleep(2)
t1.raise_exception()
t1.join()
The article this came from can currently be found here:
https://www.geeksforgeeks.org/python-different-ways-to-kill-a-thread/
I have 3 threads. 1 threads collects data and returns it
var1 = Thread1.start()
Thread 2 and thread 3 use this variable var1 to do routine.
I'm not sure If I do it right. because sometimes var1 is returned and it's not an empty list, I store it in variable of each thread, use list comprehension to extract data. In debugging logs I see elements that must be in thread 3, but debugging logger of that thread returns nothing.
In Thread 2 and thread 3 algoritm:
def __init__(self):
self.lock = threading.RLock()
def do_smth2(self,var1):
self.lock.acquire()
var1_2 = var1
self.lock.release()
def do_smth3(self,var3)
self.lock.acquire()
var1_3 = var1
self.lock.release()
in main
object = thread1.start()
thread2.start(object)
thread3.start(object)
Thread 2 and thread 3 run in the same time, and I use time.sleep(3) if var1_3 or var1_2 is None (this is a list type) or when len(var1_3) <0
EDIT
class Application:
def __init__(self):
self.logger = RootLogger()
self.logger.set_config(__name__, sys_log)
self.adapter = Adapter()
self.transit_listener = TransitListener()
def run(self):
#start listeners
transits_list = self.transit_listener.start()
self.adapter.start(transits_list)
# start REST service
RestWebService().run()
Thread 1
class TransitListener:
def __init__(self):
self.interval = session_interval
def _transits_data(self):
# while polling is running change interval after 1st cycle
while datetime.now() >= session_interval:
result = self.connector.query(self.statement,
fetch=True)
self.logger.debug(result)
# store result
self.transits_queue.put(result)
self.logger.debug(self.interval)
time.sleep(5)
# change interval
self._interval_granularity()
self.logger.debug(self.interval)
def start(self):
self.worker = Thread(target=self._transits_data)
self._configure()
self.logger.info("Starting thread 'transists listener'...")
try:
self.worker.start()
if self.worker.is_alive():
self.logger.info("Thread 'transits listener' started")
# return result from queue
return self.transits_queue.get()
Thread 2 and 3
class Adapter:
def __init__(self):
self.logger = RootLogger()
self.logger.set_config(name=__name__, logfile=epp_log)
self.lock = RLock()
self.threads = []
def _session_start(self, transits):
while datetime.now() >= session_interval:
self.lock.acquire()
transit_list = transits
self.lock.release()
self.logger.debug(f"ENTRIES {transit_list}")
def _session_stop(self, transits):
while datetime.now() >= session_interval:
self.lock.acquire()
transit_list = transits
self.lock.release()
self.logger.debug(f"EXITS{transit_list}")
def start(self, transits):
# prepare SQL tables
# define priority of threads
# 1st
session_start_thread = Thread(target=self._session_start, args=(transits,))
self.threads.append(session_start_thread)
# 2nd
session_stop_thread = Thread(target=self._session_stop, args=(transits,))
self.threads.append(session_stop_thread)
self.threads[0].start()
self.threads[1].start()
I have the following function:
def getSuggestengineResult(suggestengine, seed, tablename):
table = getTable(tablename)
for keyword_result in results[seed][suggestengine]:
i = 0
while True:
try:
allKeywords.put_item(
Item={
'keyword': keyword_result
}
)
break
except ProvisionedThroughputExceededException as pe:
if (i > 9):
addtoerrortable(keyword_result)
print(pe)
break
sleep(1)
i = i + 1
print("ProvisionedThroughputExceededException in getSugestengineResult")
The function gets started in more then one thread. I have this process and if the process works, the function should be ready in the thread. Otherwise it should try again 9 times. Now my problem:
the "print("ProvisionedThroughputExceededException in getSugestengineResult")" Never got printed. Just the exception as pe gets printed. So there is my problem? Are all the threads working on the same "i"? Or is it never possible to get to the print? I dont know what I am doin wrong ...
You have to use a specific counter if you want all your thread to have the same counter :
from multiprocessing import Lock, Process, Value
class ThreadCounter(object):
def __init__(self, initval=0):
self.val = Value('i', initval)
self.lock = Lock()
def increment(self):
with self.lock:
self.val.value += 1
def value(self):
with self.lock:
return self.val
then you can pass the counter to your function
counter=ThreadCounter(0)
def getSuggestengineResult(suggestengine, seed, tablename,counter):
...
except ProvisionedThroughputExceededException as pe:
if (counter.value() > 9):
...
counter.increment()
...
This counter will be shared with the other threads
Im using the following code to multithread urlib2. However what is the best way to limit the number of threads that it consumes ??
class ApiMultiThreadHelper:
def __init__(self,api_calls):
self.q = Queue.Queue()
self.api_datastore = {}
self.api_calls = api_calls
self.userpass = '#####'
def query_api(self,q,api_query):
self.q.put(self.issue_request(api_query))
def issue_request(self,api_query):
self.api_datastore.update({api_query:{}})
for lookup in ["call1","call2"]:
query = api_query+lookup
request = urllib2.Request(query)
request.add_header("Authorization", "Basic %s" % self.userpass)
f = urllib2.urlopen(request)
response = f.read()
f.close()
self.api_datastore[api_query].update({lookup:response})
return True
def go(self):
threads = []
for i in self.api_calls:
t = threading.Thread(target=self.query_api, args = (self.q,i))
t.start()
threads.append(t)
for t in threads:
t.join()
You should use a thread pool. Here's my implementation I've made years ago (Python 3.x friendly):
import traceback
from threading import Thread
try:
import queue as Queue # Python3.x
except ImportError:
import Queue
class ThreadPool(object):
def __init__(self, no=10):
self.alive = True
self.tasks = Queue.Queue()
self.threads = []
for _ in range(no):
t = Thread(target=self.worker)
t.start()
self.threads.append(t)
def worker(self):
while self.alive:
try:
fn, args, kwargs = self.tasks.get(timeout=0.5)
except Queue.Empty:
continue
except ValueError:
self.tasks.task_done()
continue
try:
fn(*args, **kwargs)
except Exception:
# might wanna add some better error handling
traceback.print_exc()
self.tasks.task_done()
def add_job(self, fn, args=[], kwargs={}):
self.tasks.put((fn, args, kwargs))
def join(self):
self.tasks.join()
def deactivate(self):
self.alive = False
for t in self.threads:
t.join()
You can also find a similar class in multiprocessing.pool module (don't ask me why it is there). You can then refactor your code like this:
def go(self):
tp = ThreadPool(20) # <-- 20 thread workers
for i in self.api_calls:
tp.add_job(self.query_api, args=(self.q, i))
tp.join()
tp.deactivate()
Number of threads is now defined a priori.
I am trying to write a method that counts down to a given time and unless a restart command is given, it will execute the task. But I don't think Python threading.Timer class allows for timer to be cancelable.
import threading
def countdown(action):
def printText():
print 'hello!'
t = threading.Timer(5.0, printText)
if (action == 'reset'):
t.cancel()
t.start()
I know the above code is wrong somehow. Would appreciate some kind guidance over here.
You would call the cancel method after you start the timer:
import time
import threading
def hello():
print "hello, world"
time.sleep(2)
t = threading.Timer(3.0, hello)
t.start()
var = 'something'
if var == 'something':
t.cancel()
You might consider using a while-loop on a Thread, instead of using a Timer.
Here is an example appropriated from Nikolaus Gradwohl's answer to another question:
import threading
import time
class TimerClass(threading.Thread):
def __init__(self):
threading.Thread.__init__(self)
self.event = threading.Event()
self.count = 10
def run(self):
while self.count > 0 and not self.event.is_set():
print self.count
self.count -= 1
self.event.wait(1)
def stop(self):
self.event.set()
tmr = TimerClass()
tmr.start()
time.sleep(3)
tmr.stop()
I'm not sure if I understand correctly. Do you want to write something like in this example?
>>> import threading
>>> t = None
>>>
>>> def sayHello():
... global t
... print "Hello!"
... t = threading.Timer(0.5, sayHello)
... t.start()
...
>>> sayHello()
Hello!
Hello!
Hello!
Hello!
Hello!
>>> t.cancel()
>>>
The threading.Timer class does have a cancel method, and although it won't cancel the thread, it will stop the timer from actually firing. What actually happens is that the cancel method sets a threading.Event, and the thread actually executing the threading.Timer will check that event after it's done waiting and before it actually executes the callback.
That said, timers are usually implemented without using a separate thread for each one. The best way to do it depends on what your program is actually doing (while waiting for this timer), but anything with an event loop, like GUI and network frameworks, all have ways to request a timer that is hooked into the eventloop.
Im not sure if best option but for me is woking like this:
t = timer_mgr(.....) append to list "timers.append(t)" and then after all created you can call:
for tm in timers:#threading.enumerate():
print "********", tm.cancel()
my timer_mgr() class is this:
class timer_mgr():
def __init__(self, st, t, hFunction, id, name):
self.is_list = (type(st) is list)
self.st = st
self.t = t
self.id = id
self.hFunction = hFunction
self.thread = threading.Timer(t, self.handle_function, [id])
self.thread.name = name
def handle_function(self, id):
if self.is_list:
print "run_at_time:", datetime.now()
self.hFunction(id)
dt = schedule_fixed_times(datetime.now(), self.st)
print "next:", dt
self.t = (dt-datetime.now()).total_seconds()
else:
self.t = self.st
print "run_every", self.t, datetime.now()
self.hFunction(id)
self.thread = threading.Timer(self.t, self.handle_function, [id])
self.thread.start()
def start(self):
self.thread.start()
def cancel(self):
self.thread.cancel()
Inspired by above post.
Cancelable and Resetting Timer in Python. It uses thread.
Features: Start, Stop, Restart, callback function.
Input: Timeout, sleep_chunk values, and callback_function.
Can use or inherit this class in any other program. Can also pass arguments to the callback function.
Timer should respond in middle also. Not just after completion of full sleep time. So instead of using one full sleep, using small chunks of sleep and kept checking event object in loop.
import threading
import time
class TimerThread(threading.Thread):
def __init__(self, timeout=3, sleep_chunk=0.25, callback=None, *args):
threading.Thread.__init__(self)
self.timeout = timeout
self.sleep_chunk = sleep_chunk
if callback == None:
self.callback = None
else:
self.callback = callback
self.callback_args = args
self.terminate_event = threading.Event()
self.start_event = threading.Event()
self.reset_event = threading.Event()
self.count = self.timeout/self.sleep_chunk
def run(self):
while not self.terminate_event.is_set():
while self.count > 0 and self.start_event.is_set():
# print self.count
# time.sleep(self.sleep_chunk)
# if self.reset_event.is_set():
if self.reset_event.wait(self.sleep_chunk): # wait for a small chunk of timeout
self.reset_event.clear()
self.count = self.timeout/self.sleep_chunk # reset
self.count -= 1
if self.count <= 0:
self.start_event.clear()
#print 'timeout. calling function...'
self.callback(*self.callback_args)
self.count = self.timeout/self.sleep_chunk #reset
def start_timer(self):
self.start_event.set()
def stop_timer(self):
self.start_event.clear()
self.count = self.timeout / self.sleep_chunk # reset
def restart_timer(self):
# reset only if timer is running. otherwise start timer afresh
if self.start_event.is_set():
self.reset_event.set()
else:
self.start_event.set()
def terminate(self):
self.terminate_event.set()
#=================================================================
def my_callback_function():
print 'timeout, do this...'
timeout = 6 # sec
sleep_chunk = .25 # sec
tmr = TimerThread(timeout, sleep_chunk, my_callback_function)
tmr.start()
quit = '0'
while True:
quit = raw_input("Proceed or quit: ")
if quit == 'q':
tmr.terminate()
tmr.join()
break
tmr.start_timer()
if raw_input("Stop ? : ") == 's':
tmr.stop_timer()
if raw_input("Restart ? : ") == 'r':
tmr.restart_timer()