I'm creating a script to monitor my mouse/keyboard activity. The intent is to update a timestamp whenever I move the mouse or press a button the keyboard. I've threaded the mouse and keyboard check methods and the main thread will check if we have passed the timeout/inactive duration and click at a specified location if we have.
Unfortunately, the keyboard monitoring is not working. The pynput.keyboard.Listener object seems to never join().
I'm not particularly comfortable with multithreading but I think I need it for this script. Please share a better way if there is one. I want to be able to run this script/class as a thread in another script later.
from pynput.keyboard import Listener
import pyautogui as gui
import threading, time
from datetime import datetime, timedelta
class activity(threading.Thread):
def __init__(self, timeout: int = 60):
self.stop_flag = False
self.timeout = timedelta(seconds=timeout)
self.last_timestamp = datetime.now()
def update_timestamp(self):
self.last_timestamp = datetime.now()
print('timestamp updated')
# For monitoring if the keyboard is active
def keybd_monitoring(self, lock: threading.Lock) -> None:
with Listener(on_release=lambda x: True) as listener:
listener.join()
lock.acquire()
self.update_timestamp()
lock.release()
print('Keyboard pressed')
if self.stop_flag:
return
# For monitoring if the mouse is active
def mouse_monitoring(self, lock: threading.Lock) -> None:
last_position = gui.position()
while not self.stop_flag:
time.sleep(3)
curr_position = gui.position()
if last_position != curr_position:
last_position = curr_position
lock.acquire()
self.update_timestamp()
lock.release()
print('Mouse Moved')
def stop(self):
self.stop_flag = True
# For monitoring if the mouse/keyboard have been used in the last TIMEOUT seconds
def run(self):
try:
width, height = gui.size()
lock = threading.Lock()
mouse = threading.Thread(target=self.mouse_monitoring, args=(lock,))
keybd = threading.Thread(target=self.keybd_monitoring, args=(lock,))
mouse.start()
keybd.start()
while not self.stop_flag:
time.sleep(.1)
if datetime.now() > self.last_timestamp + self.timeout:
curr_position = gui.position()
gui.click(int(width*.6),height)
gui.moveTo(curr_position)
finally:
self.stop()
if mouse.is_alive():
mouse.join()
if keybd.is_alive():
keybd.join()
if __name__ == '__main__':
act = activity()
act.run()
I've made it work without the monitoring functions being in a class. I'm still curious if it could work within a class.
from pynput.keyboard import Listener
import pyautogui as gui
import threading, time
from datetime import datetime, timedelta
stop_flag = False
timeout = timedelta(seconds=60)
last_timestamp = datetime.now()
lock = threading.Lock()
def update_timestamp(key=None):
lock.acquire()
global last_timestamp
last_timestamp = datetime.now()
lock.release()
return stop_flag
# For monitoring if the keyboard is active
def keybd_monitoring(lock: threading.Lock) -> None:
with Listener(on_release=update_timestamp) as listener:
listener.join()
# For monitoring if the mouse is active
def mouse_monitoring(lock: threading.Lock) -> None:
last_position = gui.position()
while not stop_flag:
time.sleep(3)
curr_position = gui.position()
if last_position != curr_position:
last_position = curr_position
update_timestamp()
def stop():
global stop_flag
stop_flag = True
# For monitoring if the mouse/keyboard have been used in the last TIMEOUT seconds
def activity():
try:
width, height = gui.size()
mouse = threading.Thread(target=mouse_monitoring, args=(lock,))
keybd = threading.Thread(target=keybd_monitoring, args=(lock,))
mouse.start()
keybd.start()
while not stop_flag:
time.sleep(1)
if datetime.now() > last_timestamp + timeout:
curr_position = gui.position()
gui.click(int(width*.6),height)
gui.moveTo(curr_position)
update_timestamp()
finally:
stop()
if mouse.is_alive():
mouse.join()
if keybd.is_alive():
keybd.join()
if __name__ == '__main__':
activity()
Background: To me it seems clear, that the conecpt of callbacks is flexible, but I also thought, it makes code much faster. However, the following example works, but it cannot show that time can be saved using a callback Ref1:
import time
from time import sleep
def callbackFunc(delay):
time.sleep(delay)
print("callback: message 3 delay " + str(delay))
def saysomething(delay, callback):
print("saysomething: message 2 delay " + str(delay))
callback(delay) # hier muss ich schon wissen, dass nur "delay" benötigt wird...
time.sleep(2)
if __name__ == '__main__':
t0 = time.time()
print("main: message 1.")
saysomething(2, callbackFunc)
print("main: message 4.")
print("\ntime: ",time.time() - t0)
Output
main: message 1.
saysomething: message 2 delay 2
callback: message 3 delay 2
main: message 4.
time: 4.01
So how can I achive something like this
main: message 1.
saysomething: message 2 delay 2
callback: message 3 delay 2
main: message 4.
time: 2 !!!!!!!!!!!!!
Perhaps it was even possible to switch the order of messages 3 & 4? Or do I get anything wrong?
Perhaps, these answers here and here and the following code from here which does not use callbacks but shows asynchronous behavior help?
Re-establishing the premise
Before we start, let's clarify the statement in your post:
A callback does not make the code itself faster. The program does end earlier because the function that accepts the callback does not block.
Also, callbacks are generally executed synchronously in the function that accepts the callback, so your example would still take 4 seconds. A more reasonable starting point would be:
import time
def callbackFunc(delay):
# time.sleep(delay) # -
print("callback: message 3 delay " + str(delay))
def saysomething(delay, callback):
print("saysomething: message 2 delay " + str(delay))
time.sleep(2) # +
callback(delay)
# time.sleep(2) # -
if __name__ == '__main__':
t0 = time.time()
print("main: message 1.")
saysomething(2, callbackFunc)
print("\ntime: ", time.time() - t0, "\n") # +
saysomething(2, callbackFunc) # +
print("\ntime: ", time.time() - t0, "\n") # +
print("main: message 4.")
print("\ntime: ", time.time() - t0)
Output:
main: message 1.
saysomething: message 2 delay 2
callback: message 3 delay 2
time: 2.00
saysomething: message 2 delay 2
callback: message 3 delay 2
time: 4.00
main: message 4.
time: 4.00
Why the example doesn't work
sleep() suspends execution of the calling thread, so you can't call it on the main thread if you want to show asynchronous behaviour.
You can use threads or an event loop to show asynchronous behaviour.
An example using an event loop
Here's an example using the built-in event loop:
import asyncio # +
import time
def callbackFunc(delay):
print("callback: message 3 delay " + str(delay))
def saysomething(delay, callback):
print("saysomething: message 2 delay " + str(delay))
# time.sleep(2) # -
# callback(delay) # -
asyncio.get_event_loop().call_later(2, callback, delay) # +
def wait_for_callbacks(): # +
def run_until_complete(loop):
loop.call_soon(lambda: run_until_complete(loop) if loop._scheduled else loop.stop())
loop = asyncio.get_event_loop()
run_until_complete(loop)
loop.run_forever()
if __name__ == '__main__':
t0 = time.time()
print("main: message 1.")
saysomething(2, callbackFunc)
print("\ntime: ", time.time() - t0, "\n")
saysomething(2, callbackFunc)
print("\ntime: ", time.time() - t0, "\n")
print("main: message 4.")
wait_for_callbacks() # +
print("\ntime: ", time.time() - t0)
Output:
main: message 1.
saysomething: message 2 delay 2
time: 0.000
saysomething: message 2 delay 2
time: 0.000
main: message 4.
callback: message 3 delay 2
callback: message 3 delay 2
time: 2.000
An example using threads
Here's an example using the built-in OS threads:
import threading # +
import time
def threadify(func): # +
def _func(*args, **kwargs):
thread = threading.Thread(target=func, args=args, kwargs=kwargs)
thread.start()
return thread
return _func
def callbackFunc(delay):
print("callback: message 3 delay " + str(delay))
#threadify # +
def saysomething(delay, callback):
print("saysomething: message 2 delay " + str(delay))
time.sleep(2)
callback(delay)
def wait_for_callbacks(): # +
for thread in threading.enumerate():
if thread is not threading.current_thread():
thread.join()
if __name__ == '__main__':
t0 = time.time()
print("main: message 1.")
saysomething(2, callbackFunc)
print("\ntime: ", time.time() - t0, "\n")
saysomething(2, callbackFunc)
print("\ntime: ", time.time() - t0, "\n")
print("main: message 4.")
wait_for_callbacks() # +
print("\ntime: ", time.time() - t0)
Output: (marginally slower than using the built-in event loop due to overhead of OS threads)
main: message 1.
saysomething: message 2 delay 2
time: 0.000
saysomething: message 2 delay 2
time: 0.000
main: message 4.
callback: message 3 delay 2
callback: message 3 delay 2
time: 2.00
You are correct about callbacks being a way of providing "flexibilty" but it has really nothing to do with "speed". The typical use case for a callback is the situation where function 1 is calling function 2 that is performing some function whose completion occurs asynchronously so that function 2 will be returning more or less immediately back but function 1 still needs to arrange for a notification to occur when that asynchronous completion occurs. Therefore, function 1 passes to function 2 a callback function that will be invoked with agreed-upon arguments when that completion occurs.
In your case your asynchronous event is the expiration of a time interval and your callback function is passed what that delay time interval was. Now it turns out that Python comes with a sched.scheduler class that allows you to schedule "events" to be run in the future by either specifying an absolute time value or a delay, which will be added to the current time to compute the absolute time at which the event is to be run. This event is just a callback function to which you can specify any arguments you wish. The problem with this class, in my opinion, is that you have to first enter all the events that you will want to be running and then call a run method that will block until all the events are run. Much better would be to just specify a future event you want to run (that is, a callback) and continue without blocking and this event will run asynchronously in another thread. And so I have heavily modified the sched.scheduler class to create a Scheduler class. Your code would then look like:
import time
from scheduler import Scheduler
def callbackFunc(msg_no, delay):
print(f"callback: message number {msg_no}, delay {delay} at {time.time()}")
def saysomething(msg_no, delay, callback):
print(f"saysomething: message {msg_no}, delay {delay} at {time.time()}")
scheduler.enter(delay, callbackFunc, args=(msg_no, delay,))
time.sleep(2)
if __name__ == '__main__':
scheduler = Scheduler()
saysomething(1, 1, callbackFunc)
saysomething(2, 2, callbackFunc)
saysomething(3, 3, callbackFunc)
Prints:
saysomething: message 1, delay 1 at 1644584120.865646
callback: message number 1, delay 1 at 1644584121.8778687
saysomething: message 2, delay 2 at 1644584122.8747876
saysomething: message 3, delay 3 at 1644584124.8790839
callback: message number 2, delay 2 at 1644584124.8790839
callback: message number 3, delay 3 at 1644584127.9029477
And the Scheduler class:
"""
Modified sched.schedule class.
"""
import time
import heapq
from collections import namedtuple
import threading
class Event(namedtuple('Event', 'start_time, priority, action, args, kwargs')):
__slots__ = []
def __eq__(s, o): return (s.start_time, s.priority) == (o.start_time, o.priority)
def __lt__(s, o): return (s.start_time, s.priority) < (o.start_time, o.priority)
def __le__(s, o): return (s.start_time, s.priority) <= (o.start_time, o.priority)
def __gt__(s, o): return (s.start_time, s.priority) > (o.start_time, o.priority)
def __ge__(s, o): return (s.start_time, s.priority) >= (o.start_time, o.priority)
Event.start_time.__doc__ = ('''Numeric type compatible with the return value from time.monotonic.''')
Event.priority.__doc__ = ('''Events scheduled for the same time will be executed
in the order of their priority.''')
Event.action.__doc__ = ('''Executing the event means executing
action(*args, **kwargs)''')
Event.args.__doc__ = ('''args is a sequence holding the positional
arguments for the action.''')
Event.kwargs.__doc__ = ('''kwargs is a dictionary holding the keyword
arguments for the action.''')
_sentinel = object()
class Scheduler:
def __init__(self, daemon=False):
"""
Initialize a new instance.
If daemon is True, the scheduler thread will run as a daemon so it will be possible
for the main thread to terminate with scheduled events yet to run.
Regardless of how the daemon argument is set, when a new event is added a new
scheduler thread will be started if the previous thread has terminated.
"""
self._queue = []
self._daemon=daemon
self._running = False
self._got_event = threading.Condition()
self._queue_exhausted = threading.Event()
self._queue_exhausted.set()
self._thread = None
def __del__(self):
if not self._daemon and self._thread:
self._thread.join()
def enterabs(self, start_time, action, args=(), kwargs=_sentinel, priority=1):
"""Enter a new event in the queue at an absolute time.
Returns an ID for the event which can be used to remove it,
if necessary.
"""
if kwargs is _sentinel:
kwargs = {}
event = Event(start_time, priority, action, args, kwargs)
with self._got_event:
heapq.heappush(self._queue, event)
self._queue_exhausted.clear()
if not self._running:
if self._thread:
self._thread.join() # tidy up
self._running = True
self._thread = threading.Thread(target=self._run, daemon=self._daemon).start()
else:
self._got_event.notify()
return event # The ID
def enter(self, delay, action, args=(), kwargs=_sentinel, priority=1):
"""A variant that specifies the time as a relative time.
This is actually the more commonly used interface.
"""
start_time = time.monotonic() + delay
return self.enterabs(start_time, action, args, kwargs, priority)
def cancel(self, event):
"""Remove an event from the queue.
This must be presented the ID as returned by enter().
If the event is not in the queue, this raises ValueError.
"""
with self._got_event:
self._queue.remove(event)
heapq.heapify(self._queue)
def empty(self):
"""Check whether the queue is empty."""
with self._got_event:
return not self._queue
def running(self):
"""Check whether the scheduler is running."""
with self._got_event:
return self._running
def _run(self):
"""Execute events until the queue is empty."""
# localize variable access to minimize overhead
# and to improve thread safety
got_event = self._got_event
q = self._queue
delayfunc = time.sleep
timefunc = time.monotonic
pop = heapq.heappop
queue_exhausted = self._queue_exhausted
while True:
try:
while True:
with got_event:
if not q:
self._running = False
queue_exhausted.set()
return
start_time, priority, action, args, kwargs = q[0]
now = timefunc()
if start_time > now:
# Wait for either the time to elapse or a new Event to be added:
got_event.wait(timeout=(start_time - now))
continue
pop(q)
action(*args, **kwargs)
delayfunc(0) # Let other threads run
except:
pass
#property
def queue(self):
"""An ordered list of upcoming events.
Events are named tuples with fields for:
start_time, priority, action, argss, kwargs
"""
# Use heapq to sort the queue rather than using 'sorted(self._queue)'.
# With heapq, two events scheduled at the same time will show in
# the actual order they would be retrieved.
with self._got_event:
events = self._queue[:]
return list(map(heapq.heappop, [events]*len(events)))
def wait_for_queue_empty(self):
"""Wait for the queue to become empty."""
return self._queue_exhausted.wait()
This is irrelevant w.r.t. the SWIG callbacks used by the routing library.
This isn't as much of a question as something I'm interested in.
I do quite a bit of asynchronous coding in Python, and there's a bit of code that I'm frequently writing over and over while I'm waiting for threads to stop (if I'm trying to exit cleanly).
while not class_containing_threads.stopped:
pass
else:
do_something()
do_something_else()
do_some_other_thing()
Although I'm sure there's a nice decorator that one can write to make this happen, I'm not too sure how I would go about writing it without ultimately making my code more complicated than it needs to be.
Basically, I wish there were something along the lines of:
when condition:
do_something()
where the thread is effectively halted while we wait for some event to occur.
To demonstrate what I mean, here's some working code that shows how much I actually end up writing the same thing over and over
import threading
import random
import time
class ClassContainingThreads:
def __init__(self):
# Just stating what stuff can be found here
self._coordinating_thread = None
self._worker_thread_1 = None
self._worker_thread_2 = None
self._worker_thread_3 = None
self._stopping = False
self._stopped = False
def run(self):
# Main method to get everything running
self._coordinating_thread = threading.Thread(target=self._run)
self._coordinating_thread.start()
def stop(self):
# Used to stop everything
self._stopping = True
#property
def stopped(self):
# Lets you know when things have stopped
return self._stopped
#property
def all_workers_running(self):
# Lets you know whether all the workers are running
return self._all_workers_are_alive()
def _run(self):
# Coordinating thread getting worker threads to start
self._worker_thread_1 = threading.Thread(
target=self._important_function_1)
self._worker_thread_2 = threading.Thread(
target=self._important_function_2)
self._worker_thread_3 = threading.Thread(
target=self._important_function_3)
self._worker_thread_1.start()
self._worker_thread_2.start()
self._worker_thread_3.start()
# Coincidentally, the block appears here
while not self._stopping:
pass
else:
while self._any_workers_are_alive():
pass
else:
self._stopping = False
self._stopped = True
def _important_function_1(self):
print(f'Thread 1 started')
# Coincidentally, the block appears here
while not self._stopping:
pass
else:
print('Thread 1 received stop signal')
# Emulating some process that takes some unknown time to stop
delay_long = random.random() * 5
delay_start = time.time()
while not (time.time() - delay_start) > delay_long:
pass
else:
print(f'Thread 1 stopped')
def _important_function_2(self):
print(f'Thread 2 started')
# Coincidentally, the block appears here
while not self._stopping:
pass
else:
print('Thread 2 received stop signal')
# Emulating some process that takes some unknown time to stop
delay = random.random() * 5
delay_start = time.time()
while not (time.time() - delay_start) > delay:
pass
else:
print(f'Thread 2 stopped')
def _important_function_3(self):
print(f'Thread 3 started')
# Coincidentally, the block appears here
while not self._stopping:
pass
else:
print('Thread 3 received stop signal')
# Emulating some process that takes some unknown time to stop
delay = random.random() * 5
delay_start = time.time()
while not (time.time() - delay_start) > delay:
pass
else:
print(f'Thread 3 stopped')
def _any_workers_are_alive(self):
# Check whether any workers are alive
if (self._worker_thread_1.is_alive() or
self._worker_thread_2.is_alive() or
self._worker_thread_3.is_alive()):
return True
else:
return False
def _all_workers_are_alive(self):
# Check whether all workers are alive
if (self._worker_thread_1.is_alive() and
self._worker_thread_2.is_alive() and
self._worker_thread_3.is_alive()):
return True
else:
return False
if __name__ == '__main__':
# Just booting everything up
print('Program started')
class_containing_threads = ClassContainingThreads()
class_containing_threads.run()
# Block I'm interested in appears here
while not class_containing_threads.all_workers_running:
pass
else:
# and here
while not input("Type 'exit' to exit > ") == "exit":
pass
else:
class_containing_threads.stop()
# and here
while not class_containing_threads.stopped:
pass
else:
print('Program stopped')
exit() # I know this is pointless here
Also, critiques are welcome.
The pattern of repeatedly checking a flag is a form of busy wait. This is an extremely wasteful pattern, as the task checking the flag will do so very, very often.
Concrete alternatives depend on the concurrency pattern used, but usually come in the form of signals, events or locks – these are generally known as "synchronisation primitives".
For example, threading provides a threading.Event that can be "waited for" and "triggered". The desired operation when condition: is simply event.wait() – this automatically pauses the current thread until the event is triggered. Another thread can trigger this condition via event.set().
Thanks to the feedback, I've rewritten the code snippet into something that uses the threading.Thread.join() method and threading.Event object. It's much simpler now, and hopefully doesn't involve any unintentional busy waiting.
import threading
import random
import time
class ClassContainingThreads:
def __init__(self, blocking_event):
# Just stating what stuff can be found here
self._blocking_event = blocking_event
self._coordinating_thread = None
self._worker_thread_1 = None
self._worker_thread_2 = None
self._worker_thread_3 = None
self._stopping = False
self._stopped = False
def run(self):
# Main method to get everything running
self._coordinating_thread = threading.Thread(target=self._run)
self._coordinating_thread.start()
def stop(self):
# Used to stop everything
self._stopping = True
#property
def stopped(self):
return self._stopped
def _run(self):
# Coordinating thread getting worker threads to start
self._worker_thread_1 = threading.Thread(
target=self._important_function_1)
self._worker_thread_2 = threading.Thread(
target=self._important_function_2)
self._worker_thread_3 = threading.Thread(
target=self._important_function_3)
# Start the workers
self._worker_thread_1.start()
self._worker_thread_2.start()
self._worker_thread_3.start()
# Let main_thread continue when workers have started
self._blocking_event.set()
# Wait for workers to complete
self._worker_thread_1.join()
self._worker_thread_2.join()
self._worker_thread_3.join()
# Once all threads are dead
self._stopping = False
self._stopped = True
self._blocking_event.set()
def _important_function_1(self):
print(f'Thread 1 started')
# Emulating some job being done
while not self._stopping:
pass
else:
print('Thread 1 received stop signal')
# Emulating some process that takes some unknown time to stop
delay_long = random.random() * 5
delay_start = time.time()
while not (time.time() - delay_start) > delay_long:
pass
else:
print(f'Thread 1 stopped')
def _important_function_2(self):
print('Thread 2 started')
# Emulating some job being done
while not self._stopping:
pass
else:
print('Thread 2 received stop signal')
# Emulating some process that takes some unknown time to stop
delay = random.random() * 5
delay_start = time.time()
while not (time.time() - delay_start) > delay:
pass
else:
print(f'Thread 2 stopped')
def _important_function_3(self):
print('Thread 3 started')
# Emulating some job being done
while not self._stopping:
pass
else:
print('Thread 3 received stop signal')
# Emulating some process that takes some unknown time to stop
delay = random.random() * 5
delay_start = time.time()
while not (time.time() - delay_start) > delay:
pass
else:
print(f'Thread 3 stopped')
if __name__ == '__main__':
# Just booting everything up
print('Program started')
blocking_event = threading.Event()
class_containing_threads = ClassContainingThreads(blocking_event)
class_containing_threads.run()
blocking_event.wait()
while not input("Type 'exit' to exit > ") == "exit":
pass
else:
class_containing_threads.stop()
blocking_event.wait()
print('Program stopped')
exit() # I know this is pointless here
I have QMediaPlayer and QVideoWidget for playing videos with spended time QLineEdit and finish program QLineEdit. I am trying to do make a counter that show how many minutes user spended on this program and when user reachs finish program limit pop-up a dialog. For that I create a thread function:
def update_video_timer(self):
end_time = int(self.end_time.text())
start_time = 0
while start_time <= end_time:
self.spended_time.setText(str(start_time))
start_time = start_time + 1
# minutes
time.sleep(60)
# reachs limit
self.mediaPlayer.pause()
dlg = QDialog(self)
dlg.setWindowTitle("YOUR TIME HAS FINISHED!")
dlg.exec_()
I call this function when first video started to play:
from threading import Thread
Thread(target=self.update_video_timer())
But the problem is when video starts to play, program is freezing. Any help and/or improvement of my code is welcome.
Tasks in a GUI must be asynchronous and invoke synchronous tasks that consume very little time. If synchronous tasks consume a lot of time then they must be executed in another thread. In your case it is not necessary to use while + time.sleep() but a counter with a QTimer is enough and thus it is not necessary to use threads:
self.timer = QTimer(self, interval=60 * 1000)
self.timer.timeout.connect(self.on_timeout)
self.start_time = 0
self.end_time = 0
def start(self):
try:
self.end_time = int(self.end_time.text())
except ValueError as e:
print("error")
else:
self.timer.start()
def on_timeout(self):
if self.start_time <= self.end_time:
self.start_time += 1
self.spended_time.setText(str(self.start_time))
else:
self.timer.stop()
self.mediaPlayer.pause()
dlg = QDialog(self)
dlg.setWindowTitle("YOUR TIME HAS FINISHED!")
dlg.exec_()
I am working on a scraper that rotates the ips, i have created a small mvp in a notebook that works as expected:
import logging
import time
import random
import threading
from datetime import datetime
from datetime import timedelta
logging.basicConfig(
level=logging.DEBUG,
format='(%(threadName)-10s) %(message)s',
)
class Controller(object):
def __init__(self, event):
self.start_time = datetime.now()
self.event = event
def worker(self):
while True:
if self.event.is_set():
rand_sleep_time = random.randint(1, 10) / 5
logging.debug("Sleeping for %.2f secs" % rand_sleep_time)
time.sleep(rand_sleep_time)
logging.debug("Werking")
else:
time.sleep(1)
def blocker(self):
while True:
rand_sleep_time = random.randint(3, 6)
logging.debug("Sleeping for %.2f secs" % rand_sleep_time)
time.sleep(rand_sleep_time)
if datetime.now() > self.start_time + timedelta(seconds=10):
self.event.clear() # only stop the execution for when the ip is updated
logging.debug("ALL THREADS SLEEP NOW!")
time.sleep(10)
self.event.set() # you can now proceed with the computations
self.start_time = datetime.now()
start_time = datetime.now()
e = threading.Event()
e.set()
c = Controller(e)
for thread in range(NUM_THREADS):
t = threading.Thread(target=c.worker, name='Thread-Worker-{}'.format(thread+1))
t.start()
threading.Thread(target=c.blocker, name='Thread-Blocker-1').start()
So the workers above do some work, then the blocker halts all of them for a brief moment of time, while it updates the "ip", and then the workers start the work again. Taking this logic and implementing it into production, this fails (because I assume the workers do not stop). Unfortunately, I cannot include all of the code, but here is the main part. Hopefully, this is enough, as other parts are not related to the fact that the Ip-Updater does not stop other threads. The only difference in this implementation is that I have used classes, perhaps that should be changed (because the methods have self arg and I'm chaning it. But if the Ip-Updater were to successfully stop the other threads, then there should be no problem, no?):
class ThreadedNewsParser(object):
"""
This little guy parses the news with multiple threads and dynamically changes the ip of the sessions
"""
def __init__(self, update_ip_in, num_threads, date_start, date_end):
assert isinstance(num_threads, int)
assert num_threads > 0
assert any(isinstance(date_start, type_) for type_ in [datetime, date])
assert any(isinstance(date_end, type_) for type_ in [datetime, date])
self.start_time = datetime.now()
self.event = threading.Event()
self.event.set()
self.update_ip_in = update_ip_in
self.check_ip_url = 'https://httpbin.org/ip'
autolog("STARTING WORK ON IP: {}".format(session.get(self.check_ip_url).text), logging.debug)
self.num_threads = num_threads
self.date_start = date_start
self.date_end = date_end
self.dates = [date for date in date_range(date_start, date_end)]
self.p = DailyCompanyNewsParser(2008, 1, 1) # the date here does not matter
def worker(self):
while len(self.dates) > 0:
if self.event.is_set():
print("THREAD WERKING!")
pause = random.randint(1, 5) / 5
autolog('THREAD SLEEPING %.2f' % pause, logging.debug)
time.sleep(pause)
if len(self.dates) > 0:
date = self.dates.pop(0)
self.p.get_news_for_all_stocks(verbose=True, date_=date)
else:
print("THREAD SLEEPING")
time.sleep(10) # so that the threads do not check if the event is set instantaneously
def ip_updater(self): # this is the blocker
while len(self.dates) > 0:
autolog("IP_UPDATER SLEEPING FOR: {}".format(self.update_ip_in / 4), logging.debug)
time.sleep(self.update_ip_in / 4) # do not check the condition every instance
if datetime.now() > self.start_time + timedelta(seconds=self.update_ip_in):
print("ALL THREADS SLEEP NOW!")
autolog("ALL THREADS SLEEP NOW!", logging.info)
self.event.clear() # Make all other threads sleep so that we can update the IP
time.sleep(10)
get_new_ip()
self.start_time = datetime().now()
# autolog("Obtained new IP address: {}".format(session.get(self.check_ip_url).text), logging.debug)
autolog("ALL THREADS WAKE UP NOW!", logging.info)
print("ALL THREADS WAKE UP NOW!")
self.event.set()
def run(self):
for thread in range(self.num_threads):
t = threading.Thread(target=self.worker, name='Thread-Worker-{}'.format(thread+1))
t.start()
threading.Thread(target=self.ip_updater, name='Thread-IPUpdater-1').start()
Rewriting everything so that event and start_time are global variables does not solve the issue.. For example:
class ThreadedNewsParser(object):
"""
This little guy parses the news with multiple threads and dynamically changes the ip of the sessions
"""
def __init__(self, update_ip_in, num_threads, date_start, date_end):
assert isinstance(num_threads, int)
assert num_threads > 0
assert any(isinstance(date_start, type_) for type_ in [datetime, date])
assert any(isinstance(date_end, type_) for type_ in [datetime, date])
self.update_ip_in = update_ip_in
self.check_ip_url = 'https://httpbin.org/ip'
autolog("STARTING WORK ON IP: {}".format(session.get(self.check_ip_url).text), logging.debug)
self.num_threads = num_threads
self.date_start = date_start
self.date_end = date_end
self.dates = [date for date in date_range(date_start, date_end)]
self.p = DailyCompanyNewsParser(2008, 1, 1) # the date here does not matter
def worker(self):
global event
while len(self.dates) > 0:
if event.is_set():
print("THREAD WERKING!")
pause = random.randint(1, 5) / 5
autolog('THREAD SLEEPING %.2f' % pause, logging.debug)
time.sleep(pause)
if len(self.dates) > 0:
date = self.dates.pop(0)
self.p.get_news_for_all_stocks(verbose=True, date_=date)
else:
print("THREAD SLEEPING")
time.sleep(10) # so that the threads do not check if the event is set instantaneously
def ip_updater(self): # this is the blocker
global start_time
global event
while len(self.dates) > 0:
autolog("IP_UPDATER SLEEPING FOR: {}".format(self.update_ip_in / 4), logging.debug)
time.sleep(self.update_ip_in / 4) # do not check the condition every instance
if datetime.now() > start_time + timedelta(seconds=self.update_ip_in):
print("ALL THREADS SLEEP NOW!")
autolog("ALL THREADS SLEEP NOW!", logging.info)
event.clear() # Make all other threads sleep so that we can update the IP
time.sleep(10)
get_new_ip()
start_time = datetime().now()
# autolog("Obtained new IP address: {}".format(session.get(self.check_ip_url).text), logging.debug)
autolog("ALL THREADS WAKE UP NOW!", logging.info)
print("ALL THREADS WAKE UP NOW!")
event.set()
def run(self):
for thread in range(self.num_threads):
t = threading.Thread(target=self.worker, name='Thread-Worker-{}'.format(thread+1))
t.start()
threading.Thread(target=self.ip_updater, name='Thread-IPUpdater-1').start()