We Keep Coding

worker causing wierd problems and i dont know why - python

So basically I created a script called worker which contain 2 classes:
Worker
CreateThread
this script is supposed to handle unexpected exception occurs, while also provide threading solution to running heavy function to prevent GUI freezing.
My problem is that I get unexpected behavior sometimes, for example one time I can run function with the worker class and I will get a good response and everything works fine, and second time things go wrong and I get odd errors and also GUI crashes.
I tried to change the order I manage the thread handling, and I tried addepting the code to the errors but there is a hidden problem that keeps hunting me...
that is my worker code:
# Create a worker class #
class Worker(QObject):
my_started = pyqtSignal()
my_finished = pyqtSignal()
my_error = pyqtSignal(str, str)
my_success = pyqtSignal()
def __init__(self):
super(Worker, self).__init__()
def run(self, func):
try:
self.my_started.emit()
time.sleep(0.1)
x = func()
if x is None:
self.my_success.emit()
self.my_finished.emit()
return
if x[0] == -1:
function_error = x[1]
trace_back = x[2]
print(f'\nFunction error: {function_error}')
print(trace_back)
self.my_error.emit(str(function_error), trace_back,
str(func.__name__))
self.my_finished.emit()
return
except Exception as e:
self.trace_back = str(traceback.format_exc())
print(f'\nFunction error: {e}')
print(self.trace_back)
self.my_error.emit(str(e), self.trace_back)
self.my_finished.emit()
# Creating thread manger class #
class CreateThread(Worker, QObject):
def __init__(self, atp_name):
super(CreateThread, self).__init__()
self.stop_flag = False
self.atp_name = atp_name
self.thread = QThread()
self.worker = Worker()
def start_thread(self, func, **kwargs):
self.kwargs = kwargs
self.func_name = func.__name__
if self.stop_flag:
return
if self.func_name == '<lambda>':
self.func_name = self.kwargs['func_name']
# Thread and Worker init #
self.thread = QThread()
self.worker = Worker()
act = partial(self.worker.run, func)
self.worker.moveToThread(self.thread)
self.thread.started.connect(act)
# Signal to Function connection #
self.worker.my_started.connect(self.started_signal_connection)
self.worker.my_finished.connect(self.finished_signal_connection)
self.worker.my_error.connect(self.error_signal_connection)
self.worker.my_success.connect(self.success_signal_connection)
# Handling the worker and the thread deletion #
self.worker.my_finished.connect(self.thread.quit)
self.worker.my_finished.connect(self.worker.deleteLater)
self.thread.finished.connect(self.thread.deleteLater)
self.thread.start()
# Connect signals from worker #
def started_signal_connection(self):
self.stop_flag = True
print(f"\nFunction {self.func_name} started\n")
def finished_signal_connection(self):
self.stop_flag = False
print("\nWorker finished\n")
def error_signal_connection(self, error_name, trace_back):
self.today = date.today()
self.now = datetime.now()
self.current_date = str(
self.now.strftime("%d/%m/%Y %H:%M:%S")).replace(' ',
'_').replace(
'/', '_').replace(':', '_')
self.header_name = f'{self.atp_name}_{self.func_name}_Date_{self.current_date}'
self.target_folder = 'target_folder'
if os.path.isfile(self.header_name) == False:
file_name = f'{self.target_folder}{self.header_name}.txt'
try:
with open(file_name, 'w', encoding='utf-8') as f:
f.write(f'''ATP: {self.atp_name}\nFunction name: {self.func_name}\nError: {error_name}\nDate: {self.current_date}\n\n{trace_back}''')
except Exception as e:
print(e)
def success_signal_connection(self):
self.stop_flag = False
print("\nFunction success")
Another thing I noticed is that every time I use the worker with function that use Tk - askopenfilename()
I get troubles, here is a traceback example:
line 57, in run
**x = func()**
line 64, in open_main_terminal
**self.path1, self.file_name = upload_item()**
line 62, in upload_item
**filename = askopenfilename()**
line 375, in askopenfilename
**return Open(**options).show()**
line 39, in show
**w = Frame(self.master)**
line 2741, in __init__
**Widget.__init__(self, master, 'frame', cnf, {}, extra)**
line 2296, in __init__ (widgetName, self._w) + extra + self._options(cnf))
**RuntimeError: main thread is not in main loop**

Assign Locust task sets programmatically

I am trying to do something where I assign the locust tasks dynamically based on the host value passed through the UI. In this example, if host is passed in as "hello", the test should run the hello task else it should run the world task.
from locust import HttpUser, TaskSet, task, events
class RandomTask1(TaskSet):
#task(1)
def soemthing(self):
print("Hello!")
class RandomTask2(TaskSet):
#task(1)
def soemthing(self):
print("World!")
class LoadTestUser(HttpUser):
def on_start(self):
host_config = self.host
if host_config == "hello":
tasks = {RandomTask1:1}
else:
tasks = {RandomTask2:1}
The example below does not work and I get the following error
Exception: No tasks defined on LoadTestUser. use the #task decorator or set the tasks property of the User (or mark it as abstract = True if you only intend to subclass it)
Any idea how I can achieve something like this? I have simplified this for the example, but for all intents and purposes, let's assume that the locust instance is already running and cannot be stopped or restarted and the tasks need to be assigned dynamically.
Edit:
Tried doing this:
class LoadTestUser(HttpUser):
def on_start(self):
if self.host == "hello":
self.tasks = {HelloTask: 1}
else:
self.tasks = {WorldTask: 1}
#task
def nothing(self):
pass
class HelloTask(TaskSet):
#task
def something(self):
print("Hello")
class WorldTask(TaskSet):
#task
def something(self):
print("World")
Now I see the following error:
Traceback (most recent call last):
File "/Users/user/project/venv/lib/python3.8/site-packages/locust/user/task.py", line 285, in run
self.schedule_task(self.get_next_task())
File "/Users/user/project/venv/lib/python3.8/site-packages/locust/user/task.py", line 420, in get_next_task
return random.choice(self.user.tasks)
File "/Users/user/opt/anaconda3/lib/python3.8/random.py", line 291, in choice
return seq[i]
KeyError: 0

Create a single task and put the logic in that task for what you want to have it run.
class LoadTestUser(HttpUser):
def something1(self):
print("Hello!")
def something2(self):
print("World!")
#task
def task_logic(self):
if self.host == "hello":
self.something1()
else:
self.something2()
However, you can just address the error you're getting directly. You need to have a task defined in the class even if you intend to overwrite or change the tasks with your TaskSets. There's an example in the documentation but just add a task with pass so it doesn't do anything, then your overrides should work.
class LoadTestUser(HttpUser):
def on_start(self):
host_config = self.host
if host_config == "hello":
self.tasks = {RandomTask1:1}
else:
self.tasks = {RandomTask2:1}
#task
def nothing(self):
pass
EDIT:
This should work but looks like there could be a bug in the current version of Locust where it only accepts a dictionary for tasks when Locust first starts and then only accepts a list afterward. Until it's fixed, the example in the other answer works.

import sys
from locust import HttpUser, TaskSet, task, events
class LoadTestUser(HttpUser):
def locust_class(self, name):
module = sys.modules[__name__]
return getattr(module, f"{name.capitalize()}Task")
def get_weighted_tasks(self, task_list):
new_tasks = []
for item in task_list:
if "locust_task_weight" in dir(item):
for i in range(item.locust_task_weight):
new_tasks.append(item)
return new_tasks
def get_locust_tasks(self, cls):
tasks = []
for maybe_task in cls.__dict__.values():
if hasattr(maybe_task, "locust_task_weight"):
tasks.append(maybe_task)
return tasks
def on_start(self):
task_cls = self.locust_class(self.host)
task_list = self.get_locust_tasks(task_cls)
self.tasks = self.get_weighted_tasks(task_list)
#task(1)
def nothing(self):
pass
class HelloTask(TaskSet):
#task(1)
def something(self):
print("Hello")
#task(100)
def something_else(self):
print("hello")
class WorldTask(TaskSet):
#task(1)
def something(self):
print("World")
#task(10)
def something_else(self):
print("world")

Python - "can't pickle thread.lock" error when creating a thread under a multiprocess in Windows

I'm getting stuck on what I think is a basic multiprocess and threading issue. I've got a multiprocess set up, and within this a thread. However, when I set up the thread class within the init function, I get the following error:
"TypeError: can't pickle thread.lock objects".
However, this does not happen if the thread is set up outside of the init function. Does anyone know why this is happening? Note I'm using Windows.
Some code is below to illustrate the issue. As typed below, it runs fine. However if print_hello() is called from within the DoStuff init def, then the error occurs, if it's called within the multi-process run() def then it's fine.
Can anyone point me in the right direction so it runs fine when called from init? thanks!
import multiprocessing
import threading
import time
class MyProcess(multiprocessing.Process):
def __init__(self, **kwargs):
super(MyProcess, self).__init__(**kwargs)
self.dostuff = DoStuff()
def run(self):
print("starting DoStuff")
# This works fine if the line below is uncommented and __init__ self.print_hello() is commented...
self.dostuff.print_hello()
class DoStuff(object):
def __init__(self, **kwargs):
super(DoStuff, self).__init__(**kwargs)
# If the following is uncommented, the error occurs...
# Note it also occurs if the lines in start_thead are pasted here...
# self.print_hello()
def print_hello(self):
print "hello"
self.start_thread()
def start_thread(self):
self.my_thread_instance = MyThread()
self.my_thread_instance.start()
time.sleep(0.1)
class MyThread(threading.Thread):
def __init__(self):
super(MyThread, self).__init__()
def run(self):
print("Starting MyThread")
if __name__ == '__main__':
mp_target = MyProcess() # Also pass the pipe to transfer data
# mp_target.daemon = True
mp_target.start()
time.sleep(0.1)

It looks like there is no simple answer, and it appears to be a restriction of Windows (Win 7, python 3.6 in my case); on Windows it looks like you need to start the process before you can start the worker thread inside the owned object.
There appears to be no such restriction on Unix (CentOS 7, python 2.7.5).
As an experiment I modified your code as follows; this version checks the OS and starts either the process first, or the thread first:
import multiprocessing
import threading
import time
import os
class MyProcess(multiprocessing.Process):
def __init__(self, **kwargs):
super(MyProcess, self).__init__(**kwargs)
self.dostuff = DoStuff(self)
def run(self):
print("MyProcess.run()")
print("MyProcess.ident = " + repr(self.ident))
if os.name == 'nt':
self.dostuff.start_thread()
class DoStuff(object):
def __init__(self, owner, **kwargs):
super(DoStuff, self).__init__(**kwargs)
self.owner = owner
if os.name != 'nt':
self.start_thread()
def start_thread(self):
print("DoStuff.start_thread()")
self.my_thread_instance = MyThread(self)
self.my_thread_instance.start()
time.sleep(0.1)
class MyThread(threading.Thread):
def __init__(self, owner):
super(MyThread, self).__init__()
self.owner = owner
def run(self):
print("MyThread.run()")
print("MyThread.ident = " + repr(self.ident))
print("MyThread.owner.owner.ident = " + repr(self.owner.owner.ident))
if __name__ == '__main__':
mp_target = MyProcess() # Also pass the pipe to transfer data
mp_target.daemon = True
mp_target.start()
time.sleep(0.1)
... and got the following on Windows, where the process starts first:
MyProcess.run()
MyProcess.ident = 14700
DoStuff.start_thread()
MyThread.run()
MyThread.ident = 14220
MyThread.owner.owner.ident = 14700
... and the following on Linux, where the thread is started first:
DoStuff.start_thread()
MyThread.run()
MyThread.ident = 140316342347520
MyThread.owner.owner.ident = None
MyProcess.run()
MyProcess.ident = 4358
If it were my code I'd be tempted to always start the process first, then create the thread within that process; the following version works fine for me across both platforms:
import multiprocessing
import threading
import time
class MyProcess(multiprocessing.Process):
def __init__(self, **kwargs):
super(MyProcess, self).__init__(**kwargs)
self.dostuff = DoStuff()
def run(self):
print("MyProcess.run()")
self.dostuff.start_thread()
class DoStuff(object):
def __init__(self, **kwargs):
super(DoStuff, self).__init__(**kwargs)
def start_thread(self):
self.my_thread_instance = MyThread()
self.my_thread_instance.start()
time.sleep(0.1)
class MyThread(threading.Thread):
def __init__(self):
super(MyThread, self).__init__()
def run(self):
print("MyThread.run()")
if __name__ == '__main__':
mp_target = MyProcess() # Also pass the pipe to transfer data
mp_target.daemon = True
mp_target.start()
time.sleep(0.1)

add_callback with thread pool, but Exception about "Cannot write() after finish()."

I'm using thread pool while using Tornado to do some work. This is the code:
common/thread_pool.py
import tornado.ioloop
class Worker(threading.Thread):
def __init__(self, queue):
threading.Thread.__init__(self)
self._queue = queue
def run(self):
logging.info('Worker start')
while True:
content = self._queue.get()
if isinstance(content, str) and content == 'quit':
break
#content: (func, args, on_complete)
func = content[0]
args = content[1]
on_complete = content[2]
resp = func(args)
tornado.ioloop.IOLoop.instance().add_callback(lambda: on_complete(resp))
#i dont know is correct to call this
#self._queue.task_done()
logging.info('Worker stop')
class WorkerPool(object):
_workers = []
def __init__(self, num):
self._queue = Queue.Queue()
self._size = num
def start(self):
logging.info('WorkerPool start %d' % self._size)
for _ in range(self._size):
worker = Worker(self._queue)
worker.start()
self._workers.append(worker)
def stop(self):
for worker in self._workers:
self._queue.put('quit')
for worker in self._workers:
worker.join()
logging.info('WorkerPool stopd')
def append(self, content):
self._queue.put(content)
gateway.py
import tornado.ioloop
import tornado.web
from common import thread_pool
workers = None
class MainServerHandler(tornado.web.RequestHandler):
#tornado.web.asynchronous
def get(self):
start_time = time.time()
method = 'get'
content = (self.handle, (method, self.request, start_time), self.on_complete)
workers.append(content)
#tornado.web.asynchronous
def post(self):
start_time = time.time()
method = 'post'
content = (self.handle, (method, self.request, start_time), self.on_complete)
workers.append(content)
def handle(self, args):
method, request, start_time = args
#for test, just return
return 'test test'
def on_complete(self, res):
logging.debug('on_complete')
self.write(res)
self.finish()
return
def main(argv):
global workers
workers = thread_pool.WorkerPool(conf_mgr.thread_num)
workers.start()
application = tornado.web.Application([(r"/", MainServerHandler)])
application.listen(8888)
tornado.ioloop.IOLoop.instance().start()
if __name__ == "__main__":
main(sys.argv[1:])
When I make many concurrent requests, I get this error:
ERROR: 2014-09-15 18:04:03: ioloop.py:435 * 140500107065056 Exception in callback <tornado.stack_context._StackContextWrapper object at 0x7fc8b4d6b9f0>
Traceback (most recent call last):
File "/home/work/nlp_arch/project/ps/se/nlp-arch/gateway/gateway/../third-party/tornado-2.4.1/tornado/ioloop.py", line 421, in _run_callback
callback()
File "/home/work/nlp_arch/project/ps/se/nlp-arch/gateway/gateway/../common/thread_pool.py", line 39, in <lambda>
tornado.ioloop.IOLoop.instance().add_callback(lambda: on_complete(resp))
File "/home/work/nlp_arch/project/ps/se/nlp-arch/gateway/gateway/gateway.py", line 92, in on_complete
self.write(res)
File "/home/work/nlp_arch/project/ps/se/nlp-arch/gateway/gateway/../third-party/tornado-2.4.1/tornado/web.py", line 489, in write
raise RuntimeError("Cannot write() after finish(). May be caused "
RuntimeError: Cannot write() after finish(). May be caused by using async operations without the #asynchronous decorator.
But I didn't call write after finish. I'm also using the #asynchronous decorator. At the same time, in the logs I see that write/finish is called by same thread.

The issue is with the way you're adding the callback to the I/O loop. Add it like this:
tornado.ioloop.IOLoop.instance().add_callback(on_complete, resp)
And the errors will go away.
You're seeing this strange behavior because when you use a lambda function, you're creating a closure in the local scope of the function, and the variables used in that closure get bound at the point the lambda is executed, not when its created. Consider this example:
funcs = []
def func(a):
print a
for i in range(5):
funcs.append(lambda: func(i))
for f in funcs:
f()
Output:
4
4
4
4
4
Because your worker method is running in a while loop, on_complete ends up getting redefined several times, which also changes the value of on_complete inside the lambda. That means if one worker thread sets on_complete for a handler A, but then gets another task and sets on_complete for handler B prior to the callback set for handler A running, both callbacks end up up running handler B's on_complete.
If you really wanted to use a lambda, you could also avoid this by binding on_complete in the local scope of the lambda:
tornado.ioloop.IOLoop.instance().add_callback(lambda on_complete=on_complete: on_complete(resp))
But just adding the function and its argument directly is much nicer.

Tornado: Thread does not start using #run_on_executor in coroutine

I have the following problem with this little tornado-test:
class SimpleIOLoopTests(tornado.testing.AsyncTestCase):
def setUp(self):
super(SimpleIOLoopTests, self).setUp()
def test_executor_future(self):
self.executor = ThreadPoolExecutor(2)
#run_on_executor
def wait_and_return_a_value():
time.sleep(2)
return 20
#coroutine
def async_compare(callback):
val = yield wait_and_return_a_value()
assert_that(val, equal_to(20))
callback()
async_compare(self.stop)
self.wait()
The point is the the test simply loops until a timeout occurs. Debugging the code it looks as if the executor-future is created as done() and, hence, no even started by the io_loop.
What am I doing wrong here? Help with this issue is really appreciated
btw: the same happens if I create a trivial future using the #return_future decorator like this one (for which the it is accidently true that is is already done)
#return_future
def get_value(callback):
callback(10)
thanks & regards
markus

Problem is that executors must "live" in a class where the io_loop and the executor is defined (this can be seen when you check the #run_on_executor decorator).
def test_executor_future(self):
class Executor():
def __init__(self, io_loop=None):
self.io_loop = io_loop or IOLoop.instance()
self.executor = ThreadPoolExecutor(2)
#tornado.concurrent.run_on_executor
def wait_and_return_a_value(self):
return 20
def destroy(self):
self.executor.shutdown(1)
#tornado.gen.coroutine
def async_compare(callback):
executor = Executor()
val = yield executor.wait_and_return_a_value()
assert_that(val, equal_to(20))
executor.destroy()
callback()
async_compare(self.stop)
self.wait()