I need to run as many threads of class Observer as there are elements in list dirlist.
When I run it python console it works all right.
class Observer(Thread):
def run(self):
naptime = random.randint(1,10)
print(self.name + ' starting, running for %ss.' % naptime)
time.sleep(naptime)
print(self.name + ' done')
observers = {}
for d in dirlist:
observers[d] = Observer()
observers[d].start()
But when I try to do it from a Master thread which is supposed to spawn the Observer threads I get errors.
class Master(Thread):
def __init__(self, dirlist):
self.dirlist = dirlist
def run(self):
observers = {}
for d in dirlist:
observers[d] = Observer()
observers[d].start()
while True:
time.sleep(3600)
master_thread = Master(dirlist)
master_thread.start()
The call to Master.start results in:
RuntimeError: thread.__init__() not called
This looks strange to me.
I am unable to understand whats the difference between both cases.
Can anybody figure out a solution to my problem ?
Somehow following doesn't produce an error, and I don't understand why.
class Master(Thread):
def set(self, dirlist):
self.dirlist = dirlist
def run(self):
observers = {}
for d in dirlist:
observers[d] = Observer()
observers[d].start()
while True:
time.sleep(3600)
master_thread = Master()
master_thread.set(dirlist)
master_thread.start()
>>> master_thread.start()
RuntimeError: thread.__init__() not called
Make sure to call Thread.__init__() in your Master.__init__:
class Master(Thread):
def __init__(self, dirlist):
super(Master, self).__init__()
self.dirlist = dirlist
Well i know it's late to answer but, what the hell, i am a newbie in python but this same thing was happening to me, so i went back to read the python tutorial and it has some differences with what we both we're trying, hope it helps.
instead of this
import threading
class Master(Thread):
def set(self, dirlist):
self.dirlist = dirlist
def run(self):
observers = {}
for d in dirlist:
...
class according to the python tutorial:
class Master(threading.Thread):
this line is missing:
threading.Thread.__init__(self)
so it will end up being:
import threading
class Master(threading.Thread):
def __init__(self, dirlist):
threading.Thread.__init__(self)
self.dirlist = dirlist
def run(self):
observers = {}
for d in dirlist:
...
and that should work, at least work for me.
I hope it was helpfull.
And your second try using the set method works, because you are not overriding the
__init__ method
from Thread therefore the original init method from the parent class is used it runs as it's supposed.
Error is clear, you should call thread.__init__():
def __init__(self, dirlist):
super(Master, self).__init__()
self.dirlist = dirlist
Related
I am trying to create a simple callback that can be registered to an object from another thread. The initial object that calls the callback is running on its own thread in this case.
This is best illustrated through the following example:
from pprint import pprint
import sys
import weakref
import threading
import time
class DummyController(object):
def __init__(self):
self.name = "fortytwo"
def callback(self):
print("I am number : " + self.name)
class SomeThread(threading.Thread):
def __init__(self, listener):
threading.Thread.__init__(self)
self.listener = listener
def run(self):
time.sleep(1)
dummy = DummyController()
self.listener.register_callback(dummy.callback)
time.sleep(5)
del dummy
class Listener(threading.Thread):
def __init__(self):
threading.Thread.__init__(self)
self.runner = weakref.WeakMethod(self.default_callback)
self.counter = 20
def default_callback(self):
print("Not implemented")
def register_callback(self, function):
self.runner = weakref.WeakMethod(function)
def run(self):
while self.counter:
try:
self.runner()()
except Exception as e:
pprint(e)
self.counter -= 1
time.sleep(1)
listen = Listener()
some = SomeThread(listen)
listen.start()
some.start()
Now the above code works just fine. But I am concerned about thread-safety here. Reading through weakref docs, it isn't very clear if weakref is really thread safe or not, except for the line:
Changed in version 3.2: Added support for thread.lock, threading.Lock, and code objects.
I might be simply not reading that right. Do I need to add locking, or is everything actually fine and pretty thread safe?
Many thanks
OK, I understand. This is not a problem about thread safe, but just a problem about weak reference.
There is an executable example:
from pprint import pprint
import sys
import weakref
import threading
import time
import gc
class SomeThread(threading.Thread):
def __init__(self, listener):
threading.Thread.__init__(self)
self.listener = listener
def run(self):
class test: # simplify this example.
def callback(self, count):
print(count)
time.sleep(1)
dummy = test()
self.listener.register_callback(dummy.callback)
time.sleep(5)
del dummy
gc.collect() # add this line to do garbage collecting.
class Listener(threading.Thread):
def __init__(self):
threading.Thread.__init__(self)
self.runner = weakref.WeakMethod(self.default_callback)
self.counter = 20
def default_callback(self):
print("Not implemented")
def register_callback(self, function):
self.runner = weakref.WeakMethod(function)
def run(self):
while self.counter:
try:
self.runner()(self.counter)
except Exception as e:
pprint(e)
self.counter -= 1
time.sleep(1)
listen = Listener()
some = SomeThread(listen)
listen.start()
some.start()
output:
TypeError('default_callback() takes 1 positional argument but 2 were given',)
TypeError('default_callback() takes 1 positional argument but 2 were given',)
18
17
16
15
TypeError("'NoneType' object is not callable",)
TypeError("'NoneType' object is not callable",)
TypeError("'NoneType' object is not callable",)
If you explicitly call gc.collect(), callback loses its last strong reference and then it becomes None. As you will never know when will gc collect garbage, there is a potential issue.
It is no matter you use thread or not, just a normal behave of weak reference.
BTW, be careful that exiting SomeThread.run will also implicitly del dummy, you can test it by removing del dummy and moving gc.collect() into try block.
I'm getting stuck on what I think is a basic multiprocess and threading issue. I've got a multiprocess set up, and within this a thread. However, when I set up the thread class within the init function, I get the following error:
"TypeError: can't pickle thread.lock objects".
However, this does not happen if the thread is set up outside of the init function. Does anyone know why this is happening? Note I'm using Windows.
Some code is below to illustrate the issue. As typed below, it runs fine. However if print_hello() is called from within the DoStuff init def, then the error occurs, if it's called within the multi-process run() def then it's fine.
Can anyone point me in the right direction so it runs fine when called from init? thanks!
import multiprocessing
import threading
import time
class MyProcess(multiprocessing.Process):
def __init__(self, **kwargs):
super(MyProcess, self).__init__(**kwargs)
self.dostuff = DoStuff()
def run(self):
print("starting DoStuff")
# This works fine if the line below is uncommented and __init__ self.print_hello() is commented...
self.dostuff.print_hello()
class DoStuff(object):
def __init__(self, **kwargs):
super(DoStuff, self).__init__(**kwargs)
# If the following is uncommented, the error occurs...
# Note it also occurs if the lines in start_thead are pasted here...
# self.print_hello()
def print_hello(self):
print "hello"
self.start_thread()
def start_thread(self):
self.my_thread_instance = MyThread()
self.my_thread_instance.start()
time.sleep(0.1)
class MyThread(threading.Thread):
def __init__(self):
super(MyThread, self).__init__()
def run(self):
print("Starting MyThread")
if __name__ == '__main__':
mp_target = MyProcess() # Also pass the pipe to transfer data
# mp_target.daemon = True
mp_target.start()
time.sleep(0.1)
It looks like there is no simple answer, and it appears to be a restriction of Windows (Win 7, python 3.6 in my case); on Windows it looks like you need to start the process before you can start the worker thread inside the owned object.
There appears to be no such restriction on Unix (CentOS 7, python 2.7.5).
As an experiment I modified your code as follows; this version checks the OS and starts either the process first, or the thread first:
import multiprocessing
import threading
import time
import os
class MyProcess(multiprocessing.Process):
def __init__(self, **kwargs):
super(MyProcess, self).__init__(**kwargs)
self.dostuff = DoStuff(self)
def run(self):
print("MyProcess.run()")
print("MyProcess.ident = " + repr(self.ident))
if os.name == 'nt':
self.dostuff.start_thread()
class DoStuff(object):
def __init__(self, owner, **kwargs):
super(DoStuff, self).__init__(**kwargs)
self.owner = owner
if os.name != 'nt':
self.start_thread()
def start_thread(self):
print("DoStuff.start_thread()")
self.my_thread_instance = MyThread(self)
self.my_thread_instance.start()
time.sleep(0.1)
class MyThread(threading.Thread):
def __init__(self, owner):
super(MyThread, self).__init__()
self.owner = owner
def run(self):
print("MyThread.run()")
print("MyThread.ident = " + repr(self.ident))
print("MyThread.owner.owner.ident = " + repr(self.owner.owner.ident))
if __name__ == '__main__':
mp_target = MyProcess() # Also pass the pipe to transfer data
mp_target.daemon = True
mp_target.start()
time.sleep(0.1)
... and got the following on Windows, where the process starts first:
MyProcess.run()
MyProcess.ident = 14700
DoStuff.start_thread()
MyThread.run()
MyThread.ident = 14220
MyThread.owner.owner.ident = 14700
... and the following on Linux, where the thread is started first:
DoStuff.start_thread()
MyThread.run()
MyThread.ident = 140316342347520
MyThread.owner.owner.ident = None
MyProcess.run()
MyProcess.ident = 4358
If it were my code I'd be tempted to always start the process first, then create the thread within that process; the following version works fine for me across both platforms:
import multiprocessing
import threading
import time
class MyProcess(multiprocessing.Process):
def __init__(self, **kwargs):
super(MyProcess, self).__init__(**kwargs)
self.dostuff = DoStuff()
def run(self):
print("MyProcess.run()")
self.dostuff.start_thread()
class DoStuff(object):
def __init__(self, **kwargs):
super(DoStuff, self).__init__(**kwargs)
def start_thread(self):
self.my_thread_instance = MyThread()
self.my_thread_instance.start()
time.sleep(0.1)
class MyThread(threading.Thread):
def __init__(self):
super(MyThread, self).__init__()
def run(self):
print("MyThread.run()")
if __name__ == '__main__':
mp_target = MyProcess() # Also pass the pipe to transfer data
mp_target.daemon = True
mp_target.start()
time.sleep(0.1)
I'm quite new to Python and hope the answer to this is obvious to most of you.
I'm creating a class in Python that represents a ScanFolder.
In the __init__ of that class, I start a watchdog.observers
This observer will fire a watchdog.events.PatternMatchingEventHandler whenever a file is changed under the watched directory.
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
import sys
import time
from watchdog.observers import Observer
import watchdog.events
path = sys.argv[1] if len(sys.argv) > 1 else '.'
class MyEventHandler(watchdog.events.PatternMatchingEventHandler):
def on_any_event(self, event):
print(event.src_path, event.event_type)
class ScanFolder:
'Class defining a scan folder'
def __init__(self, path):
self.path = path
self.documents = dict() # key = document label value = Document reference
self.event_handler = MyEventHandler(patterns=["*.jpg", "*.jpeg", "*.png", "*.bmp", "*.pdf"],
ignore_patterns=[],
ignore_directories=True)
self.observer = Observer()
self.observer.schedule(self.event_handler, self.path, recursive=False)
self.observer.start()
def stop(self):
self.observer.stop()
self.observer.join()
scan_folder = ScanFolder(path)
try:
while True:
time.sleep(1)
"""Here, I'll act on my scan_folder object that lists the discovered files"""
except KeyboardInterrupt:
log.warning("Ouch !!! Keyboard interrupt received.")
scan_folder.stop()
My problem is the following:
How can I have my scan_folder object modified by my scan_folder.event_handler() ?
Actually, I would like to populate the scan_folder.documents dictionary wherever a file is detected in the scan folder.
Thank you very much and sorry for my ignorance.
There are a lot of way to do it: but the simplest way is set a bound method of ScanFolder as on_any_event callback function of watchdog.events.PatternMatchingEventHandler. So your code become:
class ScanFolder:
'Class defining a scan folder'
def __init__(self, path):
self.path = path
self.documents = dict() # key = document label value = Document reference
self.event_handler = watchdog.events.PatternMatchingEventHandler(patterns=["*.jpg", "*.jpeg", "*.png", "*.bmp", "*.pdf"],
ignore_patterns=[],
ignore_directories=True)
self.event_handler.on_any_event = self.on_any_event
self.observer = Observer()
self.observer.schedule(self.event_handler, self.path, recursive=False)
self.observer.start()
def on_any_event(self, event):
print(event.src_path, event.event_type)
print("Complete ScanFolder() access")
def stop(self):
self.observer.stop()
self.observer.join()
Other way could be derive ScanFolder from watchdog.events.PatternMatchingEventHandler .... But injecting function is one of the power of python
I'm trying to create my own threading class in Python2.7. I want it to be able to stop that thread with my own class function. Currently I have something like this:
class loop(threading.Thread):
def __init__(self, myvar):
super(loop, self).__init__()
self.terminate = False
self.myvar = myvar
def run(self):
while not self.terminate:
do.smthng.useful(self.myvar)
def change(self, newvar):
self.myvar = newvar #Doesnt work, in run() my old var is still being used
def stoploop(self):
self.terminate = True #Also not working
l = loop(1)
l.start()
time.sleep(1)
l.change(2) #thread still using "1"
time.sleep(1)
l.stoploop() #doesnt stop
I've read some posts here about this, but it wasnt what I needed.
Any help would be appreciated.
Thank you.
EDIT:
As some of the commenters already stated, this part of code looks like to be really working! Problem is in another place of my project. I've found it, but can't solve it. Maybe some of you could help.
So, my project uses Apache Thrift library and the server is in python.
Server.py:
loo = loop(0)
handler = ServHandler(loo)
processor = serv.Processor(handler)
transport = TSocket.TServerSocket('0.0.0.0', port=9090)
tfactory = TTransport.TBufferedTransportFactory()
pfactory = TBinaryProtocol.TBinaryProtocolFactory()
server = TProcessPoolServer.TProcessPoolServer(processor, transport, tfactory, pfactory)
print 'Starting the server...'
server.serve()
ServHandler.py:
class ServHandler:
def __init__(self, loo):
self.loo = loo
def terminate(self): #Function that can be called remotely
self.loo.stoploop() #Doesn't work
In above case thread isn't terminated and I don't why. There's no error, object exists, but it sets self.terminate value somewhere else. The object id seems to be the same as well as memory address, but it just looks like object is different although loop init function is called only once...
Below is the example, when the loop is terminated successfully.
ServHandler.py:
class ServHandler:
def __init__(self, loo):
self.loo = None
def terminate(self): #Function that can be called remotely
self.loo.stoploop() #Does work!!!!!!
def create(self):
self.loo = loop(0) #Function that can be called remotely
When I create loop object remotely, I can terminate it remotely. But it doesn't fit me. There should be a thread created before thrift server is served and multiple users have to be able to change vars/terminate/etc of that thread. How can I achieve this?
Thank you!
Not a answer per sae, but a useful debug code for the OP
from time import sleep
from threading import Thread
class loop(Thread):
def __init__(self, myvar):
Thread.__init__(self)
self.terminate = False
self.myvar = myvar
def run(self):
while self.terminate is False:
print('Run says myvar is:',self.myvar)
sleep(0.5)
def change(self, newvar):
self.myvar = newvar
def stoploop(self):
self.terminate = True
l = loop(1)
l.start()
sleep(1)
l.change(2)
sleep(1)
l.stoploop()
print('Final product:',l.myvar)
sleep(2)
print('Is the thread alive:',l.isAlive())
Tried your code with some debugging prints, and it's working?
Following code produced:
[torxed#archie ~]$ python test.py
Run says myvar is: 1
Run says myvar is: 1
Run says myvar is: 2 <-- Proves that change() does change `myvar`
Run says myvar is: 2
Final product: 2 <-- Also the global scope knows about the change
Is the thread alive: False <-- And the thread got terminated as intended
However, these are not bulletproof ideas when fetching data or dealing with thread-returns for a number of reasons (even tho i use this method myself from time to time), you should consider using thread.join which should be used in combination with l.toplooop() like so:
l = loop(1)
l.start()
l.change(2)
l.stoploop()
ret = l.join()
Also when updating data you should aquire locks on your data so collisions don't occur, have a look at semaphore objects.
Is it what you need?
import threading
import time
class Worker(threading.Thread):
def __init__(self):
threading.Thread.__init__(self)
self.state = threading.Condition()
self.variable = 10
self.paused = False
def run(self):
while True:
with self.state:
if self.paused:
self.state.wait()
self.do_stuff()
def do_stuff(self):
time.sleep(.1)
print self.variable
def resume(self):
with self.state:
self.paused = False
self.state.notify()
def pause(self):
with self.state:
self.paused = True
loop = Worker()
loop.start()
time.sleep(1)
loop.pause()
loop.variable = 11
print 'CHANGED!'
loop.resume()
time.sleep(1)
I'm getting segfaults when calling a dbus method inside a thread. This is my scenario: I have a program Service1 that exposes a method test. A second program Service2 exposes a method expose. As this method does some serious numerical computation, I pass some params from expose to a running thread reader. This thread, in turn, calls the method test of Service1 when it ends its work. I'm geting segfaults in last dbus call.
The code:
# Service1.py
class Service1(Object):
def __init__(self, bus):
name = BusName('com.example.Service1', bus)
path = '/'
super(Service1, self).__init__(name, path)
#method(dbus_interface='com.example.Service1',
in_signature='s', out_signature='s')
def test(self, test):
print 'test being called'
return test
dbus_loop = DBusGMainLoop()
dsession = SessionBus(mainloop=dbus_loop)
loop = gobject.MainLoop()
gobject.threads_init()
im = Service1(dsession)
loop.run()
# Service2.py
dbus_loop = DBusGMainLoop()
dsession = SessionBus(mainloop=dbus_loop)
class Service2(Object):
def __init__(self, bus):
name = BusName('com.example.Service2', bus)
super(Service2, self).__init__(name, '/')
self.queue = Queue()
self.db = bus.get_object('com.example.Service1', '/')
self.dbi = dbus.Interface(self.db, dbus_interface='com.example.Service1')
#method(dbus_interface='com.example.Service2',
in_signature='', out_signature='')
def expose(self):
print 'calling expose'
self.queue.put(('params',))
def reader(self):
while True:
val = self.queue.get()
dd = self.dbi.test('test')
print dd
self.queue.task_done()
gobject.threads_init()
loop = gobject.MainLoop()
im = Service2(dsession)
reader = threading.Thread(target=im.reader)
reader.start()
loop.run()
To test, run Service1.py, Service2.py and later this snippet:
dbus_loop = DBusGMainLoop()
session = SessionBus(mainloop=dbus_loop)
proxy = session.get_object('com.example.Service2', '/')
test_i = dbus.Interface(proxy, dbus_interface='com.example.Service2')
test_i.expose()
Service2.py should crash after running this code a few times. But why?
gobject.threads_init() is not enough, you need to call dbus.mainloop.glib.threads_init() to make dbus-glib thread safe.
In Service1.py, Try calling gobject.threads_init() before assigning dbus_loop to DBusGMainLoop().