I'm in doubt, so please help me.
Let's say I have a class like this:
class class_1():
def __init__(self):
self.a = 0
self.b = False
def run(self):
while True:
self.worker()
if self.a > 10:
self.b = True
def worker(self):
self.a = self.a + 1
time.sleep(1)
def get_a(self):
return self.a
def get_b(self):
return self.b
I would like to start instance of a class in a thread (and at some point have several of instances), and pull values from instance(s) to main thread or to separate thread.
something like this:
def run_instance_1():
global one = class_1
one.run()
def pull_1():
global one
w = one.get_a()
z = one.get_b()
print ('one.a = ' + str(w))
print ('one.b = ' + str(z))
if __name__ == '__main__':
t1 = threading.Thread(name='process 1', target=run_instance_1)
t2 = threading.Thread(name='process 2', target=pull_1)
t1.start()
t2.start()
My question is: if I instantiate class in global variable like above, will it really run in a separate thread? Basic concept for this is to obtain data from "another" thread. Idea for instantiating class in a global variable is to be acessible from any thred. Am I on to something or totaly off?
You can share the same object between threads, but instead of using global variables, I'd much rather pass the instance to both threads:
def run_instance_1(one):
one.run()
def pull_1(one):
w = one.get_a()
z = one.get_b()
print ('one.a = ' + str(w))
print ('one.b = ' + str(z))
if __name__ == '__main__':
one = class_1()
t1 = threading.Thread(name='process 1', target=run_instance_1, args=(one,))
# alternatively: t1 = threading.Thread(target=one.run)
t2 = threading.Thread(name='process 2', target=pull_1, args=(one,))
t1.start()
t2.start()
That way you make sure that the instance is created before both threads start and make the functions reusable. Another potential problem is that the class attributes are modified in the thread, and those changes are not atomic. You're highly unlikely to get this in this particular case, but consider using locks for shared access to mutable data.
Related
My code is listening for file changes in a folder, in class A. When a change occurs, then I trigger a function of class B, which is a field in class A.
class A:
def __init__(self, b):
...
self.handler = b
def run(self):
# listen for changes in a folder using watchdog.observers.Observer
self.observer.schedule(self.handler, self.input_directory, recursive=True)
self.observer.start()
try:
while not self.stopped:
time.sleep(self.scanning_frequency)
except:
self.observer.stop()
self.observer.join()
class B(FileSystemEventHandler):
...
def on_any_event(self, event):
# a change occurred and handled here.
Now what I want to test is that when a file is copied to this folder, then on_any_event should be triggered. This is how I tried to do that:
def test_file_watcher(self):
# arrange
b = B()
a = A(b)
a.handler.on_any_event = MagicMock()
shutil.copy(# copy file to the watched folder)
p1 = Process(target=a.run)
p1.start()
time.sleep(folder scanning frequency + 1 second)
a.stop() # stops watching the folder
assert a.handler.on_any_event.called
p1.join()
However this assertion turns out to be false all the time. Where am I doing wrong exactly? Also would it be possible to achieve this by also mocking B completely?
Edit: I think the reason could be that I am using a different process, therefore a.handler.on_any_event.called is always false. But I couldn't figure out how to solve this.
Workaround for your test in a multiprocessing context
I agree with you that multiprocessing causes the failure of the test. I have found a workaround that can help you to do the test in a strange way, but that you can adapt for your needs.
The workaround is based on the use of Sharing Global Variables in Multiprocessing by multiprocessing.Value (see the documentation).
To do this I have defined 2 sharing variables and 2 functions as below:
from multiprocessing import Value
shared_value = Value('i', 0)
stopped = Value('i',0)
# this function is used to substitute method on_any_event() of class B
def on_any_event_spy(event):
shared_value.value += 1
# this is the target for Process p1
def f(a: A):
# substitution of on_any_event method with on_any_event_spy()
a.handler.on_any_event = on_any_event_spy
a.run()
Furthermore I have to modify the run() and stop() methods of class A.
New method stop() of class A:
def stop(self):
stopped.value = 1
Method run() of class A (change only the condition of the while):
def run(self):
# listen for changes in a folder using watchdog.observers.Observer
self.observer.schedule(self.handler, self.input_directory, recursive=True)
self.observer.start()
try:
#while not self.stopped: # <------ comment this instruction
while stopped.value == 0: # <----- add this instruction
time.sleep(self.scanning_frequency)
except:
self.observer.stop()
self.observer.join()
The test method becomes:
class MyTestCase(unittest.TestCase):
def test_file_watcher(self):
# arrange
b = B()
a = A(b)
shutil.copy( # copy file to the watched folder)
# I have changed yor target value and add args
p1 = Process(target=f, args=(a, ))
p1.start()
time.sleep(a.scanning_frequency + 1)
a.stop() # stops watching the folder
# the shared_value value MUST BE > 0
self.assertGreater(shared_value.value, 0)
p1.join()
if __name__ == '__main__':
unittest.main()
How to mock B completely
The previous paragraph of this answer tells that the real problem of this test is the multiprocessing, but if you want mock B completely as you ask in your question, try to change your test_file_watcher() as following:
def test_file_watcher(self):
# arrange
#b = B() # <---------------- ------------- comment this instruction
b = Mock(wraps=B()) # <--------------------- add this instruction
a = A(b)
#a.handler.on_any_event = MagicMock() # <--- comment this instruction
shutil.copy(# copy file to the watched folder)
p1 = Process(target=a.run)
p1.start()
time.sleep(folder scanning frequency + 1 second)
a.stop() # stops watching the folder
#assert a.handler.on_any_event.called# <---- comment this instruction
assert b.on_any_event.called <---- add this instruction
p1.join()
I hope that with the instruction:
b = Mock(wraps=B())
you will wrap B completely as you ask in your question and this can be useful for future more traditionally tests.
I'm trying to use returned data from one function into multiple other functions. But I don't want the first function to run each time; which is happening in my case.
#Function lab
def func_a():
print('running function a')
data = 'test'
return data
def func_b():
print(func_a())
def func_c():
print(func_a())
def func_d():
print(func_a())
if __name__ == '__main__':
func_a()
func_b()
func_c()
func_d()
Each time that whole function_a runs. But I just want the returned data from "func_a" in other functions.
IIUC, you could alleviate this with a simple class.
I hold the state of the class which runs func_a in a variable called output. I can then reference this output variable once the class has finished running as much as I like in all other functions without having to re-run func_a.
Hope this helps!
class FunctionA:
def __init__(self):
self.output = None
def run_function(self):
print('running function a')
data = 'test'
self.output = data
def func_b():
print(func_a.output)
def func_c():
print(func_a.output)
def func_d():
print(func_a.output)
if __name__ == '__main__':
func_a = FunctionA()
func_a.run_function()
func_b()
func_c()
func_d()
>>> running function a
>>> test
>>> test
>>> test
Your func_a does two things. To make this clear, let's call it, print_and_return_data.
There are several ways to to break apart the two things print_and_return_data does. One way is to split up the two behaviors into smaller sub-methods:
def print_and_return_data():
print('running function a') # keeping the old print behavior
data = 'test'
return data
into:
def print_run():
print('running function a') # keeping the old print behavior
def return_data():
return 'test'
def print_and_return_data():
print_run()
return return_data()
So that other functions only use what they need:
def func_b():
print(return_data())
Another way is to change print_and_return_data to behave differently the first time it's called from the following times it's called (I don't recommend this because functions changing based on how many times it's been called can be confusing):
context = {'has_printed_before': False}
def print_and_return_data():
if not context['has_printed_before']:
print('running function a')
context['has_printed_before'] = True
data = 'test'
return data
def func_b():
print(print_and_return_data())
if __name__ == '__main__':
func_a() # prints
func_b() # won't print
One way to avoid "functions behaving differently when they're called" is to pass the variation (the "context") in as an argument:
def return_data(also_print=False):
if also_print:
print('running function a')
data = 'test'
return data
def func_b():
print(return_data())
if __name__ == '__main__':
func_a(also_print=True) # prints
func_b() # won't print
I have an annoying problem in Python concerning the heritage of class.
The code is the following:
import time
from threading import Thread
class Main:
def __init__(self):
self.var = 0
self.fils_on()
def fils_on(self):
self.temp = Fils()
self.temp.start()
def fils_off(self):
self.temp.stop()
del self.temp
class Fils(Thread, Main):
def __init__(self):
Main.__init__(self)
Thread.__init__(self)
self.encore = True
def run(self):
i = 0
while self.encore:
chaine = str(i)
print chaine
print "var: ", self.var
i += 1
time.sleep(1)
def stop(self):
self.encore = False
if __name__ == "__main__":
main = Main()
My problem is when I call "Main", I don't manage to print the self.var variable from Fils class.
Someone know why?
And how can I fix it?
I don't understand what you are trying to achieve but it seems you are trying to experiment with threads, The above implementation exceeds recursion depth because you have a cyclic dependency, __init__ from Main class depends on self.fils_on() and it construct Fils() which trigger Fils.__init__() which in turn calls Main.__init__(self) and again the whole process continues.
Here I tried to modify your code, in which I removed the inheritance between Fils and Mains and moved var inside Fils.
import time
from threading import Thread
class Main:
def __init__(self):
self.fils_on()
def fils_on(self):
self.temp = Fils()
self.temp.start()
def fils_off(self):
self.temp.stop()
del self.temp
class Fils(Thread):
def __init__(self):
Thread.__init__(self)
self.encore = True
self.var = 0
def run(self):
i = 0
while self.encore:
chaine = str(i)
print chaine
print "var: ", self.var
i += 1
time.sleep(1)
def stop(self):
self.encore = False
if __name__ == "__main__":
main = Main()
I'm trying to create my own threading class in Python2.7. I want it to be able to stop that thread with my own class function. Currently I have something like this:
class loop(threading.Thread):
def __init__(self, myvar):
super(loop, self).__init__()
self.terminate = False
self.myvar = myvar
def run(self):
while not self.terminate:
do.smthng.useful(self.myvar)
def change(self, newvar):
self.myvar = newvar #Doesnt work, in run() my old var is still being used
def stoploop(self):
self.terminate = True #Also not working
l = loop(1)
l.start()
time.sleep(1)
l.change(2) #thread still using "1"
time.sleep(1)
l.stoploop() #doesnt stop
I've read some posts here about this, but it wasnt what I needed.
Any help would be appreciated.
Thank you.
EDIT:
As some of the commenters already stated, this part of code looks like to be really working! Problem is in another place of my project. I've found it, but can't solve it. Maybe some of you could help.
So, my project uses Apache Thrift library and the server is in python.
Server.py:
loo = loop(0)
handler = ServHandler(loo)
processor = serv.Processor(handler)
transport = TSocket.TServerSocket('0.0.0.0', port=9090)
tfactory = TTransport.TBufferedTransportFactory()
pfactory = TBinaryProtocol.TBinaryProtocolFactory()
server = TProcessPoolServer.TProcessPoolServer(processor, transport, tfactory, pfactory)
print 'Starting the server...'
server.serve()
ServHandler.py:
class ServHandler:
def __init__(self, loo):
self.loo = loo
def terminate(self): #Function that can be called remotely
self.loo.stoploop() #Doesn't work
In above case thread isn't terminated and I don't why. There's no error, object exists, but it sets self.terminate value somewhere else. The object id seems to be the same as well as memory address, but it just looks like object is different although loop init function is called only once...
Below is the example, when the loop is terminated successfully.
ServHandler.py:
class ServHandler:
def __init__(self, loo):
self.loo = None
def terminate(self): #Function that can be called remotely
self.loo.stoploop() #Does work!!!!!!
def create(self):
self.loo = loop(0) #Function that can be called remotely
When I create loop object remotely, I can terminate it remotely. But it doesn't fit me. There should be a thread created before thrift server is served and multiple users have to be able to change vars/terminate/etc of that thread. How can I achieve this?
Thank you!
Not a answer per sae, but a useful debug code for the OP
from time import sleep
from threading import Thread
class loop(Thread):
def __init__(self, myvar):
Thread.__init__(self)
self.terminate = False
self.myvar = myvar
def run(self):
while self.terminate is False:
print('Run says myvar is:',self.myvar)
sleep(0.5)
def change(self, newvar):
self.myvar = newvar
def stoploop(self):
self.terminate = True
l = loop(1)
l.start()
sleep(1)
l.change(2)
sleep(1)
l.stoploop()
print('Final product:',l.myvar)
sleep(2)
print('Is the thread alive:',l.isAlive())
Tried your code with some debugging prints, and it's working?
Following code produced:
[torxed#archie ~]$ python test.py
Run says myvar is: 1
Run says myvar is: 1
Run says myvar is: 2 <-- Proves that change() does change `myvar`
Run says myvar is: 2
Final product: 2 <-- Also the global scope knows about the change
Is the thread alive: False <-- And the thread got terminated as intended
However, these are not bulletproof ideas when fetching data or dealing with thread-returns for a number of reasons (even tho i use this method myself from time to time), you should consider using thread.join which should be used in combination with l.toplooop() like so:
l = loop(1)
l.start()
l.change(2)
l.stoploop()
ret = l.join()
Also when updating data you should aquire locks on your data so collisions don't occur, have a look at semaphore objects.
Is it what you need?
import threading
import time
class Worker(threading.Thread):
def __init__(self):
threading.Thread.__init__(self)
self.state = threading.Condition()
self.variable = 10
self.paused = False
def run(self):
while True:
with self.state:
if self.paused:
self.state.wait()
self.do_stuff()
def do_stuff(self):
time.sleep(.1)
print self.variable
def resume(self):
with self.state:
self.paused = False
self.state.notify()
def pause(self):
with self.state:
self.paused = True
loop = Worker()
loop.start()
time.sleep(1)
loop.pause()
loop.variable = 11
print 'CHANGED!'
loop.resume()
time.sleep(1)
I'm working on a project in Python using the "thread" module.
How can I make a global variable (in my case I need it to be True or False) that all the threads in my project (about 4-6) can access?
We can define the variable outside the thread classes and declare it global inside the methods of the classes.
Please see below trivial example which prints AB alternatively. Two variables flag and val are shared between two threads Thread_A and Thread_B. Thread_A prints val=20 and then sets val to 30. Thread_B prints val=30, since val is modified in Thread_A. Thread_B then sets val to 20 which is again used in Thread_A. This demonstrates that variable val is shared between two threads. Similarly variable flag is also shared between two threads.
import threading
import time
c = threading.Condition()
flag = 0 #shared between Thread_A and Thread_B
val = 20
class Thread_A(threading.Thread):
def __init__(self, name):
threading.Thread.__init__(self)
self.name = name
def run(self):
global flag
global val #made global here
while True:
c.acquire()
if flag == 0:
print "A: val=" + str(val)
time.sleep(0.1)
flag = 1
val = 30
c.notify_all()
else:
c.wait()
c.release()
class Thread_B(threading.Thread):
def __init__(self, name):
threading.Thread.__init__(self)
self.name = name
def run(self):
global flag
global val #made global here
while True:
c.acquire()
if flag == 1:
print "B: val=" + str(val)
time.sleep(0.5)
flag = 0
val = 20
c.notify_all()
else:
c.wait()
c.release()
a = Thread_A("myThread_name_A")
b = Thread_B("myThread_name_B")
b.start()
a.start()
a.join()
b.join()
Output looks like
A: val=20
B: val=30
A: val=20
B: val=30
A: val=20
B: val=30
A: val=20
B: val=30
Each thread prints the value which was modified in another thread.
With no clue as to what you are really trying to do, either go with nio's approach and use locks, or consider condition variables:
From the docs
# Consume one item
cv.acquire()
while not an_item_is_available():
cv.wait()
get_an_available_item()
cv.release()
# Produce one item
cv.acquire()
make_an_item_available()
cv.notify()
cv.release()
You can use this to let one thread tell another a condition has been met, without having to think about the locks explicitly. This example uses cv to signify that an item is available.
How about using a threading.Event object per this description?
For example in the script below, worker1 and worker2 share an Event, and when worker2 changes its value this is seen by worker1:
import time
from threading import Thread, Event
shared_bool = Event()
def worker1(shared_bool):
while True:
if shared_bool.is_set():
print("value is True, quitting")
return
else:
print("value is False")
time.sleep(1)
def worker2(shared_bool):
time.sleep(2.5)
shared_bool.set()
t1 = Thread(target=worker1, args=(shared_bool, ))
t2 = Thread(target=worker2, args=(shared_bool, ))
t1.start()
t2.start()
t1.join()
t2.join()
Prints out:
value is False
value is False
value is False
value is True, quitting