I have a python service which may be ended by exception - I'm ok with it. But the problem is that it spawned an child process which continues to run even it's parent got a fail.
import multiprocessing as mp
from time import sleep
import os
import atexit
import psutil
#atexit.register # Doesn't fired at exception
def goodbye():
current_process = psutil.Process()
children = current_process.children(recursive=True)
for child in children:
print('Kill child with pid {}'.format(child.pid))
try:
child.terminate()
except:
pass
print("You are now leaving the Python sector.")
def func(): # Child process
while True:
ppid = os.getppid()
print("Parent process id:", ppid)
if ppid == 1:
print("Parent process has terminated")
break
sleep(1)
t = mp.Process(target=func, args=())
t.start()
print(9 + "0") # Exception here
print("I'm ok")
And service continues to work (formally) until it got a kick from outside:
Parent process id: 29118
Traceback (most recent call last):
File "stestcp.py", line 32, in <module>
print(9 + "0") # Exception here
TypeError: unsupported operand type(s) for +: 'int' and 'str'
Parent process id: 29118
Parent process id: 29118
Parent process id: 29118
Parent process id: 29118
Parent process id: 29118
^CError in atexit._run_exitfuncs:
Traceback (most recent call last):
File "/usr/lib/python3.6/multiprocessing/popen_fork.py", line 28, in poll
Process Process-1:
pid, sts = os.waitpid(self.pid, flag)
KeyboardInterrupt
Traceback (most recent call last):
File "/usr/lib/python3.6/multiprocessing/process.py", line 258, in _bootstrap
self.run()
File "/usr/lib/python3.6/multiprocessing/process.py", line 93, in run
self._target(*self._args, **self._kwargs)
File "stestcp.py", line 27, in func
sleep(1)
KeyboardInterrupt
Kill child with pid 29119
You are now leaving the Python sector.
The question is - is there any way to call some global fallback function (like atexit) when program failed with exception?
Thanks to Andrej Kesely, solution found. Working example now looks like:
import multiprocessing as mp
from time import sleep
import sys
import os
import atexit
import psutil
class ExitHooks(object):
def __init__(self):
self.exit_code = None
self.exception = None
def hook(self):
self._orig_exit = sys.exit
sys.exit = self.exit
sys.excepthook = self.exc_handler
def exit(self, code=0):
self.exit_code = code
self._orig_exit(code)
def exc_handler(self, exc_type, exc, *args): # Called at exception
self.exception = exc
goodbye()
hooks = ExitHooks()
hooks.hook()
#atexit.register # Doesn't fired at exception
def goodbye():
if hooks.exit_code is not None:
print("death by sys.exit(%d)" % hooks.exit_code)
elif hooks.exception is not None:
print("death by exception: %s" % hooks.exception)
else:
print("natural death")
current_process = psutil.Process()
children = current_process.children(recursive=True)
for child in children:
print('Kill child with pid {}'.format(child.pid))
try:
child.terminate()
except:
pass
print("You are now leaving the Python sector.")
def func(): # Child process
while True:
ppid = os.getppid()
print("Parent process id:", ppid)
if ppid == 1:
print("Parent process has terminated")
break
sleep(1)
t = mp.Process(target=func, args=())
t.start()
sleep(2)
print(9 + "0") # Exception here
Related
I have a python function and want to run it as a separate process with multiprocessing package.
def run(ctx: Context):
print("hello world!")
return ctx
afterward running it as a separate process with the following script:
import multiprocessing
p = multiprocessing.Process(target=run, args=(ctx, ))
p.start()
p.join()
Now, I need to capture live stdout and stderr of the above process. Is there any way like as:
import subprocess
proc = subprocess.Popen(['python','fake_utility.py'],stdout=subprocess.PIPE)
while True:
line = proc.stdout.readline()
if not line:
break
But I need to pass the function not running a command with Popen. Do you know how can I read stdout when I run my function in a separate process?
My approach would be to create a custom context manager that can temporarily replace sys.stdout and sys.stderr with io.String() instances to capture the output and return this. For this you need to make the target of your Process a new function that can setup the context manager and return the results, for which a multiprocessing.Queue is used (this, by the way, would be needed anyway if you expect run to return its result back to the main process):
from multiprocessing import Process, Queue
from io import StringIO
import sys
class CaptureOutput:
def __enter__(self):
self._stdout_output = ''
self._stderr_output = ''
self._stdout = sys.stdout
sys.stdout = StringIO()
self._stderr = sys.stderr
sys.stderr = StringIO()
return self
def __exit__(self, *args):
self._stdout_output = sys.stdout.getvalue()
sys.stdout = self._stdout
self._stderr_output = sys.stderr.getvalue()
sys.stderr = self._stderr
def get_stdout(self):
return self._stdout_output
def get_stderr(self):
return self._stderr_output
def run(ctx):
print("hello world!")
print("It works!", file=sys.stderr)
raise Exception('Oh oh!') # Comment out to have a successful completion
return ctx
def worker(ctx, queue):
import traceback
with CaptureOutput() as capturer:
try:
result = run(ctx)
except Exception as e:
result = e
print(traceback.format_exc(), file=sys.stderr)
queue.put((result, capturer.get_stdout(), capturer.get_stderr()))
if __name__ == '__main__':
queue = Queue()
ctx = None # for demo purposes
p = Process(target=worker, args=(ctx, queue))
p.start()
# Must do this call before call to join:
result, stdout_output, stderr_output = queue.get()
p.join()
print('stdout:', stdout_output)
print('stderr:', stderr_output)
Prints:
stdout: hello world!
stderr: It works!
Traceback (most recent call last):
File "C:\Booboo\test\test.py", line 44, in worker
result = run(ctx)
File "C:\Booboo\test\test.py", line 36, in run
raise Exception('Oh oh!') # Comment out to have a successful completion
Exception: Oh oh!
I am trying to run a code where I am working with Queue and threads.
Below is the code snippet:
import threading
from Queue import Queue
def function():
results = []
for i in range(68):
return_list = function2(i)
results.append(return_list)
if return_list:
print("True returned")
else:
print("Returned false")
return results
def function2(i):
print("In function 2 with: " + str(i))
results = []
working_queue = Queue()
for _ in range(25):
worker = threading.Thread(target=function3, args=(working_queue, results))
worker.setDaemon(True)
worker.start()
for p in range(150):
working_queue.put(p)
working_queue.join()
return results
def function3(working_queue, results):
while True:
try:
current_item = working_queue.get()
print("Processing:" + str(current_item))
results.append("True")
except Exception as e:
print("An exception in function 3: " + str(e))
finally:
working_queue.task_done()
if __name__ == "__main__":
results = function()
print(str(results))
The code raises the following exception:
Traceback (most recent call last):
File "C:/pythonErrors/stackoverflow.py", line 45, in <module>
function()
File "C:/pythonErrors/stackoverflow.py", line 8, in function
return_list = function2(i)
File "C:/pythonErrors/stackoverflow.py", line 24, in function2
worker.start()
File "C:\Python27\lib\threading.py", line 736, in start
_start_new_thread(self.__bootstrap, ())
thread.error: can't start new thread
How can we delete the previously created and completed threads. So that with each for loop execution in function2() new threads are not created.
The requirement is to create only 25 threads for the whole process (currently total 68x25 threads are being created)
Since, the requirement was to work with 25 threads in total; creation of threads should be the part of outermost function (i.e. function() here).
Below is the solution:
import threading
from Queue import Queue
def function():
results = []
working_queue = Queue()
for _ in range(25):
worker = threading.Thread(target=function3, args=(working_queue, results))
worker.setDaemon(True)
worker.start()
for i in range(68):
return_list = function2(i, working_queue)
working_queue.join()
results.append(return_list)
if return_list:
print("True returned")
else:
print("Returned false")
return results
def function2(i, working_queue):
print("In function 2 with: " + str(i))
for p in range(150):
working_queue.put(p)
def function3(working_queue, results):
while True:
try:
current_item = working_queue.get()
print("Processing:" + str(current_item))
results.append("True")
except Exception as e:
print("An exception in function 3: " + str(e))
finally:
working_queue.task_done()
if __name__ == "__main__":
results = function()
print(str(results))
When i run the code below, i got an exception
# System
import time
import logging
import sys
import os
import threading
# cv2 and helper:
import cv2
class inic_thread(threading.Thread):
def __init__(self, threadID, name, counter):
threading.Thread.__init__(self)
self.threadID = threadID
self.name = name
self.counter = counter
def run(self):
print "Starting " + self.name
if self.counter == 1: capture_continuos()
elif self.counter == 2: face_search()
def capture_continuos():
#os.system('python capture_continuos.py')
while(1):
print 'a'
def face_search():
# atributes
pool = []
path_pool = './pool/'
while(1):
pool_get = os.listdir(path_pool)
if len(pool_get) > 0:
#print(str(len(pool_get))+' images in the pool')
for image in pool_get:
print(image)
os.system('python face_search.py -i '+str(image))
else:
print('Empty Pool')
try:
capture_continuos = inic_thread(1, "capture_continuos_1", 1)
face_search_2 = inic_thread(2, "face_search_2", 2)
capture_continuos.start()
face_search_2.start()
except:
print("Error: unable to start thread")
But it don't make sense to me, because one of the threads run normal, (face_search) but the other one give this exception.
Starting capture_continuos_1
Exception in thread capture_continuos_1:
Traceback (most recent call last):
File "/usr/lib/python2.7/threading.py", line 801, in __bootstrap_inner
self.run()
File "main.py", line 44, in run
if self.counter == 1: capture_continuos()
TypeError: 'inic_thread' object is not callable
What i'm doing wrong?
I run in a Raspberry Pi 3 model B with Ubuntu MATE 14.04; Python 2.7.12
At the bottom of your script you redefine variable capture_continuos assigning thread object to it.
Also as was mentioned to terminate thread it's better to call os._exit() instead of sys.exit().
I am using Python 3.5.2 on windows.
I would like to run a python script but guarantee that it will not take more than N seconds. If it does take more than N seconds, an exception should be raised, and the program should exit. Initially I had thought I could just launch a thread at the beginning that waits for N seconds before throwing an exception, but this only manages to throw an exception to the timer thread, not to the parent thread. For example:
import threading
import time
def may_take_a_long_time(name, wait_time):
print("{} started...".format(name))
time.sleep(wait_time)
print("{} finished!.".format(name))
def kill():
time.sleep(3)
raise TimeoutError("No more time!")
kill_thread = threading.Thread(target=kill)
kill_thread.start()
may_take_a_long_time("A", 2)
may_take_a_long_time("B", 2)
may_take_a_long_time("C", 2)
may_take_a_long_time("D", 2)
This outputs:
A started...
A finished!.
B started...
Exception in thread Thread-1:
Traceback (most recent call last):
File "C:\Program Files\Python35\lib\threading.py", line 914, in _bootstrap_inner
self.run()
File "C:\Program Files\Python35\lib\threading.py", line 862, in run
self._target(*self._args, **self._kwargs)
File "timeout.py", line 11, in kill
raise TimeoutError("No more time!")
TimeoutError: No more time!
B finished!.
C started...
C finished!.
D started...
D finished!.
Is this even remotely possible? I realize I could do something like this:
import threading
import time
def may_take_a_long_time(name, wait_time, thread):
if not thread.is_alive():
return
print("{} started...".format(name))
time.sleep(wait_time)
print("{} finished!.".format(name))
def kill():
time.sleep(3)
raise TimeoutError("No more time!")
kill_thread = threading.Thread(target=kill)
kill_thread.start()
may_take_a_long_time("A", 2, kill_thread)
may_take_a_long_time("B", 2, kill_thread)
may_take_a_long_time("C", 2, kill_thread)
may_take_a_long_time("D", 2, kill_thread)
But this method fails if, for example, may_take_a_long_time("B", 60, kill_thread) was called.
So I guess my TL;DR question is, what's the best way to put a time limit on the main thread itself?
You can use _thread.interrupt_main (this module is called thread in Python 2.7):
import time, threading, _thread
def long_running():
while True:
print('Hello')
def stopper(sec):
time.sleep(sec)
print('Exiting...')
_thread.interrupt_main()
threading.Thread(target = stopper, args = (2, )).start()
long_running()
If the parent thread is the root thread, you may want to try os._exit(0).
import os
import threading
import time
def may_take_a_long_time(name, wait_time):
print("{} started...".format(name))
time.sleep(wait_time)
print("{} finished!.".format(name))
def kill():
time.sleep(3)
os._exit(0)
kill_thread = threading.Thread(target=kill)
kill_thread.start()
may_take_a_long_time("A", 2)
may_take_a_long_time("B", 2)
may_take_a_long_time("C", 2)
may_take_a_long_time("D", 2)
The cleanest way to do this is to have a queue that the parent thread checks on a regular basis. If the child wants to kill the parent, it sends a message (e.g. "DIE") to the queue. The parent checks the queue and if it sees the message it dies.
q = queue.Queue()
bc = queue.Queue() # backchannel
def worker():
while True:
key = q.get()
try:
process_key(key)
except ValueError as e:
bc.put('DIE')
q.task_done()
# Start the threads
for i in range(args.threads):
threading.Thread(target=worker, daemon=True).start()
for obj in object_farm():
q.put(obj)
try:
back = bc.get(block=False)
except queue.Empty:
pass
else:
print("Data received on backchannel:",back)
if back=='DIE':
raise RuntimeError("time to die")
#!/usr/bin/env python
#coding=utf-8
import sys,os,threading
import Queue
keyword = sys.argv[1]
path = sys.argv[2]
class keywordMatch(threading.Thread):
def __init__(self,queue):
threading.Thread.__init__(self)
self.queue = queue
def run(self):
while True:
line = self.queue.get()
if keyword in line:
print line
queue.task_done()
def main():
concurrent = 100 # Number of threads
queue = Queue.Queue()
for i in range(concurrent):
t = keywordMatch(True)
t.setDaemon(True)
t.start()
allfiles = os.listdir(path)
for files in allfiles:
pathfile = os.path.join(path,files)
fp = open(pathfile)
lines = fp.readlines()
for line in lines:
queue.put(line.strip())
queue.join()
if __name__ == '__main__':
main()
This program is for searching the keyword in a directory,
but there occurs an error:
Exception in thread Thread-100:
Traceback (most recent call last):
File "/usr/local/Cellar/python/2.7.3/Frameworks/Python.framework/Versions/2.7/lib/python2.7/threading.py", line 551, in __bootstrap_inner
self.run()
File "du.py", line 17, in run
line = self.queue.get()
AttributeError: 'bool' object has no attribute 'get'
How can I get rid of the error?
You're instantiating the thread with t = keywordMatch(True), and then in __init__ you're taking this argument and saving it as self.queue - so naturally self.queue is going to be a bool. If you want there to be a Queue instance there, you should pass it in.
In main() you wrote:
t = keywordMatch(True)
The keywordMatch class's __init__ does this:
def __init__(self,queue):
self.queue = queue
So now self.queue is True! Later, trying to do self.queue.get fails because it isn't a queue at all.