I am writing a small program which has a heartbeat process and an echo process. I implemented this with a multiprocessing library, but it doesn't seem to work.
from multiprocessing import Process
import os
import time
def ticking():
while True:
time.sleep(1)
print 'ticking'
def echo():
while True:
a = raw_input('please type something')
print 'echo: ' + a
if __name__ == '__main__':
p = Process(target=ticking, args=())
p.start()
p.join()
p = Process(target=echo, args=())
p.start()
p.join()
You create a process that will run forever and join() to it. The second process will never get created, because the join() will stall your main process forever.
If this is how you want to proceed, then you should for example first create both processes and then join them:
if __name__ == '__main__':
p1 = Process(target=ticking, args=())
p1.start()
p2 = Process(target=echo, args=())
p2.start()
p1.join()
p2.join()
For create a daemon you can use this function:
def daemonize():
"""UNIX double fork mechanism."""
try:
pid = os.fork()
if pid > 0:
# exit first parent
sys.exit(0)
except OSError as err:
sys.stderr.write('_Fork #1 failed: {0}\n'.format(err))
sys.exit(1)
# decouple from parent environment
os.chdir('/')
os.setsid()
os.umask(0)
# do second fork
try:
pid = os.fork()
if pid > 0:
# exit from second parent
sys.exit(0)
except OSError as err:
sys.stderr.write('_Fork #2 failed: {0}\n'.format(err))
sys.exit(1)
# redirect standard file descriptors
sys.stdout.flush()
sys.stderr.flush()
si = open(os.devnull, 'r')
so = open(os.devnull, 'w')
se = open(os.devnull, 'w')
os.dup2(si.fileno(), sys.stdin.fileno())
os.dup2(so.fileno(), sys.stdout.fileno())
os.dup2(se.fileno(), sys.stderr.fileno())
Related
I have created a 3 process in python. I have attached a code.
Now I want to stop the execution of running p2,p3 process because I got an error due to p1 process.I have idea to add p2.terminate(),I don't know where to add in this case. Thanks in advance.
def table(a):
try:
for i in range(100):
print(i,'x',a,'=',a*i)
except:
print("error")
processes = []
p1= multiprocessing.Process(target = table,args=['s'])
p2= multiprocessing.Process(target = table,args=[5])
p3= multiprocessing.Process(target = table,args=[2])
p1.start()
p2.start()
p3.start()
processes.append(p1)
processes.append(p2)
processes.append(p3)
for process in processes:
process.join()```
To stop any given process once one of the process terminates due to an error, first set up your target table() to exit with an appropriate exitcode > 0
def table(args):
try:
for i in range(100):
print(i,'x', a ,'=', a*i)
except:
sys.exit(1)
sys.exit(0)
Then you can start your processes and poll the processes to see if any one has terminated.
#!/usr/bin/env python3
# coding: utf-8
import multiprocessing
import time
import logging
import sys
logging.basicConfig(level=logging.INFO, format='[%(asctime)-15s] [%(processName)-10s] %(message)s', datefmt='%Y-%m-%d %H:%M:%S')
def table(args):
try:
for i in range(5):
logging.info('{} x {} = {}'.format(i, args, i*args))
if isinstance(args, str):
raise ValueError()
time.sleep(5)
except:
logging.error('Done in Error Path: {}'.format(args))
sys.exit(1)
logging.info('Done in Success Path: {}'.format(args))
sys.exit(0)
if __name__ == '__main__':
p1 = multiprocessing.Process(target=table, args=('s',))
p2 = multiprocessing.Process(target=table, args=(5,))
p3 = multiprocessing.Process(target=table, args=(2,))
processes = [p1, p2, p3]
for process in processes:
process.start()
while True:
failed = []
completed = []
for process in processes:
if process.exitcode is not None and process.exitcode != 0:
failed.append(process)
if failed:
for process in processes:
if process not in failed:
logging.info('Terminating Process: {}'.format(process))
process.terminate()
break
if len(completed) == len(processes):
break
time.sleep(1)
Essentially, you are using terminate() to stop the remaining processes that are still running.
to stop a all cores when one core has faced with error, i use this code block:
processes = []
for j in range(0, n_core):
p = multiprocessing.Process(target=table, args=('some input',))
processes.append(p)
time.sleep(0.1)
p.start()
flag = True
while flag:
flag = False
for p in processes:
if p.exitcode == 1:
for z in processes:
z.kill()
sys.exit(1)
elif p.is_alive():
flag = True
for p in processes:
p.join()
First, I have modified function table to throw an exception that is not caught when the argument passed to it is 's' and to delay .1 seconds otherwise before printing to give the main process a chance to realize that the sub-process through an exception and can cancel the other processes before they have started printing. Otherwise, the other processes will have completed before you can cancel them. Here I am using a process pool, which supports a terminate method that conveniently terminates all submitted, uncompleted tasks without having to cancel each one individually (although that is also an option).
The code creates a multiprocessing pool of size 3 since that is the number of "tasks" being submitted and then uses method apply_async to submit the 3 tasks to run in parallel (assuming you have at least 3 processors). apply_sync returns an AsyncResult instance whose get method can be called to wait for the completion of the submitted task and to get the return value from the worker function table, which is None for the second and third tasks submitted and of no interest, or will throw an exception if the worker function had an uncaught exception, which is the case with the first task submitted:
import multiprocessing
import time
def table(a):
if a == 's':
raise Exception('I am "s"')
time.sleep(.1)
for i in range(100):
print(i,'x',a,'=',a*i)
# required for Windows:
if __name__ == '__main__':
pool = multiprocessing.Pool(3) # create a pool of 3 processes
result1 = pool.apply_async(table, args=('s',))
result2 = pool.apply_async(table, args=(5,))
result3 = pool.apply_async(table, args=(2,))
try:
result1.get() # wait for completion of first task
except Exception as e:
print(e)
pool.terminate() # kill all processes in the pool
else:
# wait for all submitted tasks to complete:
pool.close()
pool.join()
"""
# or alternatively:
result2.get() # wait for second task to finish
result3.get() # wait for third task to finish
"""
Prints:
I am "s"
In my development, I have shared file which need to write mutliple process but I found it write only header part(only first line i.e test_data). I am able to create same behavior using following script. Any Input what am I missing?
#! /usr/bin/env python
import multiprocessing
def check_proc(procs):
"""checking cpu utilization befor starting process"""
for proc in procs:
proc.start()
while(len(multiprocessing.active_children())>10):
continue
for proc in procs:
proc.join()
while (len(multiprocessing.active_children())>5):
time.sleep(1)
def create_file():
f_data = open("temp", "w")
f_data.write("test data\n")
return f_data
def write_data(f_data, number, lock):
lock.acquire()
f_data.write(number)
lock.release()
def main():
f_data = create_file()
procs = []
lock = multiprocessing.Lock()
for i in range(0, 20, 1):
proc = multiprocessing.Process(target=write_data,
args=(f_data, str(i), lock))
procs.append(proc)
check_proc(procs)
if __name__ == "__main__":
main()
Expected:
it should have number
Python2.7 and linux enviroment
I have gone through stack overflow question and block and found following link. But I am not able to relate or use in above code. Any input.
http://ptspts.blogspot.com/2013/08/how-to-send-unix-file-descriptors.html
How to force a function and all it's child processes to timeout on Linux?
For example, how could multiprocessed_func be forced to finish after 10s:
import time
def multiprocessed_func(seconds):
# Assume this a long running function which uses
# multiprocessing internally and returns None.
time.sleep(seconds)
try:
multiprocessed_func(600)
except:
print('took too long')
Borrowing from the psutil docs, we could inspect the current process and terminate or kill all the child processes after a given time.
def terminate_children(grace_period):
procs = psutil.Process().children()
for p in procs:
p.terminate()
gone, still_alive = psutil.wait_procs(procs, timeout=grace_period)
for p in still_alive:
p.kill()
raise TimeoutError
try:
multiprocessed_func(long_run=600)
time.sleep(10) # then timeout
terminate_children(grace_period=2)
except TimeoutError:
print('timed out')
pass
Full example:
import multiprocessing
import time
import psutil
def slow_worker(long_run):
print('started')
time.sleep(long_run)
print('finished')
def multiprocessed_func(long_run):
jobs = []
for i in range(5):
p = multiprocessing.Process(target=slow_worker, args=(long_run,))
jobs.append(p)
p.start()
print('starting', p.pid)
def on_terminate(proc):
print('terminating {}, exit code {}'.format(proc, proc.returncode))
def terminate_children(grace_period):
procs = psutil.Process().children()
for p in procs:
p.terminate()
gone, still_alive = psutil.wait_procs(procs, timeout=grace_period,
callback=on_terminate)
for p in still_alive:
p.kill()
raise TimeoutError
try:
multiprocessed_func(long_run=600)
time.sleep(10)
terminate_children(grace_period=2)
except TimeoutError:
print('timed out')
pass
If terminating all the child processes in the current process is excessive because there are additional multiprocessed methods in the current process that need to be preserved, then we could wrap multiprocessed_func in another process.
def safe_run(timeout, grace_period):
try:
multiprocessed_func(long_run=600)
time.sleep(timeout)
terminate_children(grace_period)
except TimeoutError:
pass
timeout, grace_period = 10, 2
p = multiprocessing.Process(target=safe_run, args=(timeout, grace_period,))
p.start()
p.join()
p.terminate()
time.sleep(2)
if p.is_alive():
p.kill()
I am trying to use the multiprocessing library in Python to process "tests" concurrently. I have a list of tests stored in the variable test_files. I want to workers to remove a test from test_files and call the process_test function of them. However when I run this code, both processes run the same test. It seems that I am not accessing test_files in a thread safe manner. What am I doing wrong?
Code
def process_worker(lock, test_files)
# Keep going until we run out of tests
while True:
test_file = None
# Critical section of code
lock.acquire()
try:
if len(test_files) != 0:
test_file = test_files.pop()
finally:
lock.release()
# End critical section of code
# If there is another test in the queue process it
if test_file is not None:
print "Running test {0} on worker {1}".format(test_file, multiprocessing.current_process().name)
process_test(test_file)
else:
# No more tests to process
return
# Mutex for workers
lock = multiprocessing.Lock()
# Declare our workers
p1 = multiprocessing.Process(target = process_worker, name = "Process 1", args=(lock, test_files))
p2 = multiprocessing.Process(target = process_worker, name = "Process 2", args=(lock, test_files))
# Start processing
p1.start()
p2.start()
# Block until both workers finish
p1.join()
p2.join()
Output
Running test "BIT_Test" on worker Process 1
Running test "BIT_Test" on worker Process 2
Trying to share a list like this not the right approach here. You should use a process-safe data structure, like multiprocessing.Queue, or better yet, use a multiprocessing.Pool and let it handle the queuing for you. What you're doing is perfectly suited for Pool.map:
import multiprocessing
def process_worker(test_file):
print "Running test {0} on worker {1}".format(test_file, multiprocessing.current_process().name)
process_test(test_file)
p = multiprocessing.Pool(2) # 2 processes in the pool
# map puts each item from test_files in a Queue, lets the
# two processes in our pool pull each item from the Queue,
# and then execute process_worker with that item as an argument.
p.map(process_worker, test_files)
p.close()
p.join()
Much simpler!
You could also use multiprocessing.Manager
import multiprocessing
def process_worker(lock, test_files):
# Keep going until we run out of tests
while True:
test_file = None
# Critical section of code
lock.acquire()
try:
if len(test_files) != 0:
test_file = test_files.pop()
finally:
lock.release()
# End critical section of code
# If there is another test in the queue process it
if test_file is not None:
print "Running test %s on worker %s" % (test_file, multiprocessing.current_process().name)
#process_test(test_file)
else:
# No more tests to process
return
# Mutex for workers
lock = multiprocessing.Lock()
manager = multiprocessing.Manager()
test_files = manager.list(['f1', 'f2', 'f3'])
# Declare our workers
p1 = multiprocessing.Process(target = process_worker, name = "Process 1", args=(lock, test_files))
p2 = multiprocessing.Process(target = process_worker, name = "Process 2", args=(lock, test_files))
# Start processing
p1.start()
p2.start()
# Block until both workers finish
p1.join()
p2.join()
I have a python script (unix-like, based on RHEL), called MyScript, that has two functions, called A and B. I'd like them to run in different, independent processes (detach B and A):
Start script MyScript
Execute function A
Spawn a new process, passing data from function A to B
While function B runs, continue with function A
When function A completes, exit MyScript even if B is still running
I thought I should use multiprocessing to create a daemon process, but the documentation suggests that's not the right usecase. So, I decided to spawn a child process and child^2 process (the child's child), and then force the child to terminate. While this workaround appears to work, it seems really ugly.
Can you help me make it more pythonic? Does the subprocess module have a method that will operate on a function? Sample code below.
import multiprocessing
import time
import sys
import os
def parent_child():
p = multiprocessing.current_process()
print 'Starting parent child:', p.name, p.pid
sys.stdout.flush()
cc = multiprocessing.Process(name='childchild', target=child_child)
cc.daemon = False
cc.start()
print 'Exiting parent child:', p.name, p.pid
sys.stdout.flush()
def child_child():
p = multiprocessing.current_process()
print 'Starting child child:', p.name, p.pid
sys.stdout.flush()
time.sleep(30)
print 'Exiting child child:', p.name, p.pid
sys.stdout.flush()
def main():
print 'starting main', os.getpid()
d = multiprocessing.Process(name='parentchild', target=parent_child)
d.daemon = False
d.start()
time.sleep(5)
d.terminate()
print 'exiting main', os.getpid()
main()
Here is just a random version of your original code that moves the functionality into a single call spawn_detached(callable). It keeps the detached process running even after the program exits:
import time
import os
from multiprocessing import Process, current_process
def spawn_detached(callable):
p = _spawn_detached(0, callable)
# give the process a moment to set up
# and then kill the first child to detach
# the second.
time.sleep(.001)
p.terminate()
def _spawn_detached(count, callable):
count += 1
p = current_process()
print 'Process #%d: %s (%d)' % (count, p.name, p.pid)
if count < 2:
name = 'child'
elif count == 2:
name = callable.func_name
else:
# we should now be inside of our detached process
# so just call the function
return callable()
# otherwise, spawn another process, passing the counter as well
p = Process(name=name, target=_spawn_detached, args=(count, callable))
p.daemon = False
p.start()
return p
def operation():
""" Just some arbitrary function """
print "Entered detached process"
time.sleep(15)
print "Exiting detached process"
if __name__ == "__main__":
print 'starting main', os.getpid()
p = spawn_detached(operation)
print 'exiting main', os.getpid()