I have implemented below code for multiprocessing that can handle multiple request concurrently but I'm getting below error. For that i use producer and consumer concept where producing putting process in queue and consumer consume that process and do some JOB.
Traceback (most recent call last):
p.start()
File "/usr/lib/python2.7/multiprocessing/process.py", line 130, in start
self._popen = Popen(self)
File "/usr/lib/python2.7/multiprocessing/forking.py", line 121, in __init__
self.pid = os.fork()
OSError: [Errno 12] Cannot allocate memory
queue = Queue()
lock = Lock()
producers = []
consumers = []
for frame in frames:
`enter code here`producers.extend([Process(target=self.producer, args=(queue, lock, frame)) for i in xrange(cpu_count())])
for i in range(50):
p = Process(target=self.consumer, args=(queue, lock))
p.daemon = True
consumers.append(p)
for p in producers:
#time.sleep(random.randint(0, 5))
p.start()
for c in consumers:
#time.sleep(random.randint(0, 5))
c.start()
# Like threading, we have a join() method that synchronizes our program
for p in producers:
p.join()
u_end = time.time()
print u_start, u_end
print('Parent process exiting...')
Related
I'm writing a script for comparing many DNA genomes with each other, and I'm trying to use multiprocessing to have it run faster. All the processes are appending to a common list, genome_score_avgs.
This is my main process:
if __name__ == "__main__":
start = time.perf_counter()
with Manager() as manager:
genome_score_avgs = manager.list()
processes = [Process(target=compareGenomes, args=(chunk, genome_score_avgs,)) for chunk in divideGenomes('TEST_DIR')]
for p in processes:
p.start()
for p in processes:
p.join()
print(genome_score_avgs)
print(*createTimeline(genome_score_avgs), sep='\n')
print(f'Finished in {time.perf_counter() - start} seconds')
This is the error that I'm getting:
Traceback (most recent call last):
File "/Library/Frameworks/Python.framework/Versions/3.9/lib/python3.9/multiprocessing/managers.py", line 801, in _callmethod
conn = self._tls.connection
AttributeError: 'ForkAwareLocal' object has no attribute 'connection'
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/Users/ayushpal/Coding/PythonStuff/C4DInter/main.py", line 59, in <module>
print(*createTimeline(genome_score_avgs), sep='\n')
File "/Users/ayushpal/Coding/PythonStuff/C4DInter/main.py", line 42, in createTimeline
min_score = min(score_avgs, key=lambda x: x[2])
File "<string>", line 2, in __getitem__
File "/Library/Frameworks/Python.framework/Versions/3.9/lib/python3.9/multiprocessing/managers.py", line 805, in _callmethod
self._connect()
File "/Library/Frameworks/Python.framework/Versions/3.9/lib/python3.9/multiprocessing/managers.py", line 792, in _connect
conn = self._Client(self._token.address, authkey=self._authkey)
File "/Library/Frameworks/Python.framework/Versions/3.9/lib/python3.9/multiprocessing/connection.py", line 507, in Client
c = SocketClient(address)
File "/Library/Frameworks/Python.framework/Versions/3.9/lib/python3.9/multiprocessing/connection.py", line 635, in SocketClient
s.connect(address)
FileNotFoundError: [Errno 2] No such file or directory
<ListProxy object, typeid 'list' at 0x7fc04ea36bb0; '__str__()' failed>
I read in a similar Stack Overflow question that the main process is ending earlier than the other processes, which destroys the shared list, and that I should use p.join() for all the processes. This is what I'm doing , however, it's still giving the same error. What should I do?
EDIT 1:
this is the code for CompareGenomes():
def compareGenomes(genome_pairings, genome_score_avgs):
scores = []
for genome1, genome2 in genome_pairings:
print(genome1, genome2)
for i, seq in enumerate(genome1.protein_seqs):
for j, seq2 in enumerate(genome2.protein_seqs[i::]):
alignment = align.globalxx(seq, seq2)
scores.append(alignment)
top_scores = []
for i in range(len(genome1.protein_seqs)):
top_scores.append(max(scores, key=lambda x: x[0][2] / len(x[0][1])))
scores.remove(max(scores, key=lambda x: x[0][2] / len(x[0][1])))
avg_score = sum([i[0][2] / len(i[0][1]) for i in top_scores]) / len(top_scores)
with open(f'alignments/{genome1.name}x{genome2.name}.txt', 'a') as file:
file.writelines([format_alignment(*i[0]) for i in top_scores])
genome_score_avgs.append((genome1, genome2, avg_score))
The error is happening because you are using the managed list after you have closed the manager. Once that happens, the process that the manager spawns is closed as well, and therefore your managed list will no longer work. You need to use the list inside the with block like below:
if __name__ == "__main__":
start = time.perf_counter()
with Manager() as manager:
genome_score_avgs = manager.list()
processes = [Process(target=compareGenomes, args=(chunk, genome_score_avgs,)) for chunk in divideGenomes('TEST_DIR')]
for p in processes:
p.start()
for p in processes:
p.join()
print(genome_score_avgs)
print(*createTimeline(genome_score_avgs), sep='\n')
print(f'Finished in {time.perf_counter() - start} seconds')
I have a producer/consumer system using Process and Queue from multiprocessing. The processes do catch the KeyboardInterrupt, and the final result dictionary is returned successfully. However, the processes keeps on spamming this error message:
Traceback (most recent call last):
File "/home/ubuntu/anaconda3/envs/nlp/lib/python3.7/multiprocessing/queues.py", line 242, in _feed
send_bytes(obj)
File "/home/ubuntu/anaconda3/envs/nlp/lib/python3.7/multiprocessing/connection.py", line 200, in send_bytes
self._send_bytes(m[offset:offset + size])
File "/home/ubuntu/anaconda3/envs/nlp/lib/python3.7/multiprocessing/connection.py", line 404, in _send_bytes
self._send(header + buf)
File "/home/ubuntu/anaconda3/envs/nlp/lib/python3.7/multiprocessing/connection.py", line 368, in _send
n = write(self._handle, buf)
BrokenPipeError: [Errno 32] Broken pipe
Which would indicate to me that somewhere there is something trying to be put on a Queue, unsuccessfully.
import signal
import sys
from multiprocessing import Process, Queue
from typing import Iterable
def do_work(task):
pass
def pull_worker(
worker_num, work_queue: Queue, result_queue: Queue
):
run = True
def signal_handle(_signal, frame):
nonlocal run
run = False
result_queue.put(None) # Signal to the consumer that the worker is finished
result_queue.close()
result_queue.join_thread()
work_queue.close()
work_queue.join_thread()
sys.exit(0)
signal.signal(signal.SIGINT, signal_handle)
signal.signal(signal.SIGTERM, signal_handle)
signal.signal(signal.SIGHUP, signal_handle)
while run:
try:
task = work_queue.get_nowait()
except work_queue.Empty:
break
else:
result = do_work(task)
if run:
result_queue.put((task, result))
result_queue.put(None) # Signal to the consumer that the worker is finished
def save_consumer(result_queue, final_result_queue, n_workers):
"""
Puts all results into the final result queue when all workers have finished
"""
n_finished_producers = 0
results = {}
def signal_handle(_signal, frame):
print("Caught keyboard interrupt in consumer")
signal.signal(signal.SIGINT, signal_handle)
signal.signal(signal.SIGTERM, signal_handle)
signal.signal(signal.SIGHUP, signal_handle)
while n_finished_producers < n_workers:
result = result_queue.get()
if result is None:
n_finished_producers += 1
continue
(task, task_result) = result
results[task] = task_result
final_result_queue.put(results)
final_result_queue.close()
final_result_queue.join_thread()
def main(
tasks: Iterable[str],
n_workers: int,
):
n_workers = min(len(tasks), n_workers)
# For tasks to be done
work_queue = Queue()
# For results as they are fetched from the workers
result_queue = Queue()
# For the final, single, result dictionary when workers are shut down
final_result_queue = Queue()
for task in tasks:
work_queue.put(task)
consumer = Process(
target=save_consumer, args=(result_queue, final_result_queue, n_workers)
)
consumer.start()
producers = []
for worker_num in range(n_workers):
proc = Process(
target=pull_worker,
args=(worker_num, work_queue, result_queue)
)
proc.start()
producers.append(proc)
try:
for proc in producers:
proc.join()
except (KeyboardInterrupt, BaseException, SystemExit) as e:
print(f"Caught interrupt in Main: {type(e)} {e}")
finally:
for proc in producers:
proc.join()
results = final_result_queue.get()
consumer.join()
return results
I wanted to set up two subprocesses in which subprocess1 keeps generating data (in type of list), and subprocess2 is in charge of processing the data sent from subprocess1.
I used multiprocessing.Manager().list() to create a shared list. But this is the error it reports:
FileNotFoundError: [WinError 2]
Code
I simplified the code as below:
ps: need to run it in terminal.
import multiprocessing as mp
import random
import time
def generator(a, b, tick): # simulating data collection,and a list will be generated at random and passed to another shared list.
counter = 0
while True:
time.sleep(1)
a.append([random.uniform(1,5), random.uniform(1,5), random.uniform(1,5), random.uniform(1,5)])
counter += 1
print('generate says', a[:])
if counter%5 == 0:
b.append(a[:])
tick.value = 1 # Telling 'printer' func to print.
for _ in a:
a.remove(_)
def printer(b, tick): # simulating data processing, and only printing data received from the 'generator' func here.
while True:
time.sleep(1)
if tick.value == 1:
time.sleep(1)
print('printer says', b[:])
tick.value = 0
for _ in b:
b.remove(_)
if __name__=='__main__':
tick=mp.Value('i', 0)
a = mp.Manager().list()
b = mp.Manager().list()
p1 = mp.Process(target=generator, args=(a, b, tick))
p2 = mp.Process(target=printer, args=(b, tick))
p1.start()
p2.start()
Error
Traceback (most recent call last):
File "d:\miniconda\lib\multiprocessing\process.py", line 297, in _bootstrap
self.run()
File "d:\miniconda\lib\multiprocessing\process.py", line 99, in run
self._target(*self._args, **self._kwargs)
File "D:\Program Files (x86)\onedrive\nilm\pi\redd\niubi.py", line 9, in generater
a.append([random.uniform(1,5),random.uniform(1,5),random.uniform(1,5),random.uniform(1,5)])
File "<string>", line 2, in append
File "d:\miniconda\lib\multiprocessing\managers.py", line 792, in _callmethod
self._connect()
File "d:\miniconda\lib\multiprocessing\managers.py", line 779, in _connect
conn = self._Client(self._token.address, authkey=self._authkey)
File "d:\miniconda\lib\multiprocessing\connection.py", line 490, in Client
c = PipeClient(address)
File "d:\miniconda\lib\multiprocessing\connection.py", line 691, in PipeClient
_winapi.WaitNamedPipe(address, 1000)
FileNotFoundError: [WinError 2] The system cannot find the file specified.
There were a few things to fix, but the primary issue is that you should include Process.join, as seen below:
import multiprocessing as mp
import random
import time
... # generator and printer definitions are unchanged
if __name__=='__main__':
manager = mp.Manager() # Create an instance of the manager
a = manager.list()
b = manager.list()
tick = mp.Value('i', 0)
p1 = mp.Process(target=generator, args=(tick, a, b))
p2 = mp.Process(target=printer, args=(tick, b))
p1.start()
p2.start()
p1.join() # Join, to ensure p1 and p2 end
p2.join()
I have a python program like below.
from multiprocessing import Lock, Process, Queue, current_process
import time
lock = Lock()
def do_job(tasks_to_accomplish, tasks_that_are_done):
while not tasks_to_accomplish.empty():
task = tasks_to_accomplish.get()
print(task)
lock.acquire()
tasks_that_are_done.put(task + ' is done by ' + current_process().name)
lock.release()
time.sleep(1)
return True
def main():
number_of_task = 10
number_of_processes = 4
tasks_to_accomplish = Queue()
tasks_that_are_done = Queue()
processes = []
for i in range(number_of_task):
tasks_to_accomplish.put("Task no " + str(i))
# creating processes
for w in range(number_of_processes):
p = Process(target=do_job, args=(tasks_to_accomplish, tasks_that_are_done))
processes.append(p)
p.start()
# completing process
for p in processes:
p.join()
# print the output
while not tasks_that_are_done.empty():
print(tasks_that_are_done.get())
return True
if __name__ == '__main__':
main()
Sometimes program run perfectly but sometimes it gets stuck and doesn't complete. When quit manually, it produces following error.
$ python3 multiprocessing_example.py
Task no 0
Task no 1
Task no 2
Task no 3
Task no 4
Task no 5
Task no 6
Task no 7
Task no 8
Task no 9
^CProcess Process-1:
Traceback (most recent call last):
File "multiprocessing_example.py", line 47, in <module>
main()
File "multiprocessing_example.py", line 37, in main
p.join()
File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/multiprocessing/process.py", line 121, in join
res = self._popen.wait(timeout)
File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/multiprocessing/popen_fork.py", line 51, in wait
return self.poll(os.WNOHANG if timeout == 0.0 else 0)
File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/multiprocessing/popen_fork.py", line 29, in poll
pid, sts = os.waitpid(self.pid, flag)
KeyboardInterrupt
Traceback (most recent call last):
File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/multiprocessing/process.py", line 249, in _bootstrap
self.run()
File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/multiprocessing/process.py", line 93, in run
self._target(*self._args, **self._kwargs)
File "multiprocessing_example.py", line 9, in do_job
task = tasks_to_accomplish.get()
File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/multiprocessing/queues.py", line 94, in get
res = self._recv_bytes()
File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/multiprocessing/connection.py", line 216, in recv_bytes
buf = self._recv_bytes(maxlength)
File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/multiprocessing/connection.py", line 407, in _recv_bytes
buf = self._recv(4)
File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/multiprocessing/connection.py", line 379, in _recv
chunk = read(handle, remaining)
KeyboardInterrupt
Can someone tell me what is the issue with the program? I am using python 3.6.
Note: Lock is not needed around a Queue.
lock.acquire()
tasks_that_are_done.put(task + ' is done by ' + current_process().name)
lock.release()
Queue
The Queue class in this module implements all the required locking semantics.
Question: ... what is the issue with the program?
You are using Queue.empty() and Queue.get(),
such leads to Deadlock on calling join() because there is no guarantee that the empty() State don't change until get()
was reaching.
Deadlock prone:
while not tasks_to_accomplish.empty():
task = tasks_to_accomplish.get()
Instead of using empty/get, Pair use for instance:
import queue
while True:
try:
task = tasks_to_accomplish.get_nowait()
except queue.Empty:
break
else:
# Handle task here
...
tasks_to_accomplish.task_done()
I am trying to execute the following script by using multiprocessing and queues,
from googlefinance import getQuotes
from yahoo_finance import Share
import multiprocessing
class Stock:
def __init__(self,symbol,q):
self.symbol = symbol
self.q = q
def current_value(self):
current_price =self.q.put(float(getQuotes(self.symbol)[0]['LastTradeWithCurrency']))
return current_price
def fundems(self):
marketcap = self.q.put(Share(self.symbol).get_market_cap())
bookvalue = self.q.put(Share(self.symbol).get_book_value())
dividend = self.q.put(Share(self.symbol).get_dividend_share())
dividend_paydate = self.q.put(Share(self.symbol).get_dividend_pay_date())
dividend_yield = self.q.put(Share(self.symbol).get_dividend_yield())
return marketcap, bookvalue, dividend, dividend_paydate, dividend_yield
def main():
q = multiprocessing.Queue()
Stock1 = Stock('aapl', q)
p1 = multiprocessing.Process(target = Stock1.current_value(), args = (q,))
p2 = multiprocessing.Process(target = Stock1.fundems(), args = (q,))
p1.start()
p2.start()
p1.join()
p2.join()
while q.empty() is False:
print q.get()
if __name__ == '__main__':
main()
I am getting the output as the following:
Process Process-2:
Traceback (most recent call last):
File "/usr/lib/python2.7/multiprocessing/process.py", line 258, in _bootstrap
self.run()
File "/usr/lib/python2.7/multiprocessing/process.py", line 114, in run
self._target(*self._args, **self._kwargs)
TypeError: 'tuple' object is not callable
139.52
732.00B
25.19
2.28
2/16/2017
1.63
Here I see that I am able to get the output which I wanted, but there was an error before that which is kind a making me confused.
Can anyone please help me understand the concept here.
Thanks in advance.
The target should be an uncalled function, you're calling the function in the parent process and trying to launch a Process with the results of the call as the target. Change:
p1 = multiprocessing.Process(target = Stock1.current_value(), args = (q,))
p2 = multiprocessing.Process(target = Stock1.fundems(), args = (q,))
to:
p1 = multiprocessing.Process(target=Stock1.current_value)
p2 = multiprocessing.Process(target=Stock1.fundems)
q is removed as an argument because the object was constructed with q, and uses its own state to access it, it doesn't receive it as an argument to each method.