We Keep Coding

Implementing Numpy into nth factorial of n

I've tried a few methods of implementing numpy into my program below, but every way I've attempted it drastically decreases performance. Is there expected with my use case, or is there a way to implement it with a performance increase instead of decrease?
import multiprocessing
import argparse
from datetime import datetime
from math import log10
parser = argparse.ArgumentParser(
formatter_class=argparse.HelpFormatter,
description="Calcs n factorial",
usage=""
)
parser.add_argument("-n", "--number", type=int, default=2)
args = parser.parse_args()
def getlog(send_end, i, threads, num, n, inc):
begin = datetime.now()
start = num-inc*i
end = num-inc*(i+1) if i < threads-1 else 0
output = sum(map(log10, range(start, end, -n)))
send_end.send(output)
final = datetime.now()
duration = final-begin
print("{},{},{},{}".format(i, duration, start, end))
def main():
n = args.number
num = int('1'*n)
threads = multiprocessing.cpu_count() if num/multiprocessing.cpu_count() > multiprocessing.cpu_count() else 1
inc = int(num/threads)
inc -= inc%n
jobs = []
pipe_list = []
for i in range(threads):
recv_end, send_end = multiprocessing.Pipe(False)
p = multiprocessing.Process(target=getlog, args=(send_end, i, threads, num, n, inc))
jobs.append(p)
pipe_list.append(recv_end)
p.start()
for proc in jobs:
proc.join()
e = sum([output.recv() for output in pipe_list])
print('%.2fe%d' % (10**(e % 1), e // 1))
if __name__ == '__main__':
start = datetime.now()
main()
end = datetime.now()
print(end-start)

Python multiprocessing returns queue is empty when actually it is not

In this program, after some iterations all the processes terminate which means that input_queue is empty as per the condition in target function. But after returning to the main function when I print the input_queue there are still items left in that queue, then why those multiple processes terminated at first place?
import cv2
import timeit
import face_recognition
import queue
from multiprocessing import Process, Queue
import multiprocessing
import os
s = timeit.default_timer()
def alternative_process_target_func(input_queue, output_queue):
while not input_queue.empty():
try:
frame_no, small_frame, face_loc = input_queue.get(False) # or input_queue.get_nowait()
print('Frame_no: ', frame_no, 'Process ID: ', os.getpid(), '----', multiprocessing.current_process())
except queue.Empty:
print('___________________________________ Breaking __________________________________________________')
break # stop when there is nothing more to read from the input
def alternative_process(file_name):
start = timeit.default_timer()
cap = cv2.VideoCapture(file_name)
frame_no = 1
fps = cap.get(cv2.CAP_PROP_FPS)
length = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
print('Frames Per Second: ', fps)
print('Total Number of frames: ', length)
print('Duration of file: ', int(length / fps))
processed_frames = 1
not_processed = 1
frames = []
process_this_frame = True
frame_no = 1
Input_Queue = Queue()
while (cap.isOpened()):
ret, frame = cap.read()
if not ret:
print('Size of input Queue: ', Input_Queue.qsize())
print('Total no of frames read: ', frame_no)
end1 = timeit.default_timer()
print('Time taken to fetch useful frames: ', end1 - start)
threadn = cv2.getNumberOfCPUs()
Output_Queue = Queue(maxsize=Input_Queue.qsize())
process_list = []
#quit = multiprocessing.Event()
#foundit = multiprocessing.Event()
for x in range((threadn - 1)):
# print('Process No : ', x)
p = Process(target=alternative_process_target_func, args=(Input_Queue, Output_Queue))#, quit, foundit
p.daemon = True
#print('I am a new process with process id of: ', os.getpid())
p.start()
process_list.append(p)
#p.join()
i = 1
for proc in process_list:
print('I am hanged here and my process id is : ', os.getpid())
proc.join()
print('I have been joined and my process id is : ', os.getpid())
i += 1
for value in range(Output_Queue.qsize()):
print(Output_Queue.get())
end = timeit.default_timer()
print('Time taken by face verification: ', end - start)
print('--------------------------------------------------------------------------------------------------')
#Here I am again printing the Input Queue which should be empty logically.
for frame in range(Input_Queue.qsize()):
frame_no, _, _ = Input_Queue.get()
print(frame_no)
break
if process_this_frame:
print(frame_no)
small_frame = cv2.resize(frame, (0, 0), fx=0.25, fy=0.25)
rgb_small_frame = small_frame[:, :, ::-1]
face_locations = face_recognition.face_locations(rgb_small_frame)
# frames.append((rgb_small_frame, face_locations))
Input_Queue.put((frame_no, rgb_small_frame, face_locations))
frame_no += 1
if processed_frames < 5:
processed_frames += 1
not_processed = 1
else:
if not_processed < 15:
process_this_frame = False
not_processed += 1
else:
processed_frames = 1
process_this_frame = True
print('-----------------------------------------------------------------------------------------------')
cap.release()
cv2.destroyAllWindows()
#chec_queues()
#compare_images()
#fps_finder()
alternative_process('user_verification_2.avi')#'hassan_checking.avi'

Your code contains while not input_queue.empty(). I guess during the work input_queue becomes empty, while loop stops and then you add something else to input_queue to process this something else. But it's too late at that moment.
Usually you work with queues like this:
while True:
element = my_queue.get()
...
To stop this loop you may count number of treated elements, use timeout argument or kill process under some condition. Another option is to use multiprocessing.Pool or concurrent.futures.ProcessPoolExecutor.

python multiprocessing is not using multiple cores

Reading the documentation: https://docs.python.org/2/library/multiprocessing.html
I decided to write a cpu intensive code and compare multiprocessing with serial computation.
First of all, if this library is using multiprocessing, then why I only see 1 python.exe process?
Secondly, why serial computation takes 12 seconds while multiprocessed one takes 22 seconds?
serial code:
from datetime import datetime
def calc_fib(ind):
fb = 1
if ind >= 3:
prev = 1
i = 2
while i < ind:
prev_tmp = fb
fb += prev
prev = prev_tmp
i += 1
return fb
def long_calc_fib(ind):
val = 0
for j in range(500):
val = calc_fib(ind)
return val
if __name__ == "__main__":
t1 = datetime.now()
for i in range(10):
tmp = long_calc_fib(10000)
t2 = datetime.now()
print str(t2 - t1)
multiprocessing pool code:
from datetime import datetime
from multiprocessing.pool import ThreadPool
def calc_fib(ind):
fb = 1
if ind >= 3:
prev = 1
i = 2
while i < ind:
prev_tmp = fb
fb += prev
prev = prev_tmp
i += 1
return fb
def long_calc_fib(ind):
val = 0
for j in range(500):
val = calc_fib(ind)
return val
if __name__ == "__main__":
t1 = datetime.now()
pool = ThreadPool(processes=10)
async_results = []
for i in range(10):
async_results.append(pool.apply_async(long_calc_fib, (10000,)))
for res in async_results:
tmp = res.get()
t2 = datetime.now()
print str(t2 - t1)

My mistake.
I must have used Pool instead of ThreadPool.
By chaning ThreadPool to Pool, I reduced the time to 3 seconds.

How to access the result of a function called in a multiprocessing process?

I am runnning this code :
import random
import multiprocessing
import time
def list_append(count, id):
out_list = []
for i in range(count):
out_list.append(random.random())
return out_list
if __name__ == "__main__":
t0 = time.clock()
size = 10000000 # Number of random numbers to add
procs = 2 # Number of processes to create
jobs = []
for i in range(0, procs):
process = multiprocessing.Process(target=list_append,args=(size, i))
jobs.append(process)
# Start the processes (i.e. calculate the random number lists)
res=[]
for j in jobs:
r= j.start()
res.append(r)
# Ensure all of the processes have finished
for j in jobs:
j.join()
print "List processing complete."
print time.clock()-t0,"seconds"
Unfortunately, at the end of it, res = [None,None] although I want it to be filled with the lists I've filled in the function list_append.

You need to use data structures that can be shared between processes:
def list_append(count, id, res):
# ^^^
out_list = []
for i in range(count):
out_list.append(random.random())
res[id] = out_list # <------
if __name__ == "__main__":
size = 10000000
procs = 2
manager = multiprocessing.Manager() # <---
res = manager.dict() # <---
jobs = []
for i in range(0, procs):
process = multiprocessing.Process(target=list_append,args=(size, i, res))
# ^^^^
jobs.append(process)
for j in jobs:
r = j.start()
for j in jobs:
j.join()
print "List processing complete."
# now `res` will contain results
As avenet commented, using multiprocessing.Pool will be simpler:
def list_append(args):
count, id = args
out_list = []
for i in range(count):
out_list.append(random.random())
return out_list
if __name__ == "__main__":
size = 10000000
procs = 2
pool = multiprocessing.Pool(procs)
res = pool.map(list_append, [(size, i) for i in range(procs)])
pool.close()
pool.join()
print "List processing complete."
# print res

python : multiprocessing managament

with multiprocessing python library I can launch multiprocess, like
import multiprocessing as mu
def worker(n)
print "worker:", n
n = int(1e4)
for i in range(n):
for j in range(n):
i*j
return
if __name__ == '__main__':
jobs = []
for i in range(5):
p = mu.Process(target=worker, args=(i,))
jobs.append(p)
p.start()
and I can get the numbers of processors (cpu cores) with
np = mu.cpu_count()
but if I have a list of process, how I can launch without overcharge the processor ?
if I have a quad core, how I can launch first 4 process? and when finish a process launch other.
References
http://pymotw.com/2/multiprocessing/basics.html

I would suggest side stepping the problem and using multiprocessing.Pool (example, api).
(modified from the example in the docs)
from multiprocessing import Pool
def f(x):
return x*x
if __name__ == '__main__':
num_proc = multiprocessing.cpu_count()
pool = Pool(processes=num_proc)
res = pool.map(f, range(10))
Alternately, you can set up a producer/consumer scheme and have a fixed number of long running sub-processes.
A third really quick and dirty way is using one mu.Queue. Note that get blocks until it gets a result back.
import multiprocessing as mu
import time
res = mu.Queue()
def worker(n):
print "worker:", n
time.sleep(1)
res.put(n)
return
if __name__ == '__main__':
jobs = []
np = mu.cpu_count()
print np
# start first round
for j in range(np):
p = mu.Process(target=worker, args=(j,))
jobs.append(p)
p.start()
# every time one finishes, start the next one
for i in range(np,15):
r = res.get()
print 'res', r
p = mu.Process(target=worker, args=(i,))
jobs.append(p)
p.start()
# get the remaining processes
for j in range(np):
r = res.get()
print 'res', r

I make this solution
import multiprocessing as mu
def worker(n):
print "worker:", n
n = int(1e4/2)
for i in range(n):
for j in range(n):
i*j
return
if __name__ == '__main__':
jobs = []
for i in range(5):
p = mu.Process(target=worker, args=(i,))
jobs.append(p)
running = []
np = mu.cpu_count()
for i in range(np):
p = jobs.pop()
running.append(p)
p.start()
while jobs != []:
for r in running:
if r.exitcode == 0:
running.remove(r)
p = jobs.pop()
p.start()
running.append(p)