I have a function that factor a number. It depends on some random condition.
So what I am trying to do it's to run multiple processors in this function and the processor that finds the factor first returns the value and all processors terminate.
What I have so far is very wrong. The processors are not terminating and I also don't know how to get the value that was returned by the function
flag = False
def rho(n, processor):
while True:
x = random.randrange(1, n-1)
x2 = x
gcd = 1
c = random.randrange(1, n-1)
while gcd == 1:
x = (x**2 + c) % n
x2 = (x2**2 + c) % n
x2 = (x2**2 + c) % n
gcd = math.gcd(abs(x - x2), n)
if gcd != n:
flag = True
print("Factor was found from "+process+" and is ", gcd)
return gcd
if __name__ == "__main__":
p1 = multiprocessing.Process(target=rho, args=(91, "process 1" ))
p2 = multiprocessing.Process(target=rho, args=(91, "process 2"))
p1.start()
p2.start()
if flag:
p1.terminate()
p2.terminate()
The output is:
Factor was found from process 2 and is 13
Factor was found from process 1 and is 7
You can use multiprocessing.Pool and it's methods map(), imap_unordered() etc. These will return also values from worker functions.
Example (I used time.sleep() to simulate some time-intesive computation):
from time import sleep
from multiprocessing import Pool
def rho(params):
n, processor = params
# your computation here
# ...
sleep(n)
print("Factor was found from " + processor + " and is 42")
return 42
if __name__ == "__main__":
with Pool() as pool:
for result in pool.imap_unordered(
rho, ((10, "process 1"), (1, "process 2"))
):
print("Result I got:", result)
break # <-- I don't want other results, so break
Prints:
Factor was found from process 2 and is 42
Result I got: 42
EDIT: Two different functions:
from time import sleep
from multiprocessing import Pool
def fn1(n, p):
sleep(n)
print("Factor was found from " + p + " and is 42")
return 42
def fn2(n, p):
sleep(n)
print("Factor was found from " + p + " and is 99")
return 99
def rho(params):
what_to_call, n, processor = params
return what_to_call(n, processor)
if __name__ == "__main__":
with Pool() as pool:
for result in pool.imap_unordered(
rho, ((fn1, 10, "process 1"), (fn2, 1, "process 2"))
):
print("Result I got:", result)
break # <-- I don't want other results, so break
Related
import threading
# The number to add.
# Change this for your experiment
n = 1000000
# Result variable
result = 0
def thread1(count):
global result
x = 0
for i in range(1, count + 1):
x = x + i
result = x
def thread2():
global result
result = result * 3
th1 = threading.Thread(target = thread1, args=[n, ])
th2 = threading.Thread(target = thread2)
th1.start()
th2.start()
correct = n * (n + 1) / 2 * 3
print("Correct result is %d." % correct)
print("Final result is %d." % result)
if result == correct:
print("CORRECT!")
else:
print("WRONG!")
When running the code with n = 10, we will get correct result. But when running the code with n= 1000000, we would get incorrect results. May I know what's wrong with n= 1000000 that leads to incorrect execution?
I'm having trouble creating tests that review the stout output of a program. The issue is that not all tests run - but this also seems to vary somewhat. Strange error; I've been playing around with flushing, thinking that that's the issue, but the problem persists.
The test code is here...
import sys
from io import StringIO
import contextlib
import unittest
import tested
class ExampleTestCase(unittest.TestCase):
def setUp(self) -> None:
f = StringIO();
with contextlib.redirect_stdout(f):
tested.main()
f.flush()
self.output = f.getvalue()
def test_n_value(self):
print("testing n...", end="")
sys.stdout.flush()
self.assertTrue('44.27715216205352' in self.output, "n value should be 44.27715216205352")
print("OK")
def test_FV_value(self):
print("testing FV...", end="")
sys.stdout.flush()
self.assertTrue('25937.424601000024' in self.output, "FV value should be 25937.424601000024")
print("OK")
def test_r_value(self):
print("testing r...", end="")
sys.stdout.flush()
self.assertTrue('0.117461894308801895' in self.output, "r value should be 0.17461894308801895")
print("OK")
def test_PV_value(self):
print("testing PV...", end="")
sys.stdout.flush()
self.assertTrue('3855.432894295314' in self.output, "PV value should be 3855.432894295314")
print("OK")
if __name__ == '__main__':
unittest.main()
The code that is being tested is here...
import math
def main():
# Calculate FV given r=0.10, n=10, PV=10000
r = 0.10
n = 10
PV = 10000
FV = PV * (1 + r) ** n
print(FV)
# Calculate PV given r=0.10, n=10, FV=10000
r = 0.10
n = 10
FV = 10000
PV = FV / (1 + r) ** n
print(PV)
# Calculate n given r=0.07, PV=10000, FV=200000
r = 0.07
FV = 200000
PV = 10000
n = math.log(FV / PV) / math.log(1 + r)
print(n)
# Calculate r given n=10, FV=100000, PV=20000
# print(math.log())
FV = 100000
PV = 20000
n = 10
r = (FV / PV) ** (1 / n) - 1
print(r)
main()
I found a solution.... (the flushes were not necessary, and the 'getvalue()' call needed to be within the with statement....
import io
import sys
import contextlib
import unittest
import tested
class ExampleTestcaset(unittest.TestCase):
def setUp(self) -> None:
f = io.StringIO()
with contextlib.redirect_stdout(f):
tested.main()
self.output = f.getvalue()
def test_n_value(self):
print("Testing n: ", end="")
self.assertTrue('44.27715216205352' in self.output, self.output + "n value should be 44.27715216205352")
print("OK")
def test_FV_value(self):
print("Testing FV: ", end="")
self.assertTrue('25937.424601000024' in self.output, "FV value should be 25937.424601000024")
print("OK")
def test_r_value(self):
print("Testing r: ", end="")
self.assertTrue('0.17461894308801895' in self.output, "r value should be 0.17461894308801895")
print("OK")
def test_PV_value(self):
print("Testing PV: ", end="")
self.assertTrue('3855.432894295314' in self.output, "PV value should be 3855.432894295314")
print("OK")
if __name__ == '__main__':
unittest.main()
I am trying to use multiprocesing library to compare the performance of my processor on 1 core vs 2 cores.
Therefore I calculate a great product using 1 loop, 2 loops on 1 core, and 2 loops on 2 cores (1 core/loop). The problem is that the value of D1.result and D2.result are 0 although they are expected to be the product of the "half/loop".
The code is the following:
import random
from multiprocessing import Process as Task, freeze_support
N = 10 ** 3
l = [random.randint(2 ** 999, 2 ** 1000 - 1) for x in range(N)]
# ---------------------------------------------------------------
class Loop:
def __init__(self):
self.result=0
def boucle(self,start,end):
self.result = l[start]
for v in l[start+1:end]:
self.result = self.result*v
# ---------------------------------------------------------------
if __name__=="__main__":
print("1 Loop without multiprocessing")
A=Loop()
sta = time.time()
ra=A.boucle(0,N)
end = time.time()
print("--> Time :", end - sta)
#----------------------------------------------------------------------
print("2 Loops without multiprocessing")
B1=Loop()
B2=Loop()
sta = time.time()
rb1 = B1.boucle(0, N//2)
rb2 = B2.boucle(N//2, N)
rb = B1.result*B2.result
end = time.time()
print("--> Time :", end - sta)
if rb - A.result == 0 :
check="OK"
else :
check="fail"
print("--> Check :", check)
# ----------------------------------------------------------------------
print("2 Loops with multiprocessing")
freeze_support()
D1=Loop()
D2=Loop()
v1 = Task(target=D1.boucle, args=(0,N//2))
v2 = Task(target=D2.boucle, args=(N//2,N))
sta = time.time()
v1.start()
v2.start()
v1.join()
v2.join()
rd = D1.result*D2.result
end = time.time()
print("D1",D1.result)
print("D2",D2.result)
print("--> Time :", end - sta)
if rd - A.result == 0 :
check="OK"
else :
check="fail"
print("--> Check :", check)
The result of this code is :
1 Loop without multiprocessing
--> Time : 0.5025153160095215
2 Loops without multiprocessing
--> Time : 0.283463716506958
--> Check : OK
2 Loops with multiprocessing
D1 0
D2 0
--> Time : 0.2579989433288574
--> Check : fail
Process finished with exit code 0
Why D1 0 and D2 0 and not the result of the loop ?
Thanks you !
The issue with this code is shown when D1 and D2 are displayed:
In multiprocessing, tasks are executed in a forked process. This process got a copy of the data.
In each forked process the value is properly computed but it is never sent back to main process.
To work around this you can:
Use shared memory to store the result, but in this case you are limited to C types. Your numbers do not fit on 64 bits (max integer size in C), so this is not a good solution.
Use a pool of process, thus data will be shared using queues and you will be able to manage real python types.
This last option requires that "boucle" function returns the result.
Here is the code:
import random
from multiprocessing import Process as Task, freeze_support, Pool
import time
N = 10 ** 3
l = [random.randint(2 ** 999, 2 ** 1000 - 1) for x in range(N)]
# ---------------------------------------------------------------
class Loop:
def __init__(self):
self.result = 0
def boucle(self, start, end):
self.result = l[start]
for v in l[start + 1:end]:
self.result = self.result * v
return self.result
# ---------------------------------------------------------------
if __name__ == "__main__":
print("1 Loop without multiprocessing")
A = Loop()
sta = time.time()
ra = A.boucle(0, N)
end = time.time()
print("--> Time :", end - sta)
# ----------------------------------------------------------------------
print("2 Loops without multiprocessing")
B1 = Loop()
B2 = Loop()
sta = time.time()
rb1 = B1.boucle(0, N // 2)
rb2 = B2.boucle(N // 2, N)
rb = B1.result * B2.result
end = time.time()
print("--> Time :", end - sta)
if rb - A.result == 0:
check = "OK"
else:
check = "fail"
print("--> Check :", check)
# ----------------------------------------------------------------------
print("2 Loops with multiprocessing")
freeze_support()
D1 = Loop()
D2 = Loop()
pool = Pool(processes=2)
with pool:
sta = time.time()
sta = time.time()
rb1 = pool.apply_async(B1.boucle, (0, N // 2))
rb2 = pool.apply_async(B2.boucle, (N // 2, N))
v1 = rb1.get()
v2 = rb2.get()
rd = v1 * v2
end = time.time()
print("D1", D1.result)
print("D2", D2.result)
print("--> Time :", end - sta)
if rd - A.result == 0:
check = "OK"
else:
check = "fail"
print("--> Check :", check)
And the result:
1 Loop without multiprocessing
--> Time : 0.3473360538482666
2 Loops without multiprocessing
--> Time : 0.18696999549865723
--> Check : OK
2 Loops with multiprocessing
D1 0
D2 0
--> Time : 0.1116642951965332
--> Check : OK
You can also use map with the pool to get the value back, but I have not tried it in this case because you only call 2 functions, and pool workers get tasks by "packets of functions - see maxtaskperchild" so it could be possible that only one worker will have taken the 2 functions for itself
I wrote 3 different codes to compare having threads vs. not having threads. Basically measuring how much time I save by using threading and the result didn't make any sense.
Here are my codes:
import time
def Function():
global x
x = 0
while x < 300000000:
x += 1
print x
e1 = time.clock()
E1 = time.time()
Function()
e2 = time.clock()
E2 = time.time()
print e2 - e1
print E2 - E1
When I ran this, I got this as output:
26.6358742929
26.6440000534
Then I wrote another function as shown below and split counting up to 300 million into counting 3, 100 millions:
import time
def Function():
global x
x = 0
while x < 100000000:
x += 1
print x
def Function2():
global x
x = 0
while x < 100000000:
x += 1
print x
def Function3():
global x
x = 0
while x < 100000000:
x += 1
print x
e1 = time.clock()
E1 = time.time()
Function()
Function2()
Function3()
e2 = time.clock()
E2 = time.time()
print e2 - e1
print E2 - E1
The output of the following function was:
26.0577638729
26.0629999638
and lastly I created 3 threads and ran each function on a single thread:
import time
import threading
e1 = time.clock()
E1 = time.time()
def Function1():
global x
x = 0
while x < 100000000:
x += 1
print x
def Function2():
global x
x = 0
while x < 100000000:
x += 1
print x
def Function3():
global x
x = 0
while x < 100000000:
x += 1
print x
new_thread1 = threading.Thread(target = Function1() , args = ())
new_thread2 = threading.Thread(target = Function2(), args = ())
new_thread3 = threading.Thread(target = Function3(), args = ())
e1 = time.clock()
E1 = time.time()
new_thread1.start()
new_thread2.start()
new_thread3.start()
e2 = time.clock()
E2 = time.time()
print e2 - e1
print E2 - E1
The out put of this one was:
0.000601416222253
0.0
These numbers make no sense to me. I'm just trying to measure how much time does threading save me. I've looked up in the documentation and using time.time
and time.clock made sense to me, but it doesn't make sense here. Also, the actual time for 1st and 2nd snippet were about 10 seconds and 3rd one about 5
you are calling it wrong ....
new_thread1 = threading.Thread(target = Function1 , args = ())
note that you should not CALL the function when you create the thread
those timings really mean nothing they are both essentially zero because all you are timing is 3 instant return function calls to start
note to get the output you will need to wait for each thread to finish (since your current code does not do this )
EDIT FOR MORE INFO
with threading you are locked by the gil to one python instruction at a time... typically this is not a problem since you are usually waiting on disk io... In your example code however it is 100% computation so threading really doesnt improve your time ... Multiprocessing may as demonstrated below
import time
import threading
import multiprocessing
def fn():
'''since all 3 functions were identical you can just use one ...'''
x = 0
while x < 100000000:
x += 1
def TEST_THREADS():
new_thread1 = threading.Thread(target = fn , args = ())
new_thread2 = threading.Thread(target = fn, args = ())
new_thread3 = threading.Thread(target = fn, args = ())
new_thread1.start()
new_thread2.start()
new_thread3.start()
new_thread1.join()
new_thread2.join()
new_thread3.join()
def TEST_NORMAL():
fn()
fn()
fn()
def TEST_MULTIPROCESSING():
new_thread1 = multiprocessing.Process(target = fn , args = ())
new_thread2 = multiprocessing.Process(target = fn, args = ())
new_thread3 = multiprocessing.Process(target = fn, args = ())
new_thread1.start()
new_thread2.start()
new_thread3.start()
new_thread1.join()
new_thread2.join()
new_thread3.join
if __name__ == "__main__":
'''It is very important to use name == __main__ guard code with threads and multiprocessing'''
import timeit
print "Time to Run 1x: %0.2fs"%(timeit.timeit(fn,number=1),)
print "NORMAL:%0.2fs"%(timeit.timeit(TEST_NORMAL,number=1),)
print "Threaded: %0.2fs"%(timeit.timeit(TEST_THREADS,number=1),)
print "Multiprocessing: %0.2fs"%(timeit.timeit(TEST_MULTIPROCESSING,number=1),)
I get the following output
Time to Run 1x: 3.71181102665
NORMAL:11.0136830117
Threaded: 23.392143814
Multiprocessing: 3.80878260515
I need to code a program that can get values from this
Very interesting question. Actually, in your case, it is very simple. You can parse the entire source string with ast module, like this
import ast
import operator
functions = {
"add": operator.add,
"abs": operator.abs,
"multiply": operator.mul
}
def recursive_evaluation(current_element):
if isinstance(current_element, ast.Module):
return recursive_evaluation(current_element.body[0].value)
elif isinstance(current_element, ast.Call):
function = functions[current_element.func.id]
args = [recursive_evaluation(item) for item in current_element.args]
return function(*args)
elif isinstance(current_element, ast.Num):
return current_element.n
else:
raise ValueError("Unknown Element " + str(current_element))
source = "abs(add(add(9465,38),multiply(add(63303,146),46)))"
print recursive_evaluation(ast.parse(source))
source = "add(1, -2)"
print recursive_evaluation(ast.parse(source))
source = "abs(add(1, -2))"
print recursive_evaluation(ast.parse(source))
Output
2928157
-1
1
Interesting problem, here is a potential solution. No doubt you could make a much more elegant solution using libraries or lambdas etc. as thefourtheye did in their answer, but this seems to work.
I have done a few test cases at the bottom, set the global verbose to True if you want debug info:
# globals
verbose = False # set to True if you want debug info printed
max_iter = 1000 # this stops infinate loops incase the code does not allow for some input
def solve(problem_str):
def multiply(arg_list):
x = 1
for i in arg_list:
x *= i
return x
def find_innermost(x_str):
a, b, c, i = [0], [0], 0, 0
while True:
i += 1
start = a[-1]+1
a.append(x_str.find('(', start)) # find next (
b.append(x_str.find(',', start)) # find next ,
c = x_str.find(')', start) # find next )
if (a[-1] > c) or (a[-1] == -1):
if (b[-2] > a[-3]) and (b[-2] < a[-2]):
return x_str[b[-2]+1:c+1]
else:
return x_str[a[-3]+1:c+1]
if i >= max_iter:
raise Exception("Infinite loop")
def do_sum(x_str):
args = [int(x) for x in x_str[x_str.find('(')+1:x_str.find(')')].split(',')]
task = x_str[:3].lower()
if task == 'add':
return sum(args)
elif task == 'sub':
return args[0] - sum(args[1:])
elif task == 'abs':
return abs(args.pop())
elif task == 'mul':
return multiply(args)
else:
print x_str + ': Task not recognised, please modify program or input'
raise Exception("Invalid input")
i = 0
while True:
i += 1
if verbose: print 'debug: problem_str:', problem_str
if problem_str.count('(') > 1:
x_str = find_innermost(problem_str)
else:
x_str = problem_str
if verbose: print '.'*6, 'x_str:\t', x_str
x = do_sum(x_str)
if verbose: print '.'*6, 'x:\t', x, '\n'
problem_str = problem_str.replace(x_str, str(x))
if problem_str.count('(') == 0:
return int(problem_str)
if i >= max_iter:
raise Exception("Infinite loop")
if __name__ == '__main__':
p1 = 'abs(add(add(9465,38),multiply(add(63303,146),46)))'
p2 = 'abs(add(multiply(95,multiply(-1,multiply(13,18875))),multiply(-1,add(18293,26))))'
p3 = 'abs(add(subtract(add(add(151,26875),122),254),subtract(237,multiply(-1,56497))))'
r1, r2, r3 = solve(p1), solve(p2), solve(p3)
print 'p1 evaluates to:', r1
print 'p2 evaluates to:', r2
print 'p3 evaluates to:', r3
Let me know if you have any questions about the code.