I'm trying to profile an instance method, so I've done something like:
import cProfile
class Test():
def __init__(self):
pass
def method(self):
cProfile.runctx("self.method_actual()", globals(), locals())
def method_actual(self):
print "Run"
if __name__ == "__main__":
Test().method()
But now problems arise when I want "method" to return a value that is computed by "method_actual". I don't really want to call "method_actual" twice.
Is there another way, something that can be thread safe? (In my application, the cProfile data are saved to datafiles named by one of the args, so they don't get clobbered and I can combine them later.)
I discovered that you can do this:
prof = cProfile.Profile()
retval = prof.runcall(self.method_actual, *args, **kwargs)
prof.dump_stats(datafn)
The downside is that it's undocumented.
An option for any arbitrary code:
import cProfile, pstats, sys
pr = cProfile.Profile()
pr.enable()
my_return_val = my_func(my_arg)
pr.disable()
ps = pstats.Stats(pr, stream=sys.stdout)
ps.print_stats()
Taken from https://docs.python.org/2/library/profile.html#profile.Profile
I was struggling with the same problem and used a wrapper function to get over direct return values. Instead of
cP.runctx("a=foo()", globals(), locales())
I create a wrapper function
def wrapper(b):
b.append(foo())
and profile the call to the wrapper function
b = []
cP.runctx("wrapper(b)", globals(), locals())
a = b[0]
extracting the result of foo's computation from the out param (b) afterwards.
I created a decorator:
import cProfile
import functools
import pstats
def profile(func):
#functools.wraps(func)
def inner(*args, **kwargs):
profiler = cProfile.Profile()
profiler.enable()
try:
retval = func(*args, **kwargs)
finally:
profiler.disable()
with open('profile.out', 'w') as profile_file:
stats = pstats.Stats(profiler, stream=profile_file)
stats.print_stats()
return retval
return inner
Decorate your function or method with it:
#profile
def somefunc(...):
...
Now that function will be profiled.
Alternatively, if you'd like the raw, unprocessed profile data (e.g. because you want to run the excellent graphical viewer RunSnakeRun on it), then:
import cProfile
import functools
import pstats
def profile(func):
#functools.wraps(func)
def inner(*args, **kwargs):
profiler = cProfile.Profile()
profiler.enable()
try:
retval = func(*args, **kwargs)
finally:
profiler.disable()
profiler.dump_stats('profile.out')
return retval
return inner
This is a minor improvement on several of the other answers on this page.
I think #detly the .runcall() is basically the best answer, but for completeness, I just wanted to take #ThomasH 's answer to be function independent:
def wrapper(b, f, *myargs, **mykwargs):
try:
b.append(f(*myargs, **mykwargs))
except TypeError:
print 'bad args passed to func.'
# Example run
def func(a, n):
return n*a + 1
b = []
cProfile.runctx("wrapper(b, func, 3, n=1)", globals(), locals())
a = b[0]
print 'a, ', a
Related
I'm trying to start and stop the line profiling of a Python function multiple times during runtime. Therefore I'd like to reset the already collected stats when starting a new profiling. Is there a way to do that?
In lack of an obvious solution I also tried replacing the line profiler lp with a fresh instance:
#!/usr/bin/env python3
from line_profiler import LineProfiler
lp = LineProfiler()
#lp
def count():
return sum(range(1_000_000))
count()
lp.print_stats()
# reset line profiler
new_lp = LineProfiler()
for f in lp.functions:
new_lp(f)
lp = new_lp
count()
lp.print_stats()
But somehow the new stats are empty, possibly because the function count() can't be wrapped twice?
I came up with the following solution based of a new profiler class. Every time a profiling is started, it creates a new instance of LineProfiler. The key is to store wrapped functions next to the originals, so that they can be reset when stopping the profiler.
from typing import Optional
from line_profiler import LineProfiler
from functools import wraps
class MyLineProfiler:
def __init__(self):
self.functions: list[list] = []
self.line_profiler: Optional[LineProfiler] = None
def __call__(self, func):
index = len(self.functions)
#wraps(func)
def wrap(*args, **kw):
return self.functions[index][1](*args, **kw)
self.functions.append([func, func])
return wrap
def start(self):
self.line_profiler = LineProfiler()
for f in self.functions:
f[1] = self.line_profiler(f[0])
def stop(self, *, print: bool = True):
for f in self.functions:
f[1] = f[0]
if self.line_profiler and print:
self.line_profiler.print_stats()
def reset(self):
self.stop(print=False)
self.start()
The wrapped functions call whatever is currently stored at functions[index][1], either the original func (when no profiling is stopped) or the decorated one (when start() was called).
It can be used as follows:
profile = MyLineProfiler()
#profile
def count():
return sum(range(1_000_000))
count()
profile.start()
count()
count()
profile.stop()
profile.start()
count()
profile.stop()
I am using the python unittest module for testing a file that takes a command line argument. The argument is a file name which is then passed into a function like so:
file_name = str(sys.argv[1])
file = open(file_name)
result = main_loop(file)
print(result)
My test is set up like so:
class testMainFile(unittest.TestCase):
def test_main_loop(self):
file = open('file_name.json')
result = main_file.main_loop(file)
self.assertEqual(result, 'Expected Result')
if __name__ == 'main':
unittest.main()
When I run the test I get an "IndexError: list index out of range".
I tried passing the argument when running the test but to no avail. How do I run my test without error?
I think you have couple of options here. Firstly go to documentation and checkout patch because i think you can get away with
from unittest.mock import patch
#patch('sys.argv', ['mock.py', 'test-value'])
def test_main_loop(self):
Options for fun:
One would be simply to override the sys.argv next to your call
def test_main_loop(self):
file = open('file_name.json')
+ orginal_argv = sys.argv
+ sys.argv = ['mock argv', 'my-test-value']
result = main_file.main_loop(file)
+ sys.argv = orginal_argv
self.assertEqual(result, 'Expected Result')
Second would be to create a simple wrapper for your function
def set_sys_argv(func: Callable):
sys.argv = ['mock.py', 'my_test_value']
def wrapper(*args, **kwargs):
func()
return wrapper
and use it with test function
#set_sys_argv
def test_main_loop(self):
We can improve it slightly and make it more generic making a decorator that accepts the values to mock
def set_sys_argv(*argv):
sys.argv = argv
def _decorator(func: Callable):
def wrapper(*args, **kwargs):
func()
return wrapper
return _decorator
and use it similarly to patch
#set_sys_argv('mock.py', 'test-value')
def test_main_loop(self):
Third would be to create a context manager, likewise:
class ReplaceSysArgv(list):
def __enter__(self):
self._argv = sys.argv
sys.argv = ['mock', 'my-test-value']
return self
def __exit__(self, *args):
sys.argv = self._argv
and use it with your code
def test_main_loop(self):
file = open('file_name.json')
with ReplaceSysArgv():
result = main_file.main_loop(file)
self.assertEqual(result, 'Expected Result')
you have to push the arguments onto sys.argv before retrieving them (if your code is pulling from command-line arguments - it's unclear to me where in your test you're using the command-line arguments but I digress)
so something like first doing
import sys
sys.argv = ['mock_filename.py', 'json_file.json']
#... continue with rest of program / test.
I tried to write a decorator to compute the time of the computation for methods in a class, but I also have a lot of properties.
I tried to write a decorator for a property as follows:
def timer(method):
import time
#property
def wrapper(*args, **kw):
start = time.time()
result = method
end = time.time()
print('Elapsed time for: {} is: {}s'.format(method.__name__,(end-start)*1000))
return result
return wrapper
I can't get the name of the property as written, moreover perhaps you would suggest to write it in another way?
You would need to stack decorators:
def timer(method):
import time
def wrapper(*args, **kwargs):
start = time.time()
result = method(*args, **kwargs) # note the function call!
end = time.time()
print('Elapsed time for: {} is: {}s'.format(method.__name__,(end-start)*1000))
return result
return wrapper
class X:
#property
#timer
def some_method(self):
# timed code
return 0
>>> x = X()
>>> x.some_method
Elapsed time for: some_method is: 0.0050067901611328125s
0
I have this minimal example:
from functools import wraps
from concurrent import futures
import random
def decorator(func):
num_process = 4
def impl(*args, **kwargs):
with futures.ProcessPoolExecutor() as executor:
fs = []
for i in range(num_process):
fut = executor.submit(func, *args, **kwargs)
fs.append(fut)
result = []
for f in futures.as_completed(fs):
result.append(f.result())
return result
return impl
#decorator
def get_random_int():
return random.randint(0, 100)
if __name__ == "__main__":
result = get_random_int()
print(result)
If we try to run this function I think we will have the following error:
_pickle.PicklingError: Can't pickle <function get_random_int at 0x7f06cee666a8>: it's not the same object as __main__.get_random_int
I think the main issue here is that the "wraps" decorator itself alters the func object and thus make it impossible to pickle. I found this rather strange. I am just wondering if there is any way to get around this behavior? I would want to use wraps if possible. Thanks!
This is because run_in_executor is calling functools.partial on the decorated function see: https://docs.python.org/3/library/asyncio-eventloop.html#asyncio-pass-keywords The picklability of partial objects is spotty (see: Are partial functions "officially" picklable?) but See this comment over here Pickling wrapped partial functions partial functions are only pickleable when the function being pickled is in the global name sapce. We know run_in_executor with a ProcessPoolExecutor will work for non wrapped functions since that pattern is documented in asyncio. To get around this I decorate a dummy function and pass the function I want to be executed in multiple processes as an argument to the decorator
from functools import wraps
from concurrent import futures
import random
def decorator(multiprocess_func):
def _decorate(func):
num_process = 4
def impl(*args, **kwargs):
with futures.ProcessPoolExecutor() as executor:
fs = []
for i in range(num_process):
fut = executor.submit(multiprocess_func, *args, **kwargs)
fs.append(fut)
result = []
for f in futures.as_completed(fs):
result.append(f.result())
return result
return impl
return _decorate
def _get_random_int():
return random.randint(0, 100)
#decorator(_get_random_int)
def get_random_int():
return _get_random_int()
if __name__ == "__main__":
result = get_random_int()
print(result)
I ultimately decided that not using a decorator was cleaner
from concurrent import futures
import random
def decorator(multiprocess_func):
num_process = 4
def impl(*args, **kwargs):
with futures.ProcessPoolExecutor(max_workers=num_process) as executor:
fs = []
for i in range(num_process):
fut = executor.submit(multiprocess_func, *args, **kwargs)
fs.append(fut)
result = []
for f in futures.as_completed(fs):
result.append(f.result())
return result
return impl
def _get_random_int():
return random.randint(0, 100)
get_random_int = decorator(_get_random_int)
if __name__ == "__main__":
result = get_random_int()
print(result)
Similar to the linked answer above about pickling wrapped partial functions.
I am using the following approach to pass in an optional argument to a decorator:
def wait(func=None, delay=1.0):
def decorator_wait(func):
def wrapper_wait(*args, **kwargs):
time.sleep(delay)
return func(*args, **kwargs)
return wrapper_wait
return decorator_wait(func) if func is not None else decorator_wait
#wait
def print_something(something):
print (something)
#wait(delay=0.2)
def print_something_else(something):
print (something)
The above code looks pretty difficult to follow though with all the nesting. Is there another approach to do the above, or is this the only method available for something like this?
You can avoid having to remember "do I need to call this or not?" by removing the func argument from the wait function, and remembering to always call your decorator-returner.
It would look like this:
def wait(delay=1.0):
def decorator_wait(func):
def wrapper_wait(*args, **kwargs):
time.sleep(delay)
return func(*args, **kwargs)
return wrapper_wait
return decorator_wait
#wait()
def print_something(something):
print (something)
#wait(delay=0.2)
def print_something_else(something):
print (something)
print_something("hello")
# 1 second delay, then 'hello'
print_something_else("there")
# 0.2 second delay, then 'there'
You just have to remember that wait will always return the decorator, so you have to use () when decorating your functions.
I think it is a little bit better:
import functools
import time
def wait(func=None, delay=1.0):
if func is None:
return lambda func: wait(func=func, delay=delay)
#functools.wraps(func) # this is good practice to use it see: https://stackoverflow.com/questions/308999/what-does-functools-wraps-do
def _wrapper(*args, **kwargs):
time.sleep(delay)
return func(*args, **kwargs)
return _wrapper
#wait
def test():
return
#wait(delay=3)
def test2():
return
You can write classes having a __call__ method, instead of writing a bunch of nested defs.
It sounds like you want a decorator Wait which haults
program execution for a few seconds.
If you don't pass in a Wait-time
then the default value is 1 seconds.
Use-cases are shown below.
##################################################
#Wait
def print_something(something):
print(something)
##################################################
#Wait(3)
def print_something_else(something_else):
print(something_else)
##################################################
#Wait(delay=3)
def print_something_else(something_else):
print(something_else)
When Wait has an argument, such as #Wait(3), then the call Wait(3)
is executed before anything else happens.
That is, the following two pieces of code are equivalent
#Wait(3)
def print_something_else(something_else):
print(something_else)
###############################################
return_value = Wait(3)
#return_value
def print_something_else(something_else):
print(something_else)
This is a problem.
if `Wait` has no arguments:
`Wait` is the decorator.
else: # `Wait` receives arguments
`Wait` is not the decorator itself.
Instead, `Wait` ***returns*** the decorator
One solution is shown below:
Let us begin by creating the following class, DelayedDecorator:
import io
class DelayedDecorator:
def __init__(i, cls, *args, **kwargs):
print("Delayed Decorator __init__", cls, args, kwargs)
i._cls = cls
i._args = args
i._kwargs = kwargs
def __call__(i, func):
print("Delayed Decorator __call__", func)
if not (callable(func)):
import io
with io.StringIO() as ss:
print(
"If only one input, input must be callable",
"Instead, received:",
repr(func),
sep="\n",
file=ss
)
msg = ss.getvalue()
raise TypeError(msg)
return i._cls(func, *i._args, **i._kwargs)
Now we can write things like:
dec = DelayedDecorator(Wait, delay=4)
#dec
def delayed_print(something):
print(something)
Note that:
dec does not not accept multiple arguments.
dec only accepts the function to be wrapped.
import inspect
class PolyArgDecoratorMeta(type):
def __call__(Wait, *args, **kwargs):
try:
arg_count = len(args)
if (arg_count == 1):
if callable(args[0]):
SuperClass = inspect.getmro(PolyArgDecoratorMeta)[1]
r = SuperClass.__call__(Wait, args[0])
else:
r = DelayedDecorator(Wait, *args, **kwargs)
else:
r = DelayedDecorator(Wait, *args, **kwargs)
finally:
pass
return r
import time
class Wait(metaclass=PolyArgDecoratorMeta):
def __init__(i, func, delay = 2):
i._func = func
i._delay = delay
def __call__(i, *args, **kwargs):
time.sleep(i._delay)
r = i._func(*args, **kwargs)
return r
The following two pieces of code are equivalent:
#Wait
def print_something(something):
print (something)
##################################################
def print_something(something):
print(something)
print_something = Wait(print_something)
We can print "something" to the console very slowly, as follows:
print_something("something")
#################################################
#Wait(delay=1)
def print_something_else(something_else):
print(something_else)
##################################################
def print_something_else(something_else):
print(something_else)
dd = DelayedDecorator(Wait, delay=1)
print_something_else = dd(print_something_else)
##################################################
print_something_else("something")
Final Notes
It may look like a lot of code, but you don't have to write the classes DelayedDecorator and PolyArgDecoratorMeta every-time. The only code you have to personally write something like as follows, which is fairly short:
from PolyArgDecoratorMeta import PolyArgDecoratorMeta
import time
class Wait(metaclass=PolyArgDecoratorMeta):
def __init__(i, func, delay = 2):
i._func = func
i._delay = delay
def __call__(i, *args, **kwargs):
time.sleep(i._delay)
r = i._func(*args, **kwargs)
return r