I am using the following approach to pass in an optional argument to a decorator:
def wait(func=None, delay=1.0):
def decorator_wait(func):
def wrapper_wait(*args, **kwargs):
time.sleep(delay)
return func(*args, **kwargs)
return wrapper_wait
return decorator_wait(func) if func is not None else decorator_wait
#wait
def print_something(something):
print (something)
#wait(delay=0.2)
def print_something_else(something):
print (something)
The above code looks pretty difficult to follow though with all the nesting. Is there another approach to do the above, or is this the only method available for something like this?
You can avoid having to remember "do I need to call this or not?" by removing the func argument from the wait function, and remembering to always call your decorator-returner.
It would look like this:
def wait(delay=1.0):
def decorator_wait(func):
def wrapper_wait(*args, **kwargs):
time.sleep(delay)
return func(*args, **kwargs)
return wrapper_wait
return decorator_wait
#wait()
def print_something(something):
print (something)
#wait(delay=0.2)
def print_something_else(something):
print (something)
print_something("hello")
# 1 second delay, then 'hello'
print_something_else("there")
# 0.2 second delay, then 'there'
You just have to remember that wait will always return the decorator, so you have to use () when decorating your functions.
I think it is a little bit better:
import functools
import time
def wait(func=None, delay=1.0):
if func is None:
return lambda func: wait(func=func, delay=delay)
#functools.wraps(func) # this is good practice to use it see: https://stackoverflow.com/questions/308999/what-does-functools-wraps-do
def _wrapper(*args, **kwargs):
time.sleep(delay)
return func(*args, **kwargs)
return _wrapper
#wait
def test():
return
#wait(delay=3)
def test2():
return
You can write classes having a __call__ method, instead of writing a bunch of nested defs.
It sounds like you want a decorator Wait which haults
program execution for a few seconds.
If you don't pass in a Wait-time
then the default value is 1 seconds.
Use-cases are shown below.
##################################################
#Wait
def print_something(something):
print(something)
##################################################
#Wait(3)
def print_something_else(something_else):
print(something_else)
##################################################
#Wait(delay=3)
def print_something_else(something_else):
print(something_else)
When Wait has an argument, such as #Wait(3), then the call Wait(3)
is executed before anything else happens.
That is, the following two pieces of code are equivalent
#Wait(3)
def print_something_else(something_else):
print(something_else)
###############################################
return_value = Wait(3)
#return_value
def print_something_else(something_else):
print(something_else)
This is a problem.
if `Wait` has no arguments:
`Wait` is the decorator.
else: # `Wait` receives arguments
`Wait` is not the decorator itself.
Instead, `Wait` ***returns*** the decorator
One solution is shown below:
Let us begin by creating the following class, DelayedDecorator:
import io
class DelayedDecorator:
def __init__(i, cls, *args, **kwargs):
print("Delayed Decorator __init__", cls, args, kwargs)
i._cls = cls
i._args = args
i._kwargs = kwargs
def __call__(i, func):
print("Delayed Decorator __call__", func)
if not (callable(func)):
import io
with io.StringIO() as ss:
print(
"If only one input, input must be callable",
"Instead, received:",
repr(func),
sep="\n",
file=ss
)
msg = ss.getvalue()
raise TypeError(msg)
return i._cls(func, *i._args, **i._kwargs)
Now we can write things like:
dec = DelayedDecorator(Wait, delay=4)
#dec
def delayed_print(something):
print(something)
Note that:
dec does not not accept multiple arguments.
dec only accepts the function to be wrapped.
import inspect
class PolyArgDecoratorMeta(type):
def __call__(Wait, *args, **kwargs):
try:
arg_count = len(args)
if (arg_count == 1):
if callable(args[0]):
SuperClass = inspect.getmro(PolyArgDecoratorMeta)[1]
r = SuperClass.__call__(Wait, args[0])
else:
r = DelayedDecorator(Wait, *args, **kwargs)
else:
r = DelayedDecorator(Wait, *args, **kwargs)
finally:
pass
return r
import time
class Wait(metaclass=PolyArgDecoratorMeta):
def __init__(i, func, delay = 2):
i._func = func
i._delay = delay
def __call__(i, *args, **kwargs):
time.sleep(i._delay)
r = i._func(*args, **kwargs)
return r
The following two pieces of code are equivalent:
#Wait
def print_something(something):
print (something)
##################################################
def print_something(something):
print(something)
print_something = Wait(print_something)
We can print "something" to the console very slowly, as follows:
print_something("something")
#################################################
#Wait(delay=1)
def print_something_else(something_else):
print(something_else)
##################################################
def print_something_else(something_else):
print(something_else)
dd = DelayedDecorator(Wait, delay=1)
print_something_else = dd(print_something_else)
##################################################
print_something_else("something")
Final Notes
It may look like a lot of code, but you don't have to write the classes DelayedDecorator and PolyArgDecoratorMeta every-time. The only code you have to personally write something like as follows, which is fairly short:
from PolyArgDecoratorMeta import PolyArgDecoratorMeta
import time
class Wait(metaclass=PolyArgDecoratorMeta):
def __init__(i, func, delay = 2):
i._func = func
i._delay = delay
def __call__(i, *args, **kwargs):
time.sleep(i._delay)
r = i._func(*args, **kwargs)
return r
Related
I have written a generator as follows:
def my_generator():
i = 0
while i < 1000000:
i += 1
yield i
Assuming the generator cannot be executed in a second and in the test function I use a timeout decorator to guarantee the function should not run more than 1 second.
#timeout(1)
def test():
for i in my_generator:
print(i)
Unfortunately, the timeout don't work as I wanted, the function print all the number from 1 to 1000000 with more than 1 second.
In the decorator, I have tried gevent and KThread, but none of them can work.
Decorator using KThread:
class KThread(threading.Thread):
"""Subclass of threading.Thread, with a kill() method."""
def __init__(self, *args, **kwargs):
threading.Thread.__init__(self, *args, **kwargs)
self.killed = False
def start(self):
"""Start the thread."""
self.__run_backup = self.run
"""Force the Thread to install our trace."""
self.run = self.__run
threading.Thread.start(self)
def __run(self):
"""Hacked run function, which installs the trace."""
sys.settrace(self.globaltrace)
self.__run_backup()
self.run = self.__run_backup
def globaltrace(self, frame, why, arg):
if why == 'call':
return self.localtrace
else:
return None
def localtrace(self, frame, why, arg):
if self.killed:
if why == 'line':
raise SystemExit()
return self.localtrace
def kill(self):
self.killed = True
def timeout(seconds):
def timeout_decorator(func):
def _new_func(oldfunc, result, oldfunc_args, oldfunc_kwargs):
result.append(oldfunc(*oldfunc_args, **oldfunc_kwargs))
def _(*args, **kwargs):
result = []
'''create new args for _new_funcbecause
we want to get the func return val to result list
'''
new_kwargs = {
'oldfunc': func,
'result': result,
'oldfunc_args': args,
'oldfunc_kwargs': kwargs
}
thd = KThread(target=_new_func, args=(), kwargs=new_kwargs)
thd.start()
thd.join(seconds)
alive = thd.isAlive()
'''kill the child thread'''
thd.kill()
if alive:
alert_exce = u'function timeout for [%d s].' % seconds
raise Timeout(alert_exce)
else:
return result[0]
_.__name__ = func.__name__
_.__doc__ = func.__doc__
return _
return timeout_decorator
Decorator using gevent:
def g_timer(timeout_seconds=None, timeout_exception=TimeoutError, exception_message=None, module_name=None):
import gevent
from gevent import monkey
monkey.patch_all()
def decorate(func):
def wrapper(*args, **kwargs):
try:
t0 = time.time()
gevent.with_timeout(timeout_seconds, func, *args, **kwargs)
elapsed = time.time() - t0
except gevent.timeout.Timeout as e:
print("exception")
return wrapper
return decorate
Here is my way,but I feel it is not very simple, any better way?
import asyncio
import time
def timer_all(f):
if asyncio.iscoroutinefunction(f):
async def wrapper(*args, **kwargs):
now = time.time()
result = await f(*args, **kwargs)
print('used {}'.format(time.time() - now))
return result
else:
def wrapper(*args, **kwargs):
now = time.time()
result = f(*args, **kwargs)
print('used {}'.format(time.time() - now))
return result
return wrapper
there is a lot of decorator, retry, add log etc,all will write this way,a bit ugly,right?
While there is no real problems with repeating the same code in specialized decorators.
Here is how I'll approach a refactoring.
I will use a class decorator that keeps the accepts a pre-call function and a post-call function,
both of which will be called with an instance of the decorator.
The result of the pre-call function will be saved to an attribute of the decorator.
This is necessary for the special timing case where a delta needs to be computed.
I guess there may be other examples that may require the return value of a pre-call function execution.
I also save the result of the decorated function executed to the result attribute of the decorator instance. This allows the post call function to read this value for logging.
Here is an example implementation:
import asyncio
class WrapAll(object):
def __init__(self, pre=lambda _: None, post=lambda _: None):
self.pre = lambda : pre(self)
self.pre_val = None
self.result = None
self.post = lambda : post(self)
def __call__(self, fn):
if asyncio.iscoroutinefunction(fn):
async def wrap(*args, **kwargs):
self.pre_val = self.pre()
self.result = await fn(*args, *kwargs)
self.post()
return self.result
else:
def wrap(*args, **kwargs):
self.pre_val = self.pre()
self.result = fn(*args, *kwargs)
self.post()
return self.result
return wrap
Timer Example
import asyncio
import time
timer = dict(
pre=lambda self: time.time(),
post=lambda self: print('used {}'.format(time.time()-self.pre_val))
)
#WrapAll(**timer)
def add(x, y):
return x + y
#WrapAll(**timer)
async def async_add(x, y):
future = asyncio.Future()
future.set_result(x+y)
await future
return future.result()
Running sync adder
>>> add(3, 4)
used 4.76837158203125e-06
7
Running async adder
>>> loop = asyncio.get_event_loop()
>>> task = asyncio.ensure_future(async_add(3, 4))
>>> try:
... loop.run_until_complete(task)
... except RuntimeError:
... pass
used 2.193450927734375e-05
Logging Example
import asyncio
import logging
FORMAT = '%(message)s'
logging.basicConfig(format=FORMAT)
logger = dict(
post=lambda self: logging.warning('subtracting {}'.format(self.result))
)
#WrapAll(**logger)
def sub(x, y):
return x - y
#WrapAll(**logger)
async def async_sub(x, y):
future = asyncio.Future()
future.set_result(x-y)
await future
return future.result()
Running sync subtractor:
>>> sub(5, 6)
subtracting -1
Running async subtractor:
>>> loop = asyncio.get_event_loop()
>>> task = asyncio.ensure_future(async_sub(5, 6))
>>> try:
... loop.run_until_complete(task)
... except RuntimeError:
... pass
subtracting -1
I have a function that returns a future. I want to create a decorator to the function which waits for the future to complete and then return the result essentially converting the async function to blocking function (which I will use in my REST API). Is there a way to do that?
def sync(fn):
def wrapped(*args, **kwargs):
return IOLoop.instance().run_sync(lambda: fn(*args, **kwargs))
return wrapped
#gen.coroutine
def my_coro():
# ...
sync_fn = sync(my_coro)
result = sync_fn()
To resolve a future you need to yield it. Something like this might work:
from tornado import gen
def blocking(func):
def new_func(*args, **kwargs):
result = yield func(*args, **kwargs)
return result
return gen.coroutine(new_func)
I need a decorator (or something that is functionally equivalent) that allows the code below to work as expected:
#timeout(1)
def outer():
inner()
#timeout(5)
def inner():
time.sleep(3)
print("Should never be printed if you call outer()")
outer()
# The outer timeout is ignored and "property" finishes
The code seems pointless, but in reality, outer calls multiple functions that take uncertain amount of time, some of which have their own timeout.
I tried timeout-decorator and two SO answers here, but none works.
Something like this:
def timeout(timeout, raise_exc=True):
"""
raise_exc - if exception should be raised on timeout
or exception inside decorated func.
Otherwise None will be returned.
"""
def decorator(func):
#wraps(func)
def wrapper(*args, **kwargs):
res = None
exc = None
def _run():
nonlocal res
nonlocal exc
try:
res = func(*args, **kwargs)
except Exception as e:
exc = e
t = threading.Thread(target=_run)
t.daemon = True
t.start()
t.join(timeout=timeout)
if raise_exc and t.is_alive():
raise TimeoutError()
elif raise_exc and (exc is not None):
raise exc
else:
return res
return wrapper
return decorator
Examples:
#timeout(0.5, raise_exc=False)
def outer():
return inner()
#timeout(2)
def inner():
time.sleep(1)
return "Shouldn't be printed"
print(outer()) # None
and
#timeout(2, raise_exc=False)
def outer():
return inner()
#timeout(2)
def inner():
time.sleep(1)
return "Should be printed"
print(outer()) # Should be printed
Note, that your task can be solved only with threads or processes, but this may lead to some non-obvious problems. I recommend you to think if your task can be solved without it. In most cases you can split your code to parts and check for timeout after each. Something like this:
def outer(arg, timeout=None):
t = Timeout(timeout)
# some operation:
time.sleep(1)
if t.is_timeout: return None
# use time left as subfunction's timeout:
return inner(arg, timeout=t.time_left)
The timeout function uses threading.Timer to set the timer and thread.interrupt_main to interrupt the main thread.
from thread import interrupt_main
from threading import Timer
from time import time, sleep
def timeout(secs):
def wrapper(func):
timer = Timer(secs, interrupt_main)
def decorated(*args, **kwargs):
timer.start()
return func(*args, **kwargs)
return decorated
return wrapper
#timeout(1)
def outer():
inner()
#timeout(5)
def inner():
sleep(3)
print("Should never be printed if you call outer()")
try:
outer()
except:
print('timed out')
I'm trying to profile an instance method, so I've done something like:
import cProfile
class Test():
def __init__(self):
pass
def method(self):
cProfile.runctx("self.method_actual()", globals(), locals())
def method_actual(self):
print "Run"
if __name__ == "__main__":
Test().method()
But now problems arise when I want "method" to return a value that is computed by "method_actual". I don't really want to call "method_actual" twice.
Is there another way, something that can be thread safe? (In my application, the cProfile data are saved to datafiles named by one of the args, so they don't get clobbered and I can combine them later.)
I discovered that you can do this:
prof = cProfile.Profile()
retval = prof.runcall(self.method_actual, *args, **kwargs)
prof.dump_stats(datafn)
The downside is that it's undocumented.
An option for any arbitrary code:
import cProfile, pstats, sys
pr = cProfile.Profile()
pr.enable()
my_return_val = my_func(my_arg)
pr.disable()
ps = pstats.Stats(pr, stream=sys.stdout)
ps.print_stats()
Taken from https://docs.python.org/2/library/profile.html#profile.Profile
I was struggling with the same problem and used a wrapper function to get over direct return values. Instead of
cP.runctx("a=foo()", globals(), locales())
I create a wrapper function
def wrapper(b):
b.append(foo())
and profile the call to the wrapper function
b = []
cP.runctx("wrapper(b)", globals(), locals())
a = b[0]
extracting the result of foo's computation from the out param (b) afterwards.
I created a decorator:
import cProfile
import functools
import pstats
def profile(func):
#functools.wraps(func)
def inner(*args, **kwargs):
profiler = cProfile.Profile()
profiler.enable()
try:
retval = func(*args, **kwargs)
finally:
profiler.disable()
with open('profile.out', 'w') as profile_file:
stats = pstats.Stats(profiler, stream=profile_file)
stats.print_stats()
return retval
return inner
Decorate your function or method with it:
#profile
def somefunc(...):
...
Now that function will be profiled.
Alternatively, if you'd like the raw, unprocessed profile data (e.g. because you want to run the excellent graphical viewer RunSnakeRun on it), then:
import cProfile
import functools
import pstats
def profile(func):
#functools.wraps(func)
def inner(*args, **kwargs):
profiler = cProfile.Profile()
profiler.enable()
try:
retval = func(*args, **kwargs)
finally:
profiler.disable()
profiler.dump_stats('profile.out')
return retval
return inner
This is a minor improvement on several of the other answers on this page.
I think #detly the .runcall() is basically the best answer, but for completeness, I just wanted to take #ThomasH 's answer to be function independent:
def wrapper(b, f, *myargs, **mykwargs):
try:
b.append(f(*myargs, **mykwargs))
except TypeError:
print 'bad args passed to func.'
# Example run
def func(a, n):
return n*a + 1
b = []
cProfile.runctx("wrapper(b, func, 3, n=1)", globals(), locals())
a = b[0]
print 'a, ', a