We Keep Coding

Why can't I interrupt a generator with timeout in python3?

I have written a generator as follows:
def my_generator():
i = 0
while i < 1000000:
i += 1
yield i
Assuming the generator cannot be executed in a second and in the test function I use a timeout decorator to guarantee the function should not run more than 1 second.
#timeout(1)
def test():
for i in my_generator:
print(i)
Unfortunately, the timeout don't work as I wanted, the function print all the number from 1 to 1000000 with more than 1 second.
In the decorator, I have tried gevent and KThread, but none of them can work.
Decorator using KThread:
class KThread(threading.Thread):
"""Subclass of threading.Thread, with a kill() method."""
def __init__(self, *args, **kwargs):
threading.Thread.__init__(self, *args, **kwargs)
self.killed = False
def start(self):
"""Start the thread."""
self.__run_backup = self.run
"""Force the Thread to install our trace."""
self.run = self.__run
threading.Thread.start(self)
def __run(self):
"""Hacked run function, which installs the trace."""
sys.settrace(self.globaltrace)
self.__run_backup()
self.run = self.__run_backup
def globaltrace(self, frame, why, arg):
if why == 'call':
return self.localtrace
else:
return None
def localtrace(self, frame, why, arg):
if self.killed:
if why == 'line':
raise SystemExit()
return self.localtrace
def kill(self):
self.killed = True
def timeout(seconds):
def timeout_decorator(func):
def _new_func(oldfunc, result, oldfunc_args, oldfunc_kwargs):
result.append(oldfunc(*oldfunc_args, **oldfunc_kwargs))
def _(*args, **kwargs):
result = []
'''create new args for _new_funcbecause
we want to get the func return val to result list
'''
new_kwargs = {
'oldfunc': func,
'result': result,
'oldfunc_args': args,
'oldfunc_kwargs': kwargs
}
thd = KThread(target=_new_func, args=(), kwargs=new_kwargs)
thd.start()
thd.join(seconds)
alive = thd.isAlive()
'''kill the child thread'''
thd.kill()
if alive:
alert_exce = u'function timeout for [%d s].' % seconds
raise Timeout(alert_exce)
else:
return result[0]
_.__name__ = func.__name__
_.__doc__ = func.__doc__
return _
return timeout_decorator
Decorator using gevent:
def g_timer(timeout_seconds=None, timeout_exception=TimeoutError, exception_message=None, module_name=None):
import gevent
from gevent import monkey
monkey.patch_all()
def decorate(func):
def wrapper(*args, **kwargs):
try:
t0 = time.time()
gevent.with_timeout(timeout_seconds, func, *args, **kwargs)
elapsed = time.time() - t0
except gevent.timeout.Timeout as e:
print("exception")
return wrapper
return decorate

Optional args, kwargs to decorator

I am using the following approach to pass in an optional argument to a decorator:
def wait(func=None, delay=1.0):
def decorator_wait(func):
def wrapper_wait(*args, **kwargs):
time.sleep(delay)
return func(*args, **kwargs)
return wrapper_wait
return decorator_wait(func) if func is not None else decorator_wait
#wait
def print_something(something):
print (something)
#wait(delay=0.2)
def print_something_else(something):
print (something)
The above code looks pretty difficult to follow though with all the nesting. Is there another approach to do the above, or is this the only method available for something like this?

You can avoid having to remember "do I need to call this or not?" by removing the func argument from the wait function, and remembering to always call your decorator-returner.
It would look like this:
def wait(delay=1.0):
def decorator_wait(func):
def wrapper_wait(*args, **kwargs):
time.sleep(delay)
return func(*args, **kwargs)
return wrapper_wait
return decorator_wait
#wait()
def print_something(something):
print (something)
#wait(delay=0.2)
def print_something_else(something):
print (something)
print_something("hello")
# 1 second delay, then 'hello'
print_something_else("there")
# 0.2 second delay, then 'there'
You just have to remember that wait will always return the decorator, so you have to use () when decorating your functions.

I think it is a little bit better:
import functools
import time
def wait(func=None, delay=1.0):
if func is None:
return lambda func: wait(func=func, delay=delay)
#functools.wraps(func) # this is good practice to use it see: https://stackoverflow.com/questions/308999/what-does-functools-wraps-do
def _wrapper(*args, **kwargs):
time.sleep(delay)
return func(*args, **kwargs)
return _wrapper
#wait
def test():
return
#wait(delay=3)
def test2():
return

You can write classes having a __call__ method, instead of writing a bunch of nested defs.
It sounds like you want a decorator Wait which haults
program execution for a few seconds.
If you don't pass in a Wait-time
then the default value is 1 seconds.
Use-cases are shown below.
##################################################
#Wait
def print_something(something):
print(something)
##################################################
#Wait(3)
def print_something_else(something_else):
print(something_else)
##################################################
#Wait(delay=3)
def print_something_else(something_else):
print(something_else)
When Wait has an argument, such as #Wait(3), then the call Wait(3)
is executed before anything else happens.
That is, the following two pieces of code are equivalent
#Wait(3)
def print_something_else(something_else):
print(something_else)
###############################################
return_value = Wait(3)
#return_value
def print_something_else(something_else):
print(something_else)
This is a problem.
if `Wait` has no arguments:
`Wait` is the decorator.
else: # `Wait` receives arguments
`Wait` is not the decorator itself.
Instead, `Wait` ***returns*** the decorator
One solution is shown below:
Let us begin by creating the following class, DelayedDecorator:
import io
class DelayedDecorator:
def __init__(i, cls, *args, **kwargs):
print("Delayed Decorator __init__", cls, args, kwargs)
i._cls = cls
i._args = args
i._kwargs = kwargs
def __call__(i, func):
print("Delayed Decorator __call__", func)
if not (callable(func)):
import io
with io.StringIO() as ss:
print(
"If only one input, input must be callable",
"Instead, received:",
repr(func),
sep="\n",
file=ss
)
msg = ss.getvalue()
raise TypeError(msg)
return i._cls(func, *i._args, **i._kwargs)
Now we can write things like:
dec = DelayedDecorator(Wait, delay=4)
#dec
def delayed_print(something):
print(something)
Note that:
dec does not not accept multiple arguments.
dec only accepts the function to be wrapped.
import inspect
class PolyArgDecoratorMeta(type):
def __call__(Wait, *args, **kwargs):
try:
arg_count = len(args)
if (arg_count == 1):
if callable(args[0]):
SuperClass = inspect.getmro(PolyArgDecoratorMeta)[1]
r = SuperClass.__call__(Wait, args[0])
else:
r = DelayedDecorator(Wait, *args, **kwargs)
else:
r = DelayedDecorator(Wait, *args, **kwargs)
finally:
pass
return r
import time
class Wait(metaclass=PolyArgDecoratorMeta):
def __init__(i, func, delay = 2):
i._func = func
i._delay = delay
def __call__(i, *args, **kwargs):
time.sleep(i._delay)
r = i._func(*args, **kwargs)
return r
The following two pieces of code are equivalent:
#Wait
def print_something(something):
print (something)
##################################################
def print_something(something):
print(something)
print_something = Wait(print_something)
We can print "something" to the console very slowly, as follows:
print_something("something")
#################################################
#Wait(delay=1)
def print_something_else(something_else):
print(something_else)
##################################################
def print_something_else(something_else):
print(something_else)
dd = DelayedDecorator(Wait, delay=1)
print_something_else = dd(print_something_else)
##################################################
print_something_else("something")
Final Notes
It may look like a lot of code, but you don't have to write the classes DelayedDecorator and PolyArgDecoratorMeta every-time. The only code you have to personally write something like as follows, which is fairly short:
from PolyArgDecoratorMeta import PolyArgDecoratorMeta
import time
class Wait(metaclass=PolyArgDecoratorMeta):
def __init__(i, func, delay = 2):
i._func = func
i._delay = delay
def __call__(i, *args, **kwargs):
time.sleep(i._delay)
r = i._func(*args, **kwargs)
return r

Python Decorator - printing description instead of name

I have the below decorator function for printing response times. Is there a way to pass description of the decorator when decorating another function with this decorator? For eg: I want to print "calling service xyz" instead of function name "call_service_xyz"
def timer(func):
"""Print the runtime of the decorated function"""
#functools.wraps(func)
def wrapper_timer(*args, **kwargs):
start_time = time.perf_counter() # 1
value = func(*args, **kwargs)
end_time = time.perf_counter() # 2
run_time = end_time - start_time # 3
logger.info(f"Finished {func.__name__!r} in {run_time:.4f} secs")
return value
return wrapper_timer
#timer("call service xyz")
def call_service_xyz():

You could print the docstring to act as the description of the function:
def timer(func):
"""Print the runtime of the decorated function"""
#functools.wraps(func)
def wrapper_timer(*args, **kwargs):
start_time = time.perf_counter() # 1
value = func(*args, **kwargs)
end_time = time.perf_counter() # 2
run_time = end_time - start_time # 3
logger.info(f"{desc} Finished in {run_time:.4f} secs")
return value
return wrapper_timer
#timer
def call_service_xyz():
"""Call service xyz."""
# ^^ this is the docstring which is
# assigned to .__doc__
Alternatively, if you wish to pass the description into timer you need to create another layer to the decorator which accepts arguments:
def timer(desc):
def decorator(func):
#functools.wraps(func)
def wrapper_timer(*args, **kw):
start_time = time.perf_counter()
value = func(*args, **kw)
end_time = time.perf_counter()
run_time = end_time - start_time
logger.info(f"{func.__doc__!r} Finished in {run_time:.4f} secs")
return value
return wrapper_timer
return decorator
Now use as you describe in your question:
#timer('call service xyz')
def xyz():
...

In python,how to use decorator compatible with normal function and coroutine function simply?

Here is my way,but I feel it is not very simple, any better way?
import asyncio
import time
def timer_all(f):
if asyncio.iscoroutinefunction(f):
async def wrapper(*args, **kwargs):
now = time.time()
result = await f(*args, **kwargs)
print('used {}'.format(time.time() - now))
return result
else:
def wrapper(*args, **kwargs):
now = time.time()
result = f(*args, **kwargs)
print('used {}'.format(time.time() - now))
return result
return wrapper
there is a lot of decorator, retry, add log etc,all will write this way,a bit ugly,right?

While there is no real problems with repeating the same code in specialized decorators.
Here is how I'll approach a refactoring.
I will use a class decorator that keeps the accepts a pre-call function and a post-call function,
both of which will be called with an instance of the decorator.
The result of the pre-call function will be saved to an attribute of the decorator.
This is necessary for the special timing case where a delta needs to be computed.
I guess there may be other examples that may require the return value of a pre-call function execution.
I also save the result of the decorated function executed to the result attribute of the decorator instance. This allows the post call function to read this value for logging.
Here is an example implementation:
import asyncio
class WrapAll(object):
def __init__(self, pre=lambda _: None, post=lambda _: None):
self.pre = lambda : pre(self)
self.pre_val = None
self.result = None
self.post = lambda : post(self)
def __call__(self, fn):
if asyncio.iscoroutinefunction(fn):
async def wrap(*args, **kwargs):
self.pre_val = self.pre()
self.result = await fn(*args, *kwargs)
self.post()
return self.result
else:
def wrap(*args, **kwargs):
self.pre_val = self.pre()
self.result = fn(*args, *kwargs)
self.post()
return self.result
return wrap
Timer Example
import asyncio
import time
timer = dict(
pre=lambda self: time.time(),
post=lambda self: print('used {}'.format(time.time()-self.pre_val))
)
#WrapAll(**timer)
def add(x, y):
return x + y
#WrapAll(**timer)
async def async_add(x, y):
future = asyncio.Future()
future.set_result(x+y)
await future
return future.result()
Running sync adder
>>> add(3, 4)
used 4.76837158203125e-06
7
Running async adder
>>> loop = asyncio.get_event_loop()
>>> task = asyncio.ensure_future(async_add(3, 4))
>>> try:
... loop.run_until_complete(task)
... except RuntimeError:
... pass
used 2.193450927734375e-05
Logging Example
import asyncio
import logging
FORMAT = '%(message)s'
logging.basicConfig(format=FORMAT)
logger = dict(
post=lambda self: logging.warning('subtracting {}'.format(self.result))
)
#WrapAll(**logger)
def sub(x, y):
return x - y
#WrapAll(**logger)
async def async_sub(x, y):
future = asyncio.Future()
future.set_result(x-y)
await future
return future.result()
Running sync subtractor:
>>> sub(5, 6)
subtracting -1
Running async subtractor:
>>> loop = asyncio.get_event_loop()
>>> task = asyncio.ensure_future(async_sub(5, 6))
>>> try:
... loop.run_until_complete(task)
... except RuntimeError:
... pass
subtracting -1

Can't get the parameters of a func using threading lib in a python decoration?

Below is the python codes,The decoration is ok without thread created through a class inherit from threading.thread. for example create a thread by giving the target func as a paremeter to threading.thread()
import threading ,time
from time import sleep, ctime
import functools
def find(func):
#functools.wraps(func)
def wrapper(*args,**kwargs):
print("ags:%s,%s\n" % (args,kwargs))
return func(*args, **kwargs)
return wrapper
#find
def now() :
return str( time.strftime( '%Y-%m-%d %H:%M:%S' , time.localtime() ) )
class myThread (threading.Thread) :
"""docstring for myThread"""
#find
def __init__(self, nloop, nsec) :
super(myThread, self).__init__()
self.nloop = nloop
self.nsec = nsec
#find
def run(self):
print('start loop', self.nloop, 'at:', ctime())
sleep(self.nsec)
print('loop', self.nloop, 'done at:', ctime())
#find
def main():
thpool=[]
print('starting at:', now())
for i in range(10):
thpool.append(myThread(i,2))
for th in thpool:
th.start()
for th in thpool:
th.join()
print('all Done at:', now())
if __name__ == '__main__':
main()
I got a error info as below:
File "F:\question\multithreadfmclass.py", line 15, in wrapper
print("ags:%s,%s\n" % (args,kwargs))
File "D:\ProgramFiles\Python352\lib\threading.py", line 813, in __repr__
assert self._initialized, "Thread.__init__() was not called"
AssertionError: Thread.__init__() was not called
How to remove the bugs? tks in advance.

You are printing the Thread object before it is initialized which is impossible. It is not easy to find this mistake because of your decorator but if you call func first, then it works:
def find(func):
#functools.wraps(func)
def wrapper(*args, **kwargs):
result = func(*args, **kwargs)
print("ags:%s,%s\n" % (args, kwargs))
return result
return wrapper