I have written a generator as follows:
def my_generator():
i = 0
while i < 1000000:
i += 1
yield i
Assuming the generator cannot be executed in a second and in the test function I use a timeout decorator to guarantee the function should not run more than 1 second.
#timeout(1)
def test():
for i in my_generator:
print(i)
Unfortunately, the timeout don't work as I wanted, the function print all the number from 1 to 1000000 with more than 1 second.
In the decorator, I have tried gevent and KThread, but none of them can work.
Decorator using KThread:
class KThread(threading.Thread):
"""Subclass of threading.Thread, with a kill() method."""
def __init__(self, *args, **kwargs):
threading.Thread.__init__(self, *args, **kwargs)
self.killed = False
def start(self):
"""Start the thread."""
self.__run_backup = self.run
"""Force the Thread to install our trace."""
self.run = self.__run
threading.Thread.start(self)
def __run(self):
"""Hacked run function, which installs the trace."""
sys.settrace(self.globaltrace)
self.__run_backup()
self.run = self.__run_backup
def globaltrace(self, frame, why, arg):
if why == 'call':
return self.localtrace
else:
return None
def localtrace(self, frame, why, arg):
if self.killed:
if why == 'line':
raise SystemExit()
return self.localtrace
def kill(self):
self.killed = True
def timeout(seconds):
def timeout_decorator(func):
def _new_func(oldfunc, result, oldfunc_args, oldfunc_kwargs):
result.append(oldfunc(*oldfunc_args, **oldfunc_kwargs))
def _(*args, **kwargs):
result = []
'''create new args for _new_funcbecause
we want to get the func return val to result list
'''
new_kwargs = {
'oldfunc': func,
'result': result,
'oldfunc_args': args,
'oldfunc_kwargs': kwargs
}
thd = KThread(target=_new_func, args=(), kwargs=new_kwargs)
thd.start()
thd.join(seconds)
alive = thd.isAlive()
'''kill the child thread'''
thd.kill()
if alive:
alert_exce = u'function timeout for [%d s].' % seconds
raise Timeout(alert_exce)
else:
return result[0]
_.__name__ = func.__name__
_.__doc__ = func.__doc__
return _
return timeout_decorator
Decorator using gevent:
def g_timer(timeout_seconds=None, timeout_exception=TimeoutError, exception_message=None, module_name=None):
import gevent
from gevent import monkey
monkey.patch_all()
def decorate(func):
def wrapper(*args, **kwargs):
try:
t0 = time.time()
gevent.with_timeout(timeout_seconds, func, *args, **kwargs)
elapsed = time.time() - t0
except gevent.timeout.Timeout as e:
print("exception")
return wrapper
return decorate
Related
I am using the following approach to pass in an optional argument to a decorator:
def wait(func=None, delay=1.0):
def decorator_wait(func):
def wrapper_wait(*args, **kwargs):
time.sleep(delay)
return func(*args, **kwargs)
return wrapper_wait
return decorator_wait(func) if func is not None else decorator_wait
#wait
def print_something(something):
print (something)
#wait(delay=0.2)
def print_something_else(something):
print (something)
The above code looks pretty difficult to follow though with all the nesting. Is there another approach to do the above, or is this the only method available for something like this?
You can avoid having to remember "do I need to call this or not?" by removing the func argument from the wait function, and remembering to always call your decorator-returner.
It would look like this:
def wait(delay=1.0):
def decorator_wait(func):
def wrapper_wait(*args, **kwargs):
time.sleep(delay)
return func(*args, **kwargs)
return wrapper_wait
return decorator_wait
#wait()
def print_something(something):
print (something)
#wait(delay=0.2)
def print_something_else(something):
print (something)
print_something("hello")
# 1 second delay, then 'hello'
print_something_else("there")
# 0.2 second delay, then 'there'
You just have to remember that wait will always return the decorator, so you have to use () when decorating your functions.
I think it is a little bit better:
import functools
import time
def wait(func=None, delay=1.0):
if func is None:
return lambda func: wait(func=func, delay=delay)
#functools.wraps(func) # this is good practice to use it see: https://stackoverflow.com/questions/308999/what-does-functools-wraps-do
def _wrapper(*args, **kwargs):
time.sleep(delay)
return func(*args, **kwargs)
return _wrapper
#wait
def test():
return
#wait(delay=3)
def test2():
return
You can write classes having a __call__ method, instead of writing a bunch of nested defs.
It sounds like you want a decorator Wait which haults
program execution for a few seconds.
If you don't pass in a Wait-time
then the default value is 1 seconds.
Use-cases are shown below.
##################################################
#Wait
def print_something(something):
print(something)
##################################################
#Wait(3)
def print_something_else(something_else):
print(something_else)
##################################################
#Wait(delay=3)
def print_something_else(something_else):
print(something_else)
When Wait has an argument, such as #Wait(3), then the call Wait(3)
is executed before anything else happens.
That is, the following two pieces of code are equivalent
#Wait(3)
def print_something_else(something_else):
print(something_else)
###############################################
return_value = Wait(3)
#return_value
def print_something_else(something_else):
print(something_else)
This is a problem.
if `Wait` has no arguments:
`Wait` is the decorator.
else: # `Wait` receives arguments
`Wait` is not the decorator itself.
Instead, `Wait` ***returns*** the decorator
One solution is shown below:
Let us begin by creating the following class, DelayedDecorator:
import io
class DelayedDecorator:
def __init__(i, cls, *args, **kwargs):
print("Delayed Decorator __init__", cls, args, kwargs)
i._cls = cls
i._args = args
i._kwargs = kwargs
def __call__(i, func):
print("Delayed Decorator __call__", func)
if not (callable(func)):
import io
with io.StringIO() as ss:
print(
"If only one input, input must be callable",
"Instead, received:",
repr(func),
sep="\n",
file=ss
)
msg = ss.getvalue()
raise TypeError(msg)
return i._cls(func, *i._args, **i._kwargs)
Now we can write things like:
dec = DelayedDecorator(Wait, delay=4)
#dec
def delayed_print(something):
print(something)
Note that:
dec does not not accept multiple arguments.
dec only accepts the function to be wrapped.
import inspect
class PolyArgDecoratorMeta(type):
def __call__(Wait, *args, **kwargs):
try:
arg_count = len(args)
if (arg_count == 1):
if callable(args[0]):
SuperClass = inspect.getmro(PolyArgDecoratorMeta)[1]
r = SuperClass.__call__(Wait, args[0])
else:
r = DelayedDecorator(Wait, *args, **kwargs)
else:
r = DelayedDecorator(Wait, *args, **kwargs)
finally:
pass
return r
import time
class Wait(metaclass=PolyArgDecoratorMeta):
def __init__(i, func, delay = 2):
i._func = func
i._delay = delay
def __call__(i, *args, **kwargs):
time.sleep(i._delay)
r = i._func(*args, **kwargs)
return r
The following two pieces of code are equivalent:
#Wait
def print_something(something):
print (something)
##################################################
def print_something(something):
print(something)
print_something = Wait(print_something)
We can print "something" to the console very slowly, as follows:
print_something("something")
#################################################
#Wait(delay=1)
def print_something_else(something_else):
print(something_else)
##################################################
def print_something_else(something_else):
print(something_else)
dd = DelayedDecorator(Wait, delay=1)
print_something_else = dd(print_something_else)
##################################################
print_something_else("something")
Final Notes
It may look like a lot of code, but you don't have to write the classes DelayedDecorator and PolyArgDecoratorMeta every-time. The only code you have to personally write something like as follows, which is fairly short:
from PolyArgDecoratorMeta import PolyArgDecoratorMeta
import time
class Wait(metaclass=PolyArgDecoratorMeta):
def __init__(i, func, delay = 2):
i._func = func
i._delay = delay
def __call__(i, *args, **kwargs):
time.sleep(i._delay)
r = i._func(*args, **kwargs)
return r
I like the default python multiprocessing.Pool, but it's still a pain that it isn't easy to show the current progress being made during the pool's execution. In leui of that, I attempted to create my own, custom multiprocess pool mapper, and it looks like this;
from multiprocessing import Process, Pool, cpu_count
from iterable_queue import IterableQueue
def _proc_action(f, in_queue, out_queue):
try:
for val in in_queue:
out_queue.put(f(val))
except (KeyboardInterrupt, EOFError):
pass
def progress_pool_map(f, ls, n_procs=cpu_count()):
in_queue = IterableQueue()
out_queue = IterableQueue()
err = None
try:
procs = [Process(target=_proc_action, args=(f, in_queue, out_queue)) for _ in range(n_procs)]
[p.start() for p in procs]
for elem in ls:
in_queue.put(elem)
in_queue.close()
bar = 0
for _ in ls:
elem = next(out_queue)
bar += 1
if bar % 1000 == 0:
print(bar)
yield elem
out_queue.close()
except (KeyboardInterrupt, EOFError) as e:
in_queue.close()
out_queue.close()
print("Joining processes")
[p.join() for p in procs]
print("Closing processes")
[p.close() for p in procs]
err = e
if err:
raise err
It works fairly well, and prints a value to the console for every 1000 items processed. The progress display itself is something I can worry about in future. Right now, however, my issue is that when cancelled, the operation does anything but fail gracefully. When I try to interrupt the map, it hangs on Joining Processes, and never makes it to Closing Processes. If I try hitting Ctrl+C again, it causes an infinite spew of BrokenPipeErrors to fill the console until I send an EOF and stop my program.
Here's iterable_queue.py, for reference;
from multiprocessing.queues import Queue
from multiprocessing import get_context, Value
import queue
class QueueClosed(Exception):
pass
class IterableQueue(Queue):
def __init__(self, maxsize=0, *, ctx=None):
super().__init__(
maxsize=maxsize,
ctx=ctx if ctx is not None else get_context()
)
self.closed = Value('b', False)
def close(self):
with self.closed.get_lock():
if not self.closed.value:
self.closed.value = True
super().put((None, False))
# throws BrokenPipeError in another thread without this sleep in between
# terrible hack, must fix at some point
import time; time.sleep(0.01)
super().close()
def __iter__(self):
return self
def __next__(self):
try:
return self.get()
except QueueClosed:
raise StopIteration
def get(self, *args, **kwargs):
try:
result, is_open = super().get(*args, **kwargs)
except OSError:
raise QueueClosed
if not is_open:
super().put((None, False))
raise QueueClosed
return result
def __bool__(self):
return bool(self.closed.value)
def put(self, val, *args, **kwargs):
with self.closed.get_lock():
if self.closed.value:
raise QueueClosed
super().put((val, True), *args, **kwargs)
def get_nowait(self):
return self.get(block=False)
def put_nowait(self):
return self.put(block=False)
def empty_remaining(self, block=False):
try:
while True:
yield self.get(block=block)
except (queue.Empty, QueueClosed):
pass
def clear(self):
for _ in self.empty_remaining():
pass
def __enter__(self):
return self
def __exit__(self, *args):
self.close()
Here is my way,but I feel it is not very simple, any better way?
import asyncio
import time
def timer_all(f):
if asyncio.iscoroutinefunction(f):
async def wrapper(*args, **kwargs):
now = time.time()
result = await f(*args, **kwargs)
print('used {}'.format(time.time() - now))
return result
else:
def wrapper(*args, **kwargs):
now = time.time()
result = f(*args, **kwargs)
print('used {}'.format(time.time() - now))
return result
return wrapper
there is a lot of decorator, retry, add log etc,all will write this way,a bit ugly,right?
While there is no real problems with repeating the same code in specialized decorators.
Here is how I'll approach a refactoring.
I will use a class decorator that keeps the accepts a pre-call function and a post-call function,
both of which will be called with an instance of the decorator.
The result of the pre-call function will be saved to an attribute of the decorator.
This is necessary for the special timing case where a delta needs to be computed.
I guess there may be other examples that may require the return value of a pre-call function execution.
I also save the result of the decorated function executed to the result attribute of the decorator instance. This allows the post call function to read this value for logging.
Here is an example implementation:
import asyncio
class WrapAll(object):
def __init__(self, pre=lambda _: None, post=lambda _: None):
self.pre = lambda : pre(self)
self.pre_val = None
self.result = None
self.post = lambda : post(self)
def __call__(self, fn):
if asyncio.iscoroutinefunction(fn):
async def wrap(*args, **kwargs):
self.pre_val = self.pre()
self.result = await fn(*args, *kwargs)
self.post()
return self.result
else:
def wrap(*args, **kwargs):
self.pre_val = self.pre()
self.result = fn(*args, *kwargs)
self.post()
return self.result
return wrap
Timer Example
import asyncio
import time
timer = dict(
pre=lambda self: time.time(),
post=lambda self: print('used {}'.format(time.time()-self.pre_val))
)
#WrapAll(**timer)
def add(x, y):
return x + y
#WrapAll(**timer)
async def async_add(x, y):
future = asyncio.Future()
future.set_result(x+y)
await future
return future.result()
Running sync adder
>>> add(3, 4)
used 4.76837158203125e-06
7
Running async adder
>>> loop = asyncio.get_event_loop()
>>> task = asyncio.ensure_future(async_add(3, 4))
>>> try:
... loop.run_until_complete(task)
... except RuntimeError:
... pass
used 2.193450927734375e-05
Logging Example
import asyncio
import logging
FORMAT = '%(message)s'
logging.basicConfig(format=FORMAT)
logger = dict(
post=lambda self: logging.warning('subtracting {}'.format(self.result))
)
#WrapAll(**logger)
def sub(x, y):
return x - y
#WrapAll(**logger)
async def async_sub(x, y):
future = asyncio.Future()
future.set_result(x-y)
await future
return future.result()
Running sync subtractor:
>>> sub(5, 6)
subtracting -1
Running async subtractor:
>>> loop = asyncio.get_event_loop()
>>> task = asyncio.ensure_future(async_sub(5, 6))
>>> try:
... loop.run_until_complete(task)
... except RuntimeError:
... pass
subtracting -1
For tracing race conditions in the code I need to display the caller name and how long the operations take.
The code below outputs:
2018-02-06 19:00:11.418800: get_data() called by **wrappe**r() (0:00:00.001010)
test result: PASS
How can I make it output the caller name instead of the decorator function name e.g.:
2018-02-06 19:05:47.617679: get_data() called by **test_get_data**() (0:00:00.034116)
test result: PASS
The code:
from datetime import datetime
import time, inspect
class Timer():
def __init__(self):
self.time_start = datetime.now()
def stop(self):
return (datetime.now() - self.time_start)
class Test2Decorators():
def caller(callee):
def timer(callee):
def wrapper(*args, **kwargs):
get_data_timer = Timer()
result = callee(*args, **kwargs)
print ' (%s)' % get_data_timer.stop()
return result
return wrapper
#timer
def wrapper(*args, **kwargs):
curframe = inspect.currentframe()
calframe = inspect.getouterframes(curframe, 2)
print '%s: %s() called by %s()' % (datetime.now(), callee.__name__, calframe[1][3]),
return callee(*args, **kwargs)
return wrapper
#caller
def get_data(self, product_id = None, product_name = None):
return 'test result: PASS'
def test_get_data():
print class_inst.get_data('beer', '32445256')
time.sleep(1)
class_inst = Test2Decorators()
test_get_data()
I just figured this out this way:
from datetime import datetime
import inspect
import time
class Timer():
def __init__(self):
self.time_start = datetime.now()
def stop(self):
return (datetime.now() - self.time_start)
class Test2Decorators():
def caller(callee):
def timer(callee):
def wrapper(*args, **kwargs):
get_data_timer = Timer()
result = callee(*args, **kwargs)
print '%s()' % inspect.stack()[1][3],
print ' (%s)' % get_data_timer.stop()
return result
return wrapper
#timer
def wrapper(*args, **kwargs):
print '%s: %s() called by' % (datetime.now(), callee.__name__),
return callee(*args, **kwargs)
return wrapper
#caller
def get_data(self, product_id = None, product_name = None):
return 'test result: PASS'
def test_get_data():
while True:
print class_inst.get_data('beer', '32445256')
time.sleep(1)
class_inst = Test2Decorators()
test_get_data()
Python's futures package allows us to enjoy ThreadPoolExecutor and ProcessPoolExecutor for doing tasks in parallel.
However, for debugging it is sometimes useful to temporarily replace the true parallelism with a dummy one, which carries out the tasks in a serial way in the main thread, without spawning any threads or processes.
Is there anywhere an implementation of a DummyExecutor?
Something like this should do it:
from concurrent.futures import Future, Executor
from threading import Lock
class DummyExecutor(Executor):
def __init__(self):
self._shutdown = False
self._shutdownLock = Lock()
def submit(self, fn, *args, **kwargs):
with self._shutdownLock:
if self._shutdown:
raise RuntimeError('cannot schedule new futures after shutdown')
f = Future()
try:
result = fn(*args, **kwargs)
except BaseException as e:
f.set_exception(e)
else:
f.set_result(result)
return f
def shutdown(self, wait=True):
with self._shutdownLock:
self._shutdown = True
if __name__ == '__main__':
def fnc(err):
if err:
raise Exception("test")
else:
return "ok"
ex = DummyExecutor()
print(ex.submit(fnc, True))
print(ex.submit(fnc, False))
ex.shutdown()
ex.submit(fnc, True) # raises exception
locking is probably not needed in this case, but can't hurt to have it.
Use this to mock your ThreadPoolExecutor
class MockThreadPoolExecutor():
def __init__(self, **kwargs):
pass
def __enter__(self):
return self
def __exit__(self, exc_type, exc_value, exc_traceback):
pass
def submit(self, fn, *args, **kwargs):
# execute functions in series without creating threads
# for easier unit testing
result = fn(*args, **kwargs)
return result
def shutdown(self, wait=True):
pass
if __name__ == "__main__":
def sum(a, b):
return a + b
with MockThreadPoolExecutor(max_workers=3) as executor:
future_result = list()
for i in range(5):
future_result.append(executor.submit(sum, i + 1, i + 2))