I have multiple functions in my script which does a REST API api requests.As i need to handle the error scenarios i have put a retry mechanism as below.
no_of_retries = 3
def check_status():
for i in range(0,no_of_retries):
url = "http://something/something"
try:
result = requests.get(url, auth=HTTPBasicAuth(COMMON_USERNAME, COMMON_PASSWORD)).json()
if 'error' not in result:
return result
else:
continue
except Exception as e:
continue
return None
I have several different methods which does similar operation. How can we do it better way to avoid duplication may be using decorators.
You can use a decorator like this and handle your own exception.
def retry(times, exceptions):
"""
Retry Decorator
Retries the wrapped function/method `times` times if the exceptions listed
in ``exceptions`` are thrown
:param times: The number of times to repeat the wrapped function/method
:type times: Int
:param Exceptions: Lists of exceptions that trigger a retry attempt
:type Exceptions: Tuple of Exceptions
"""
def decorator(func):
def newfn(*args, **kwargs):
attempt = 0
while attempt < times:
try:
return func(*args, **kwargs)
except exceptions:
print(
'Exception thrown when attempting to run %s, attempt '
'%d of %d' % (func, attempt, times)
)
attempt += 1
return func(*args, **kwargs)
return newfn
return decorator
#retry(times=3, exceptions=(ValueError, TypeError))
def foo1():
print('Some code here ....')
print('Oh no, we have exception')
raise ValueError('Some error')
foo1()
if you do not mind installing a library you could use the tenacity (github.com/jd/tenacity) module. one of their examples:
import random
from tenacity import retry, stop_after_attempt
# #retry # retry forever
#retry(stop=stop_after_attempt(3))
def do_something_unreliable():
if random.randint(0, 10) > 1:
raise IOError("Broken sauce, everything is hosed!!!111one")
else:
return "Awesome sauce!"
print(do_something_unreliable())
this also allows you to specify the number of tries or seconds you want to keep retrying.
for your case this might look something like this (not tested!):
#retry(stop=stop_after_attempt(3))
def retry_get():
result = requests.get(
url, auth=HTTPBasicAuth(COMMON_USERNAME, COMMON_PASSWORD)).json()
if 'error' not in result:
raise RequestException(result)
The third-party retry module is now widely accepted for this. You can also pass the list of exceptions to retry for, number of retries, delays, maximum delay, exponential back-off, etc.
$ pip install retry
Example usage:
from retry import retry
#retry(ZeroDivisionError, tries=3, delay=2)
def make_trouble():
'''Retry on ZeroDivisionError, raise error after 3 attempts, sleep 2 seconds between attempts.'''
Production level example
import logging
import time
import functools
import traceback
LOG_FORMAT = "%(asctime)s - %(levelname)s - %(pathname)s - %(funcName)s - %(lineno)d -msg: %(message)s"
logging.basicConfig(level=logging.INFO, format=LOG_FORMAT)
def retry(retry_num, retry_sleep_sec):
"""
retry help decorator.
:param retry_num: the retry num; retry sleep sec
:return: decorator
"""
def decorator(func):
"""decorator"""
# preserve information about the original function, or the func name will be "wrapper" not "func"
#functools.wraps(func)
def wrapper(*args, **kwargs):
"""wrapper"""
for attempt in range(retry_num):
try:
return func(*args, **kwargs) # should return the raw function's return value
except Exception as err: # pylint: disable=broad-except
logging.error(err)
logging.error(traceback.format_exc())
time.sleep(retry_sleep_sec)
logging.error("Trying attempt %s of %s.", attempt + 1, retry_num)
logging.error("func %s retry failed", func)
raise Exception('Exceed max retry num: {} failed'.format(retry_num))
return wrapper
return decorator
usage
# this means try your function 5 times, each time sleep 60 seconds
#retry(5, 60)
def your_func():
pass
Formal reference: https://peps.python.org/pep-0318/
I would recommend using the retry library like #MohitC mentioned. However, if you're restricted to import 3rd party libraries like I was, you're welcome to try my version:
import time
def retry(tries= -1, delay=0, max_delay=None, backoff=1, exceptions=Exception, log=False):
"""Retry Decorator with arguments
Args:
tries (int): The maximum number of attempts. Defaults to -1 (infinite)
delay (int, optional): Delay between attempts (seconds). Defaults to 0
max_delay (int, optional): The maximum value of delay (seconds). Defaults to None (Unlimited)
backoff (int, optional): Multiplier applied to delay between attempts (seconds). Defaults to 1 (No backoff)
exceptions (tuple, optional): Types of exceptions to catch. Defaults to Exception (all)
log (bool, optional): Print debug logs. Defaults to False
"""
def retry_decorator(func):
def retry_wrapper(*args, **kwargs):
nonlocal tries, delay, max_delay, backoff, exceptions, log
while tries:
try:
return func(*args, **kwargs)
except exceptions:
tries -= 1
# Reached to maximum tries
if not tries:
raise
# Log the retry logs for the given function
if log:
print(f"Retrying {func.__name__} in {delay} seconds")
# Apply delay between requests
time.sleep(delay)
# Adjust the next delay according to backoff
delay *= backoff
# Adjust maximum delay duration
if max_delay is not None:
delay = min(delay, max_delay)
return retry_wrapper
return retry_decorator
Example Uses:
Simple:
#retry(10, delay=5)
def do_something(params):
# Example func to retry
pass
Advanced:
#retry(10, delay=1, backoff=2, max_delay=10, exceptions=(TimeoutError), log=True)
def do_something(params):
# Example func to retry only for TimeoutErrors
pass
Instead of using decorators, the probably better solution is to move the request to its own function, arriving at a structure similar to this:
no_of_retries = 3
def make_request(url):
for i in range(0,no_of_retries):
try:
result = requests.get(url, auth=HTTPBasicAuth(COMMON_USERNAME, COMMON_PASSWORD)).json()
if 'error' not in result:
return result
else:
continue
except Exception as e:
continue
return result
def check_status():
result = make_request("http://something/status")
def load_file():
result = make_request("http://something/file")
This way, you avoid duplicate code while encapsulating the request. If you were to use a decorator, you would need to wrap the whole load_file() method which would prevent you from further processing the request's result within this function.
I have created a custom retry function. The function will retry if the first number is less than the second.
CODE:
import time
# CUSTOM EXCEPTION
class custom_error(Exception):
pass
# RETRY FUNCTION.
def retry(func, retries=3):
print(func)
def retry_wrapper(*args, **kwargs):
print(args)
n = args[0]
u = args[1]
print(n, u)
attempts = 0
while attempts < retries:
try:
if n > u:
return func(*args, **kwargs)
else:
raise custom_error
except custom_error:
print("error")
time.sleep(2)
attempts += 1
return retry_wrapper
#retry
def akash(a, b):
c = a / b
return c
# CALLING THE FUNCTION
a = akash(1, 2)
print(a)
OUTPUT:
<function akash at 0x00000187C3A66B00>
(1, 2)
1 2
error
error
error
Using functools on top of mrkiril's answer:
from functools import wraps, partial
def retry(f=None, times=10):
if f is None:
return partial(retry, times=times)
#wraps(f)
def wrap(*args, **kwargs):
attempt = 0
while attempt < times:
try:
return f(*args, **kwargs)
except:
print(f"{f.__name__}, attempt {attempt} of {times}")
attempt += 1
return f(*args, **kwargs)
return wrap
Then, wrap your function like the following:
import random
#retry
def foo():
if random.randint(0, 5) != 0:
raise Exception
I have got this function for logging in existing code. There is this duplication due to if asyncio.iscoroutinefunction(func).
Is there any better way to avoid this duplication along with also keeping async wrapper).
def log_it():
def decorator(func):
if asyncio.iscoroutinefunction(func):
async def wrapper(*args, **kwargs):
name = func.__name__
event = args[0].__class__.__name__
try:
result = await func(*args, **kwargs)
except Exception as e:
LOGGER.error('Error in processing: {} - {}'.format(name, event))
LOGGER.exception(e)
raise e
return result
else:
def wrapper(*args, **kwargs):
name = func.__name__
event = args[0].__class__.__name__
try:
result = func(*args, **kwargs)
except Exception as e:
LOGGER.error('Error in processing: {} - {}'.format(name, event))
LOGGER.exception(e)
raise e
return result
return wrapper
return decorator
My question is pretty strange i know, but i have to do something like this:
def on_update(self, context):
self.possibleContext=[r"\/file\/",r"\/anyFile\/"]
def inner(function, *args, **kwargs):
self.handlers.append(function)
#i define result, is the dictionary returned by telegram request
function(result, *args, **kwargs)
return update
return inner
There is the decorator, under the polling function that should call every decorator
def polling(self, timeout: int = 0, *args, **kwargs):
print(f"Started {self.bot['first_name']} on TeleLib V.{self.version}")
try:
update_id = self.getUpdates()[0]['update_id']
except IndexError:
update_id = None
while True:
try:
for funct in self.handlers:
#Here i should call the decorator, not the function, because the decorator call the function giving it an argument
except Error1:
#here some telegram errors exceptions
except Error2:
#ecc
for update in self.getUpdates(offset=update_id, timeout=timeout):
update_id = update['update_id'] + 1
self.polling()
return update
and here a sample function:
#client.on_update('ciikk')
def funzione(update):
client.sendMessage(update['message']['chat']['id'],update['message']['text'])
I'm using an interceptor to check the validity of a token passed by the user in my tornado application.
def token_authenticate():
def wrapper(self, transforms, *args, **kwargs):
def _throw_error(self):
print 'writing basic auth'
if self._headers_written:
raise Exception('headers have already been written')
self.write(json.dumps({'auth': 'false'}))
self.finish()
return False
request = self.request
try:
token = request.arguments.get('token')[0]
if not token:
return _throw_error(self)
session = Instance().get_session()
user_token = session.query(UserToken)\
.filter(UserToken.token == token)\
.filter(UserToken.expires > datetime.utcnow())\
.one()
if user_token:
self.token = user_token
self.user = user_token.user
else:
print 'no user token'
return _throw_error(self)
except Exception, e:
print 'exception ' + e
return _throw_error(self)
return True
return wrapper
def interceptor(func):
def classwrapper(cls):
def wrapper(old):
def inner(self, transforms, *args, **kwargs):
log.debug('Invoking wrapper %s', func)
ret = func(self, transforms, *args, **kwargs)
if ret:
return old(self, transforms, *args, **kwargs)
else:
return ret
return inner
cls._execute = wrapper(cls._execute)
return cls
return classwrapper
## HANDLER
#interceptor(token_authenticate())
class SampleAuthenticatedRequestHandler(BaseHandler):
def get(self):
self.write({'response': self.user.as_dict()})
The request receives an empty response when the token is missing/invalid.
> curl localhost:8888/test -I -v
* Adding handle: conn: 0x7fb18b004000
* Adding handle: send: 0
* Adding handle: recv: 0
* Curl_addHandleToPipeline: length: 1
* - Conn 0 (0x7fb18b004000) send_pipe: 1, recv_pipe: 0
* About to connect() to localhost port 8888 (#0)
* Trying ::1...
* Connected to localhost (::1) port 8888 (#0)
> HEAD /test HTTP/1.1
> User-Agent: curl/7.30.0
> Host: localhost:8888
> Accept: */*
>
* Empty reply from server
* Connection #0 to host localhost left intact
curl: (52) Empty reply from server
Is there something missing? Is this the best way to abstract authentication away from my handler? I was hoping to replicate this model for different kinds of authentication (token based, session based etc).
RequestHandler._execute is an internal method and this will break in Tornado 4.0. prepare() and the HTTP verb methods get()/post()/etc are the supported methods to be overridden in subclasses. I don't see any reason that this code would return an empty response, although the server logs (which you did not post) might have something useful.
This is a working version of the above function. Just for anyone else who's having the same trouble -
def token_authenticate():
"""
This is a basic authentication interceptor which
protects the desired URIs and requires
authentication as per configuration
"""
def wrapper(self, transforms, *args, **kwargs):
request = self.request
try:
token = request.arguments.get('token')[0]
if not token:
return False
user_token = get_user(token)
if user_token:
self.token = user_token
self.user = user_token.user
return True
except:
pass
return False
return wrapper
def interceptor(func):
"""
This is a class decorator which is helpful in configuring
one or more interceptors which are able to intercept, inspect,
process and approve or reject further processing of the request
"""
def classwrapper(cls):
def wrapper(old):
def inner(self, transforms, *args, **kwargs):
ret = func(self, transforms, *args, **kwargs)
if ret:
return old(self, transforms, *args, **kwargs)
else:
self._transforms = transforms
return self._unauthorized()
return inner
cls._execute = wrapper(cls._execute)
return cls
return classwrapper
retVal = None
retries = 5
success = False
while retries > 0 and success == False:
try:
retVal = graph.put_event(**args)
success = True
except:
retries = retries-1
logging.info('Facebook put_event timed out. Retrying.')
return success, retVal
In the code above, how can I wrap this whole thing up as a function and make it so that any command (in this example, 'graph.put_event(**args)') can be passed in as a parameter to be executed within the function?
To directly answer your question:
def foo(func, *args, **kwargs):
retVal = None
retries = 5
success = False
while retries > 0 and success == False:
try:
retVal = func(*args, **kwargs)
success = True
except:
retries = retries-1
logging.info('Facebook put_event timed out. Retrying.')
return success, retVal
This can then be called as such:
s, r = foo(graph.put_event, arg1, arg2, kwarg1="hello", kwarg2="world")
As an aside, given the above task, I would write it along the lines of:
class CustomException(Exception): pass
# Note: untested code...
def foo(func, *args, **kwargs):
retries = 5
while retries > 0:
try:
return func(*args, **kwargs)
except:
retries -= 1
# maybe sleep a short while
raise CustomException
# to be used as such
try:
rv = foo(graph.put_event, arg1, arg2, kwarg1="hello", kwarg2="world")
except CustomException:
# handle failure
def do_event(evt, *args, **kwargs):
...
retVal = evt(*args, **kwargs)
...