I want to increase the coverage of scraping the content and avoid getting timeout, but however I want if the page raise timeout error then retry again, but if the second round of timeouts increase the timeout counter retry the URL again with a max of 3 times, and if it doesn't increase remove from dict
async def _worker(self, i):
while True:
url = await self.fetching.get()
if url is None:
return
#logger.info(f'Fetch worker {i} is fetching a URL: {url}')
try:
site = await self._fetch_rss(url)
await self._processing(site)
except asyncio.TimeoutError as err:
while self.timeouts[url] < 1:
logger.info("adding {}".format(url))
site = await self.fetch(url)
await self.process(site)
self.timeouts[url] += 1
In general the pattern I use for "try n times and error if I never have a success" is the following:
for _ in range(MAX_TRIES):
try:
# do something
pass
except Exception:
# handle exception
pass
else:
break
else: # A lesser-known syntax; else after for is only run if the for *wasn't* broken out of
# We never broke, which means we never had a success, so handle the failure here
pass
Related
I had the following idea. Is it possible to implement a retry routine in python? Here is a simple example of what I have done. I would like to have a more flexible solution. Independent of the function. So switch removeFile with any other function and get rid of the while loop in the main.
import os
import time
def removeFile(file):
try:
os.remove(file)
print("removed : "+file)
return True
except PermissionError:
print("could not delete file "+file+" ; will try again")
return False
if __name__ == "__main__":
file = "some_path/file.ext"
sucess = False
maxCount = 5
count = 0
while not sucess:
sucess = removeFile(file)
count += 1
if count == maxCount:
sucess = True
print("could not delete file "+file+" ; permission denied.")
time.sleep(5)
Thanks to #shmee I got a new approach, using a decorator.
def retry(ExceptionToCheck, tries=4, delay=3, backoff=2, logger=None):
"""Retry calling the decorated function using an exponential backoff.
http://www.saltycrane.com/blog/2009/11/trying-out-retry-decorator-python/
original from: http://wiki.python.org/moin/PythonDecoratorLibrary#Retry
:param ExceptionToCheck: the exception to check. may be a tuple of
exceptions to check
:type ExceptionToCheck: Exception or tuple
:param tries: number of times to try (not retry) before giving up
:type tries: int
:param delay: initial delay between retries in seconds
:type delay: int
:param backoff: backoff multiplier e.g. value of 2 will double the delay
each retry
:type backoff: int
:param logger: logger to use. If None, print
:type logger: logging.Logger instance
"""
def deco_retry(f):
#wraps(f)
def f_retry(*args, **kwargs):
mtries, mdelay = tries, delay
while mtries > 1:
try:
return f(*args, **kwargs)
except ExceptionToCheck:
msg = "%s, Retrying in %d seconds..." % (str(ExceptionToCheck), mdelay)
if logger:
#logger.exception(msg) # would print stack trace
logger.warning(msg)
else:
print(msg)
time.sleep(mdelay)
mtries -= 1
mdelay *= backoff
return f(*args, **kwargs)
return f_retry # true decorator
return deco_retry
and now the only thing left to do is decorate our little remove function:
#retry(PermissionError, tries=5, delay=2,backoff=2)
def removeFile(f):
os.remove(f)
I use this generic template code for re-tries at work to handle all cases and avoid code failures.
N_try = 10
for n_try in range(0,N_try):
success = None
try:
#code execution
output = None
output = CodeExecution(input)
except ex_name_1 as (errno, strerror):
success = False
print("EXCEPTION CAUGHT: Try {} of {}:".format(n_try, N_try))
print("exc_n={}, exc_str={}".format(errno, strerror))
except ex_name_2 as (errno, strerror):
success = False
print("EXCEPTION CAUGHT: Try {} of {}:".format(n_try, N_try))
print("exc_n={}, exc_str={}".format(errno, strerror))
except:
print("UNKNOWN EXCEPTION CAUGHT: Try {} of {}".format(n_try, N_try))
success = False
if success is True:
if CheckOutpup(output) is True:
success = True
break
else:
success = False
print("CHECK OUTPUT: Try {n_try} of {N_try} FAILED".format{n_try, N_try})
else:
print("EXECUTION: Try {n_try} of {N_try} FAILED".format{n_try, N_try})
success = False
if success is False:
print("Failed execution after {N_try} tries".format{N_try})
Just put the while loop into the method
import os
import time
def removeFile(file):
sucess = False
maxCount = 5
count = 0
while not sucess:
try:
os.remove(file)
print("removed : "+file)
sucess = True
except PermissionError:
print("could not delete file "+file+" ; will try again")
sucess = False
count += 1
if count == maxCount:
sucess = True
print("could not delete file "+file+" ; permission denied.")
time.sleep(5)
if __name__ == "__main__":
file = "some_path/file.ext"
removeFile(file)
for i in range(0, 650):
s = ticket[i]
try:
response = resource.get(path='ticket/%s' % s[0]) # Get ticket data from RT server
except urllib2.URLError, e: # If connection fails
resource = RTResource(url, user, pwd, CookieAuthenticator) # Reconnect to RT server
count -= 1 # Count re-connection attempts
if count < 0:
print "Connection failed at ticket %s" % s[0]
print "Got %s tickets out of %s" % {i + 1, len(ticket) + 1}
wb.save(fname)
sys.exit(1)
print 'Trying again...'
i -= 1
continue
count = 10
...more code here...
The above code executes well but will skip an iteration when exception is thrown. I am trying to decreament the value of i and then continuing the loop so that when exception is thrown, the loop will repeat for same value of i. When a value of i is skipped I lose one ticket from RT server. How do I fix it?
You ... can't do that in python. You can't affect the value of the iterator - it's using it's own internal value for each step in the loop, not paying attention to your attempts to override. If you have to succeed for each iteration I use something like this:
while True:
# code here
if success:
break
And place that inside your for loop. Or better, extract a method to simplify readability, but that's another post.
(Besides the correct point raised by g.d.d.c. about the inability to decrement the loop counter the specific way you've gone, )this sort of stuff is exactly the motivation for finally. You should probably organize your code as follows:
try - the part that's supposed to run but might not
except - the part to do only if there was a problem
else (optional) - the part to do only if there wasn't a problem
finally - stuff to do in any case
An alternative to embedding a while loop in your for loop, as suggested by g.d.d.c, is to simply use a while loop instead of a for loop, like so:
i = 0
while i < 650:
s = ticket[i]
try:
response = resource.get(path='ticket/%s' % s[0]) # Get ticket data from RT server
except urllib2.URLError, e: # If connection fails
resource = RTResource(url, user, pwd, CookieAuthenticator) # Reconnect to RT server
count -= 1 # Count re-connection attempts
if count < 0:
print "Connection failed at ticket %s" % s[0]
print "Got %s tickets out of %s" % {i + 1, len(ticket) + 1}
wb.save(fname)
sys.exit(1)
print 'Trying again...'
continue
count = 10
i += 1
...more code here...
Is there a command such as break and continue which could repeat recent iteration?
For example, when exception is thrown.
for i in range(0,500):
try:
conn = getConnection(url+str(i))
doSomething(conn)
except:
repeat
Let's have an iteration where i variable's value is 6. During this iteration some connection error occurred. I want to repeat this iteration.
Is there a command which can do that?
Of course I can do this:
i=0
while i!=500:
try:
conn = getConnection(url+str(i))
doSomething(conn)
i+=1
except:
pass
No, there is no command to "rewind" a for-loop in Python.
You could use a while True: loop inside the for-loop:
for i in range(500):
while True:
try:
conn = getConnection(url+str(i))
doSomething(conn)
except Exception: # Replace Exception with something more specific.
continue
else:
break
or without the else::
for i in range(500):
while True:
try:
conn = getConnection(url+str(i))
doSomething(conn)
break
except Exception: # Replace Exception with something more specific.
continue
But I personally think that your proposed solution is better because it avoids an indentation level.
for i in range(500):
while True
try:
conn = getConnection(url+str(i))
break
except Exception: # still allows to quit with KeyboardInterrupt
continue
do_your_stuff()
This looks bit risky, however, you should at least enable some logging inside a while block.
If you expect to use it in more places, you might write a simple decorator:
def keep_trying(fn, *args, **kwargs):
def inner(*args, **kwargs):
while True:
try:
return fn(*args, **kwargs)
except Exception:
continue
return inner
# later you can use it simple like this:
for i in range(500):
conn = keep_trying(getConnection)(url+str(i))
You can use generators :
def process_connections(n_connections, url, max_tries=50):
i = 0
try_count = 0
while i < n_connections:
try:
conn = getConnection(url+str(i))
yield conn
except:
try_count += 1
if try_count > max_tries:
raise Exception("Unable to connect after %s tries" % max_tries)
else:
i += 1 # increments only if no exception
And you perform your operations :
for conn in process_connections(500, url):
do_something(conn)
You can use nested for loops to put a cap on the number of times you retry the operation. This is bascially the sam as #PierreAlex's generator answer but without the extra function definition.
for i in range(500):
for retry in range(10):
try:
conn = getConnection(url+str(i))
doSomething(conn)
except Exception: # Replace Exception with something more specific.
time.sleep(1)
else:
print "iteration", i, "failed"
Why not just use an if statement?
n=6
i=0
while i!=500:
failed = False;
try:
conn = getConnection(url+str(i))
doSomething(conn)
i+=1
except:
#handle error
failed = True;
#try again if n-th case failed first time
if(i == n and failed):
try:
conn = getConnection(url+str(i))
doSomething(conn)
except:
#handle error
Here is one. You would need to add a logging or alert system to let you know that something is stuck:
state = "" #state of the loop
# If there is no error continue. If there is error, remain in loop
while True:
if state != "error":
try:
1/0 # command
break # no error so break out of loop
except:
state = "error" #declare error so maintain loop
continue
elif state == "error": # maintain loop
continue
In my code below, it stops executing when I hit an exception. How can I get it to re-enter the try statement from where the exception left off? Maybe I am looking for a different way to tackle this without a try-except statement?
import requests
from requests import exceptions
contains_analyst = []
try:
for x in data:
r = requests.get(str(x), timeout=10, verify=False)
if "analyst" in r.text:
contains_analyst.append("Analyst")
print "Analyst # %s" % x
else:
contains_analyst.append("NOPERS")
print "Nopers"
except exceptions.RequestException:
contains_analyst.append("COULD NOT CONNECT")
You should put the try/except around only the part whose error you want to trap. In your example, it looks like you want something more like this:
for x in data:
try:
r = requests.get(str(x), timeout=10, verify=False)
except exceptions.RequestException:
contains_analyst.append("COULD NOT CONNECT")
else:
if "analyst" in r.text:
contains_analyst.append("Analyst")
print "Analyst # %s" % x
else:
contains_analyst.append("NOPERS")
print "Nopers"
Here I use the else clause of the try block to handle the case where no exception is raised (see documentation). In many cases, if you don't need to do anything else after the exception, you could just return at that point and put the following no-exception code in the main function body, reducing indentation a bit:
for x in data:
try:
r = requests.get(str(x), timeout=10, verify=False)
except exceptions.RequestException:
contains_analyst.append("COULD NOT CONNECT")
return contains_analyst
# execution reaches here if no exception
if "analyst" in r.text:
contains_analyst.append("Analyst")
print "Analyst # %s" % x
else:
contains_analyst.append("NOPERS")
print "Nopers"
Of course, whether it makes sense to return at that point depends on the surrounding context of your code.
The program is like this：
HEADER CODE
urllib2.initialization()
try:
while True:
urllib2.read(somebytes)
urllib2.read(somebytes)
urllib2.read(somebytes)
...
except Exception, e:
print e
FOOTER CODE
My question is when error occurs (timeout, connection reset by peer, etc), how to restart from urllib2.initialization() instead of existing main program and restarting from HEADER CODE again？
You could wrap your code in a "while not done" loop:
#!/usr/bin/env python
HEADER CODE
done=False
while not done:
try:
urllib2.initialization()
while True:
# I assume you have code to break out of this loop
urllib2.read(somebytes)
urllib2.read(somebytes)
urllib2.read(somebytes)
...
except Exception, e: # Try to be more specific about the execeptions
# you wish to catch here
print e
else:
# This block is only executed if the try-block executes without
# raising an exception
done=True
FOOTER CODE
How about just wrap it in another loop?
HEADER CODE
restart = True
while restart == True:
urllib2.initialization()
try:
while True:
restart = False
urllib2.read(somebytes)
urllib2.read(somebytes)
urllib2.read(somebytes)
...
except Exception, e:
restart = True
print e
FOOTER CODE
Simple way with attempts restrictions
HEADER CODE
attempts = 5
for attempt in xrange(attempts):
urllib2.initialization()
try:
while True:
urllib2.read(somebytes)
urllib2.read(somebytes)
urllib2.read(somebytes)
...
except Exception, e:
print e
else:
break
FOOTER CODE