I had the following idea. Is it possible to implement a retry routine in python? Here is a simple example of what I have done. I would like to have a more flexible solution. Independent of the function. So switch removeFile with any other function and get rid of the while loop in the main.
import os
import time
def removeFile(file):
try:
os.remove(file)
print("removed : "+file)
return True
except PermissionError:
print("could not delete file "+file+" ; will try again")
return False
if __name__ == "__main__":
file = "some_path/file.ext"
sucess = False
maxCount = 5
count = 0
while not sucess:
sucess = removeFile(file)
count += 1
if count == maxCount:
sucess = True
print("could not delete file "+file+" ; permission denied.")
time.sleep(5)
Thanks to #shmee I got a new approach, using a decorator.
def retry(ExceptionToCheck, tries=4, delay=3, backoff=2, logger=None):
"""Retry calling the decorated function using an exponential backoff.
http://www.saltycrane.com/blog/2009/11/trying-out-retry-decorator-python/
original from: http://wiki.python.org/moin/PythonDecoratorLibrary#Retry
:param ExceptionToCheck: the exception to check. may be a tuple of
exceptions to check
:type ExceptionToCheck: Exception or tuple
:param tries: number of times to try (not retry) before giving up
:type tries: int
:param delay: initial delay between retries in seconds
:type delay: int
:param backoff: backoff multiplier e.g. value of 2 will double the delay
each retry
:type backoff: int
:param logger: logger to use. If None, print
:type logger: logging.Logger instance
"""
def deco_retry(f):
#wraps(f)
def f_retry(*args, **kwargs):
mtries, mdelay = tries, delay
while mtries > 1:
try:
return f(*args, **kwargs)
except ExceptionToCheck:
msg = "%s, Retrying in %d seconds..." % (str(ExceptionToCheck), mdelay)
if logger:
#logger.exception(msg) # would print stack trace
logger.warning(msg)
else:
print(msg)
time.sleep(mdelay)
mtries -= 1
mdelay *= backoff
return f(*args, **kwargs)
return f_retry # true decorator
return deco_retry
and now the only thing left to do is decorate our little remove function:
#retry(PermissionError, tries=5, delay=2,backoff=2)
def removeFile(f):
os.remove(f)
I use this generic template code for re-tries at work to handle all cases and avoid code failures.
N_try = 10
for n_try in range(0,N_try):
success = None
try:
#code execution
output = None
output = CodeExecution(input)
except ex_name_1 as (errno, strerror):
success = False
print("EXCEPTION CAUGHT: Try {} of {}:".format(n_try, N_try))
print("exc_n={}, exc_str={}".format(errno, strerror))
except ex_name_2 as (errno, strerror):
success = False
print("EXCEPTION CAUGHT: Try {} of {}:".format(n_try, N_try))
print("exc_n={}, exc_str={}".format(errno, strerror))
except:
print("UNKNOWN EXCEPTION CAUGHT: Try {} of {}".format(n_try, N_try))
success = False
if success is True:
if CheckOutpup(output) is True:
success = True
break
else:
success = False
print("CHECK OUTPUT: Try {n_try} of {N_try} FAILED".format{n_try, N_try})
else:
print("EXECUTION: Try {n_try} of {N_try} FAILED".format{n_try, N_try})
success = False
if success is False:
print("Failed execution after {N_try} tries".format{N_try})
Just put the while loop into the method
import os
import time
def removeFile(file):
sucess = False
maxCount = 5
count = 0
while not sucess:
try:
os.remove(file)
print("removed : "+file)
sucess = True
except PermissionError:
print("could not delete file "+file+" ; will try again")
sucess = False
count += 1
if count == maxCount:
sucess = True
print("could not delete file "+file+" ; permission denied.")
time.sleep(5)
if __name__ == "__main__":
file = "some_path/file.ext"
removeFile(file)
Related
I want to increase the coverage of scraping the content and avoid getting timeout, but however I want if the page raise timeout error then retry again, but if the second round of timeouts increase the timeout counter retry the URL again with a max of 3 times, and if it doesn't increase remove from dict
async def _worker(self, i):
while True:
url = await self.fetching.get()
if url is None:
return
#logger.info(f'Fetch worker {i} is fetching a URL: {url}')
try:
site = await self._fetch_rss(url)
await self._processing(site)
except asyncio.TimeoutError as err:
while self.timeouts[url] < 1:
logger.info("adding {}".format(url))
site = await self.fetch(url)
await self.process(site)
self.timeouts[url] += 1
In general the pattern I use for "try n times and error if I never have a success" is the following:
for _ in range(MAX_TRIES):
try:
# do something
pass
except Exception:
# handle exception
pass
else:
break
else: # A lesser-known syntax; else after for is only run if the for *wasn't* broken out of
# We never broke, which means we never had a success, so handle the failure here
pass
I'm trying to make do with Python's lack of a switch statement and make code more efficient by using a dictionary, but I can't quite get what I'm looking for. Here's a simplified version of the code I'm working with
def zero():
print("Yo")
def one():
print("Hey")
options = {
0: zero,
1: one,
}
while True:
response = int(input("Number: "))
try:
options.get(response)()
except TypeError:
print("Not a valid response")
and what I would like to see is some way to break the loop such as 2: break that exits the loop. Currently I'm using sys.exit(0), but was wondering if it was possible to use the break keyword
You could define a LoopBreak exception, raise that in a two function, and catch it in the loop to break:
class LoopBreak(Exception):
pass
def zero():
print("Yo")
def one():
print("Hey")
def two():
raise LoopBreak
options = {
0: zero,
1: one,
2: two
}
while True:
response = int(input("Number: "))
try:
options.get(response)()
except TypeError:
print("Not a valid response")
except LoopBreak:
break
As a point of interest, this is similar to the pattern used natively by Python to stop generators; they raise a StopIteration exception when they run out of values to yield.
EDIT: As #mfripp correctly notes below, this will mask any TypeErrors that are raised during execution of zero or one. I would change the main loop to this instead (so you don't have to rely on TypeError):
while True:
response = int(input("Number: "))
action = options.get(response)
if action is None:
print("Not a valid response")
continue
try:
action()
except LoopBreak:
break
There are a few ways you could do this.
Here's a more robust version of #BingsF's clever answer (this one won't mask exceptions raised within the selected function):
class LoopBreak(Exception):
pass
def zero():
print("Yo")
def one():
print("Hey")
def two():
raise LoopBreak
options = {
0: zero,
1: one,
2: two
}
while True:
try:
response = int(input("Number: "))
action = options[response]
except (ValueError, KeyError):
print("Not a valid response")
continue
try:
action()
except LoopBreak:
break
Or you could specify a special flag in your dictionary that will force a break:
def zero():
print("Yo")
def one():
print("Hey")
options = {
0: zero,
1: one,
2: False
}
while True:
try:
response = int(input("Number: "))
action = options[response]
except (ValueError, KeyError):
print("Not a valid response")
continue
if action is False:
break
else:
action()
Or use a special return value to force a break:
def zero():
print("Yo")
# returns None by default
def one():
print("Hey")
# returns None by default
def two():
return False
options = {
0: zero,
1: one,
2: two
}
while True:
try:
response = int(input("Number: "))
action = options[response]
except (ValueError, KeyError):
print("Not a valid response")
continue
if action() is False:
break
The following code might be more "Pythonic". It could be infinitesimally slower than the approaches above, because it has to check all the if statements instead of looking up the function in a dictionary via a hash. But it may be easier to read and maintain.
while True:
try:
response = int(input("Number: "))
except ValueError:
response = -1 # error flag
if response == 0:
print("Yo")
elif response == 1:
print("Hey")
elif response == 2:
break
else:
print("Not a valid response")
This is all you need:
while True:
response = int(input("Number: "))
if response not in options:
print("Not a valid response")
else if response == 2:
break
else:
options[response]() # invoke the corresponding function
Incidentally, storing functions in a dictionary and having to invoke them like this isn't exactly Pythonic. It's a lot nicer to simply explicitly enumerate the behaviour you need with successive ifs.
I have a situation where I need to return the first item found in a list, How do I return it when I use recursion ?:
def get_hostname (ifname):
try :
# Do something to get hostname
return hostname
except IOError:
return -1
def get_hostname_r(lst):
if not lst:
return False
if get_hostname(lst[0]) != -1 :
print 'Found ', get_hostname(lst[0])
return get_hostname(lst[0]) # DOESNT WORK
else :
print 'Not found ', get_hostname(lst[0])
get_hostname_r(lst[1:])
print 'return = ', get_hostname_r(['eth1','eth2','eth3','eth4','eth5' ])
I understand that the return goes back to the calling stack but Iam looking here for the best practices without using a global variable to get the value?
You can simply return the value, the returned value is handed over the whole recursion stack:
def get_hostname_r(lst):
if not lst:
return False
if get_hostname(lst[0]) != -1 :
print 'Found ', get_hostname(lst[0])
return get_hostname(lst[0])
else:
print 'Not found ', get_hostname(lst[0])
return get_hostname_r(lst[1:])
But easier to read is a for-loop:
def get_hostname_r(interfaces):
for interface in interfaces:
result = get_hostname(interface)
if result != -1:
return result
return False
First, if you can't do anything about the IO error, don't mask it by returning -1 ("What does -1 mean? Why didn't I get a host name back?"). Just document that get_hostname might raise an IOError.
def get_hostname(ifname):
# Do stuff that might raise an IOError
return hostname
The same goes for the recursive version. Either return a valid hostname, or raise an exception (or let an uncaught exception continue).
def get_hostname_r(lst):
if not lst:
raise IOError("Hostname not found")
try:
hostname = get_hostname(lst[0])
print >>sys.stderr, 'Found {0}'.format(hostname)
return hostname
except IOError:
print 'Not found with {0}'.format(lst[0])
return get_hostname_r(lst[1:])
Of course, recursion isn't really the best way to write this; use a simple for loop instead to iterate over lst.
def get_hostname_iter(lst):
for ifname in lst:
try:
return get_hostname(ifname)
except IOError:
continue
raise IOError("Hostname not found")
I want to run a line of code if any of the exceptions are met, but not if the try succeeds, a bit like the opposite of else when using try/except.
Currently I have exceptionOccured to set to True if an exception occurs, but I'm guessing there should be a more pythonic way of doing this.
Here's the current code I have, it was an attempt to edit values in a dictionary from a list, and create the keys if they don't exist. How would I redo the exceptions so that exceptionOccured is not needed?
dictionaryValue = {"data": dict.fromkeys( [0, 1, 2, 3] ), "data2": {0: "test",1:"test2"}}
reducedDictionary = dictionaryValue
valueList = ["data", 1, 64, "Testing", "value"]
canOverwriteKeys = True
for i in valueList[:-2]:
exceptionOccured = False
try:
if type( reducedDictionary ) != dict:
raise ValueError()
elif reducedDictionary.get( i, False ) == False:
raise KeyError()
except ValueError:
print "not dictionary"
reducedDictionary = {}
exceptionOccured = True
except KeyError:
print "key doesn't exist"
exceptionOccured = True
if exceptionOccured or ( type( reducedDictionary[i] ) != dict and canOverwriteKeys ):
print "setting key value"
reducedDictionary[i] = {}
reducedDictionary = reducedDictionary[i]
reducedDictionary[valueList[-2]] = valueList[-1]
print dictionaryValue
Edit: Improved the code based on the answers, thanks :)
def editDictionary( dictionaryName, listOfValues, canOverwriteKeys=True ):
reducedDictionary = dictionaryName
for i in valueList[:-2]:
if type( reducedDictionary ) != dict:
reducedDictionary = {}
try:
if reducedDictionary.get( i, False ) == False:
raise ValueError()
elif type( reducedDictionary[i] ) != dict:
if not canOverwriteKeys:
return
raise KeyError()
except( ValueError, KeyError ):
reducedDictionary[i] = {}
except:
print "Something went wrong"
return
reducedDictionary = reducedDictionary[i]
reducedDictionary[valueList[-2]] = valueList[-1]
Just catch both exceptions in one handler:
try:
# ...
except (ValueError, KeyError) as e:
if isinstance(e, ValueError):
print "not dictionary"
reducedDictionary = {}
else:
print "key doesn't exist"
print "setting key value"
reducedDictionary[i] = {}
If your exception handlers are more complex, you could also use a function:
def handle_common_things():
# things common to all exception handlers
try:
# ...
except Exception1:
# do exception-specific things
handle_common_things()
except Exception2:
# do exception-specific things
handle_common_things()
I'd probably go with Martijn's answer, but you can also wrap your try/except block in another layer of try/except (and in real code I'd frown on using a bare except like this, or more likely define a new exception that's thrown from inside any except: clauses that you want to detect)
def exception_test(val):
try:
try:
result = 1.0 / val
except ZeroDivisionError:
print "Divide by zero"
raise
else:
print "1 / {0} = {1}".format(val, result)
except:
print "there was an exception thrown."
>>> exception_test(2.0)
1 / 2.0 = 0.5
>>> exception_test(0)
Divide by zero
there was an exception thrown.
Is there a command such as break and continue which could repeat recent iteration?
For example, when exception is thrown.
for i in range(0,500):
try:
conn = getConnection(url+str(i))
doSomething(conn)
except:
repeat
Let's have an iteration where i variable's value is 6. During this iteration some connection error occurred. I want to repeat this iteration.
Is there a command which can do that?
Of course I can do this:
i=0
while i!=500:
try:
conn = getConnection(url+str(i))
doSomething(conn)
i+=1
except:
pass
No, there is no command to "rewind" a for-loop in Python.
You could use a while True: loop inside the for-loop:
for i in range(500):
while True:
try:
conn = getConnection(url+str(i))
doSomething(conn)
except Exception: # Replace Exception with something more specific.
continue
else:
break
or without the else::
for i in range(500):
while True:
try:
conn = getConnection(url+str(i))
doSomething(conn)
break
except Exception: # Replace Exception with something more specific.
continue
But I personally think that your proposed solution is better because it avoids an indentation level.
for i in range(500):
while True
try:
conn = getConnection(url+str(i))
break
except Exception: # still allows to quit with KeyboardInterrupt
continue
do_your_stuff()
This looks bit risky, however, you should at least enable some logging inside a while block.
If you expect to use it in more places, you might write a simple decorator:
def keep_trying(fn, *args, **kwargs):
def inner(*args, **kwargs):
while True:
try:
return fn(*args, **kwargs)
except Exception:
continue
return inner
# later you can use it simple like this:
for i in range(500):
conn = keep_trying(getConnection)(url+str(i))
You can use generators :
def process_connections(n_connections, url, max_tries=50):
i = 0
try_count = 0
while i < n_connections:
try:
conn = getConnection(url+str(i))
yield conn
except:
try_count += 1
if try_count > max_tries:
raise Exception("Unable to connect after %s tries" % max_tries)
else:
i += 1 # increments only if no exception
And you perform your operations :
for conn in process_connections(500, url):
do_something(conn)
You can use nested for loops to put a cap on the number of times you retry the operation. This is bascially the sam as #PierreAlex's generator answer but without the extra function definition.
for i in range(500):
for retry in range(10):
try:
conn = getConnection(url+str(i))
doSomething(conn)
except Exception: # Replace Exception with something more specific.
time.sleep(1)
else:
print "iteration", i, "failed"
Why not just use an if statement?
n=6
i=0
while i!=500:
failed = False;
try:
conn = getConnection(url+str(i))
doSomething(conn)
i+=1
except:
#handle error
failed = True;
#try again if n-th case failed first time
if(i == n and failed):
try:
conn = getConnection(url+str(i))
doSomething(conn)
except:
#handle error
Here is one. You would need to add a logging or alert system to let you know that something is stuck:
state = "" #state of the loop
# If there is no error continue. If there is error, remain in loop
while True:
if state != "error":
try:
1/0 # command
break # no error so break out of loop
except:
state = "error" #declare error so maintain loop
continue
elif state == "error": # maintain loop
continue