I am trying to create two set of databases every time inside my python script for the same of which I have written the below set of code which looks redundant to me since I am initializing the variable ext 2 times and hence if anyone can suggest some better alternatives, that would be really helpful.
def create_datasets(database, ext):
try:
dataset = "bq --location=US mk -d " + database + ext
try:
return_cd, out, err = run_sys_command(dataset)
except Exception as e:
print(e)
except Exception as e:
print(e)
raise
ext = ''
create_datasets(database, ext)
ext = '_stg'
create_datasets(database, ext)
Use a loop?
for ext in ['', '_stg']:
create_datasets(database, ext)
About your function:
def create_datasets(database, ext):
try:
dataset = f"bq --location=US mk -d {database}{ext}"
return_cd, out, err = run_sys_command(dataset)
except Exception as e: # <- you should catch sub exception!
print(e)
Any exception Exception raised in your function is caught and handled by the inner try block. The outer therefore seems redundant.
def create_datasets(database, ext):
try:
dataset = "bq --location=US mk -d " + database + ext
return_cd, out, err = run_sys_command(dataset)
except Exception as e:
print(e)
Related
What my function does:
creates a war file by processing the folder contents (/tmp/A, /tmp/B and so on)
Does some file path and folder path manipulations to get the final version from the war file.
store the file name in one variable and the version in another.
Push the war file to the Repository using curl.
I'm using multiple try & except blocks to catch the exception for each action and looks very un-pythonic.
Is there an elegant and simple way to approach this ? thanks in advance.
import shutil
import traceback
import subprocess
import os
import glob
def my_function(path_a, path_b, tmp_dir)
try:
<shutil.copy to the tmp dir>
except:
traceback.print_exc()
try:
war_process = subprocess.run([WAR GENERATION COMMAND], check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
print(war_process.stdout.decode("utf-8"))
except subprocess.CalledProcessError as e:
exit_code = e.returncode
stderror = e.stderr
print(exit_code, stderror)
print(war_process.stderr.decode("utf-8"))
try:
output_folder = os.path.join("/tmp/dir/work", FILE_PATH, ARTIFACT_DATE, FILE_WO_EXTENSION)
except:
traceback.print_exc()
try:
file_name = list(glob.glob(os.path.join(output_folder, "*.war")))
except:
traceback.print_exc()
try:
file_path = os.path.join(output_folder, file_name)
except:
traceback.print_exc()
try:
os.rename(file_path, file_path.split('war')[0] + ".tgz")
except:
traceback.print_exc()
try:
file_version = os.path.basename(file_path)
except:
traceback.print_exc()
cmd = "curl -u username -T ....)"
try:
curl_output = subprocess.run([cmd], shell=True, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
print(curl_output.stdout.decode("utf-8"))
except subprocess.CalledProcessError as er:
print(proc_c.stderr.decode("utf-8"))
exit_c = er.returncode
std = er.stderr
print(exit_c, std)
You can write try once, then handle all the exceptions later:
try:
output_folder = os.path.join("/tmp/dir/work", FILE_PATH, ARTIFACT_DATE, FILE_WO_EXTENSION)
file_name = list(glob.glob(os.path.join(output_folder, "*.war")))
file_path = os.path.join(output_folder, file_name)
os.rename(file_path, file_path.split('war')[0] + ".tgz")
except FooException:
print('foo')
except BarException:
print('bar')
First of all never use bare except in your code. Read bullets 6 to 11 in PEP8:Programming Recommendations.
My suggestion is to use this code instead:
def my_function(path_a, path_b, tmp_dir)
try:
<shutil.copy to the tmp dir>
except:
traceback.print_exc()
try:
war_process = subprocess.run([WAR GENERATION COMMAND], check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
print(war_process.stdout.decode("utf-8"))
output_folder = os.path.join("/tmp/dir/work", FILE_PATH, ARTIFACT_DATE, FILE_WO_EXTENSION)
file_name = list(glob.glob(os.path.join(output_folder, "*.war")))
file_path = os.path.join(output_folder, file_name)
os.rename(file_path, file_path.split('war')[0] + ".tgz")
file_version = os.path.basename(file_path)
except subprocess.CalledProcessError as e:
exit_code = e.returncode
stderror = e.stderr
print(exit_code, stderror)
print(war_process.stderr.decode("utf-8"))
except Exception as e:
print(f'The program caught an exception {e}')
traceback.print_exc()
cmd = "curl -u username -T ....)"
try:
curl_output = subprocess.run([cmd], shell=True, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
print(curl_output.stdout.decode("utf-8"))
except subprocess.CalledProcessError as er:
print(proc_c.stderr.decode("utf-8"))
exit_c = er.returncode
std = er.stderr
print(exit_c, std)
The second and the third try/except blocks must stay separated because both catch the same exception.
Also, if any of the blocks you created here catch a specific exception in this list, you should behave them like you behave the subprocess.CalledProcessError .
Best practice is to write one try block with multiple excepts in which each except block catches a specific exception.
You don't need to put a try/except block after every statement. It would be better to put multiple statements in a try/except block
def my_function(path_a, path_b, tmp_dir)
try:
<shutil.copy to the tmp dir>
war_process = subprocess.run([WAR GENERATION COMMAND], check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
print(war_process.stdout.decode("utf-8"))
except subprocess.CalledProcessError as e:
exit_code = e.returncode
stderror = e.stderr
print(exit_code, stderror)
print(war_process.stderr.decode("utf-8"))
try:
output_folder = os.path.join("/tmp/dir/work", FILE_PATH, ARTIFACT_DATE, FILE_WO_EXTENSION)
file_name = list(glob.glob(os.path.join(output_folder, "*.war")))
file_path = os.path.join(output_folder, file_name)
os.rename(file_path, file_path.split('war')[0] + ".tgz")
file_version = os.path.basename(file_path)
except:
traceback.print_exc()
cmd = "curl -u username -T ....)"
try:
curl_output = subprocess.run([cmd], shell=True, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
print(curl_output.stdout.decode("utf-8"))
except subprocess.CalledProcessError as er:
print(proc_c.stderr.decode("utf-8"))
exit_c = er.returncode
std = er.stderr
print(exit_c, std)
```
is there any solution in python which lets a function execute after the previous one was finished?
Here is one of the ideas I'm using now. But it is not solving the problem when files are larger and the program needs more time.
def copy_to_jumphost(self):
try:
if self.connect():
stdin, stdout, stderr = self.client.exec_command('sshpass -p %s scp -r %s#%s:%s/' % (self.password,self.username,self.hostname,self.log_path) + self.lista[self.file_number].rstrip() + ' ' + '/home/%s/' % (self.username) + self.lista[self.file_number].rstrip())
except (AttributeError, TypeError) as e:
print("Error occurred:", e)
try:
if self.connect():
if self.copy_to_jumphost():
ftp_client = self.client.open_sftp()
ftp_client.get(filepath, self.localpath)
print("Success! \nFile coppied to %s" %(self.localpath))
else:
time.sleep(5)
ftp_client = self.client.open_sftp()
ftp_client.get(filepath, self.localpath)
print("Success but needed some time! \nFile coppied to %s" %(self.localpath))
except (AttributeError, TypeError) as e:
print("Error occurred:", e)
Perfect situation for me will be if in else statement there is a solution to wait for finishing the copy_to_jumphost() function, because time.sleep(5) will fail if I will need to copy larger files.
Following is the code snippet from our code base
# global library function
def check_call_noout(params, acceptable_exit_codes = (0,), shellCommand=False):
FNULL = open('/dev/null', 'w')
sts = 1
try:
if shellCommand:
p = subprocess.Popen(params, stdout=FNULL, stderr=FNULL,shell=True)
else:
p = subprocess.Popen(params, stdout=FNULL, stderr=FNULL)
sts = os.waitpid(p.pid, 0)[1]
except:
raise
finally:
FNULL.close()
exit_code = sts >> 8
if exit_code not in acceptable_exit_codes:
raise subprocess.CalledProcessError(exit_code, params)
# driver code
try:
cmd = ["/bin/tar", "--acls", "--selinux", "--xattrs", "-czf a.tar.gz", "./a.xml", "--exclude","\"lost+found\""]
check_call_noout(cmd,(0,1),False)
except subprocess.CalledProcessError as e:
print e.output, e.returncode
except Exception as e:
print(type(e).__name__, e)
I want to print the params argument value passed into subprocess.CalledProcessError object which is raised inside the library function and caught in my driver code.
However, I cannot change anything in the library function check_call_noout()
If i understand correctly, getting the __dict__ attribute of subprocess.CalledProcessError class would do:
try:
subprocess.run([...], check=True)
except subprocess.CalledProcessError as e:
print(e.__dict__)
You can also use vars function which will call __dict__ internally:
try:
subprocess.run([...], check=True)
except subprocess.CalledProcessError as e:
print(vars(e))
I'm implementing a fast copy function for Python (because ain't nobody got time for shutil) however I'm getting a syntax error E901 on this line except (IOError, os.error), why: Here's the full code:
class CTError(Exception):
def __init__(self, errors):
self.errors = errors
try:
O_BINARY = os.O_BINARY
except:
O_BINARY = 0
READ_FLAGS = os.O_RDONLY | O_BINARY
WRITE_FLAGS = os.O_WRONLY | os.O_CREAT | os.O_TRUNC | O_BINARY
BUFFER_SIZE = 128 * 1024
def copyfile(src, dst):
try:
fin = os.open(src, READ_FLAGS)
stat = os.fstat(fin)
fout = os.open(dst, WRITE_FLAGS, stat.st_mode)
for x in iter(lambda: os.read(fin, BUFFER_SIZE), ""):
os.write(fout, x)
finally:
try:
os.close(fin)
except:
pass
try:
os.close(fout)
except:
pass
def copytree(src, dst, symlinks=False, ignore=[]):
names = os.listdir(src)
if not os.path.exists(dst):
os.makedirs(dst)
errors = []
for name in names:
if name in ignore:
continue
srcname = os.path.join(src, name)
dstname = os.path.join(dst, name)
try:
if symlinks and os.path.islink(srcname):
linkto = os.readlink(srcname)
os.symlink(linkto, dstname)
elif os.path.isdir(srcname):
copytree(srcname, dstname, symlinks, ignore)
else:
copyfile(srcname, dstname)
# XXX What about devices, sockets etc.?
except (IOError, os.error), why: #XXX Here's the bug!
errors.append((srcname, dstname, str(why)))
except CTError, err:
errors.extend(err.errors)
if errors:
raise CTError(errors)
Why is this syntax invalid?
In Python 2, this fragment's syntax appears valid (CPython 2.7.10 accepts it on my machine).
In Python 3, this syntax isn't valid. The syntax,
except <TYPEEXPR>, <VAR>:
was deprecated. It was replaced with,
except <TYPEEXPR> as <VAR>:
in Python 3. E.g.,
except (IOError, os.error) as why:
This syntax is valid in Python 2 (it was added in 2.6, I believe), and I find it easier to read, so I'd recommend using it in Python 2 over the deprecated syntax as well, especially since it's forward-compatible and Python ≤2.5 usage is pretty small nowadays, and probably not worth supporting; I believe most libraries have dropped support.
I recently switched from urlib2 to requests and I'm not sure how to deal with exceptions. What is best practice? My current code looks like this, but is not doing any good:
try:
response = requests.get(url)
except requests.ConnectionError , e:
logging.error('ConnectionError = ' + str(e.code))
return False
except requests.HTTPError , e:
logging.error('HTTPError = ' + str(e.reason))
return False
except requests.Timeout, e:
logging.error('Timeout')
return False
except requests.TooManyRedirects:
logging.error('TooManyRedirects')
return False
except Exception:
import traceback
logging.error('generic exception: ' + traceback.format_exc())
return False
Since it looks bad as a comment, have you tried:
try:
# some code
except Exception as e:
print e