Following is the code snippet from our code base
# global library function
def check_call_noout(params, acceptable_exit_codes = (0,), shellCommand=False):
FNULL = open('/dev/null', 'w')
sts = 1
try:
if shellCommand:
p = subprocess.Popen(params, stdout=FNULL, stderr=FNULL,shell=True)
else:
p = subprocess.Popen(params, stdout=FNULL, stderr=FNULL)
sts = os.waitpid(p.pid, 0)[1]
except:
raise
finally:
FNULL.close()
exit_code = sts >> 8
if exit_code not in acceptable_exit_codes:
raise subprocess.CalledProcessError(exit_code, params)
# driver code
try:
cmd = ["/bin/tar", "--acls", "--selinux", "--xattrs", "-czf a.tar.gz", "./a.xml", "--exclude","\"lost+found\""]
check_call_noout(cmd,(0,1),False)
except subprocess.CalledProcessError as e:
print e.output, e.returncode
except Exception as e:
print(type(e).__name__, e)
I want to print the params argument value passed into subprocess.CalledProcessError object which is raised inside the library function and caught in my driver code.
However, I cannot change anything in the library function check_call_noout()
If i understand correctly, getting the __dict__ attribute of subprocess.CalledProcessError class would do:
try:
subprocess.run([...], check=True)
except subprocess.CalledProcessError as e:
print(e.__dict__)
You can also use vars function which will call __dict__ internally:
try:
subprocess.run([...], check=True)
except subprocess.CalledProcessError as e:
print(vars(e))
Related
I am trying to create two set of databases every time inside my python script for the same of which I have written the below set of code which looks redundant to me since I am initializing the variable ext 2 times and hence if anyone can suggest some better alternatives, that would be really helpful.
def create_datasets(database, ext):
try:
dataset = "bq --location=US mk -d " + database + ext
try:
return_cd, out, err = run_sys_command(dataset)
except Exception as e:
print(e)
except Exception as e:
print(e)
raise
ext = ''
create_datasets(database, ext)
ext = '_stg'
create_datasets(database, ext)
Use a loop?
for ext in ['', '_stg']:
create_datasets(database, ext)
About your function:
def create_datasets(database, ext):
try:
dataset = f"bq --location=US mk -d {database}{ext}"
return_cd, out, err = run_sys_command(dataset)
except Exception as e: # <- you should catch sub exception!
print(e)
Any exception Exception raised in your function is caught and handled by the inner try block. The outer therefore seems redundant.
def create_datasets(database, ext):
try:
dataset = "bq --location=US mk -d " + database + ext
return_cd, out, err = run_sys_command(dataset)
except Exception as e:
print(e)
What my function does:
creates a war file by processing the folder contents (/tmp/A, /tmp/B and so on)
Does some file path and folder path manipulations to get the final version from the war file.
store the file name in one variable and the version in another.
Push the war file to the Repository using curl.
I'm using multiple try & except blocks to catch the exception for each action and looks very un-pythonic.
Is there an elegant and simple way to approach this ? thanks in advance.
import shutil
import traceback
import subprocess
import os
import glob
def my_function(path_a, path_b, tmp_dir)
try:
<shutil.copy to the tmp dir>
except:
traceback.print_exc()
try:
war_process = subprocess.run([WAR GENERATION COMMAND], check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
print(war_process.stdout.decode("utf-8"))
except subprocess.CalledProcessError as e:
exit_code = e.returncode
stderror = e.stderr
print(exit_code, stderror)
print(war_process.stderr.decode("utf-8"))
try:
output_folder = os.path.join("/tmp/dir/work", FILE_PATH, ARTIFACT_DATE, FILE_WO_EXTENSION)
except:
traceback.print_exc()
try:
file_name = list(glob.glob(os.path.join(output_folder, "*.war")))
except:
traceback.print_exc()
try:
file_path = os.path.join(output_folder, file_name)
except:
traceback.print_exc()
try:
os.rename(file_path, file_path.split('war')[0] + ".tgz")
except:
traceback.print_exc()
try:
file_version = os.path.basename(file_path)
except:
traceback.print_exc()
cmd = "curl -u username -T ....)"
try:
curl_output = subprocess.run([cmd], shell=True, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
print(curl_output.stdout.decode("utf-8"))
except subprocess.CalledProcessError as er:
print(proc_c.stderr.decode("utf-8"))
exit_c = er.returncode
std = er.stderr
print(exit_c, std)
You can write try once, then handle all the exceptions later:
try:
output_folder = os.path.join("/tmp/dir/work", FILE_PATH, ARTIFACT_DATE, FILE_WO_EXTENSION)
file_name = list(glob.glob(os.path.join(output_folder, "*.war")))
file_path = os.path.join(output_folder, file_name)
os.rename(file_path, file_path.split('war')[0] + ".tgz")
except FooException:
print('foo')
except BarException:
print('bar')
First of all never use bare except in your code. Read bullets 6 to 11 in PEP8:Programming Recommendations.
My suggestion is to use this code instead:
def my_function(path_a, path_b, tmp_dir)
try:
<shutil.copy to the tmp dir>
except:
traceback.print_exc()
try:
war_process = subprocess.run([WAR GENERATION COMMAND], check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
print(war_process.stdout.decode("utf-8"))
output_folder = os.path.join("/tmp/dir/work", FILE_PATH, ARTIFACT_DATE, FILE_WO_EXTENSION)
file_name = list(glob.glob(os.path.join(output_folder, "*.war")))
file_path = os.path.join(output_folder, file_name)
os.rename(file_path, file_path.split('war')[0] + ".tgz")
file_version = os.path.basename(file_path)
except subprocess.CalledProcessError as e:
exit_code = e.returncode
stderror = e.stderr
print(exit_code, stderror)
print(war_process.stderr.decode("utf-8"))
except Exception as e:
print(f'The program caught an exception {e}')
traceback.print_exc()
cmd = "curl -u username -T ....)"
try:
curl_output = subprocess.run([cmd], shell=True, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
print(curl_output.stdout.decode("utf-8"))
except subprocess.CalledProcessError as er:
print(proc_c.stderr.decode("utf-8"))
exit_c = er.returncode
std = er.stderr
print(exit_c, std)
The second and the third try/except blocks must stay separated because both catch the same exception.
Also, if any of the blocks you created here catch a specific exception in this list, you should behave them like you behave the subprocess.CalledProcessError .
Best practice is to write one try block with multiple excepts in which each except block catches a specific exception.
You don't need to put a try/except block after every statement. It would be better to put multiple statements in a try/except block
def my_function(path_a, path_b, tmp_dir)
try:
<shutil.copy to the tmp dir>
war_process = subprocess.run([WAR GENERATION COMMAND], check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
print(war_process.stdout.decode("utf-8"))
except subprocess.CalledProcessError as e:
exit_code = e.returncode
stderror = e.stderr
print(exit_code, stderror)
print(war_process.stderr.decode("utf-8"))
try:
output_folder = os.path.join("/tmp/dir/work", FILE_PATH, ARTIFACT_DATE, FILE_WO_EXTENSION)
file_name = list(glob.glob(os.path.join(output_folder, "*.war")))
file_path = os.path.join(output_folder, file_name)
os.rename(file_path, file_path.split('war')[0] + ".tgz")
file_version = os.path.basename(file_path)
except:
traceback.print_exc()
cmd = "curl -u username -T ....)"
try:
curl_output = subprocess.run([cmd], shell=True, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
print(curl_output.stdout.decode("utf-8"))
except subprocess.CalledProcessError as er:
print(proc_c.stderr.decode("utf-8"))
exit_c = er.returncode
std = er.stderr
print(exit_c, std)
```
is there any solution in python which lets a function execute after the previous one was finished?
Here is one of the ideas I'm using now. But it is not solving the problem when files are larger and the program needs more time.
def copy_to_jumphost(self):
try:
if self.connect():
stdin, stdout, stderr = self.client.exec_command('sshpass -p %s scp -r %s#%s:%s/' % (self.password,self.username,self.hostname,self.log_path) + self.lista[self.file_number].rstrip() + ' ' + '/home/%s/' % (self.username) + self.lista[self.file_number].rstrip())
except (AttributeError, TypeError) as e:
print("Error occurred:", e)
try:
if self.connect():
if self.copy_to_jumphost():
ftp_client = self.client.open_sftp()
ftp_client.get(filepath, self.localpath)
print("Success! \nFile coppied to %s" %(self.localpath))
else:
time.sleep(5)
ftp_client = self.client.open_sftp()
ftp_client.get(filepath, self.localpath)
print("Success but needed some time! \nFile coppied to %s" %(self.localpath))
except (AttributeError, TypeError) as e:
print("Error occurred:", e)
Perfect situation for me will be if in else statement there is a solution to wait for finishing the copy_to_jumphost() function, because time.sleep(5) will fail if I will need to copy larger files.
I have this code block that it should give out the CIK number when the stock ticker is supplied:
def lookup_cik(ticker, name=None):
good_read = False
ticker = ticker.strip().upper()
url = 'http://www.sec.gov/cgi-bin/browse-edgar?action+getcompany&CIK=(cik)&count=10&output=xml'.format(cik=ticker)
try:
xmlFile = urlopen ( url )
try:
xmlData = xmlFile.read()
good_read = True
finally:
xmlFile.close()
except HTTPError as e:
print( "HTTP Error:", e.code )
except URLError as e:
print( "URL Error:", e.reason )
except TimeoutError as e:
print( "Timeout Error:", e.reason )
except socket.timeout:
print( "Socket Timeout Error" )
if not good_read:
print( "Unable to lookup CIK for ticker:", ticker )
return
try:
root = ET.fromstring(xmlData)
except ET.ParseError as perr:
print( "XML Parser Error:", perr )
try:
cikElement = list(root.iter( "CIK" ))[0]
return int(cikElement.text)
except StopIteration:
pass
However when it try to input a Stock ticker i get
>>> lookup_cik(BDX)
Traceback (most recent call last):
File "<pyshell#34>", line 1, in <module>
lookup_cik(BDX)
NameError: name 'BDX' is not defined
I know that it is a NameError but i have never met an issue where the function does not recognize the supposedly inputted argument data the stock ticker which in our example is BDX.
Your function expects a string, so pass in one:
lookup_cik("BDX")
Without the quotes Python parses that as a name, but you never bound anything to that name (assigned to it).
Note that you'll also get a UnboundLocalError: local variable 'root' referenced before assignment exception if there was a parse error. You probably want to exit the function at that point:
try:
root = ET.fromstring(xmlData)
except ET.ParseError as perr:
print( "XML Parser Error:", perr )
return
You'll most likely get a parse error, because you never actually interpolate the ticker anywhere in the string; you are missing a {cik} placeholder:
url = 'http://www.sec.gov/cgi-bin/browse-edgar?action+getcompany&CIK=(cik)&count=10&output=xml'.format(cik=ticker)
You probably meant to use CIK={cik} there. A quick experiment directly calling the site also shows you need to use action=getcompany (= instead of +):
url = 'http://www.sec.gov/cgi-bin/browse-edgar?action=getcompany&CIK={cik}&count=10&output=xml'.format(cik=ticker)
Because you use list() on root.iter(), the whole expression will not raise StopIteration (list() catches that). Instead, the expression could raise a IndexError instead.
I'd use next() there instead:
cikElement = next(root.iter("CIK"), None)
return cikElement and int(cikElement.text)
or better still, just use Element.find():
cikElement = root.find("CIK")
return cikElement and int(cikElement.text)
I recently switched from urlib2 to requests and I'm not sure how to deal with exceptions. What is best practice? My current code looks like this, but is not doing any good:
try:
response = requests.get(url)
except requests.ConnectionError , e:
logging.error('ConnectionError = ' + str(e.code))
return False
except requests.HTTPError , e:
logging.error('HTTPError = ' + str(e.reason))
return False
except requests.Timeout, e:
logging.error('Timeout')
return False
except requests.TooManyRedirects:
logging.error('TooManyRedirects')
return False
except Exception:
import traceback
logging.error('generic exception: ' + traceback.format_exc())
return False
Since it looks bad as a comment, have you tried:
try:
# some code
except Exception as e:
print e