Is it possible to evaluate multiple conditions in a try block in Python. Below is the case.
I have 2 conditions below.
Connect to sql server to read data into two dataframes. There is a timeout in the code, if the connection takes more than 15 seconds the code should raise an exception and exit.
Check if these two dataframe have data.If either one of the dataframes is empty, exit the code, if not continue the code in the else block.
I am currently thinking of doing like this. Is there more elegant way.
try:
#Condition 1
except:
#Condition 1
try:
#Condition 2
except:
#Condition 2
else:
#Condition 3
def connect_to_server(table):
# Connection Code
server = ''
username = ''
password = ''
database = ''
cnxn = pyodbc.connect('DRIVER={ODBC Driver 17 for SQL Server};SERVER='+server+';DATABASE='+database+';ENCRYPT=yes;UID='+username+';PWD='+ password)
cnxn.timeout = 5
cursor = cnxn.cursor()
try:
cnxn.execute('SELECT * FROM ' +table)
except Exception as my_timeout_exception:
raise my_timeout_exception
def read_database(table):
server = ''
username = ''
password = ''
database = ''
cnxn = pyodbc.connect('DRIVER={ODBC Driver 17 for SQL Server};SERVER='+server+';DATABASE='+database+';ENCRYPT=yes;UID='+username+';PWD='+ password)
cursor = cnxn.cursor()
df = pd.read_sql('SELECT * FROM ' +table, cnxn)
if df.empty:
print("dataframe is empty")
else:
return df
try:
using_con = connect_to_server('table')
df = read_database('table')
except my_timeout_exception:
handle_error_1
#break
except empty_df_exception:
handle_error_2
#break
else:
print("z")
Fortunately pyodbc offers its own error classes so no need to create a custom one however we do create a custom error for an empty dataframe:
import pyodbc
import pandas as pd
class MyEmptyDfError(Exception):
def __init__(self):
message = "Dataframe is empty"
super().__init__(message)
def connect_to_server():
# Connection Code
server = ''
username = ''
password = ''
database = ''
cnxn = pyodbc.connect('DRIVER={ODBC Driver 17 for SQL Server};SERVER='+server+';DATABASE='+database+';ENCRYPT=yes;UID='+username+';PWD='+ password)
return cnxn
def read_database(table, cnxn):
cnxn.timeout = 5
df = pd.read_sql('SELECT * FROM ' + table, cnxn)
if df.empty:
raise MyEmptyDfError
return df
try:
conn = connect_to_server()
df = read_database("test_table", conn)
except pyodbc.OperationalError as e:
print(e)
except MyEmptyDfError as e:
print(e)
finally:
print("Final code reached")
Here if server connection triggers an error it propagates up to your code and is captured as OperatioalError (which I think is the error triggered if it times out)
I think this should work (haven't been able to test it yet)
rIf you just want to capture different error conditions then you can include your code to connect to server and then test for data in the try statement. you then specify the error you want to catch in the except statement:
def connect_to_server(db):
# Connection Code
if connection_timed_out:
raise my_timeout_exception #either a custom error you have created or propagate standard error from connection timeout
return connection
def read_database(conn):
#read db into dataframe code
if dataframe_isempty:
raise empty_df_exception #either a custom error you have created or propagate standard error from dataframe reading
return dataframe
try:
using_con = connect_to_server(db)
df = read_database(using_con)
except my_timeout_exception:
handle_error_1
break
except empty_df_exception:
handle_error_2
break
else:
continue_code
You can actually include both exceptions in one except statement if the handling code is the same (eg just a break statement).
Related
I have a Python Azure function. In the code, I have a Connection timeout error exception from pyodbc connection.
def connection():
username = ''
server = ''
password = ''
database = ''
connection= pyodbc.connect('DRIVER={ODBC Driver 17 for SQL Server};SERVER='+server+';DATABASE='+database+';ENCRYPT=yes;UID='+username+';PWD='+ password)
return connection
When the connection times out the code fails with a connection timeout error in the logs. But I need the function to return the error value instead of showing the error message in the logs. Is there any way to achieve this in Python? Currently, I am using a try-except statement.
try:
conn = connection()
# .....
except pyodbc.OperationalError as e:
raise e
Edit:
Helper code:
def connection():
username = ''
server = ''
password = ''
database = ''
connection= pyodbc.connect('DRIVER={ODBC Driver 17 for SQL Server};SERVER='+server+';DATABASE='+database+';ENCRYPT=yes;UID='+username+';PWD='+ password)
return conctn
def read_table(table, cnxn):
conctn.timeout = 3
df = pd.read_sql('SELECT * FROM ' + table, cnxn)
if df.empty:
raise customerror
return df
Main code:
try:
conn = connection()
df1 = read_table("table_name", conn)
except pyodbc.OperationalError as e:
return e
except customerror as e:
return e
else:
.....
When the connection times out, the main code fails with the below error.
SELECT * FROM table_name': ('HYT00', '[HYT00] [Microsoft][ODBC Driver 17 for SQL Server]Query timeout expired (0) (SQLExecDirectW)
When the query times out I do not want the code to fail with the above error message instead I want to exit/terminate the code with a custom time out error message.
You're describing, vaguely, something like...
def connection():
username = ''
server = ''
password = ''
database = ''
try:
return (
pyodbc.connect('DRIVER={ODBC Driver 17 for SQL Server};SERVER='+server+';DATABASE='+database+';ENCRYPT=yes;UID='+username+';PWD='+ password),
None
)
except pyodbc.OperationalError as e:
return (
None,
e
)
Then, an operational error will never cause a crash, and your calling code can be something like...
conn, e = connection()
if e:
# do something with the error
else:
# do something with the connection
I am trying to execute the mentioned code and I got the following error:
Not able to get the exact cause of the error
2023-02-10 14:44:27,611 ERROR : No results. Previous SQL was not a query.
Traceback (most recent call last):
File "<ipython-input-1-00966ada7c84>", line 30, in <module>
for row in rows:
pyodbc.ProgrammingError: No results. Previous SQL was not a query.
Code:
import pyodbc
import pandas as pd
import sqlalchemy
from arcgis.geocoding import geocode
from arcgis.gis import GIS
import logging
logging.basicConfig(filename="C:\\Users\\addr_errors.log",filemode='w', level= logging.ERROR,format= '%(asctime)s %(levelname)s : %(message)s')
try:
gis = GIS('home')
# Connect to the SQL Server database
conn = pyodbc.connect(
"DRIVER=;"
"SERVER=;"
"DATABASE=;"
"Trusted_Connection=yes;")
# Create a cursor from the connection
cursor = conn.cursor()
# Execute a SELECT statement to get the rows to update
rows = cursor.execute("SELECT top 1 * FROM [dbo].[Aff_SC] where pflag is null ORDER BY AffiliationID")
if not rows:
print("No results found")
else:
for row in rows:
# Get the values from the current row
address = row.OldAddress
vaffliationid = row.AffiliationID
print(address)
print(vaffliationid)
# Geocode the address
result = geocode(address, as_featureset=True)
#print(result)
if result is not None:
try:
best_match = result.features[0]
print(best_match)
except IndexError:
best_match = None
print(vaffliationid)
update_query = f"UPDATE [dbo].[Aff_SC] SET pflag = 1 WHERE OldAddress = '{address}' and AffiliationID = '{vaffliationid}'"
cursor.execute(update_query)
if best_match is not None:
# Get the standardized address
standardized_address = best_match.attributes["Match_addr"]
print("standardized_address")
print(standardized_address)
#print(vaffliationid)
update_query = f"UPDATE [dbo].[Aff_SC] SET NewAddress = '{standardized_address}' , pflag = 1 WHERE OldAddress = '{address}' and AffiliationID = '{vaffliationid}'"
cursor.execute(update_query)
# Commit the changes to the database
conn.commit()
# Close the cursor and the connection
cursor.close()
conn.close()
except Exception as e:
logging.exception(e)
#finally:
logging.shutdown()
#close the log file, overwriting the logfile worked after closing the handlers
handlers = logging.getLogger().handlers[:]
for handler in handlers:
handler.close()
logging.getLogger().removeHandler(handler)
Tried to run the print statements in the blocks of query and looks fine to me
I'm a newbie to python and would require your help in building this utility.
Use case: I need to build a python utility which compares and does basic data validation like row count, count of columns on those tables between sql server and snowflake tables. The list of the tables needs to be extracted by reading and looping an excel file (list of sql server tables v/s snowflake tables listed there.). The difference is to be written in a separate file.
Code :
# -------------- Import packages needed ----------------------------
import sys, os, pyodbc, datetime, collections
import pandas as pd
import snowflake.connector as sf
import sqlalchemy as sa
#import SNCC_Conn as sfconn
pd.set_option("display.max_rows", 999)
# set params for Snowflake Connection
sncc_auth = 'externalbrowser'
sncc_user = 'xxx'
sncc_warehouse = 'xxx'
sncc_db = 'xxx'
sncc_sch = 'SFSCHEMA'
sncc_tbl = 'TABLE_1'
sncc_qry = 'SELECT COUNT(*) FROM '+sncc_sch+'.'+sncc_tbl+''
#sncc_qry1 = 'SELECT COUNT(COLUMN_NAME) FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_NAME ='+sncc_tbl''
sf_qry = r'' + sncc_qry
# set params for SQL Connection TST .
sql_srvr = 'xxxx'
sql_db = 'xxx'
sql_user = 'xxx'
sql_pwd = 'xxx'
sql_driver = '{ODBC Driver 17 for SQL Server}'
sql_sch = 'SQLSCHEMA'
sql_tbl = 'TABLE_1'
ms_sql_qry = 'SELECT COUNT(*) FROM '+sql_sch+'.' +sql_tbl+''
#ms_sql_qry1 = 'SELECT COUNT(*) FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_SCHEMA = 'allegro' and TABLE_NAME = 'trade''
fileName = 'SQL_SF_Comparison'
# --------------------------- Snowflake Connection ---------------------------
try:
sf_conn = sf.connect(authenticator='externalbrowser',
user='xxxx',
account='xxx',
warehouse='xxx',
database='xxx',
schema ='',
role=''
)
except Exception as e:
print('Connection Failed. Please try again.')
print('Error: ' + str(e) )
quit()
print('Snowflake Connection established!')
print(sf_qry)
try:
# excute the query
sf_conn.execute(sf_qry)
# Fetch all snowflake results into a Pandas Dataframe
sf_df = sf_conn.fetch_pandas_all()
# Make all Dataframe Columns Uppercase
sf_df.columns = map(str.upper, sf_df.columns)
# Print out results on screen during development phase.
print(sf_df)
print(sf_df.columns)
print('Snowflake Dataframe Load Successful.')
except Exception as e:
print('Snowflake Dataframe load Unsuccessful. Please try again.')
print('Error: ' + str(e) )
# # --------------------------- SQL Server Connection ---------------------------
try:
# single '\' provides a concat to the DRIVE, SERVER, DATABASE, trusted connection lines, as if a single line of code.
sql_conn = pyodbc.connect('DRIVER='+sql_driver+';SERVER=tcp:'+sql_srvr+';PORT=1433;DATABASE='+sql_db+';UID='+sql_user+';PWD='+ sql_pwd+'') # Using Windows User Account for authentication.
cursor = sql_conn.cursor()
print('SQL Server Connection established!')
print(ms_sql_qry)
except Exception as e:
print('Connection Failed. Please try again.')
print('Error: ' + str(e) )
try:
# Query results and place them in variable
# cursor.execute(sql_qry)
sql_qry = pd.read_sql_query(ms_sql_qry,sql_conn)
# Put results into a Data Frame from Pandas
sql_df = pd.DataFrame(sql_qry)
# Make all Dataframe Columns Uppercase
sql_df.columns = map(str.upper, sql_df.columns)
# Print out results during development phase.
print(sql_df)
print(sql_df.columns)
print('SQL Server Dataframe Load Successful')
print('Comparing SQL to SNCC Dataframes')
#/********************* COMPARISON SCRIPT **************/
#sql_df.compare(sncc_df)
# Compare the two DataFrames and produce results from Source (sql_df) that do not match Target (sf_df).
df_diff = sql_df[sf_df != sql_df]
# print out results of differences during development phase.
print(df_diff)
# Export out to CSV using a variable for the name of the file, future state.
df_diff.to_csv(r'D:\PythonResults\DataDiff_' + fileName + '.csv', index = False)
print('Datafram output from comparison outputed to PythonResults folder in Documents as DataDiff_' + fileName + 'csv.')
except pyodbc.Error as e:
# Message stating export unsuccessful.
print("MSSQL Dataframe load unsuccessful.")
finally:
sf_conn.close()
print("Connection to Snowflake closed")
sql_conn.commit()
sql_conn.close()
print("Connection to MSSQL Server closed")
File Data and file name :
Tables.xlsx
Help me in completing the code in extracting the list of tables from the excel file and loop the data and load them to datframes and compare them
See pyodbc Connecting to Microsoft Excel for details on reading from Excel.
psycopg2.DatabaseError: could not receive data from server: Connection timed out
you can use this code in your :
def retrieve_data(db, table_name):
try:
comm ="SELECT * FROM {};".format(table_name)
with db.connect() as conn:
column_of_sql_table = conn.execute(comm).fetchall()
return pd.DataFrame(column_of_sql_table)
except Exception as e:
print(e)
return pd.DataFrame()
df = retrieve_data(db, table_name)
if not df.empty :
< do what ever you want>
else:
< rise error>
Wrap it inside try/except block.
like:
try:
conn = conn = dbapi.connect(user=<>,password=<>,host=<>,database=<>)
except psycopg2.DatabaseError:
<whatever code>
I am using Python and mySQL, and there is a long lag between queries. As a result, I get an 'MySQL connection has gone away' error, that is wait_timeout is exceeded.
This has been discussed e.g. in
Gracefully handling "MySQL has gone away"
but this does not specifically answer my query.
So my approach to handling this -
I have wrapped all my sql execute statements in a method as -
def __execute_sql(self,sql,cursor):
try:
cursor.execute(sql)
except MySQLdb.OperationalError, e:
if e[0] == 2006:
self.logger.do_logging('info','DB', "%s : Restarting db" %(e))
self.start_database()
I have several places in the code which calls this query. The thing is, I also have several cursors, so the method invocations look like-
self.__execute_sql(sql,self.cursor_a)
self.__execute_sql(sql,self.cursor_b)
and so on
I need a way to gracefully re-execute the query after the db has been started. I could wrap the calls in an if statement, and re-execute so it would be
def __execute_sql(self,sql,cursor):
try:
cursor.execute(sql)
return 1
except MySQLdb.OperationalError, e:
if e[0] == 2006:
self.logger.do_logging('info','DB', "%s : Restarting db" %(e))
self.start_database()
return 0
and then
if (self.__execute_sql(sql,self.cursor_a) == 0):
self.__execute_sql(sql,self.cursor_a)
But this is clunky. Is there a better way to do this?
Thanks!!!
I tried Crasched's approach, which got me to a new OperationalError:
OperationalError: (2013, 'Lost connection to MySQL server during query')
My final solution was to first try the ping, and if another OperationalError was raised, to reconnect and recreate the cursor with the new connection, like so:
try:
self.connection.ping(True)
except MySQLdb.OperationalError:
self.connection = MySQLdb.connect(
self.db_host,
self.db_user,
self.db_passwd,
self.db_dbase,
self.db_port)
# reconnect your cursor as you did in __init__ or wherever
self.cursor = self.connection(
MySQLdb.cursors.DictCursor)
Back in business!
Python 2.7, MySQL 5.5.41
I had the same problem and wanted to wrap the exception to capture it but instead I solved it by using the following.
Before calling the execute, call
self.con.ping(TRUE)
http://www.neotitans.com/resources/python/mysql-python-connection-error-2006.html
http://mysql-python.sourceforge.net/MySQLdb.html
I can no longer find the original source material I found this out from, but this solved the problem immediately.
I was running into mystifying "MySQL server has gone away" errors, and here is my solution.
This solution will let you retry through MySQL errors, handle pretty much any type of query, include query variables in the query str or in a separate tuple, and collect and return all of the success and error messages you encounter along the way:
def execute_query(query_str, values=None):
# defaults
num_affected_rows = 0
result_rows = None
success = False
message = "Error executing query: {}".format(query_str)
# run the query
try:
mysql_conn = get_existing_mysql_connection()
cur = mysql_conn.cursor()
if values == None or len(values) < 1:
num_affected_rows = cur.execute(query_str)
else:
num_affected_rows = cur.execute(query_str, values)
result_rows = cur.fetchall() # only relevant to select, but safe to run with others
cur.close()
mysql_conn.commit()
success = True
message = "Mysql success for query: {}".format(query_str)
except BaseException as e:
message = "Mysql error: {}; for query: {}".format(repr(e), query_str)
return (success, num_affected_rows, result_rows, message)
def execute_query_with_retry(query_str, values=None, num_tries=3, message=""):
# defaults
success = False
num_affected_rows = 0
result_rows = None
this_message = "Error executing query: {}".format(query_str)
# should we still try?
if num_tries < 1:
this_message = "Ran out of tries for query: {}".format(query_str)
return (False, 0, None, message + '; ' + this_message)
num_tries_after_this = num_tries - 1
# try to execute query
try:
(success, num_affected_rows, result_rows, this_message) = execute_query(query_str, values)
except BaseException as e:
success = False
# handle success or failure
if success == True:
return (True, num_affected_rows, result_rows, message + '; ' + this_message)
else:
open_new_mysql_connection() # reconnect using password etc.
return(execute_query_with_retry(query_str, values=values, num_tries=num_tries_after_this, message=(message + '; ' + this_message)))