We Keep Coding

Evaluating multiple conditions in try block in Python

Is it possible to evaluate multiple conditions in a try block in Python. Below is the case.
I have 2 conditions below.
Connect to sql server to read data into two dataframes. There is a timeout in the code, if the connection takes more than 15 seconds the code should raise an exception and exit.
Check if these two dataframe have data.If either one of the dataframes is empty, exit the code, if not continue the code in the else block.
I am currently thinking of doing like this. Is there more elegant way.
try:
#Condition 1
except:
#Condition 1
try:
#Condition 2
except:
#Condition 2
else:
#Condition 3
def connect_to_server(table):
# Connection Code
server = ''
username = ''
password = ''
database = ''
cnxn = pyodbc.connect('DRIVER={ODBC Driver 17 for SQL Server};SERVER='+server+';DATABASE='+database+';ENCRYPT=yes;UID='+username+';PWD='+ password)
cnxn.timeout = 5
cursor = cnxn.cursor()
try:
cnxn.execute('SELECT * FROM ' +table)
except Exception as my_timeout_exception:
raise my_timeout_exception
def read_database(table):
server = ''
username = ''
password = ''
database = ''
cnxn = pyodbc.connect('DRIVER={ODBC Driver 17 for SQL Server};SERVER='+server+';DATABASE='+database+';ENCRYPT=yes;UID='+username+';PWD='+ password)
cursor = cnxn.cursor()
df = pd.read_sql('SELECT * FROM ' +table, cnxn)
if df.empty:
print("dataframe is empty")
else:
return df
try:
using_con = connect_to_server('table')
df = read_database('table')
except my_timeout_exception:
handle_error_1
#break
except empty_df_exception:
handle_error_2
#break
else:
print("z")

Fortunately pyodbc offers its own error classes so no need to create a custom one however we do create a custom error for an empty dataframe:
import pyodbc
import pandas as pd
class MyEmptyDfError(Exception):
def __init__(self):
message = "Dataframe is empty"
super().__init__(message)
def connect_to_server():
# Connection Code
server = ''
username = ''
password = ''
database = ''
cnxn = pyodbc.connect('DRIVER={ODBC Driver 17 for SQL Server};SERVER='+server+';DATABASE='+database+';ENCRYPT=yes;UID='+username+';PWD='+ password)
return cnxn
def read_database(table, cnxn):
cnxn.timeout = 5
df = pd.read_sql('SELECT * FROM ' + table, cnxn)
if df.empty:
raise MyEmptyDfError
return df
try:
conn = connect_to_server()
df = read_database("test_table", conn)
except pyodbc.OperationalError as e:
print(e)
except MyEmptyDfError as e:
print(e)
finally:
print("Final code reached")
Here if server connection triggers an error it propagates up to your code and is captured as OperatioalError (which I think is the error triggered if it times out)
I think this should work (haven't been able to test it yet)

rIf you just want to capture different error conditions then you can include your code to connect to server and then test for data in the try statement. you then specify the error you want to catch in the except statement:
def connect_to_server(db):
# Connection Code
if connection_timed_out:
raise my_timeout_exception #either a custom error you have created or propagate standard error from connection timeout
return connection
def read_database(conn):
#read db into dataframe code
if dataframe_isempty:
raise empty_df_exception #either a custom error you have created or propagate standard error from dataframe reading
return dataframe
try:
using_con = connect_to_server(db)
df = read_database(using_con)
except my_timeout_exception:
handle_error_1
break
except empty_df_exception:
handle_error_2
break
else:
continue_code
You can actually include both exceptions in one except statement if the handling code is the same (eg just a break statement).

How should I do exception handling in my database python code for no data fetched from server or connection time out

psycopg2.DatabaseError: could not receive data from server: Connection timed out

you can use this code in your :
def retrieve_data(db, table_name):
try:
comm ="SELECT * FROM {};".format(table_name)
with db.connect() as conn:
column_of_sql_table = conn.execute(comm).fetchall()
return pd.DataFrame(column_of_sql_table)
except Exception as e:
print(e)
return pd.DataFrame()
df = retrieve_data(db, table_name)
if not df.empty :
< do what ever you want>
else:
< rise error>

Wrap it inside try/except block.
like:
try:
conn = conn = dbapi.connect(user=<>,password=<>,host=<>,database=<>)
except psycopg2.DatabaseError:
<whatever code>

How to modularize a code containing 'with' in Python

I have this code that I am using to get information from a mysql database
def query_result_connect(_query):
with SSHTunnelForwarder((ssh_host, ssh_port),
ssh_password=ssh_password,
ssh_username=ssh_user,
remote_bind_address=('127.0.0.1', 3306)) as server:
connection = mdb.connect(user=sql_username,
passwd=sql_password,
db=sql_main_database,
host='127.0.0.1',
port=server.local_bind_port)
cursor = connection.cursor()
cursor.execute(_query)
connection.commit()
try:
y = pd.read_sql(_query, connection)
return y
except TypeError as e:
x = cursor.fetchall()
return x
I would like to create a function that includes the following part.
with SSHTunnelForwarder((ssh_host, ssh_port),
ssh_password=ssh_password,
ssh_username=ssh_user,
remote_bind_address=('127.0.0.1', 3306)) as server:
connection = mdb.connect(user=sql_username,
passwd=sql_password,
db=sql_main_database,
host='127.0.0.1',
port=server.local_bind_port)
and execute it in the query_result_connect() function. The problem is that I don't know how to include more code within the 'with' statement. The code should look something like this:
# Maybe introduce some arguments
def db_connection():
with SSHTunnelForwarder((ssh_host, ssh_port),
ssh_password=ssh_password,
ssh_username=ssh_user,
remote_bind_address=('127.0.0.1', 3306)) as server:
connection = mdb.connect(user=sql_username,
passwd=sql_password,
db=sql_main_database,
host='127.0.0.1',
port=server.local_bind_port)
# Maybe return something
def query_result_connect(_query):
# call the db_connection() function somehow.
# Write the following code in a way that is within the 'with' statement of the db_connection() function.
cursor = connection.cursor()
cursor.execute(_query)
connection.commit()
try:
y = pd.read_sql(_query, connection)
return y
except TypeError as e:
x = cursor.fetchall()
return x
Thank you

What's about to make "do_connection" to be a context manager itself?
#contextmanager
def do_connection():
# prepare connection
# yield connection
# close connection (__exit__). Perhaps you even want to call "commit" here.
Then, you will use it like this:
with do_connection() as connection:
cursor = connection.cursor()
...
It is a common approach to use context managers for creating DB connections.

You could make you own Connection class, that works like a conext manager.
__enter__ sets up ssh tunnel and db connection.
__exit__, tries to close the cursor, db connection and the ssh tunnel.
from sshtunnel import SSHTunnelForwarder
import psycopg2, traceback
class MyDatabaseConnection:
def __init__(self):
self.ssh_host = '...'
self.ssh_port = 22
self.ssh_user = '...'
self.ssh_password = '...'
self.local_db_port = 59059
def _connect_db(self, dsn):
try:
self.con = psycopg2.connect(dsn)
self.cur = self.con.cursor()
except:
traceback.print_exc()
def _create_tunnel(self):
try:
self.tunnel = SSHTunnelForwarder(
(self.ssh_host, self.ssh_port),
ssh_password=self.ssh_password,
ssh_username=self.ssh_user,
remote_bind_address=('localhost', 5959),
local_bind_address=('localhost', self.local_db_port)
)
self.tunnel.start()
if self.tunnel.local_bind_port == self.local_db_port:
return True
except:
traceback.print_exc()
def __enter__(self):
if self._create_tunnel():
self._connect_db(
"dbname=mf port=%s host='localhost' user=mf_usr" %
self.local_db_port
)
return self
def __exit__(self, *args):
for c in ('cur', 'con', 'tunnel'):
try:
obj = getattr(self, c)
obj.close()
obj = None
del obj
except:
pass
with MyDatabaseConnection() as db:
print(db)
db.cur.execute('Select count(*) from platforms')
print(db.cur.fetchone())
Out:
<__main__.MyDatabaseConnection object at 0x1017cb6d0>
(8,)
Note:
I am connecting to Postgres, but that should work using mysql as well. Probably you need to adjust to match your own needs.

Enter several parameters in SQL stored procedures from python with sqlalchemy

I can successfully connect to SQL Server from my jupyter notebook with this script:
from sqlalchemy import create_engine
import pyodbc
import csv
import time
import urllib
params = urllib.parse.quote_plus('''DRIVER={SQL Server Native Client 11.0};
SERVER=SV;
DATABASE=DB;
TRUSTED_CONNECTION=YES;''')
engine = create_engine("mssql+pyodbc:///?odbc_connect=%s" % params)
And I can successfully execute SQL stored procedures without parameters from jupyter notebook with the following function :
def execute_stored_procedure(engine, procedure_name):
res = {}
connection = engine.raw_connection()
try:
cursor = connection.cursor()
cursor.execute("EXEC "+procedure_name)
cursor.close()
connection.commit()
res['status'] = 'OK'
except Exception as e:
res['status'] = 'ERROR'
res['error'] = e
finally:
connection.close()
return res
How please could I transform this previous function for stored procedures having several parameters (two in my case) ?

Solution of my problem, working only for stored procedures with 0 or 2 parameters (just edit the 10th line if you want another number of parameters):
def execute_stored_procedure(engine, procedure_name,params_dict=None):
res = {}
connection = engine.raw_connection()
try:
cursor = connection.cursor()
if params_dict is None:
cursor.execute("EXEC "+procedure_name)
else:
req = "EXEC "+procedure_name
req += ",".join([" #"+str(k)+"='"+str(v)+"'" for k,v in params_dict.items()])
cursor.execute(req)
cursor.close()
connection.commit()
res['status'] = 'OK'
except Exception as e:
res['status'] = 'ERROR'
res['error'] = e
finally:
connection.close()
return res

When to use DRCP and sessionpool in cx_oracle python?

For a specific use case - in which I have 100 databases and 1 database is the central database, now my app connects to that one central database which spawns connections to any of the 100 databases as per the request of the user to run some query on any of them.
In this case does using DRCP makes same as I dont want the connection to be killed if the user is running the query at the same time I dont want too many connections to be opened to the db which I control by creating a profile on the database which limits the number of active sessions to some low number say 5 for that specific user(read_only_user) using that specific profile(read_only_profile).
Right now I am using the standard open a connection per request model. But Im not sure if thats the best way to go about it.
import cx_Oracle
import logging, time
class Database(object):
'''
Use this method to for DML SQLS :
Inputs - Sql to be executed. Data related to that sql
Returns - The last inserted, updated, deleted ID.
'''
def __init__(self, user, password, host, port, service_name, mode, *args):
#mode should be 0 if not cx_Oracle.SYSDBA
self.user = user
self.password = password
self.host = host
self.port = port
self.user = user
self.service_name = service_name
self.logger = logging.getLogger(__name__)
try:
self.mode = mode
except:
self.mode = 0
self.logger.info(" Mode is not mentioned while creating database object")
self.connection = None
dsn = cx_Oracle.makedsn(self.host, self.port, self.service_name)
self.connect_string = self.user + '/' + self.password + '#' + dsn
try:
self.connection = cx_Oracle.connect(self.connect_string, mode=self.mode,
threaded=True)
self.connection.stmtcachesize = 1000
self.connection.client_identifier = 'my_app'
self.cursor = self.connection.cursor()
self.idVar = self.cursor.var(cx_Oracle.NUMBER)
except cx_Oracle.DatabaseError, exc:
error, = exc
self.logger.exception(
'Exception occured while trying to create database object : %s',
error.message)
raise exc
def query(self, q):
try:
self.cursor.execute(q)
return self.cursor.fetchall()
except cx_Oracle.DatabaseError, exc:
error, = exc
self.logger.info(
"Error occured while trying to run query: %s, error : %s", q,
error.message)
return error.message
def dml_query(self, sql):
try:
self.cursor.execute(sql)
self.connection.commit()
return 1
except Exception as e:
self.logger.exception(e)
return 0
def dml_query_with_data(self, sql, data):
"""
Use this method to for DML SQLS :
Inputs - Sql to be executed. Data related to that sql
Returns - The last inserted, updated, deleted ID.
"""
try:
self.cursor.execute(sql, data)
self.connection.commit()
return 1
except Exception as e:
self.logger.exception(e)
return 0
def update_output(self, clob, job_id, flag):
try:
q = "Select output from my_table where job_id=%d" % job_id
self.cursor.execute(q)
output = self.cursor.fetchall()
#Checking if we already have some output in the clob for that job_id
if output[0][0] is None:
if flag == 1:
self.cursor.execute("""UPDATE my_table
SET OUTPUT = :p_clob
,job_status=:status WHERE job_id = :p_key""",
p_clob=clob, status="COMPLETED", p_key=job_id)
else:
self.cursor.execute("""UPDATE my_table
SET OUTPUT = :p_clob
,job_status=:status WHERE job_id = :p_key""",
p_clob=clob, status="FAILED", p_key=job_id)
else:
self.cursor.execute("""UPDATE my_table
SET OUTPUT = OUTPUT || ',' || :p_clob
WHERE job_id = :p_key""", p_clob=clob, p_key=job_id)
self.connection.commit()
rows_updated = self.cursor.rowcount
return rows_updated
except Exception as e:
self.logger.exception(e)
return 0
def __del__(self):
try:
if self.connection is not None:
self.connection.close()
except Exception as e:
self.logger.exception(
"Exception while trying to close database connection object : %s", e)
'''
if __name__ == '__main__':
db = Database('test', 'test', 'my_host', '1000', 'my_db', 0)
columns = db.query('select * from my-table')
print columns
'''
This is my database class, and I create an object whenever I need a connect to the DB. And the init and del method take care of constructing and destructing the object.
Should I be using DRCP/ sessionPool to improve performance.
What if there are too many users waiting coz all the connections in DRCP are taken?
Can I have sessionPool per database (for the 100 databases, each database can take atmost 5 connections at a time for that read_only_user)