Given the following Python code:
# Use impyla package to access Impala
from impala.dbapi import connect
import logging
def process():
conn = connect(host=host, port=port) # Mocking host and port
try:
cursor = conn.cursor()
# Execute query and fetch result
except:
loggin.error("Task failed with some exception")
finally:
cursor.close() # Exception here!
conn.close()
The connection to Impala was created. But there was an exception in cursor.close() due to Impala timeout.
What is the proper way to close the cursor and conn given the latent exception?
You have to nest the try-blocks:
def process():
conn = connect(host=host, port=port) # Mocking host and port
try:
cursor = conn.cursor()
try:
# Execute query and fetch result
finally:
# here cursor may fail
cursor.close()
except:
loggin.error("Task failed with some exception")
finally:
conn.close()
To avoid such try-finally-blocks, you can use the with-statement:
def process():
conn = connect(host=host, port=port) # Mocking host and port
try:
with conn.cursor() as cursor:
# Execute query and fetch result
# cursor is automatically closed
except:
loggin.error("Task failed with some exception")
finally:
conn.close()
Related
I have continuous Python script that parses certain websites or XML's every 30 second and adds records o database if it turns out, there is something new.
First, I was just connecting to database every time which I knew, wasn't ideal way how to do it. i had something like this:
def job():
try:
cnx = mysql.connector.connect(user=DB_USER, password=DB_PASSWORD, host='XYZ', database=DB_NAME)
cursor = cnx.cursor()
# CALLS OF PARSERS:
run_parser1(cnx, cursor)
run_parser2(cnx, cursor)
# etc...
except Exception as e:
cursor.close()
cnx.close()
schedule.every(30).seconds.do(job)
while 1:
schedule.run_pending()
time.sleep(1)
Now I edited my code so connection is open until there is an exception either in connecting to database or in parsing:
try:
cnx = mysql.connector.connect(user=DB_USER, password=DB_PASSWORD, host='XYZ', database=DB_NAME)
cursor = cnx.cursor()
except Exception as e:
cursor.close()
cnx.close()
def job():
try:
alert_list = CAPParser(getxml()).as_dict()
# CALL OF PARSERS:
run_parser1(cnx, cursor)
run_parser2(cnx, cursor)
# etc...
except Exception as e:
cursor.close()
cnx.close()
schedule.every(30).seconds.do(job)
while 1:
schedule.run_pending()
time.sleep(1)
However, there is problem. While I know it's secure way to do it, it means that I need to restart script several times per day. There are a lot of exceptions either from lost database connection or unavailable URL's of parsed sites or files.
Any advice how to find better solution?
You could try to create a function which verifies if the cnx is open and if it's not, recreate it. Something like:
def job():
global cnx
global cursor
if not cnx.is_connected(): # Uses ping to verify connection
try:
cnx = mysql.connector.connect(user=DB_USER, password=DB_PASSWORD, host='XYZ', database=DB_NAME)
cursor = cnx.cursor()
except Exception as e:
cursor.close()
cnx.close()
# You can do this try in a while with a delay to keep retrying to connect
... # job here
I am doing the code in Lambda function(Python 3.6) using RDS instance. The problem is if I don't do conn.close() occurs an increase in the number of connections in RDS. That will lead to errors afterward in API response[Internal server error].
Thing is we tried in finally block but not getting a proper output which is given below.
try:
conn = pymysql.connect(rds_host, user=name, passwd=password, db=db_name, port=3268, connect_timeout=30)
except:
logger.error("ERROR: Unexpected error: Could not connect to MySql instance.")
sys.exit()
def handler(event, context):
try:
with conn.cursor() as cur:
cur.execute('SELECT * from OrderMaster')
cur.close()
return {'statusCode':200,"headers":{"access-control-allow-origin":"{}".format(headers)},"body":json.dumps(out)}
except Exception as e:
logger.error("error")
finally:
conn.close()
I cannot think of a reason to open the connection outside of your handler function but close it inside of it.
Open the connection using the with statement inside of your handler function.
def handler(event, context):
try:
with pymysql.connect(rds_host, user=name, passwd=password, db=db_name, port=3268, connect_timeout=30) as conn
with conn.cursor() as cur:
cur.execute('SELECT * from OrderMaster')
return {'statusCode':200,"headers":{"access-control-allow-origin":"{}".format(headers)},"body":json.dumps(out)}
except Exception as e:
logger.error("error")
You also don't need to close your cursor because your with statement does that for you. That goes also for the connection.
def handler(event, context):
try:
with pymysql.connect(rds_host, user=name, passwd=password, db=db_name, port=3268, connect_timeout=30) as cur:
cur.execute('SELECT * from OrderMaster')
return {'statusCode':200,"headers":{"access-control-allow-origin":"{}".format(headers)},"body":json.dumps(out)}
except Exception as e:
logger.error("error")
finally:
cur.close()
I'm developing a authentication script in Python. I need PyMySQL to check if user exists, how can I do that?
I read the documention but didn't find answer to my question.
def check():
connection = pymysql.connect(host='localhost',
user='user',
password='passwd',
db='db',
charset='utf8mb4',
cursorclass=pymysql.cursors.DictCursor)
userexist = # PyMySQL check code here
if userexist=='True':
return 'good'
It's not clear whether you want to check for the pymysql user or a user from the database. This code will address both instances.
pymysql will raise an exception: pymysql.err.OperationalError if the user does not exist. You therefore need a try, except clause like this:
def check():
try:
connection = pymysql.connect(host='localhost',
user='user',
password='passwd',
db='db',
charset='utf8mb4',
cursorclass=pymysql.cursors.DictCursor)
except pymysql.err.OperationalError:
print('User with entered connection credentials does not exists')
return False
try:
with connection.cursor() as cursor:
cursor.execute("select...") # Fill with your select statement
result = cursor.fetchone()
if result is None:
connection.close()
return False
finally:
connection.close()
return 'good'
I have a problem with my python mysql connection which I need help with.
My setup is two Pi's running servers on each one. One Pi (SolartPi) has Mysql database collecting data. The other pi (OfficePi) is connecting to the solarPi database to retrieve and update data over a network connection.
My main script works all ok until I have to reboot the SolarPi for a maintenance or power problem and the connection to the OfficePi is lost. The python script on the officePi then goes into a fault loop "2006, MYSQL Server has gone away" Below is a sample of this script.
import MySQLdb
connSolar = MySQLdb.connect("192.xxx.x.x", "external", "xxxxx", "xxxxx")
#eternal connection to solar pi database
cursSolar = connSolar.cursor()
while 1:
try:
cursSolar.execute("SELECT * FROM dashboard")
connSolar.commit()
for reading in cursSolar.fetchall():
heatingDemand = reading[2] #get heating demand from dB
print heatingDemand
except (MySQLdb.Error, MySQLdb.Warning) as e:
print (e)
connSolar.close()
So I tried rewriting this with the script from stackoverflow and a web site as shown below, but this now terminates the program when SolarPi is rebooted with the following error
_mysql_exceptions.OperationalError: (2003, 'Can\'t connect to MySQL server on \'192.xxx.x.x' (111 "Connection refused")')
import MySQLdb
class DB:
con = None
def connect(self):
self.conn = MySQLdb.connect("192.xxx.x.x", "xxxxx", "xxxxxx", "house") #eternal connection to solar pi database
def query(self, sql):
try:
cursor = self.conn.cursor()
cursor.execute(sql)
except (AttributeError, MySQLdb.OperationalError):
self.connect()
cursor = self.conn.cursor()
cursor.execute(sql)
return cursor
while 1:
db = DB()
sql = "SELECT * FROM dashboard"
cur = db.query(sql)
for reading in cur.fetchall():
heatingDemand = reading[2] #get heating demand from dB
print heatingDemand
Is there a way for the OfficePi to keep trying to connect to SolarPi mysql database when it has shut down.
Change your code to check a valid connection each loop otherwise pass:
import MySQLdb
class DB:
def connect(self):
try:
self.conn = MySQLdb.connect("192.xxx.x.x", "xxxxx", "xxxxxx", "house")
except (MySQLdb.Error, MySQLdb.Warning) as e:
print (e)
self.conn = None
return self.conn
def query(self, sql):
try:
cursor = self.conn.cursor()
cursor.execute(sql)
except (AttributeError, MySQLdb.OperationalError):
self.connect()
cursor = self.conn.cursor()
cursor.execute(sql)
return cursor
while 1:
db = DB()
conn = db.connect()
if conn:
sql = "SELECT * FROM dashboard"
cur = db.query(sql)
for reading in cur.fetchall():
heatingDemand = reading[2] #get heating demand from dB
print heatingDemand
i have to connect to mysql server and grab some data for ever
so i have two way
1)connect to mysql the grab data in a while
conn = mysql.connector.connect(user='root',password='password',host='localhost',database='db',charset='utf8',autocommit=True)
cursor = conn.cursor(buffered=True)
while True:
cursor.execute("statments")
sqlData = cursor.fetchone()
print(sqlData)
sleep(0.5)
this working good but if script crashed due to mysql connection problem script goes down
2)connect to mysql in while
while True:
try:
conn = mysql.connector.connect(user='root',password='password',host='localhost',database='db',charset='utf8',autocommit=True)
cursor = conn.cursor(buffered=True)
cursor.execute("statments")
sqlData = cursor.fetchone()
print(sqlData)
cursor.close()
conn.close()
sleep(0.5)
except:
print("recoverable error..")
both code working good but my question is which is better?!
Among these two, better way will be to use a single connection but create a new cursor for each statement because creation of new connection takes time but creating a new cursor is fast. You may update the code as:
conn = mysql.connector.connect(user='root',password='password',host='localhost',database='db',charset='utf8',autocommit=True)
while True:
try:
cursor = conn.cursor(buffered=True)
cursor.execute("statments")
sqlData = cursor.fetchone()
print(sqlData)
except Exception: # Catch exception which will be raise in connection loss
conn = mysql.connector.connect(user='root',password='password',host='localhost',database='db',charset='utf8',autocommit=True)
cursor = conn.cursor(buffered=True)
finally:
cursor.close()
conn.close() # Close the connection
Also read Defining Clean-up Actions regarding the usage of try:finally block.