For a specific use case - in which I have 100 databases and 1 database is the central database, now my app connects to that one central database which spawns connections to any of the 100 databases as per the request of the user to run some query on any of them.
In this case does using DRCP makes same as I dont want the connection to be killed if the user is running the query at the same time I dont want too many connections to be opened to the db which I control by creating a profile on the database which limits the number of active sessions to some low number say 5 for that specific user(read_only_user) using that specific profile(read_only_profile).
Right now I am using the standard open a connection per request model. But Im not sure if thats the best way to go about it.
import cx_Oracle
import logging, time
class Database(object):
'''
Use this method to for DML SQLS :
Inputs - Sql to be executed. Data related to that sql
Returns - The last inserted, updated, deleted ID.
'''
def __init__(self, user, password, host, port, service_name, mode, *args):
#mode should be 0 if not cx_Oracle.SYSDBA
self.user = user
self.password = password
self.host = host
self.port = port
self.user = user
self.service_name = service_name
self.logger = logging.getLogger(__name__)
try:
self.mode = mode
except:
self.mode = 0
self.logger.info(" Mode is not mentioned while creating database object")
self.connection = None
dsn = cx_Oracle.makedsn(self.host, self.port, self.service_name)
self.connect_string = self.user + '/' + self.password + '#' + dsn
try:
self.connection = cx_Oracle.connect(self.connect_string, mode=self.mode,
threaded=True)
self.connection.stmtcachesize = 1000
self.connection.client_identifier = 'my_app'
self.cursor = self.connection.cursor()
self.idVar = self.cursor.var(cx_Oracle.NUMBER)
except cx_Oracle.DatabaseError, exc:
error, = exc
self.logger.exception(
'Exception occured while trying to create database object : %s',
error.message)
raise exc
def query(self, q):
try:
self.cursor.execute(q)
return self.cursor.fetchall()
except cx_Oracle.DatabaseError, exc:
error, = exc
self.logger.info(
"Error occured while trying to run query: %s, error : %s", q,
error.message)
return error.message
def dml_query(self, sql):
try:
self.cursor.execute(sql)
self.connection.commit()
return 1
except Exception as e:
self.logger.exception(e)
return 0
def dml_query_with_data(self, sql, data):
"""
Use this method to for DML SQLS :
Inputs - Sql to be executed. Data related to that sql
Returns - The last inserted, updated, deleted ID.
"""
try:
self.cursor.execute(sql, data)
self.connection.commit()
return 1
except Exception as e:
self.logger.exception(e)
return 0
def update_output(self, clob, job_id, flag):
try:
q = "Select output from my_table where job_id=%d" % job_id
self.cursor.execute(q)
output = self.cursor.fetchall()
#Checking if we already have some output in the clob for that job_id
if output[0][0] is None:
if flag == 1:
self.cursor.execute("""UPDATE my_table
SET OUTPUT = :p_clob
,job_status=:status WHERE job_id = :p_key""",
p_clob=clob, status="COMPLETED", p_key=job_id)
else:
self.cursor.execute("""UPDATE my_table
SET OUTPUT = :p_clob
,job_status=:status WHERE job_id = :p_key""",
p_clob=clob, status="FAILED", p_key=job_id)
else:
self.cursor.execute("""UPDATE my_table
SET OUTPUT = OUTPUT || ',' || :p_clob
WHERE job_id = :p_key""", p_clob=clob, p_key=job_id)
self.connection.commit()
rows_updated = self.cursor.rowcount
return rows_updated
except Exception as e:
self.logger.exception(e)
return 0
def __del__(self):
try:
if self.connection is not None:
self.connection.close()
except Exception as e:
self.logger.exception(
"Exception while trying to close database connection object : %s", e)
'''
if __name__ == '__main__':
db = Database('test', 'test', 'my_host', '1000', 'my_db', 0)
columns = db.query('select * from my-table')
print columns
'''
This is my database class, and I create an object whenever I need a connect to the DB. And the init and del method take care of constructing and destructing the object.
Should I be using DRCP/ sessionPool to improve performance.
What if there are too many users waiting coz all the connections in DRCP are taken?
Can I have sessionPool per database (for the 100 databases, each database can take atmost 5 connections at a time for that read_only_user)
Related
This question already has an answer here:
MySQLdb raises "execute() first" error even though I execute before calling fetchall
(1 answer)
Closed 3 years ago.
I have a connection function that return cursor, I am using pymysql to connect with my db, and I have another file that use the connection function to execute sql statement but when I fetch the row it return an error raise err.ProgrammingError("execute() first")
Here is the connection function that extend from my database config file
def kasaa():
try:
conn = mysql.connect()
cursor = conn.cursor(pymysql.cursors.DictCursor)
return cursor
except Exception as e:
print(e)
But when I call the kasaa() inside an employee class that is where it raised an error of execute() first
def get_multiple_info(self,employees_ids):
"""Get info of multiple Employees in database"""
try:
for employee_id in employees_ids:
kasaa().execute(
"SELECT * FROM ospos_employees WHERE ospos_employees.deleted = 0 AND ospos_employees.person_id = %s ",
employee_id
)
row = kasaa().fetchone()
return row()
except Exception as e:
print(e)
finally:
kasaa().close()
The code in kasaa() is executed twice, so fetchone() is actually called without execute(). Get the cursor and use it to call the other functions
def get_multiple_info(self, employees_ids):
"""Get info of multiple Employees in database"""
cursor = None
try:
for employee_id in employees_ids:
cursor = kasaa()
cursor.execute(
"SELECT * FROM ospos_employees WHERE ospos_employees.deleted = 0 AND ospos_employees.person_id = %s ",
employee_id
)
row = cursor.fetchone()
return row()
except Exception as e:
print(e)
finally:
cursor.close()
I'm learning python since last few weeks. For better learning, I decided to work on some project. So here is my Class for MySQL connection and demo example as well. Can you please tell me. What other improvement can be possible for following code?
Structure?
What else I can do to optimize code?
And Please forgive. If I'm doing some silly mistakes in code. (I'm learning)
#!/usr/bin/python
import pymysql
# select (table, parameter)
# insert (table, data)
# update (table, id, data)
# delete (table, id)
class MySQL:
def __init__(self):
self.sort_by = ""
self.order = ""
# initiate database connection.
self.connection = pymysql.connect(host='localhost',
user='root',
password='',
db='sherlock',
charset='utf8mb4')
self.cursor = self.connection.cursor(pymysql.cursors.DictCursor)
# this function is for selecting any feild on any table.(feilds veriable is optinal)
def select(self, table, *feilds):
flds = "" #differnt name for feilds veriable.
if not feilds:
flds = '*'
else:
for f in feilds:
if not flds:
flds = f
else:
flds += ",`%s`" % f
sql = "SELECT %s FROM `%s` " % (flds, table)
if self.sort_by:
sql = sql +"order by "+ str(self.sort_by) +" "+ str(self.order)
print sql
self.cursor.execute(sql)
result = self.cursor.fetchall()
return result
# This function is for data sorting for Mysql; but optinal.
# example : SELECT * FROM `users` order by id asc
def order_by(self, sort_by="", order="", *args, **kwargs):
self.sort_by = sort_by
self.order = order
# this function is for closing Mysql connection
def close(self):
self.connection.close()
########### END OF MySQL CLASS #############
sql = MySQL()
# sql.order_by function should be called before the sql.select() function.
sql.order_by("email")
# this will select all the feilds from `users` table.
# you can specify whichever feilds you want to return. like : sql.select("users", "id, email")
result = sql.select("users", "password")
for email in result:
print email["password"]
sql.close()
I have a class that working with db operations like below :
class DepartmentOperations(DatabaseOperations):
def __init__(self):
try:
self._connection = Database.create_connection()
self._cursor = self._connection.cursor()
self.isactive = True
except ConnectionException as ex:
print(ex.args)
def get_id(self, department_name):
if(self.isactive):
try:
self._cursor.execute("select BolumId from BOLUMLER where BolumAdi = %s" , department_name)
row = self._cursor.fetchone()
if row is not None:
return row[0]
else:
return 0
except:
raise DbException("Kayıt Getirirken Hata OLuştu...")
finally:
self._connection.close()
self._cursor.close()
self.isactive = False
else:
try:
self._connection = Database.create_connection()
self._cursor = self._connection.cursor()
self.isactive = True
except ConnectionException as ex:
print(ex.args)
try:
self._cursor.execute("select BolumId from BOLUMLER where BolumAdi = %s" , department_name)
row = self._cursor.fetchone()
if row is not None:
return row[0]
else:
return 0
except:
raise DbException("Kayıt Getirirken Hata OLuştu...")
finally:
self._connection.close()
self._cursor.close()
self.isactive = False
def add(self, department_name):
if(self.isactive):
try:
self._cursor.execute("insert into BOLUMLER values (%s)",(department_name))
self._connection.commit()
except:
raise DbException("Veri kayıt ederken hata oluştu.")
finally:
self._connection.close()
self._cursor.close()
self.isactive = False
else:
try:
self._connection = Database.create_connection()
self._cursor = self._connection.cursor()
self.isactive = True
except ConnectionException as ex:
print(ex.args)
try:
self._cursor.execute("insert into BOLUMLER values (%s)",(department_name))
self._connection.commit()
except:
raise DbException("Veri kayıt ederken hata oluştu.")
finally:
self._connection.close()
self._cursor.close()
self.isactive = False
When i instantiate this class and use it, works for the first but not second time because as u see in the code in finally block i close the connection . I delete finally block the methods work good but when i close the connection . How can i manage connections ?
Best way is don't keep the connection open if you are in a web application, instead you can use with statement :
Like this:
with pymssql.connect(server, user, password, "tempdb") as conn:
with conn.cursor(as_dict=True) as cursor:
cursor.execute('SELECT * FROM persons WHERE salesrep=%s', 'John Doe')
for row in cursor:
print("ID=%d, Name=%s" % (row['id'], row['name']))
This way, the connection will open and close in the context.
You can check if connection still active or not:
Use try/except and if the db connection is closed, reopen it.
You can use a custom pool:
def pool(ctor, limit=None):
local_pool = multiprocessing.Queue()
n = multiprocesing.Value('i', 0)
#contextlib.contextmanager
def pooled(ctor=ctor, lpool=local_pool, n=n):
# block iff at limit
try: i = lpool.get(limit and n.value >= limit)
except multiprocessing.queues.Empty:
n.value += 1
i = ctor()
yield i
lpool.put(i)
return pooled
example:
def do_something():
try:
with connection.cursor() as cursor:
# Create a new record
sql = "INSERT INTO `users` (`email`, `password`) VALUES (%s, %s)"
cursor.execute(sql, ('webmaster#python.org', 'very-secret'))
connection.commit()
with connection.cursor() as cursor:
# Read a single record
sql = "SELECT `id`, `password` FROM `users` WHERE `email`=%s"
cursor.execute(sql, ('webmaster#python.org',))
result = cursor.fetchone()
print(result)
finally:
connection.close()
and then
my_pool = pool(lambda: do_something())
with my_pool() as my_obj:
my_obj.do_something()
PYDAL:
I recommend you to use pydal, Extended documentation here
General usage is something like this: (Almost for every database usage):
from pydal import DAL
db = DAL('mysql://username:password#localhost/test', pool_size=150)
class DALHandler(Object):
def __init__(self, db):
self.db = db
def on_start(self):
self.db._adapter.reconnect()
def on_success(self):
self.db.commit()
def on_failure(self):
self.db.rollback()
def on_end(self):
self.db._adapter.close()
I have a database class in python which I use to query the database.
class Database():
def __init__(self, user, password, host, port, service_name, mode, *args):
#mode should be 0 if not cx_Oracle.SYSDBA
self.user = user
self.password = password
self.host = host
self.port = port
self.user = user
self.service_name = service_name
self.logger = logging.getLogger(__name__)
self.mode = 0
self.connection = None
self.connect_string = self.user + '/' + self.password + '#' + dsn
try:
self.connection = cx_Oracle.connect(self.connect_string, mode=self.mode, threaded=True)
self.connection.stmtcachesize = 1000
self.connection.client_identifier = 'my_app_scheduler'
self.cursor = self.connection.cursor()
self.cursor.arraysize = 10000
self.idVar = self.cursor.var(cx_Oracle.NUMBER)
except cx_Oracle.DatabaseError, exc:
error, = exc
self.logger.exception('Exception occured while trying to create database object : %s', error.message)
raise exc
def query(self, q):
try:
self.cursor.execute(q)
return self.cursor.fetchall(), self.cursor.rowcount
except cx_Oracle.DatabaseError, exc:
raise exc
And this is the code to manipulate the fetched data and convert it.
output, rowcount = db_run_query.query(sql_text)
#self.logger.debug('output : %s, type : %s', output, type(output))
end_time=time.time()
time_taken=end_time - start_time
self.logger.debug('Rowcount : %s, time_taken : %s', rowcount, time_taken)
column_name = [d[0] for d in db_run_query.cursor.description]
result = [dict(zip(column_name, row)) for row in output]
#Convert everything to string : Eg: datetime
try:
for each_dict in result:
for key in each_dict:
if isinstance(each_dict[key], cx_Oracle.LOB):
self.logger.debug('%s', each_dict[key].size())
each_dict[key]=each_dict[key].read()
#self.logger.debug('%s %s %s %s %s %s %s', key, each_dict, type(key), type(each_dict[key]), type(each_dict), temp_each_dict, type(temp_each_dict))
else:
each_dict[key]=str(each_dict[key])
except Exception as e:
self.logger.debug(e)
So without self.cursor.arraysize = 10000
and for a query like select clob_value from table it was able to fetch the data and logged Rowcount : 4901, time_taken : 0.196296930313 but was giving me an error like
LOB variable no longer valid after subsequent fetch
but when I mention the arraysize parameter the error goes away. ( Is arraysize only for lob columns coz it works fine for select other_column from table where rownum<20000 <- other_column in varchar)
Why does that happen?
Turns out CLOBs and fetchall don't place nice together:
Internally, Oracle uses LOB locators which are allocated based on the
cursor array size. Thus, it is important that the data in the LOB
object be manipulated before another internal fetch takes place. The
safest way to do this is to use the cursor as an iterator. In
particular, do not use the fetchall() method.
Avoiding cursor.fetchall() and using it like an iterator (e.g. for row in cursor: ...) and i was able to get around this problem.
I have the class below to handle my Postgres DB and I'm running into trouble with multiple inserts where foreign keys are involved. If I insert first in a parent table and then a child table I get a foreign key violation error although I think I have all the deferrable things in place. (autocommit is not enabled)
The constraint on the foreign key is set as follows:
CONSTRAINT tblorganisations_status_tblorganisations_fkey FOREIGN KEY (org_id)
REFERENCES organisations.tblorganisations (org_id) MATCH SIMPLE
ON UPDATE NO ACTION ON DELETE NO ACTION DEFERRABLE INITIALLY IMMEDIATE;
The code that calls the class:
postgres = Postgresql("organisations")
r = postgres.insert(self.db_table, data, return_cols='org_id')
self.org_id = r['org_id']
postgres.insert('tblorganisations_status',
{'org_id': self.org_id,
'org_status_id': 'NEW_CGM'})
postgres.commit()
And the class:
class Postgresql():
conn = None
cur = None
last_result = None
def __init__(self, schema=None):
reload(sys) # Reload does the trick!
sys.setdefaultencoding("utf-8")
self.log = Log()
self.connect()
if schema is not None:
self.schema = schema
self.set_default_schema(schema)
def connection_string(self):
return 'host=%s port=%s dbname=%s user=%s password=%s' % \
(get_config('DATABASE', 'host'),
get_config('DATABASE', 'port'),
get_config('DATABASE', 'dbname'),
get_config('DATABASE', 'user'),
get_config('DATABASE', 'password'))
def connect(self):
try:
self.conn = psycopg2.connect(self.connection_string())
self.conn.set_session(isolation_level='read uncommitted', deferrable=True)
self.cur = self.conn.cursor(cursor_factory=RealDictCursor)
except Exception, e:
self.log.error(e.message)
raise
def set_default_schema(self, schema):
try:
self.cur.execute("SET search_path TO %s,public;", (schema, ))
except Exception, e:
self.log.error(e.message)
raise
def commit(self):
self.conn.commit()
self.close()
def rollback(self):
self.conn.rollback()
self.close()
def close(self):
self.cur.close()
self.conn.close()
def insert(self, table, data, return_cols=None, **kwargs):
data = self.cleanup_data(table, data)
fields = data.keys()
if self.schema is not None:
table = self.schema + '.' + table
sql = "INSERT INTO " + table + " ("
sql += ",".join(fields) + ") VALUES (" + ",".join(["%s"]*len(fields)) + ")"
if return_cols:
sql += " RETURNING " + return_cols
sql += ";"
if 'debug' in kwargs:
raise Exception(sql % tuple(data.values()))
try:
self.log.event('POSTGRES: ' + (sql % tuple(data.values())))
self.cur.execute(sql, data.values())
if return_cols:
result = self.cur.fetchone()
return result
except Exception, e:
self.log.error(e.message)
self.conn.rollback()
self.close()
raise
`
I figured it out myself. Apparently psycopg2 behaves this way because I declared the connection and class variables outside __init__.