I am working on a simple convenience class to use with the with operator so that I can establish exclusive access to a sqlite3 database for a extended writing session in a concurrent system.
Here is the class:
import sqlite3 as sql
class ExclusiveSqlConnection(object):
"""meant to be used with a with statement to ensure proper closure"""
def __enter__(self, path):
self.con = sql.connect(path, isolation_level = "EXCLUSIVE")
self.con.execute("BEGIN EXCLUSIVE")
self.cur = self.con.cursor()
return self
def __exit__(self):
self.con.commit()
self.con.close()
However, when I run this I get the error:
with sql_lib.ExclusiveSqlConnection(self.output) as c:
TypeError: object.__new__() takes no parameters
The constructor (__init__ method) for your ExclusiveSqlConnection needs to take a path parameter.
On the other hand, __enter__ takes no parameter other than self.
import sqlite3 as sql
class ExclusiveSqlConnection(object):
"""meant to be used with a with statement to ensure proper closure"""
def __init__(self, path):
self.path = path
def __enter__(self):
self.con = sql.connect(self.path, isolation_level = "EXCLUSIVE")
self.con.execute("BEGIN EXCLUSIVE")
self.cur = self.con.cursor()
return self
def __exit__(self, exception_type, exception_val, trace):
self.con.commit()
self.con.close()
Related
I am currently working on a huge project, which constantly executes queries. My problem is, that my old code always created a new database connection and cursor, which decreased the speed immensivly. So I thought it's time to make a new database class, which looks like this at the moment:
class Database(object):
_instance = None
def __new__(cls):
if cls._instance is None:
cls._instance = object.__new__(cls)
try:
connection = Database._instance.connection = mysql.connector.connect(host="127.0.0.1", user="root", password="", database="db_test")
cursor = Database._instance.cursor = connection.cursor()
except Exception as error:
print("Error: Connection not established {}".format(error))
else:
print("Connection established")
return cls._instance
def __init__(self):
self.connection = self._instance.connection
self.cursor = self._instance.cursor
# Do database stuff here
The queries will use the class like so:
def foo():
with Database() as cursor:
cursor.execute("STATEMENT")
I am not absolutly sure, if this creates the connection only once regardless of how often the class is created. Maybe someone knows how to initialize a connection only once and how to make use of it in the class afterwards or maybe knows if my solution is correct. I am thankful for any help!
Explanation
The keyword here is clearly class variables. Taking a look in the official documentation, we can see that class variables, other than instance variables, are shared by all class instances regardless of how many class instances exists.
Generally speaking, instance variables are for data unique to each instance and class variables are for attributes and methods shared by all instances of the class:
So let us asume you have multiple instances of the class. The class itself is defined like below.
class Dog:
kind = "canine" # class variable shared by all instances
def __init__(self, name):
self.name = name # instance variable unique to each instance
In order to better understand the differences between class variables and instance variables, I would like to include a small example here:
>>> d = Dog("Fido")
>>> e = Dog("Buddy")
>>> d.kind # shared by all dogs
"canine"
>>> e.kind # shared by all dogs
"canine"
>>> d.name # unique to d
"Fido"
>>> e.name # unique to e
"Buddy"
Solution
Now that we know that class variables are shared by all instances of the class, we can simply define the connection and cursor like shown below.
class Database(object):
connection = None
cursor = None
def __init__(self):
if Database.connection is None:
try:
Database.connection = mysql.connector.connect(host="127.0.0.1", user="root", password="", database="db_test")
Database.cursor = Database.connection.cursor()
except Exception as error:
print("Error: Connection not established {}".format(error))
else:
print("Connection established")
self.connection = Database.connection
self.cursor = Database.cursor
As a result, the connection to the database is created once at the beginning and can then be used by every further instance.
Kind of like this. It's a cheap way of using a global.
class Database(object):
connection = None
def __init__(self):
if not Database.connection:
Database.connection = mysql.connector.connect(host="127.0.0.1", user="root", password="", database="db_test")
def query(self,sql):
cursor = Database.connection.cursor()
cursor.execute(sql)
# Do database stuff here
This too does work and you are guaranteed to always have one instance of the database
def singleton(class_):
instances = {}
def get_instance(*args, **kwargs):
if class_ not in instances:
instances[class_] = class_(*args, **kwargs)
return instances[class_]
return get_instance
#singleton
class SingletonDatabase:
def __init__(self) -> None:
print('Initializing singleton database connection... ', random.randint(1, 100))
The Reason you have to do all this is if you just create
a connection once and leave it at that you then
will end up trying to use a connection which is dropped
so you create a connection and attach it to your app
then whenever you get a new request check if the connection
still exists, with before request hook if not then recreate the
connection and proceeed.
on create_app
def create_app(self):
if not app.config.get('connection_created'):
app.database_connection = Database()
app.config['connection_created'] = True
on run app
#app.before_request
def check_database_connection(self):
if not app.config.get('connection_created') or not app.database_connection:
app.database_connection = Database()
app.config['connection_created'] = True
this will insure that your application always runs with an active connection
and that it gets created only once per app
if connection is dropped on any subsequent call then it gets recreated again...
I'm working on building a simple report generator that will run a query, put the data into a formatted excel sheet and email it.
I am trying to get better at proper coding practices (cohesion, coupling, etc) and I am wondering if this is the proper way to build this or if there's a better way. It feels somewhat redundant to have to pass the arguments of the Extractor twice: once to the main class and then again to the subclass.
Should I be using nested classes? **kwargs? Or is this correct?
from typing_extensions import ParamSpec
import pandas as pd
from sqlalchemy import create_engine
from O365 import Account
from jinjasql import JinjaSql
class Emailer:
pass
class Extractor:
'''Executes a sql query and returns the data.'''
def __init__(self, query: str, database: str, connection: dict, param_query: bool = False) -> None:
self.query = query
self.database = database
# self.conn_params = connection
self.engine = create_engine(connection)
# self.engine = Reactor().get_engine(**self.conn_params)
def parse_query(self) -> None:
'''If the query needs parameterization, do that here.'''
pass
def run_query(self) -> None:
'''Run a supplied query. Expects the name of the query.'''
# TODO: Make this check if it's a query or a query name.
with open(self.query, "r") as f:
query = f.read()
return pd.read_sql_query(query, self.engine)
class ReportGenerator:
'''Main class'''
def __init__(self, query: str, connection: dict, param_query: bool = False) -> None:
self.extractor = Extractor(query, connection, param_query)
self.emailer = Emailer()
def build_report(self) -> None:
pass
This isn't a bad solution, but I think it can be improved.
If you'll continue this correctly, ReportGenerator should never know about query/connection and other members of Extractor, thus it shouldn't be passed to his constructor.
What you can do instead is create an Extractor and an Emailer before the creation of the ReportGenerator and pass them to the constructor.
I have a project written in Python 2.7 where the main program needs frequent access to a sqlite3 db for writing logs, measurement results, getting settings,...
At the moment I have a db module with functions such as add_log(), get_setting(), and each function in there basically looks like:
def add_log(logtext):
try:
db = sqlite3.connect(database_location)
except sqlite3.DatabaseError as e:
db.close() # try to gracefully close the db
return("ERROR (ADD_LOG): While opening db: {}".format(e))
try:
with db: # using context manager to automatically commit or roll back changes.
# when using the context manager, the execute function of the db should be used instead of the cursor
db.execute("insert into logs(level, source, log) values (?, ?, ?)", (level, source, logtext))
except sqlite3.DatabaseError as e:
return("ERROR (ADD_LOG): While adding log to db: {}".format(e))
return "OK"
(some additional code and comments removed).
It seems I should write a class extends the base sqlite connection object function so that the connection is created only once (at the beginning of the main program), and then this object contains the functionality such as
class Db(sqlite3.Connection):
def __init__(self, db_location = database_location):
try:
self = sqlite3.connect(db_location)
return self
except sqlite3.DatabaseError as e:
self.close() # try to gracefully close the db
def add_log(self, logtext):
self.execute("insert into logs(level, source, log) values (?, ?, ?)", (level, source, logtext))
It seems this should be fairly straightforward but, I can't seem to get it working.
It seems there is some useful advise here:
Python: How to successfully inherit Sqlite3.Cursor and add my customized method but I can't seem to understand how to use a similar construct for my purpose.
You are not that far away.
First of all, a class initializer cannot return anything but None (emphasis mine):
Because __new__() and __init__() work together in constructing objects (__new__() to create it, and __init__() to customise it), no non-None value may be returned by __init__(); doing so will cause a TypeError to be raised at runtime.
Second, you overwrite the current instance self of your Db object with a sqlite3.Connection object right in the initializer. That makes subclassing SQLite's connection object a bit pointless.
You just need to fix your __init__ method to make this work:
class Db(sqlite3.Connection):
# If you didn't use the default argument, you could omit overriding __init__ alltogether
def __init__(self, database=database_location, **kwargs):
super(Db, self).__init__(database=database, **kwargs)
def add_log(self, logtext, level, source):
self.execute("insert into logs(level, source, log) values (?, ?, ?)", (level, source, logtext))
That lets you use instances of your class as context managers:
with Db() as db:
print [i for i in db.execute("SELECT * FROM logs")]
db.add_log("I LAUNCHED THAT PUG INTO SPACE!", 42, "Right there")
Maurice Meyer said in the comments of the question that methods such as execute() are cursor methods and, per the DB-API 2.0 specs, that's correct.
However, sqlite3's connection objects offer a few shortcuts to cursor methods:
This is a nonstandard shortcut that creates an intermediate cursor object by calling the cursor method, then calls the cursor’s execute method with the parameters given.
To expand on the discussion in the comments:
The remark about the default argument in my code example above was targeted at the requirement to override sqlite3.Connection's __init__ method.
The __init__ in the class Db is only needed to define the default value database_location on the database argument for the sqlite3.Connection initializer.
If you were willing to pass such a value upon every instantiation of that class, your custom connection class could look like this, and still work the same way, except for that argument:
class Db(sqlite3.Connection):
def add_log(self, logtext, level, source):
self.execute("insert into logs(level, source, log) values (?, ?, ?)", (level, source, logtext))
However, the __init__ method has nothing to do with the context manager protocol as defined in PEP 343.
When it comes to classes, this protocol requires to implement the magic methods __enter__ and __exit__
The sqlite3.Connection does something along these lines:
class Connection:
def __enter__(self):
return self
def __exit__(self, exc_type, exc_val, exc_tb):
if exc_val is None:
self.commit()
else:
self.rollback()
Note: The sqlite3.Connection is provided by a C module, hence does not have a Python class definition. The above reflects what the methods would roughly look like if it did.
Lets say you don't want to keep the same connection open all the time, but rather have a dedicated connection per transaction while maintaining the general interface of the Db class above.
You could do something like this:
# Keep this to have your custom methods available
class Connection(sqlite3.Connection):
def add_log(self, level, source, log):
self.execute("INSERT INTO logs(level, source, log) VALUES (?, ?, ?)",
(level, source, log))
class DBM:
def __init__(self, database=database_location):
self._database = database
self._conn = None
def __enter__(self):
return self._connection()
def __exit__(self, exc_type, exc_val, exc_tb):
# Decide whether to commit or roll back
if exc_val:
self._connection().rollback()
else:
self._connection().commit()
# close connection
try:
self._conn.close()
except AttributeError:
pass
finally:
self._conn = None
def _connection(self):
if self._conn is None:
# Instantiate your custom sqlite3.Connection
self._conn = Connection(self._database)
return self._conn
# add shortcuts to connection methods as seen fit
def execute(self, sql, parameters=()):
with self as temp:
result = temp.execute(sql, parameters).fetchall()
return result
def add_log(self, level, source, log):
with self as temp:
temp.add_log(level, source, log)
This can be used in a context and by calling methods on the instance:
db = DBM(database_location)
with db as temp:
print [i for i in temp.execute("SELECT * FROM logs")]
temp.add_log(1, "foo", "I MADE MASHED POTATOES")
# The methods execute and add_log are only available from
# the outside because the shortcuts have been added to DBM
print [i for i in db.execute("SELECT * FROM logs")]
db.add_log(1, "foo", "I MADE MASHED POTATOES")
For further reading on context managers refer to the official documentation. I'll also recommend Jeff Knupp's nice introduction. Also, the aforementioned PEP 343 is worth having a look at for the technical specification and rationale behind that protocol.
I'm using python3.4 to interact with oracle(11g)/sql developer.
Is it true that cx_Oracle could not deal with sqlPlus statements? It seems that the page https://sourceforge.net/p/cx-oracle/mailman/message/2932119/ said so.
So how could we execute 'spool' command by python?
The code:
import cx_Oracle
db_conn = cx_Oracle.connect(...)
cursor = db_conn.cursor()
cursor.execute('spool C:\\Users\Administrator\Desktop\mycsv.csv')
...
the error: cx_Oracle.DatabaseError: ORA-00900:
The "spool" command is very specific to SQL*Plus and is not available in cx_Oracle or any other application that uses the OCI (Oracle Call Interface). You can do something similar, however, without too much trouble.
You can create your own Connection class subclassed from cx_Oracle.Connection and your own Cursor class subclassed from cx_Oracle.Cursor that would perform any logging and have a special command "spool" that would turn it on and off at will. Something like this:
class Connection(cx_Oracle.Connection):
def __init__(self, *args, **kwargs):
self.spoolFile = None
return super(Connection, self).__init__(*args, **kwargs)
def cursor(self):
return Cursor(self)
def spool(self, fileName):
self.spoolFile = open(fileName, "w")
class Cursor(cx_Oracle.Cursor):
def execute(self, statement, args):
result = super(Cursor, self).execute(statement, args)
if self.connection.spoolFile is not None:
self.connection.spoolFile.write("Headers for query\n")
self.connection.spoolFile.write("use cursor.description")
def fetchall(self):
rows = super(Cursor, self).fetchall()
if self.connection.spoolFile is not None:
for row in rows:
self.connection.spoolFile.write("row details")
That should give you some idea on where to go with this.
I'm trying to come up with SQLiteDB object, and following is the open/close code for it.
Does this work without problem? Am I missing something important?
For close(), I use con.close() and cursor.close(), but I'm wondering if cursor.close() is necessary.
class SQLiteDB(object):
def __init__(self, dbFile, connect = True):
self.dbFile = dbFile
self.con = None
self.cursor = None
if connect:
self.open()
def __del__(self):
if self.con:
self.close()
def open(self):
self.con = sqlite3.connect(self.dbFile)
self.cursor = self.connector.cursor()
return self.con, self.cursor
def close(self):
self.con.close()
self.cursor.close()
self.cursor = None
self.con = None
What happens on Cursor.close() depends on the underlying database implementation. For SQLite it might currently work without closing, but for other implementations or a future SQLite version it might not, so I would recommend to close the Cursor object. You can find further information on Cursor.close() in PEP 249.
Also, there seems to be a typo in your code:
self.connector = sqlite3.connect(self.dbFile)
should probably be
self.con = sqlite3.connect(self.dbFile)
Otherwise your code looks fine to me. Happy coding :) .