I'm learning python since last few weeks. For better learning, I decided to work on some project. So here is my Class for MySQL connection and demo example as well. Can you please tell me. What other improvement can be possible for following code?
Structure?
What else I can do to optimize code?
And Please forgive. If I'm doing some silly mistakes in code. (I'm learning)
#!/usr/bin/python
import pymysql
# select (table, parameter)
# insert (table, data)
# update (table, id, data)
# delete (table, id)
class MySQL:
def __init__(self):
self.sort_by = ""
self.order = ""
# initiate database connection.
self.connection = pymysql.connect(host='localhost',
user='root',
password='',
db='sherlock',
charset='utf8mb4')
self.cursor = self.connection.cursor(pymysql.cursors.DictCursor)
# this function is for selecting any feild on any table.(feilds veriable is optinal)
def select(self, table, *feilds):
flds = "" #differnt name for feilds veriable.
if not feilds:
flds = '*'
else:
for f in feilds:
if not flds:
flds = f
else:
flds += ",`%s`" % f
sql = "SELECT %s FROM `%s` " % (flds, table)
if self.sort_by:
sql = sql +"order by "+ str(self.sort_by) +" "+ str(self.order)
print sql
self.cursor.execute(sql)
result = self.cursor.fetchall()
return result
# This function is for data sorting for Mysql; but optinal.
# example : SELECT * FROM `users` order by id asc
def order_by(self, sort_by="", order="", *args, **kwargs):
self.sort_by = sort_by
self.order = order
# this function is for closing Mysql connection
def close(self):
self.connection.close()
########### END OF MySQL CLASS #############
sql = MySQL()
# sql.order_by function should be called before the sql.select() function.
sql.order_by("email")
# this will select all the feilds from `users` table.
# you can specify whichever feilds you want to return. like : sql.select("users", "id, email")
result = sql.select("users", "password")
for email in result:
print email["password"]
sql.close()
Related
I'm working on an IRC bot, forked from a modular bot called Skybot.
There are two other modules that make use of the sqlite3 database by default; they have both been removed and their tables dropped, so I know that the issue is somewhere in what I'm doing.
I only call 3 db.execute() statements in the whole thing and they're all immediately committed. This thing isn't getting hammered with queries either, but the lock remains.
Relevant code:
def db_init(db):
db.execute("create table if not exists searches"
"(search_string UNIQUE PRIMARY KEY,link)")
db.commit()
return db
def get_link(db, inp):
row = db.execute("select link from searches where"
" search_string=lower(?) limit 1",
(inp.lower(),)).fetchone()
db.commit()
return row
def store_link(db, stub, search):
db.execute("insert into searches (search_string, link) VALUES (?, ?)", (search.lower(), stub))
db.commit()
return stub
If the script only has to touch db_init() and get_link() it breezes through, but if it needs to call store_link() while the database is unlocked it will do the insert, but doesn't seem to be committing it in a way that future calls to get_link() can read it until the bot restarts.
The bot's db.py:
import os
import sqlite3
def get_db_connection(conn, name=''):
"returns an sqlite3 connection to a persistent database"
if not name:
name = '%s.%s.db' % (conn.nick, conn.server)
filename = os.path.join(bot.persist_dir, name)
return sqlite3.connect(filename, isolation_level=None)
bot.get_db_connection = get_db_connection
I did adjust the isolation_level myself, that was originally timeout=10. I am fairly stumped.
EDIT: The usages of get_db_connection():
main.py (main loop):
def run(func, input):
args = func._args
if 'inp' not in input:
input.inp = input.paraml
if args:
if 'db' in args and 'db' not in input:
input.db = get_db_connection(input.conn)
if 'input' in args:
input.input = input
if 0 in args:
out = func(input.inp, **input)
else:
kw = dict((key, input[key]) for key in args if key in input)
out = func(input.inp, **kw)
else:
out = func(input.inp)
if out is not None:
input.reply(unicode(out))
...
def start(self):
uses_db = 'db' in self.func._args
db_conns = {}
while True:
input = self.input_queue.get()
if input == StopIteration:
break
if uses_db:
db = db_conns.get(input.conn)
if db is None:
db = bot.get_db_connection(input.conn)
db_conns[input.conn] = db
input.db = db
try:
run(self.func, input)
except:
traceback.print_exc()
Send conn in your functions, along with db, as mentioned. If you wrote the code yourself, you'll know where the database actually is. Conventionally you would do something like:
db = sqlite3.connect('database.db')
conn = db.cursor()
Then for general usage:
db.execute("...")
conn.commit()
Hence, in your case:
def db_init(conn,db):
db.execute("create table if not exists searches"
"(search_string UNIQUE PRIMARY KEY,link)")
conn.commit()
return db
def get_link(conn,db, inp):
row = db.execute("select link from searches where"
" search_string=lower(?) limit 1",
(inp.lower(),)).fetchone()
conn.commit()
return row
def store_link(conn,db, stub, search):
db.execute("insert into searches (search_string, link) VALUES (?, ?)", (search.lower(), stub))
conn.commit()
return stub
On the basis that you have set the isolation_level to automatic updates:
sqlite3.connect(filename, isolation_level=None)
There is no need whatsoever for the commit statements in your code
Edit:
Wrap your execute statements in try statements, so that you at least have a chance of finding out what is going on i.e.
import sqlite3
def get_db(name=""):
if not name:
name = "db1.db"
return sqlite3.connect(name, isolation_level=None)
connection = get_db()
cur = connection.cursor()
try:
cur.execute("create table if not exists searches"
"(search_string UNIQUE PRIMARY KEY,link)")
except sqlite3.Error as e:
print 'Searches create Error '+str(e)
try:
cur.execute("insert into searches (search_string, link) VALUES (?, ?)", ("my search", "other"))
except sqlite3.Error as e:
print 'Searches insert Error '+str(e)
cur.execute("select link from searches where search_string=? limit 1", ["my search"])
s_data = cur.fetchone()
print 'Result:', s_data
I have a database class in python which I use to query the database.
class Database():
def __init__(self, user, password, host, port, service_name, mode, *args):
#mode should be 0 if not cx_Oracle.SYSDBA
self.user = user
self.password = password
self.host = host
self.port = port
self.user = user
self.service_name = service_name
self.logger = logging.getLogger(__name__)
self.mode = 0
self.connection = None
self.connect_string = self.user + '/' + self.password + '#' + dsn
try:
self.connection = cx_Oracle.connect(self.connect_string, mode=self.mode, threaded=True)
self.connection.stmtcachesize = 1000
self.connection.client_identifier = 'my_app_scheduler'
self.cursor = self.connection.cursor()
self.cursor.arraysize = 10000
self.idVar = self.cursor.var(cx_Oracle.NUMBER)
except cx_Oracle.DatabaseError, exc:
error, = exc
self.logger.exception('Exception occured while trying to create database object : %s', error.message)
raise exc
def query(self, q):
try:
self.cursor.execute(q)
return self.cursor.fetchall(), self.cursor.rowcount
except cx_Oracle.DatabaseError, exc:
raise exc
And this is the code to manipulate the fetched data and convert it.
output, rowcount = db_run_query.query(sql_text)
#self.logger.debug('output : %s, type : %s', output, type(output))
end_time=time.time()
time_taken=end_time - start_time
self.logger.debug('Rowcount : %s, time_taken : %s', rowcount, time_taken)
column_name = [d[0] for d in db_run_query.cursor.description]
result = [dict(zip(column_name, row)) for row in output]
#Convert everything to string : Eg: datetime
try:
for each_dict in result:
for key in each_dict:
if isinstance(each_dict[key], cx_Oracle.LOB):
self.logger.debug('%s', each_dict[key].size())
each_dict[key]=each_dict[key].read()
#self.logger.debug('%s %s %s %s %s %s %s', key, each_dict, type(key), type(each_dict[key]), type(each_dict), temp_each_dict, type(temp_each_dict))
else:
each_dict[key]=str(each_dict[key])
except Exception as e:
self.logger.debug(e)
So without self.cursor.arraysize = 10000
and for a query like select clob_value from table it was able to fetch the data and logged Rowcount : 4901, time_taken : 0.196296930313 but was giving me an error like
LOB variable no longer valid after subsequent fetch
but when I mention the arraysize parameter the error goes away. ( Is arraysize only for lob columns coz it works fine for select other_column from table where rownum<20000 <- other_column in varchar)
Why does that happen?
Turns out CLOBs and fetchall don't place nice together:
Internally, Oracle uses LOB locators which are allocated based on the
cursor array size. Thus, it is important that the data in the LOB
object be manipulated before another internal fetch takes place. The
safest way to do this is to use the cursor as an iterator. In
particular, do not use the fetchall() method.
Avoiding cursor.fetchall() and using it like an iterator (e.g. for row in cursor: ...) and i was able to get around this problem.
I have a small problem with this class which handle my DB. It still saying:
cursor.execute(sql)
ValueError: operation parameter must be str
I tried lots of things but nothing work as i want. I looked over https://docs.python.org/3.4/library/sqlite3.html and i'm sure i do the same things.
import sqlite3
class Database():
def __init__(self):
try:
self.db = sqlite3.connect('../database.sqlite')
self.cur = self.db.cursor()
self.cur.execute('pragma foreign_keys="1"')
except sqlite3.Error as e:
raise e
def select(self,sql):
cursor = self.db.cursor()
cursor.execute(sql)
records = cursor.fetchall()
cursor.close()
return records
def insert(self,sql):
cursor = self.db.cursor()
cursor.execute(sql)
newID = cursor.lastrowid
self.db.commit()
cursor.close()
return newID
def execute(self,sql):
""" execute any SQL statement but no return value given """
cursor = self.db.cursor()
cursor.execute(sql)
self.db.commit()
cursor.close()
if __name__ == '__main__':
db = Database()
#sql = "SELECT skuref, titre_prod FROM product"
t = ("888888",)
sql= "UPDATE product SET created = 1 WHERE skuref = ?", t
db.execute(sql)
If someone can help me it would be grateful.Later i wanted to pass something like this in the main program inside a for loop
lastpost = record[0]
if created = True
sql = "UPDATE product SET created = 1 WHERE skuref = ?",(lastpost,)
db.execute(sql)
sql is a tuple containing SQL statement and the parameters.
Change as following, so that sql and parameters are passed separately, instead of being passed as a tuple:
def execute(self, sql):
""" execute any SQL statement but no return value given """
cursor = self.db.cursor()
cursor.execute(*sql) # <------
self.db.commit()
cursor.close()
With your statement
sql = "UPDATE product SET created = 1 WHERE skuref = ?",(lastpost,)
you have created a tupel like
("UPDATE product SET created = 1 WHERE skuref = ?", (lastpost,))
You have to give the arguments as parameters to the execute() function.
Also your if statement is bad: no :, = instead of == and the whole check for True is no nesesary.
Try this:
lastpost = record[0]
if created:
sql = "UPDATE product SET created = 1 WHERE skuref = ?"
db.execute(sql, lastpost)
I have a class as below that I'm using to connect to a remote SQL server instance from a linux server python web app. I define and set cursor in the init constructor and wish to use it throughout the class. How do I do this? I come form a java background and don't understand the scope and protection levels of Python fields.
import pyodbc
class SQLSeverConnection():
def __init__(self, DSN, user, password, database):
connectionString = 'DSN=%s;UID=%s;PWD=%s;DATABASE=%s;' % (DSN, user, password, database)
connection = pyodbc.connect(connectionString)
cursor = connection.cursor()
def getColumnData(self, columnName, tableName):
cursor.execute('SELECT ' columnName ' FROM ' tableName ' ORDER BY timestamp')
data = cursor.fetchall()
return data
def getColumnTitles(self, tableName):
cursor.execute('select column_name,* from information_schema.columns where table_name = 'tableName' order by ordinal_position')
columns = cursor.fetchall()
return columns
def getTableNames(self):
cursor.execute('SELECT TABLE_NAME FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_TYPE = ''BASE TABLE''')
tables = cursor.fetchall()
return tables
The answer is simple: Python's "methods" are really plain functions, and local variables are plain local variables. To set / access instance attributes, you must use the current instance, which is passed as first argument to the function (and by convention named self):
class SQLSeverConnection():
def __init__(self, DSN, user, password, database):
connectionString = 'DSN=%s;UID=%s;PWD=%s;DATABASE=%s;' % (DSN, user, password, database)
self.connection = pyodbc.connect(connectionString)
self.cursor = connection.cursor()
def getColumnData(self, columnName, tableName):
self.cursor.execute('SELECT ' columnName ' FROM ' tableName ' ORDER BY timestamp')
data = self.cursor.fetchall()
return data
def getColumnTitles(self, tableName):
self.cursor.execute('select column_name,* from information_schema.columns where table_name = 'tableName' order by ordinal_position')
columns = self.cursor.fetchall()
return columns
def getTableNames(self):
BASE_TABLE ='BASE_TABLE'
self.cursor.execute('SELECT TABLE_NAME FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_TYPE = 'BASE TABLE'')
tables = self.cursor.fetchall()
return tables
Now using a single shared cursor for all operations is brittle, you'd better instanciate a new cursor for each operation. Also, since a cursor is an iterable, you may want to return the cursor itself and let client code iterate over it, it might save some memory...
class SQLSeverConnection(object):
def __init__(self, DSN, user, password, database):
connectionString = 'DSN=%s;UID=%s;PWD=%s;DATABASE=%s;' % (DSN, user, password, database)
self.connection = pyodbc.connect(connectionString)
def getCursor(self):
return self.connection.cursor()
def getColumnData(self, columnName, tableName):
cursor = self.getCursor()
cursor.execute('SELECT ' columnName ' FROM ' tableName ' ORDER BY timestamp')
return cursor
# etc
Oh and yes: using mixCased is not pythonic, we prefer all_lower ;)
change cursor to self.cursor
def __init__(self, DSN, user, password, database):
connectionString = 'DSN=%s;UID=%s;PWD=%s;DATABASE=%s;' % (DSN, user, password, database)
connection = pyodbc.connect(connectionString)
self.cursor = connection.cursor()
def getColumnData(self, columnName, tableName):
self.cursor.execute('SELECT ' columnName ' FROM ' tableName ' ORDER BY timestamp')
data = self.cursor.fetchall()
return data
I am trying to imitate one of the code of Collective Intelligence and stumbled due to some unkown error. All the methods works fine inside the class except the one for retrieving query.
class Connect:
# Create the Database
def __init__(self,dbname):
self.con = sqlite3.connect(dbname)
# Add the row
def create(self,date,name,age):
self.con.execute('create table Profile(date text, name text,age real)')
# Lock the changes
def commit(self):
self.con.commit()
# Retrive the details
def getentry(self,table,field,value):
cursor = self.con.execute(
"select * from %s where %s = '%s'" % (table,field,value))
result_set = cursor.fetchall()
print result_set
Working Example :
To create DB
C = Connect('test.db')
To add rows
C.create('2013-03-06','Joy',34)
To make changes and lock the files
C.commit()
Getting the row
C.getentry(Profile,name,'Joy')
Error : NameError: name 'Profile' is not defined
Then with making parenthesis.
C.getentry('Profile','name','Joy')
Result = [ ]
The problem is in Connect.create. It creates the table but does not populate it as the comment above its definition appears to imply. You need to update it to something like the following:
def create(self,date,name,age):
self.con.execute( 'create table Profile(date text, name text,age real)' )
self.con.execute( "insert into Profile values('{0}','{1}','{2}')"
.format(date,name,age) )
return