We Keep Coding

How to update sqlite3 record in python using WHERE?

I am struggeling with updating a record in my sqlite3 database in python.
I have tried to follow this tutorial and modify this function from the tutorial for my needs:
def update_task(conn, task):
"""
update priority, begin_date, and end date of a task
:param conn:
:param task:
:return: project id
"""
sql = ''' UPDATE tasks
SET priority = ? ,
begin_date = ? ,
end_date = ?
WHERE id = ?'''
cur = conn.cursor()
cur.execute(sql, task)
conn.commit()
I have a table set up like this:
sql_create_answers_table = """CREATE TABLE IF NOT EXISTS answers (
id integer PRIMARY KEY,
predmet text NOT NULL UNIQUE,
tag text NOT NULL,
keyword text NOT NULL,
obsah text NOT NULL,
autor text NOT NULL,
datum text NOT NULL
);"""
db.create_table(sql_create_answers_table)
I have my update function written like this:
def update_answer(answer):
try:
conn = create_connection(db_name)
cur = conn.cursor()
sql_command = """UPDATE answers
SET predmet = ? ,
tag = ? ,
keyword = ? ,
obsah = ? ,
autor = ? ,
datum = ? ,
WHERE id = ?"""
# I have also tried this: cur.execute(sql_command, answer )
cur.execute(sql_command, (answer[1], answer[2], answer[3], answer[4], answer[5], answer[6], answer[0]) )
conn.commit()
conn.close()
finally:
unlock_db()
This is how I establishe the connection to the database:
def create_connection(db_file):
""" create a database connection to an SQLite database """
conn = None
try:
conn = sqlite3.connect(db_file)
return conn
except Error as e:
print(e)
When I call the function in my tkinter app, the original answer looks like this:
(1, 'pokus', 'žádný', 'datum', 'Text zpravy a ahoj kamo.\n', 'Vlad', '02/09/21')
Then I create a new version of the answer that looks like this:
new_answer = (1, 'pokus', 'žádný', 'datum', 'Text\n', 'Vlad', '02/15/21') # the 4th field changes
And then call the update function (db is my imported module with the function):
db.update_answer(new_answer)
I am getting an Error "
File "f:\AAA Vlada\a škola\Python\zapoctak\db_module.py", line 135, in update_answer
cur.execute(sql_command, (answer[1], answer[2], answer[3], answer[4], answer[5], answer[6], answer[0]) )
sqlite3.OperationalError: near "WHERE": syntax error
"
Other operations on my database work normally and I do not understand why this simple update function doesnt work. The "updated answer" seems to me to be made correctly and so does the update function.
I am gratefull for any help and I can provide more code if necesary.

Remove the extra , character just before WHERE clause and you should be fine.

Need Python Programming Tips

I'm learning python since last few weeks. For better learning, I decided to work on some project. So here is my Class for MySQL connection and demo example as well. Can you please tell me. What other improvement can be possible for following code?
Structure?
What else I can do to optimize code?
And Please forgive. If I'm doing some silly mistakes in code. (I'm learning)
#!/usr/bin/python
import pymysql
# select (table, parameter)
# insert (table, data)
# update (table, id, data)
# delete (table, id)
class MySQL:
def __init__(self):
self.sort_by = ""
self.order = ""
# initiate database connection.
self.connection = pymysql.connect(host='localhost',
user='root',
password='',
db='sherlock',
charset='utf8mb4')
self.cursor = self.connection.cursor(pymysql.cursors.DictCursor)
# this function is for selecting any feild on any table.(feilds veriable is optinal)
def select(self, table, *feilds):
flds = "" #differnt name for feilds veriable.
if not feilds:
flds = '*'
else:
for f in feilds:
if not flds:
flds = f
else:
flds += ",`%s`" % f
sql = "SELECT %s FROM `%s` " % (flds, table)
if self.sort_by:
sql = sql +"order by "+ str(self.sort_by) +" "+ str(self.order)
print sql
self.cursor.execute(sql)
result = self.cursor.fetchall()
return result
# This function is for data sorting for Mysql; but optinal.
# example : SELECT * FROM `users` order by id asc
def order_by(self, sort_by="", order="", *args, **kwargs):
self.sort_by = sort_by
self.order = order
# this function is for closing Mysql connection
def close(self):
self.connection.close()
########### END OF MySQL CLASS #############
sql = MySQL()
# sql.order_by function should be called before the sql.select() function.
sql.order_by("email")
# this will select all the feilds from `users` table.
# you can specify whichever feilds you want to return. like : sql.select("users", "id, email")
result = sql.select("users", "password")
for email in result:
print email["password"]
sql.close()

python and mysql connections in a class , DB open and close strange behaviour

I have written a small app that uses mysql to get a list of products that need updating on our magento website.
Python then actions these updates and marks the product in the db as complete.
My Original code (pseudo to show the overview)
class Mysqltools:
def get_products():
db = pymysql.connect(host= .... )
mysqlcursor = db.cursor(pymysql.cursors.DictCursor)
sql = select * from x where y = z
mysqlcursor.execute(sql % (z))
rows = mysqlcursor.fetchall()
mysqlcursor.close()
db.close
return rows
def write_products(sku, name, id):
db = pymysql.connect(host= .... )
mysqlcursor = db.cursor(pymysql.cursors.DictCursor)
sql = update table set sku = sku, name = name, id = id.....
mysqlcursor.execute(sql % (sku, name, id))
mysqlcursor.close()
db.close
This was working ok, but on each db connection string we were getting a pause.
I did a bit of research and did the following:
class Mysqltools:
def __init__():
self.db = pymysql.connect(host= .... )
def get_products():
mysqlcursor = self.db.cursor(pymysql.cursors.DictCursor)
sql = select * from x where y = z
mysqlcursor.execute(sql % (z))
rows = mysqlcursor.fetchall()
mysqlcursor.close()
def write_products(sku, name, id):
mysqlcursor = self.db.cursor(pymysql.cursors.DictCursor)
sql = update table set sku = sku, name = name, id = id.....
mysqlcursor.execute(sql % (sku, name, id))
mysqlcursor.close()
db.commit()
This has a MASSIVE speed improvement. However, it would only do a successful get_products on the first iteration, once it was called a second time, it was finding 0 products to update, even though performing the same SQL on the db would show a number of rows returned.
Am I doing something wrong with the connections ?
I have also tried moving the db = outside of the class and referencing it but that still gives the same issue.
UPDATE
Doing some testing, and if I remove the DictCursor from the cursor I can get the correct rows returned each time (I've just created a quick loop to keep checking for records)
Is the DictCursor doing something I am unaware of ?
** UPDATE 2 **
I've removed the DictCursor, and tried the following.
Create a while True loop which calls my get_product method.
In MySQL change some rows so that they should be found.
If I go from having 0 possible rows to find, then change some so they should be found, my code just shows 0 found, and loops stating this.
If I got from having x possible rows to find, then change it to 0 in mysql, my code continues to loop showing the x possible rows.

Ok, the answer to this is as follows:
db = pymysql.connect(host=.... user=... )
class MySqlTools:
def get_products():
mysqlcursor = db.cursor(pymysql.cursors.DictCursor)
sql = select * from x where y = z
mysqlcursor.execute(sql % (z))
rows = mysqlcursor.fetchall()
mysqlcursor.close()
db.commit()
This then allows you to re-use the db connection and remove the overhead of creating and closing a connection each and every time.
In testing, downloading 500 orders from our website and writing them to a db went from 16minutes to <3 minutes.

Database is locking but all statements are followed by commit?

I'm working on an IRC bot, forked from a modular bot called Skybot.
There are two other modules that make use of the sqlite3 database by default; they have both been removed and their tables dropped, so I know that the issue is somewhere in what I'm doing.
I only call 3 db.execute() statements in the whole thing and they're all immediately committed. This thing isn't getting hammered with queries either, but the lock remains.
Relevant code:
def db_init(db):
db.execute("create table if not exists searches"
"(search_string UNIQUE PRIMARY KEY,link)")
db.commit()
return db
def get_link(db, inp):
row = db.execute("select link from searches where"
" search_string=lower(?) limit 1",
(inp.lower(),)).fetchone()
db.commit()
return row
def store_link(db, stub, search):
db.execute("insert into searches (search_string, link) VALUES (?, ?)", (search.lower(), stub))
db.commit()
return stub
If the script only has to touch db_init() and get_link() it breezes through, but if it needs to call store_link() while the database is unlocked it will do the insert, but doesn't seem to be committing it in a way that future calls to get_link() can read it until the bot restarts.
The bot's db.py:
import os
import sqlite3
def get_db_connection(conn, name=''):
"returns an sqlite3 connection to a persistent database"
if not name:
name = '%s.%s.db' % (conn.nick, conn.server)
filename = os.path.join(bot.persist_dir, name)
return sqlite3.connect(filename, isolation_level=None)
bot.get_db_connection = get_db_connection
I did adjust the isolation_level myself, that was originally timeout=10. I am fairly stumped.
EDIT: The usages of get_db_connection():
main.py (main loop):
def run(func, input):
args = func._args
if 'inp' not in input:
input.inp = input.paraml
if args:
if 'db' in args and 'db' not in input:
input.db = get_db_connection(input.conn)
if 'input' in args:
input.input = input
if 0 in args:
out = func(input.inp, **input)
else:
kw = dict((key, input[key]) for key in args if key in input)
out = func(input.inp, **kw)
else:
out = func(input.inp)
if out is not None:
input.reply(unicode(out))
...
def start(self):
uses_db = 'db' in self.func._args
db_conns = {}
while True:
input = self.input_queue.get()
if input == StopIteration:
break
if uses_db:
db = db_conns.get(input.conn)
if db is None:
db = bot.get_db_connection(input.conn)
db_conns[input.conn] = db
input.db = db
try:
run(self.func, input)
except:
traceback.print_exc()

Send conn in your functions, along with db, as mentioned. If you wrote the code yourself, you'll know where the database actually is. Conventionally you would do something like:
db = sqlite3.connect('database.db')
conn = db.cursor()
Then for general usage:
db.execute("...")
conn.commit()
Hence, in your case:
def db_init(conn,db):
db.execute("create table if not exists searches"
"(search_string UNIQUE PRIMARY KEY,link)")
conn.commit()
return db
def get_link(conn,db, inp):
row = db.execute("select link from searches where"
" search_string=lower(?) limit 1",
(inp.lower(),)).fetchone()
conn.commit()
return row
def store_link(conn,db, stub, search):
db.execute("insert into searches (search_string, link) VALUES (?, ?)", (search.lower(), stub))
conn.commit()
return stub

On the basis that you have set the isolation_level to automatic updates:
sqlite3.connect(filename, isolation_level=None)
There is no need whatsoever for the commit statements in your code
Edit:
Wrap your execute statements in try statements, so that you at least have a chance of finding out what is going on i.e.
import sqlite3
def get_db(name=""):
if not name:
name = "db1.db"
return sqlite3.connect(name, isolation_level=None)
connection = get_db()
cur = connection.cursor()
try:
cur.execute("create table if not exists searches"
"(search_string UNIQUE PRIMARY KEY,link)")
except sqlite3.Error as e:
print 'Searches create Error '+str(e)
try:
cur.execute("insert into searches (search_string, link) VALUES (?, ?)", ("my search", "other"))
except sqlite3.Error as e:
print 'Searches insert Error '+str(e)
cur.execute("select link from searches where search_string=? limit 1", ["my search"])
s_data = cur.fetchone()
print 'Result:', s_data

python Sqlite3 parameter subs

I have a small problem with this class which handle my DB. It still saying:
cursor.execute(sql)
ValueError: operation parameter must be str
I tried lots of things but nothing work as i want. I looked over https://docs.python.org/3.4/library/sqlite3.html and i'm sure i do the same things.
import sqlite3
class Database():
def __init__(self):
try:
self.db = sqlite3.connect('../database.sqlite')
self.cur = self.db.cursor()
self.cur.execute('pragma foreign_keys="1"')
except sqlite3.Error as e:
raise e
def select(self,sql):
cursor = self.db.cursor()
cursor.execute(sql)
records = cursor.fetchall()
cursor.close()
return records
def insert(self,sql):
cursor = self.db.cursor()
cursor.execute(sql)
newID = cursor.lastrowid
self.db.commit()
cursor.close()
return newID
def execute(self,sql):
""" execute any SQL statement but no return value given """
cursor = self.db.cursor()
cursor.execute(sql)
self.db.commit()
cursor.close()
if __name__ == '__main__':
db = Database()
#sql = "SELECT skuref, titre_prod FROM product"
t = ("888888",)
sql= "UPDATE product SET created = 1 WHERE skuref = ?", t
db.execute(sql)
If someone can help me it would be grateful.Later i wanted to pass something like this in the main program inside a for loop
lastpost = record[0]
if created = True
sql = "UPDATE product SET created = 1 WHERE skuref = ?",(lastpost,)
db.execute(sql)

sql is a tuple containing SQL statement and the parameters.
Change as following, so that sql and parameters are passed separately, instead of being passed as a tuple:
def execute(self, sql):
""" execute any SQL statement but no return value given """
cursor = self.db.cursor()
cursor.execute(*sql) # <------
self.db.commit()
cursor.close()

With your statement
sql = "UPDATE product SET created = 1 WHERE skuref = ?",(lastpost,)
you have created a tupel like
("UPDATE product SET created = 1 WHERE skuref = ?", (lastpost,))
You have to give the arguments as parameters to the execute() function.
Also your if statement is bad: no :, = instead of == and the whole check for True is no nesesary.
Try this:
lastpost = record[0]
if created:
sql = "UPDATE product SET created = 1 WHERE skuref = ?"
db.execute(sql, lastpost)