I have to get the recently updated data from database. For the purpose of solving it, I have saved the last read row number into shelve of python. The following code works for a simple query like select * from rows. My code is:
from pyodbc import connect
from peewee import *
import random
import shelve
import connection
d = shelve.open("data.shelve")
db = SqliteDatabase("data.db")
class Rows(Model):
valueone = IntegerField()
valuetwo = IntegerField()
class Meta:
database = db
def CreateAndPopulate():
db.connect()
db.create_tables([Rows],safe=True)
with db.atomic():
for i in range(100):
row = Rows(valueone=random.randrange(0,100),valuetwo=random.randrange(0,100))
row.save()
db.close()
def get_last_primay_key():
return d.get('max_row',0)
def doWork():
query = "select * from rows" #could be anything
conn = connection.Connection("localhost","","SQLite3 ODBC Driver","data.db","","")
max_key_query = "SELECT MAX(%s) from %s" % ("id", "rows")
max_primary_key = conn.fetch_one(max_key_query)[0]
print "max_primary_key " + str(max_primary_key)
last_primary_key = get_last_primay_key()
print "last_primary_key " + str(last_primary_key)
if max_primary_key == last_primary_key:
print "no new records"
elif max_primary_key > last_primary_key:
print "There are some datas"
optimizedQuery = query + " where id>" + str(last_primary_key)
print query
for data in conn.fetch_all(optimizedQuery):
print data
d['max_row'] = max_primary_key
# print d['max_row']
# CreateAndPopulate() # to populate data
doWork()
While the code will work for a simple query without where clause, but the query can be anything from simple to complex, having joins and multiple where clauses. If so, then the portion where I'm adding where will fail. How can I get only last updated data from database whatever be the query?
PS: I cannot modify database. I just have to fetch from it.
Use an OFFSET clause. For example:
SELECT * FROM [....] WHERE [....] LIMIT -1 OFFSET 1000
In your query, replace 1000 with a parameter bound to your shelve variable. That will skip the top "shelve" number of rows and only grab newer ones. You may want to consider a more robust refactor eventually, but good luck.
Related
I have made a GUI in PyQt5 that allows you to deal with a database. There is an insert button which allows you to insert data into a database and then using a stored procedure whose parameter is a MySQL query in string format, it passes a select query to the stored procedure whose where clause consists of values just entered.
`
def insert(self):
try:
self.table.setRowCount(0)
QEmpID = self.lineEmpID.text() + "%"
QFName = self.lineFName.text() + "%"
QLName = self.lineLName .text() + "%"
QSalary = self.lineSalary.text() + "%"
QTask = self.lineTask.text() + "%"
mydb = mc.connect(host="localhost",username="root",password="",database="Office")
mycursor = mydb.cursor()
selectQuery = "SELECT * From Employee WHERE EmpID like '{}' and FirstName like '{}' and LastName like '{}' and Salary like '{}' and Task like '{}'".format(QEmpID, QFName,QLName,QSalary,QTask)
QEmpID = self.lineEmpID.text()
QFName = self.lineFName.text()
QLName = self.lineLName.text()
QSalary = self.lineSalary.text()
QTask = self.lineTask.text()
insertQuery = "INSERT INTO Employee Values({},'{}','{}',{},'{}')".format(QEmpID,QFName, QLName, QSalary, QTask)
mycursor.execute(insertQuery)
mydb.commit()
insertResult = mycursor.fetchall()
mycursor.callProc('fetchData',[selectQuery])
for result in mycursor.stored_results():
selectResult = result.fetchall()
for row_number,row_data in enumerate(selectResult):
self.table.insertRow(row_number)
for column_number,data in enumerate(row_data):
self.table.setItem(row_number,column_number,QTableWidgetItem(str(data)))
except mc.Error as e:
print(e)
The above is my python code for the insert function which is then connected to the insert button.
`
DELIMITER $$
CREATE DEFINER=`root`#`localhost` PROCEDURE `fetchData`(in query1 varchar(1000))
begin
set #q = query1;
PREPARE stmt from #q;
EXECUTE stmt;
DEALLOCATE PREPARE stmt;
end$$
DELIMITER ;
The above is my stored procedure which executes a query passed to it in string format.
However, when I type in the record to be inserted into the fields and then press Insert, the following shows up without any tracebacks or error reports in the IDLE Shell:
The thing is, the record does get inserted into the database and I think the issue is with the calling of stored procedure with a select query passed to it and whose result can then be populated into the QTableWidget.
I can't think of anything right now. Help is needed.
Thank you!
Say for example, I have a table of students, and I have a Python dictionary
mydict = {"fname" : "samwise", "lname" : "gamgee", "age" : 13}
How can I safely generate a Python function that can UPDATE this into my student table? (In my use-case I can safely assume that the student already exists in the table, AND I KNOW the id already)
I have created a function that achieves this functionality, but I can't help but think it's a bit crude, and perhaps open to SQL injection attacks
def sqlite_update(conn, table, data, pkeyname, pkeyval):
set_lines = ""
for k,v in data.items():
set_lines += "{} = '{}', ".format(k,v)
set_lines = set_lines[:-2] #remove space and comma from last item
sql = "UPDATE {0} SET {1} WHERE {2} = '{3}'"
statement = sql.format(table, set_lines, pkeyname, pkeyval)
conn.execute(statement)
conn.commit()
And to update I just call
sqlite_update(conn, "student", mydict, "id", 1)
As I assume you are using sqlalchemy. In this case, you can use sqlalchemy.sql.text function which escapes strings if required.
You can try to adjust your function as below.
from sqlalchemy.sql import text
def sqlite_update(conn, table, data, pkeyname, pkeyval):
set_lines = ",".join([f"{k}=:{k}" for k in data.keys()])
statement = text(f"UPDATE {table} SET {set_lines} WHERE {pkeyname} = :pkeyval")
args = dict(data)
args["pkeyval"] = pkeyval
conn.execute(statement, args)
conn.commit()
For more details, refer to sqlalchemy official documentation on text function.
EDIT
As for sqlite3 connection you can do basically the same thing as above, with slight changes.
def sqlite_update(conn, table, data, pkeyname, pkeyval):
set_lines = ",".join([f"{k}=:{k}" for k in data.keys()])
statement = f"UPDATE {table} SET {set_lines} WHERE {pkeyname} = :pkeyval"
args = dict(data)
args["pkeyval"] = pkeyval
conn.execute(statement, args)
conn.commit()
Refer to sqlite3 execute
This is indeed widely opened to SQL injection, because you build the query as a string including its values, instead of using a parameterized query.
Building a parameterized query with Python is easy:
def sqlite_update(conn, table, data, pkeyname, pkeyval):
query = f"UPDATE {table} SET " + ', '.join(
"{}=?".format(k) for k in data.keys()) + f" WHERE {pkeyname}=?"
# uncomment next line for debugging
# print(query, list(data.values()) + [pkeyval])
conn.execute(query, list(data.values()) + [pkeyval])
With your example, the query displayed by the debug print line is:
UPDATE student SET fname=?, lname=?, age=? WHERE id=?
with the following values list: ['samwise', 'gamgee', 13, 1]
But beware, to be fully protected from SQL injection, you should sanitize the table and field names to ensure they contain no dangerous characters like ;
I have a SQL database file which contains a multitude of columns, two of which are 'GEO_ID' and 'MED_INCOME'. I am trying to retrieve just the 'MED_INCOME' column data using the associated 'GEO_ID'. Here is what I thought would work:
import sqlite3 as db
def getIncome(censusID):
conn = db.connect('census.db')
c = conn.cursor()
c.execute("SELECT 'MED_INCOME' FROM censusDbTable WHERE GEO_ID = %s" % (censusID)
response = c.fetchall()
c.close()
conn.close()
return response
id = 60014001001
incomeValue = getIncome(id)
print("incomeValue: ", incomeValue)
Which results in:
incomeValue: [('MED_INCOME',)]
I thought that I had used this method before when attempting to retrieve the data from just one column, but this method does not appear to work. If I were to instead write:
c.execute("SELECT * FROM censusDbTable WHERE GEO_ID = %s" % (censusID)
I get the full row's data, so I know the ID is in the database file.
Is there something about my syntax that is causing this request to result in an empty set?
Per #Ernxst comment, I adjusted the request to:
c.execute("SELECT MED_INCOME FROM censusDbTable WHERE GEO_ID = %s" % (censusID)
Removing the quotes around the column ID, which solved the problem.
I had a question pertaining to mysql as being used in Python. Basically I have a dropdown menu on a webpage using flask, that provides the parameters to change the mysql queries. Here is a code snippet of my problem.
select = request.form.get('option')
select_2 = request.form.get('option_2')
conn = mysql.connect()
cursor = conn.cursor()
query = "SELECT * FROM tbl_user WHERE %s = %s;"
cursor.execute(query, (select, select_2))
data = cursor.fetchall()
This returns no data from the query because there are single qoutes around the first variable, i.e.
Select * from tbl_user where 'user_name' = 'Adam'
versus
Select * from tbl_user where user_name = 'Adam'.
Could someone explain how to remove these single qoutes around the columns for me? When I hard code the columns I want to use, it gives me back my desired data but when I try to do it this way, it merely returns []. Any help is appreciated.
I have a working solution dealing with pymysql, which is to rewrite the escape method in class 'pymysql.connections.Connection', which obviously adds "'" arround your string. maybe you can try in a similar way, check this:
from pymysql.connections import Connection, converters
class MyConnect(Connection):
def escape(self, obj, mapping=None):
"""Escape whatever value you pass to it.
Non-standard, for internal use; do not use this in your applications.
"""
if isinstance(obj, str):
return self.escape_string(obj) # by default, it is :return "'" + self.escape_string(obj) + "'"
if isinstance(obj, (bytes, bytearray)):
ret = self._quote_bytes(obj)
if self._binary_prefix:
ret = "_binary" + ret
return ret
return converters.escape_item(obj, self.charset, mapping=mapping)
config = {'host':'', 'user':'', ...}
conn = MyConnect(**config)
cur = conn.cursor()
I have written a small app that uses mysql to get a list of products that need updating on our magento website.
Python then actions these updates and marks the product in the db as complete.
My Original code (pseudo to show the overview)
class Mysqltools:
def get_products():
db = pymysql.connect(host= .... )
mysqlcursor = db.cursor(pymysql.cursors.DictCursor)
sql = select * from x where y = z
mysqlcursor.execute(sql % (z))
rows = mysqlcursor.fetchall()
mysqlcursor.close()
db.close
return rows
def write_products(sku, name, id):
db = pymysql.connect(host= .... )
mysqlcursor = db.cursor(pymysql.cursors.DictCursor)
sql = update table set sku = sku, name = name, id = id.....
mysqlcursor.execute(sql % (sku, name, id))
mysqlcursor.close()
db.close
This was working ok, but on each db connection string we were getting a pause.
I did a bit of research and did the following:
class Mysqltools:
def __init__():
self.db = pymysql.connect(host= .... )
def get_products():
mysqlcursor = self.db.cursor(pymysql.cursors.DictCursor)
sql = select * from x where y = z
mysqlcursor.execute(sql % (z))
rows = mysqlcursor.fetchall()
mysqlcursor.close()
def write_products(sku, name, id):
mysqlcursor = self.db.cursor(pymysql.cursors.DictCursor)
sql = update table set sku = sku, name = name, id = id.....
mysqlcursor.execute(sql % (sku, name, id))
mysqlcursor.close()
db.commit()
This has a MASSIVE speed improvement. However, it would only do a successful get_products on the first iteration, once it was called a second time, it was finding 0 products to update, even though performing the same SQL on the db would show a number of rows returned.
Am I doing something wrong with the connections ?
I have also tried moving the db = outside of the class and referencing it but that still gives the same issue.
UPDATE
Doing some testing, and if I remove the DictCursor from the cursor I can get the correct rows returned each time (I've just created a quick loop to keep checking for records)
Is the DictCursor doing something I am unaware of ?
** UPDATE 2 **
I've removed the DictCursor, and tried the following.
Create a while True loop which calls my get_product method.
In MySQL change some rows so that they should be found.
If I go from having 0 possible rows to find, then change some so they should be found, my code just shows 0 found, and loops stating this.
If I got from having x possible rows to find, then change it to 0 in mysql, my code continues to loop showing the x possible rows.
Ok, the answer to this is as follows:
db = pymysql.connect(host=.... user=... )
class MySqlTools:
def get_products():
mysqlcursor = db.cursor(pymysql.cursors.DictCursor)
sql = select * from x where y = z
mysqlcursor.execute(sql % (z))
rows = mysqlcursor.fetchall()
mysqlcursor.close()
db.commit()
This then allows you to re-use the db connection and remove the overhead of creating and closing a connection each and every time.
In testing, downloading 500 orders from our website and writing them to a db went from 16minutes to <3 minutes.