I'm attempting to run a mysql query using python.
#!/usr/bin/python
import mysql.connector
import re
db = mysql.connector.connect(host='localhost',database='planes',user='root',password='root')
cursor = db.cursor ( )
cursor.execute("SELECT * from acars where date_time_stamp between str_to_date('2016-09-01', '%Y-%m-%d') "
"and str_to_date('2016-09-30', '%Y-%m-%d')")
# Fetch rows
data = cursor.fetchall()
for row in data:
print row
p = re.compile('K[A-Z][A-Z][A-Z]')
str =
print(p.findall(row))
# close cursor
cursor.close ( )
# close connection
db.close ( )
My problem begins in the section with the "Fetch Rows" comment. I'd like to run the re.compile to search each returned data entry for instances that 4-letter strings beginning with 'K' appear. I cannot figure out what to use as the str to represent my returned data though.
Related
I have made a GUI in PyQt5 that allows you to deal with a database. There is an insert button which allows you to insert data into a database and then using a stored procedure whose parameter is a MySQL query in string format, it passes a select query to the stored procedure whose where clause consists of values just entered.
`
def insert(self):
try:
self.table.setRowCount(0)
QEmpID = self.lineEmpID.text() + "%"
QFName = self.lineFName.text() + "%"
QLName = self.lineLName .text() + "%"
QSalary = self.lineSalary.text() + "%"
QTask = self.lineTask.text() + "%"
mydb = mc.connect(host="localhost",username="root",password="",database="Office")
mycursor = mydb.cursor()
selectQuery = "SELECT * From Employee WHERE EmpID like '{}' and FirstName like '{}' and LastName like '{}' and Salary like '{}' and Task like '{}'".format(QEmpID, QFName,QLName,QSalary,QTask)
QEmpID = self.lineEmpID.text()
QFName = self.lineFName.text()
QLName = self.lineLName.text()
QSalary = self.lineSalary.text()
QTask = self.lineTask.text()
insertQuery = "INSERT INTO Employee Values({},'{}','{}',{},'{}')".format(QEmpID,QFName, QLName, QSalary, QTask)
mycursor.execute(insertQuery)
mydb.commit()
insertResult = mycursor.fetchall()
mycursor.callProc('fetchData',[selectQuery])
for result in mycursor.stored_results():
selectResult = result.fetchall()
for row_number,row_data in enumerate(selectResult):
self.table.insertRow(row_number)
for column_number,data in enumerate(row_data):
self.table.setItem(row_number,column_number,QTableWidgetItem(str(data)))
except mc.Error as e:
print(e)
The above is my python code for the insert function which is then connected to the insert button.
`
DELIMITER $$
CREATE DEFINER=`root`#`localhost` PROCEDURE `fetchData`(in query1 varchar(1000))
begin
set #q = query1;
PREPARE stmt from #q;
EXECUTE stmt;
DEALLOCATE PREPARE stmt;
end$$
DELIMITER ;
The above is my stored procedure which executes a query passed to it in string format.
However, when I type in the record to be inserted into the fields and then press Insert, the following shows up without any tracebacks or error reports in the IDLE Shell:
The thing is, the record does get inserted into the database and I think the issue is with the calling of stored procedure with a select query passed to it and whose result can then be populated into the QTableWidget.
I can't think of anything right now. Help is needed.
Thank you!
I am having a weird issue. I am using Sqlite for storing Stock data and a native sqlite3 library to interact with it. When I run on TablePlus it returns the following:
But when I run the code it returns the following:
I am unable to figure out why is it happening. The count is correct but it is pulling the wrong symbols.
The code that is doing all this is given below:
def fetch_data_by_date(day):
results = None
conn = get_connection()
print(day)
sql = "select DISTINCT(created_at), symbol,count(stock_id) as total from mentions" \
" inner join symbols on symbols.id = mentions.stock_id WHERE DATE(created_at) = '{}'" \
" group by symbol order by total DESC LIMIT 5".format(day)
print(sql)
cursor = conn.cursor()
cursor.execute(sql)
results = cursor.fetchall()
return results
Below code is for extracting the data from oracle database in csv file.
In query,For converting from Fractional decimal into date format,i have used To_Date('12/30/1899', 'MM/DD/YYYY HH24:MI:SS')+DTIMESTAMP) Decoded_Date.
And also specified the date range for extracting the data between dates.
Please help what's wrong in below code giving invalid syntax.
import csv
import cx_Oracle
dsn_tns = cx_Oracle.makedsn('hostname', 'port', sid='sid') # if needed, place an 'r' before any parameter in order to address special characters such as '\'.
conn = cx_Oracle.connect(user=r'username', password='password', dsn=dsn_tns)
cursor = conn.cursor()
csv_file = open("C:/Users/locations.csv", "w")
writer = csv.writer(csv_file, delimiter=',', lineterminator="\n", quoting=csv.QUOTE_NONNUMERIC)
r = cursor.execute("""SELECT *
FROM (SELECT LROWNUM,DTIMESTAMP,LSCENARIO,LYEAR,LPERIOD,
LENTITY,LPARENT,LVALUE,LACCOUNT,LICP,LCUSTOM1,
LCUSTOM2,STRUSERNAME,STRSERVERNAME,
LACTIVITY,DDATAVALUE,BNODATA,
(To_Date('12/30/1899', 'MM/DD/YYYY HH24:MI:SS')+DTIMESTAMP) Decoded_Date
FROM TABLE_NAME
) SUB
WHERE SUB.Decoded_Date between '23-MAR-2020' and '24-APR-2020';
""")
for row in cursor:
writer.writerow(row)
cursor.close()
conn.close()
csv_file.close()
The opening and closing parentheses should not be present. I can't test the SQL directly, of course, but this should in theory work for you!
r = cursor.execute"""
SELECT *
FROM
( SELECT LROWNUM,DTIMESTAMP,LSCENARIO,LYEAR,LPERIOD,
LENTITY,LPARENT,LVALUE,LACCOUNT,LICP,LCUSTOM1,
LCUSTOM2,STRUSERNAME,STRSERVERNAME,
LACTIVITY,DDATAVALUE,BNODATA,
To_Date('12/30/1899','MM/DD/YYYY') +
DTIMESTAMP as Decoded_Date
FROM TABLE_NAME
) SUB
WHERE SUB.Decoded_Date between to_date('23-MAR-2020', 'DD-MON-YYYY')
and to_date('24-APR-2020', 'DD-MON-YYYY')
"""
Note the changes to the last line as well. Unless you know the value of NLS_DATE_FORMAT you can't compare strings with dates directly. Note that you can also bind date values directly as in
sql = "select ... where sub.decoded_date between :1 and :2"
cursor.execute(sql, [datetime.date(2020, 3, 23), datetime.date(2020, 4, 24)])
I am trying to fetch records after a regular interval from a database table which growing with records. I am using Python and its pyodbc package to carry out the fetching of records. While fetching, how can I point the cursor to the next row of the row which was read/fetched last so that with every fetch I can only get the new set of records inserted.
To explain more,
my table has 100 records and they are fetched.
after an interval the table has 200 records and I want to fetch rows from 101 to 200. And so on.
Is there a way with pyodbc cursor?
Or any other suggestion would be very helpful.
Below is the code I am trying:
#!/usr/bin/python
import pyodbc
import csv
import time
conn_str = (
"DRIVER={PostgreSQL Unicode};"
"DATABASE=postgres;"
"UID=userid;"
"PWD=database;"
"SERVER=localhost;"
"PORT=5432;"
)
conn = pyodbc.connect(conn_str)
cursor = conn.cursor()
def fetch_table(**kwargs):
qry = kwargs['qrystr']
try:
#cursor = conn.cursor()
cursor.execute(qry)
all_rows = cursor.fetchall()
rowcnt = cursor.rowcount
rownum = cursor.description
#return (rowcnt, rownum)
return all_rows
except pyodbc.ProgrammingError as e:
print ("Exception occured as :", type(e) , e)
def poll_db():
for i in [1, 2]:
stmt = "select * from my_database_table"
rows = fetch_table(qrystr = stmt)
print("***** For i = " , i , "******")
for r in rows:
print("ROW-> ", r)
time.sleep(10)
poll_db()
conn.close()
I don't think you can use pyodbc, or any other odbc package, to find "new" rows. But if there is a 'timestamp' column in your database, or if you can add such a column (some databases allow for it to be automatically populated as the time of insertion so you don't have to change the insert queries) then you can change your query to select only the rows whose timestamp is greater than the previous timestamp. And you can keep changing the prev_timestamp variable on each iteration.
def poll_db():
prev_timestamp = ""
for i in [1, 2]:
if prev_timestamp == "":
stmt = "select * from my_database_table"
else:
# convert your timestamp str to match the database's format
stmt = "select * from my_database_table where timestamp > " + str(prev_timestamp)
rows = fetch_table(qrystr = stmt)
prev_timestamp = datetime.datetime.now()
print("***** For i = " , i , "******")
for r in rows:
print("ROW-> ", r)
time.sleep(10)
I have to get the recently updated data from database. For the purpose of solving it, I have saved the last read row number into shelve of python. The following code works for a simple query like select * from rows. My code is:
from pyodbc import connect
from peewee import *
import random
import shelve
import connection
d = shelve.open("data.shelve")
db = SqliteDatabase("data.db")
class Rows(Model):
valueone = IntegerField()
valuetwo = IntegerField()
class Meta:
database = db
def CreateAndPopulate():
db.connect()
db.create_tables([Rows],safe=True)
with db.atomic():
for i in range(100):
row = Rows(valueone=random.randrange(0,100),valuetwo=random.randrange(0,100))
row.save()
db.close()
def get_last_primay_key():
return d.get('max_row',0)
def doWork():
query = "select * from rows" #could be anything
conn = connection.Connection("localhost","","SQLite3 ODBC Driver","data.db","","")
max_key_query = "SELECT MAX(%s) from %s" % ("id", "rows")
max_primary_key = conn.fetch_one(max_key_query)[0]
print "max_primary_key " + str(max_primary_key)
last_primary_key = get_last_primay_key()
print "last_primary_key " + str(last_primary_key)
if max_primary_key == last_primary_key:
print "no new records"
elif max_primary_key > last_primary_key:
print "There are some datas"
optimizedQuery = query + " where id>" + str(last_primary_key)
print query
for data in conn.fetch_all(optimizedQuery):
print data
d['max_row'] = max_primary_key
# print d['max_row']
# CreateAndPopulate() # to populate data
doWork()
While the code will work for a simple query without where clause, but the query can be anything from simple to complex, having joins and multiple where clauses. If so, then the portion where I'm adding where will fail. How can I get only last updated data from database whatever be the query?
PS: I cannot modify database. I just have to fetch from it.
Use an OFFSET clause. For example:
SELECT * FROM [....] WHERE [....] LIMIT -1 OFFSET 1000
In your query, replace 1000 with a parameter bound to your shelve variable. That will skip the top "shelve" number of rows and only grab newer ones. You may want to consider a more robust refactor eventually, but good luck.