I need to retrieve results from my sqlite3 database 160 rows at a time, and repeat that until there are no rows left for my query, this is what I have:
conn = sqlite3.connect("C:\\Users\\%s\\AppData\\Roaming\\GridcoinResearch\\reports\\Rain.db" % user_account)
c = conn.cursor()
conn.text_factory = str
address = c.execute('select Address from NNDATA where NeuralMagnitude != 0 and NeuralMagnitude is not null and CPID in (select cpids from GRIDCOINTEAM)').fetchmany(160)
conn.text_factory = float
nn_mag = c.execute('select NeuralMagnitude from NNDATA where NeuralMagnitude != 0 and NeuralMagnitude is not null and CPID in (select cpids from GRIDCOINTEAM)').fetchmany(160)
conn.close()
while True:
if nn_mag == ():
sys.exit("Complete")
The reason for sys.exit is I have a bunch of other code to go between conn.close() and while True:, so when the last loop is done I can exit the program. Right now its doing the first pass then the cmd.exe is hanging.
EDIT: Just relaised I dont tell the loop to select the NEXT 160, oh dear!
The fetchmany attribute returns an empty list if there is no item lefts so you can just check the validation of its result. Also note that you should remove the limit from your query and the fetchall. Because the whole essence of using fetchmany is to fetch limited results from your cursor object.
chunk_size = 160
while True:
result = nn_mag.fetchmany(chunk_size)
if not result:
sys.exit("Complete")
else:
# do something with result
OK the full answer is:
conn = sqlite3.connect("C:\\Users\\%s\\AppData\\Roaming\\GridcoinResearch\\reports\\Rain.db" % user_account)
c = conn.cursor()
position = 00
while True:
conn.text_factory = str
address_db = c.execute('select Address from NNDATA where NeuralMagnitude != 0 and NeuralMagnitude is not null and CPID in (select cpids from GRIDCOINTEAM) limit {}, 160'.format(position)).fetchall()
conn.text_factory = float
rac_db = c.execute('select rac from GRIDCOINTEAM where rac != 0 and rac is not null limit {}, 160'.format(position)).fetchall()
if not address_db:
conn.close()
sys.exit("Complete")
else:
position += 160
Related
Working on a python app connecting to a movies database in mysql:
Looking to show first 5 lines from the query, if the user presses anything besides from q(will bring back to main menu) it will fetch next 5 lines and so on. So far I have :
with conn:
cursor = conn.cursor()
query = "select FilmName,ActorName from Film,Actor,FilmCast where Film.FilmID = FilmCast.CastFilmID and FilmCast.CastActorID = Actor.ActorID order by FilmName ASC,ActorName ASC LIMIT 5;"
cursor.execute(query)
result = cursor.fetchall()
print(result)
button = str(input('Press Q to quit'))
if button == 'q':
main()
Any ideas?
In that case you want to use the concept of OFFSET.
Use a counter variable to keep track of your row number on each user input, and the use that in your query.
Here's the code
offset = 0
with conn:
cursor = conn.cursor()
limit = 5
query = """select FilmName,ActorName from Film,Actor,FilmCast
where Film.FilmID = FilmCast.CastFilmID
and FilmCast.CastActorID = Actor.ActorID
order by FilmName ASC,
ActorName ASC LIMIT {0}, {1};""".format(offset, limit)
cursor.execute(query)
result = cursor.fetchall()
print(result)
button = str(input('Press Q to quit'))
if button == 'q':
main()
else:
offset += 1
I have an sql table which i want to update based on a function
this is my code so far:
def read(conn):
cursor = conn.cursor()
#cursor2 = conn.cursor()
cursor.execute("SELECT All [id],[keyword],[result],[status],[createddate] FROM [Table1].[dbo].[aa]")
index = 1
cursor.execute("set nocount on")
for row in cursor:
s = row[1]
s = re.sub(r'[^\w\s]', '', s)
a=do_func(s)
if a:
cursor.execute("update [Table1].[dbo].[aa] set status = 'completed', result = 'True' where id ={}".format(index))
else:
cursor.execute("update [Table1].[dbo].[aa] set status = 'completed', result = 'False' where id ={}".format(index))
if index == 10:
break
index += 1
i get pyodbc.ProgrammingError: No results. Previous SQL was not a query.
I added "set nocount on" but didn't solve i also tried making a second cursor but also didn't solve problem
okay, see the code: you will need to split the cursor for select and cursor for update, you cannot use both at the same time. And after update, you will need to commit. Let me know if it works.
def read(conn):
selectcursor = conn.cursor()
updatecursor = conn.cursor()
selectcursor.execute("SELECT [id],[keyword],[result],[status],[createddate] FROM [Table1].[dbo].[aa]")
index = 1
result = selectcursor.fetchall()
for row in result:
s = row[1]
s = re.sub(r'[^\w\s]', '', s)
a=do_func(s)
if a:
updatecursor.execute("update [Table1].[dbo].[aa] set status = 'completed', result = 'True' where id ={}".format(index))
updatecursor.commit()
else:
updatecursor.execute("update [Table1].[dbo].[aa] set status = 'completed', result = 'False' where id ={}".format(index))
updatecursor.commit()
if index == 10:
break
index += 1
selectcursor.close()
updatecursor.close()
write python program to create a mysql table and insert data into this table,the program is as follows:
def pre_data_db_manage(type,data):
conn = pymysql.connect(host="localhost", port=3306, user="root", passwd="********", db="facebook_info",charset="utf8")
cur = conn.cursor()
if type == "pre_davi_group_members_data":
is_exist_table_sql = "SHOW TABLES LIKE 'fb_pre_davi_group_members_posts'"
if cur.execute(is_exist_table_sql) == 0:
create_table_sql = '''CREATE TABLE fb_pre_davi_group_members_posts (id bigint not null primary key auto_increment,userID bigint,userName varchar(128),userURL varchar(256),
postTime varchar(128),postText text,postTextLength int,likesCount int,sharesCount int,commentsCount int,postTextPolarity varchar(64),postTextSubjectivity varchar(64))'''
cur.execute(create_table_sql)
r = re.compile(r'^[a-zA-Z0-9]')
for item in data:
if "'" in item["PostText"]:
item["PostText"] = item["PostText"].replace("'"," ")
if "\\" in item["PostText"]:
item["PostText"] = item["PostText"].replace("\\","\\\\")
for i in item["PostText"]:
result = r.match(i)
if result == None:
print("in re")
item['PostText'] = item['PostText'].replace(i, ' ')
if "nan" in item["SharesCount"]:
item["SharesCount"] = 0
if "nan" in item["LikesCount"]:
item["LikesCount"] = 0
if "nan" in item["CommentsCount"]:
item["CommentsCount"] = 0
if "nan" in item["PostTextLength"]:
item["PostTextLength"] = 0
item["PostTextLength"] = int(item["PostTextLength"])
item["LikesCount"] = int(item["LikesCount"])
item["SharesCount"] = int(item["SharesCount"])
item["CommentsCount"] = int(item["CommentsCount"])
if type == "pre_davi_group_members_data":
insert_sql = '''INSERT INTO fb_pre_davi_group_members_posts (userID,userName,userURL,
postTime,postText,postTextLength,likesCount,sharesCount,commentsCount,postTextPolarity,postTextSubjectivity) VALUES
({0},"{1}",'{2}','{3}','{4}',{5},{6},{7},{8},{9},{10})'''.format(item["UserID"],item["UserName"],item["UserURL"],item["PostTime"],item["PostText"],item["PostTextLength"],item["LikesCount"],item["SharesCount"],item["CommentsCount"],item["PostTextPolarity"],item["PostTextSubjectivity"])
print(insert_sql)
try:
cur.execute(insert_sql)
except Exception as e:
print("insert error")
continue
cur.close()
conn.commit()
conn.close()
and write call statement as follows:
type = "pre_davi_group_members_data"
pre_data_db_manage(type, df_list)
however,when execute this program, found that no data have been inserted into table:fb_pre_davi_group_members_posts,
in the mysql order line, write:
select count(*) from fb_pre_davi_group_members_posts;
the result is 0
could you please tell me the reason and how to solve it
This code returns an empty list. When I run it in my application I get lots of records. I have used this basic script with other SQL queries and they work fine but not this one.
The database server is Sybase SQL Anywhere 12
import pyodbc
cnxn = pyodbc.connect('DSN=dmserver')
cursor = cnxn.cursor()
cursor.execute("""select
debtor_id
,name1
,assign_id
,(select
dateformat(dateadd(minute,user_gmt_offset,string(act_date,' ',act_time)),'MM/DD/YYYY HH::NN::SS')
from
dm.dbtract
where
item_no = (select max(item_no) from dm.dbtract as d2
where
left(comments,5) = 'AC11::'
and
act_date < today(*) - 1
and
d2.debtor_id = dbtract.debtor_id)
and
dbtr.debtor_id = dbtract.debtor_id) as act_code_date_time
,(select
list(phone,'~' ORDER by item_no asc)
from
dm.dbtrphon
where
status = 'A'
and
dbtrphon.debtor_id = dbtr.debtor_id) as Active_phone_list
from
dm.dbtr
where
(select
count(*)
from
dm.dbtract
where
left(comments,5) = 'AC11::'
and
act_date < today(*) - 1
and
dbtr.debtor_id = dbtract.debtor_id) > 0
""")
while 1:
row = cursor.fetchone()
if not row:
break
print row
I have this code, the object is to read one line of serial and then add it to the database if it is valid. The code that sends the serial is terminated with \n. When this code was in two scripts it worked as it should, trying to push it together has somehow messed it up.
Here's the code:
#GKPCM Database Test
#outline open database, read serial,write string, repeat
import serial
import sqlite3
ser = serial.Serial('/dev/pts/2', 19200, timeout=0)
print ser.name # check which port was really used
db = sqlite3.connect('Data/telemetry.gkpcm')
cursor = db.cursor()
InsertQuery ="""INSERT INTO vehicletelemetry (date,time,cyclecount,rpm,speed,odometer,oiltemp,airtemp,fuellevel,enginetemp,ind1,ind2,ind3,ind4,ind5,ind6,ind7,ind8) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"""
tablename="vehicletelemetry"
cursor.execute(""" SELECT COUNT(*) FROM sqlite_master WHERE name = ? """, (tablename, ))
QUERY = cursor.fetchone()
print bool(QUERY[0]) # True if exists
if bool(QUERY[0]) !=1:
print('not in DB')
cursor.execute('''CREATE TABLE vehicletelemetry(id INTEGER PRIMARY KEY, date INTEGER,time INTEGER, cyclecount INTEGER, rpm INTEGER, speed INTEGER, odometer INTEGER, oiltemp INTEGER, airtemp INTEGER, fuellevel INTEGER, enginetemp INTEGER, ind1 BOOL, ind2 BOOL, ind3 BOOL, ind4 BOOL, ind5 BOOL, ind6 BOOL, ind7 BOOL, ind8 BOOL)''')
cursor.execute('INSERT INTO vehicletelemetry (date,time,cyclecount,rpm,speed,odometer,oiltemp,airtemp,fuellevel,enginetemp,ind1,ind2,ind3,ind4,ind5,ind6,ind7,ind8) VALUES (031514,013030,18960,3000,22,192768,210,72,98,210,0,0,0,0,0,0,0,0)')
db.commit()
else:
print('DB Table Exists')
#openfile to read
while(1):
DataLine = ser.readline()
Valid = bool(DataLine)
if Valid == True:
print(DataLine)
#Read Database for last record date
LastEntry = cursor.execute('SELECT * FROM vehicletelemetry ORDER BY id DESC LIMIT 1')
for Records in LastEntry:
LastLogDate = Records[1]
LastLogTime = Records[2]
LastLogCycles = Records[3]
DataLine1 = DataLine.strip()
DataLine2 = DataLine1.split(',')
print(DataLine2)
#Check Packet for Correct Length
if len(DataLine2) != 20:
print (len(DataLine2))
print ("Invalid Data Length")
else:
#Check Packet DataQualifiers to ensure proper package introduction and termination
if DataLine2[0] != "7887" and DataLine2[18] != "0420":
print ("Invalid Data Qulifier")
else:
#Remove Qualifiers So data can be stored
PiP = 1
DataLine3 = []
for Packet in DataLine2:
if PiP >= 1 and PiP <= 18:
DataLine3.append(DataLine2[PiP])
PiP = PiP + 1
#Compare Date Time and Cycle Count to Current Record
#print(DataLine3)
if int(DataLine2[1]) >= int(LastLogDate):
if int(DataLine2[2]) >= int(LastLogTime):
if int(DataLine2[3]) > int(LastLogCycles):
cursor.execute(InsertQuery,DataLine3)
db.commit()
print(Records,DataLine2[3],LastLogCycles,"Data Valid")
db.close()
When I run the code, it gets to checking the data for length, but even if it is the right length to continue down the script the code goes back to ser.readline(). Nothing is ever written to the database.