I end up always with "Incorrect number of bindings supplied" error of SQlite3. I tried (dataset), (dataset,),[dataset].
import requests
import json
import datetime
import sqlite3
#Get tbe Data
url = 'https://opensky-network.org//api/flights/departure?airport=EDDF&begin=1517227200&end=1517230800'
content = requests.get(url).content
dataset = json.loads(content)
#print (dataset)
#Create Table in Sqlite3
try:
db = sqlite3.connect('Flights')
cursor = db.cursor()
cursor.execute('''create table flights(icao24 VARCHAR(50), firstSeen VARCHAR(50), estDepartureAirport VARCHAR(50), lastSeen VARCHAR(50), estArrivalAirport VARCHAR(50), callsign VARCHAR(50), estDepartureAirportHorizDistance VARCHAR(50), estDepartureAirportVertDistance VARCHAR(50), estArrivalAirportHorizDistance VARCHAR(50), estArrivalAirportVertDistance VARCHAR(50), departureAirportCandidatesCount VARCHAR(50), arrivalAirportCandidatesCount VARCHAR(50))''')
except Exception as E:
print('Error:', E)
else:
print ('Table created')
#Insert Date to the Table
try:
cursor.executemany('insert into flights(icao24, firstSeen, estDepartureAirport, lastSeen, estArrivalAirport, callsign, estDepartureAirportHorizDistance, estDepartureAirportVertDistance, estArrivalAirportHorizDistance, estArrivalAirportVertDistance, departureAirportCandidatesCount, arrivalAirportCandidatesCount) values (?,?,?,?,?,?,?,?,?,?,?,?)', (dataset,))
except Exception as E:
print('Error:', E)
else:
db.commit()
print ('Data inserted')
dataset is a list of dictionaries. My assumption is that you want to insert the values from the dictionaries into the table. So, replacing (dataset,) with
(tuple(record.values()) for record in dataset) should produce the desired result (worked here):
...
#Insert Date to the Table
try:
cursor.executemany(
'''
insert into flights (
icao24,
firstSeen,
estDepartureAirport,
lastSeen,
estArrivalAirport,
callsign,
estDepartureAirportHorizDistance,
estDepartureAirportVertDistance,
estArrivalAirportHorizDistance,
estArrivalAirportVertDistance,
departureAirportCandidatesCount,
arrivalAirportCandidatesCount
) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
''',
(tuple(record.values()) for record in dataset)
)
...
(tuple(record.values()) for record in dataset) is an iterator (actually a generator, which is also an iterater) which is a suitable argument for .executemany():
... The sqlite3 module also allows using an iterator yielding parameters instead of a sequence.
It seems to me that the dictionaries are sorted in the order you need for the insert. If that is not (always) the case you should modify the generator accordingly (fetching the values explicitly by keys in the desired order).
The problem was that you were trying to insert a list of dictionaries into your database instead of inserting each dictionary separately and then using an update statement after that to put them together again. The following code should work for you now. It uses the same structure as your original code but inserts each dictionary one at a time instead of trying to do it all in one go.
# Import required modules
import requests
import json
from bs4 import BeautifulSoup
# Get the data
url = 'https://opensky-network.org//api/flights/departure?airport=EDDF&begin=1517227200&end=1517230800'
content = requests.get(url).content
dataset = json.loads(content)
# Create connection
conn = sqlite3.connect("Flights")
# Insert the data
try:
cursor = conn.cursor()
cursor.execute('''create table flights(icao24 VARCHAR(50), firstSeen VARCHAR(50), estDepartureAirport VARCHAR(50), lastSeen VARCHAR(50), estArrivalAirport VARCHAR(50), callsign VARCHAR(50), estDepartureAirportHorizDistance VARCHAR(50), estDepartureAirportVertDistance VARCHAR(50), estArrivalAirportHorizDistance VARCHAR(50), estArrivalAirportVertDistance VARCHAR(50), departureAirportCandidatesCount VARCHAR(50), arrivalAirportCandidatesCount VARCHAR(50))''')
cursor.executemany('''insert into flights(icao24, firstSeen, estDepartureAirport, lastSeen, estArrivalAirport, callsign, estDepartureAirportHorizDistance, estDepartureAirportVertDistance, estArrivalAirportHorizDistance, estArrivalAirportVertDistance,
departureAirportCandidatesCount, arrivalAirportCandidatesCount) values (?,?,?,?,?,?,?,?,?,?,?,?)''', (dataset,))
except Exception as E:
print('Error:', E)
else:
conn.commit()
print('Data inserted')
# Update the table to include the new information
try:
cursor = conn.cursor()
cursor.execute('''update flights set estDepartureAirport = ?, estDepartureAirportHorizDistance = ?, estDepartureAirportVertDistance = ?, estArrivalAirport = ?, estArrivalAirportHorizDistance = ?, estArrivalAirportVertDistance = ?, departureAirportCandidatesCount = ?, arrivalAirportCandidatesCount = ? where ICAO24 = ?''', (dataset.pop(0).get('estDepartureAirport'), dataset.pop(0).get('estDepartureAirportHorizDistance'), dataset.pop(0).get('estDepartureAirportVertDistance'), dataset.pop(0).get('estArrivalAirport'), dataset.pop(0).get('estArrivalAirportHorizDistance'), dataset.pop(0).get('estArrivalAirportVertDistance'), dataset.pop(0).get('departureAirportCandidatesCount'), dataset.pop(0).get('arrivalAirportCandidatesCount'), dataset.pop(0).get('ICAO24')))
except Exception as E:
print('Error:', E)
else:
conn.commit()
print('Data updated')
Related
I want to add data to my database through python, but I do not know what to do with the ID colum.
I have four colums and I only want to add the last three, the ID is counting up itself.
def add_data(temp, hum):
try:
dt = datetime.datetime.now().replace(microsecond=0).isoformat(' ')
statement = "INSERT INTO messstation (?id?,uhrzeit, luftfeuchtigkeit, raumtemperatur) VALUES (?, ?, ?, ?)"
data = (?id?, dt, hum, temp)
cursor.execute(statement, data)
connection.commit()
except database.error as e:
print(f"Error:{e}")
This should work if the id field is an auto-increment one:
statement = "INSERT INTO messstation (uhrzeit, luftfeuchtigkeit, raumtemperatur) VALUES (?, ?, ?)"
data = (dt, hum, temp)
I have 2 things I needed help with:
1) I am unsure as to how I can check if a table exists in python using the sqlite3 library.
2) I am unsure as to how I can save variables from the program to a database. I want to be able to check if UserDetails exists before making the database.
I've been reading around and everyone is doing stuff differently,
Here is the section of my code that is responsible for saving the variables:
connection = sqlite3.connect("UserDetails.db")
crsr = connection.cursor()
#create table
sql_command = table_creator
crsr.execute(sql_command)
#insert values into table
data_to_insert = (username, first_name, surname, age, user_salt, password_hash, date_today)
sql_command = """INSERT INTO UserDetails VALUES ((?, ?, ?, ?, ?, ?, ?), data_to_insert);"""
crsr.execute(sql_command)
connection.commit() #save changes
connection.close() #terminate connection
and in case you want to see table_creator it looks like this:
table_creator = '''CREATE TABLE `UserDetails` (
`Username` VARCHAR(8) NOT NULL,
`Firstname` VARCHAR(10) NOT NULL,
`Surname` VARCHAR(20) NOT NULL,
`Age` INT(2) NOT NULL,
`Salt` VARCHAR(10) NOT NULL,
`Hash` VARCHAR(64) NOT NULL,
`Date` DATE NOT NULL,
PRIMARY KEY (`UserName`)
);'''
I will appreciate and feedback or support.
I am still learning to code, and my CompSci teacher doesnt teach us Python specifically, so what I know is self taught.
Oh and this is the error message I get:
Traceback (most recent call)
File "c:/Users/Arslan/Project A2/login.py", line 99, in <module>
save_details()
File "c:/Users/Arslan/Project A2/login.py", line 93, in save_details
crsr.execute(sql_command)
sqlite3.OperationalError: no such column: data_to_insert
How to check if a table exists or no :
The first way :
Use this query:
SELECT name FROM sqlite_master WHERE type='table' AND name='{table_name}';
Modify {table_name} with your table to check
There are two cases :
. If the cursor equal to 0 ==> the table does not exist
Else, the table exists
The second way:
Use :
PRAGMA table_info(table_name)
example:
The third way :
Use this query :
select 1 from table
It will return the constant 1 for every row of the table if the table exists, or nothing if not.
There are many other ways, but I listed the best in my opinion.
How to save variables from the program to a database:
To insert data into sqlite3, you can use :
cursor.execute("insert into UserDetails values (?, ?, ?, ?, ?, ?, ?)", (username, firstname, surname, age, salt, hash, date))
DON'T USE (SQL injection):
cursor.execute("insert into UserDetails values ('{1}', '{2}', '{3}', '{4}', '{5}', '{6}', '{7}')".format(username, firstname, surname, age, salt, hash, date))
Don't forget :
conn.commit()
Or you can use instead of it the connection as a context manager:
with conn:
# then cursor.execute..
1) I am unsure as to how I can check if a table exists in python using the sqlite3 library.
Use CREATE TABLE IF NOT EXISTS:
table_creator = '''CREATE TABLE IF NOT EXISTS `UserDetails` (
`Username` VARCHAR(8) NOT NULL,
`Firstname` VARCHAR(10) NOT NULL,
...
);'''
2) I am unsure as to how I can save variables from the program to a database.
You can pass variables for insert with the following syntax:
data_to_insert = (username, first_name, surname, age, user_salt, password_hash, date_today)
sql_command = '''INSERT INTO UserDetails VALUES (?, ?, ?, ?, ?, ?, ?)''';
crsr.execute(sql_command, data_to_insert )
As you can see I am trying to fetch data from this API-endpoint https://api.coindesk.com/v1/bpi/currentprice.json and I have chosen few data I want to fetch and store it in SQLite.
When I try to save it in a database it gives me this error:
Traceback (most recent call last):
File "bitcoin.py", line 41, in <module>
cur.execute("INSERT INTO COINS (Identifier, symbol, description) VALUES (?, ?, ?);", to_db)
sqlite3.ProgrammingError: Binding 1 has no name, but you supplied a dictionary (which has only names).
How can I store the some of the data from API-endpoint into the database?
I'm doing this to learn programming and still new to this so hopefully, you can guide me in the right way.
Here is what I have tried so far:
import requests
import sqlite3
con = sqlite3.connect("COINS.db")
cur = con.cursor()
cur.execute('DROP TABLE IF EXISTS COINS')
cur.execute(
"CREATE TABLE COINS (Identifier INTEGER PRIMARY KEY, symbol TEXT, description TEXT);"
)
r = requests.get('https://api.coindesk.com/v1/bpi/currentprice.json')
to_db = r.json() # I do not have to do it in json, CSV would also be another
# solution but the data that is been stored cannot be static.
# It has to automatically fetch the data from API-endpoint
cur.execute("INSERT INTO COINS (Identifier, symbol, description) VALUES (?, ?, ?);", to_db)
con.commit()
con.close()
import requests
import sqlite3
con = sqlite3.connect("COINS.db")
cur = con.cursor()
cur.execute('DROP TABLE IF EXISTS COINS')
cur.execute(
"CREATE TABLE COINS (id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENTUNIQUE,
symbol TEXT, description TEXT);")
r = requests.get('https://api.coindesk.com/v1/bpi/currentprice.json')
to_db = r.json()
des=to_db['bpi']['USD']['description']
code=to_db['bpi']['USD']['code']
cur.execute("INSERT INTO COINS (symbol, description) VALUES (?, ?);",
(des,code))
con.commit()
con.close()
Check full code
I have to read data from Excel and insert it into Table...
For this I am using Python 2.7, pymssql and xlrd modules...
My sql connection is working fine and I am also able to read data from Excel properly.
My table structure :
CREATE TABLE MONTHLY_BUDGET
(
SEQUENCE INT IDENTITY,
TRANSACTION_DATE VARCHAR(100),
TRANSACTION_REMARKS VARCHAR(1000),
WITHDRAWL_AMOUNT VARCHAR(100),
DEPOSIT_AMOUNT VARCHAR(100),
BALANCE_AMOUNT VARCHAR(100)
)
My excel values are like this :
02/01/2015 To RD Ac no 147825000874 7,000.00 - 36,575.74
I am having problem while inserting multiple values in the table... I am not sure how to do this...
import xlrd
import pymssql
file_location = 'C:/Users/praveen/Downloads/OpTransactionHistory03-01-2015.xls'
#Connecting SQL Server
conn = pymssql.connect (host='host',user='user',password='pwd',database='Practice')
cur = conn.cursor()
# Open Workbook
workbook = xlrd.open_workbook(file_location)
# Open Worksheet
sheet = workbook.sheet_by_index(0)
for rows in range(13,sheet.nrows):
for cols in range(sheet.ncols):
cur.execute(
" INSERT INTO MONTHLY_BUDGET VALUES (%s, %s, %s, %s, %s)", <--- Not sure!!!
[(sheet.cell_value(rows,cols))])
conn.commit()
Error :
ValueError: 'params' arg () can be only a tuple or a dictionary.
The docs are here : http://pymssql.org/en/stable/pymssql_examples.html
The exception you are getting says that the "'params' arg() can be only a tuple or a dictionary" but you're passing in a list. Also, your parameter list appears to be a single tuple instead of a list with 4 values. Try changing
cur.execute(
" INSERT INTO MONTHLY_BUDGET VALUES (?, ?, ?, ?, ?)", <--- Not sure!!!
[(sheet.cell_value(rows,cols))])
to
cur.execute(
" INSERT INTO MONTHLY_BUDGET VALUES (?, ?, ?, ?, ?)", <--- Not sure!!!
(sheet.cell_value(rows,cols)))
... or maybe
cur.execute(
" INSERT INTO MONTHLY_BUDGET VALUES (?, ?, ?, ?, ?)", <--- Not sure!!!
((sheet.cell_value(rows,cols))))
NB: untested. I've always changed how the bind variables in your SQL are being called.
I need to update a row if a record already exists or create a new one if it dosen't. I undersant ON DUPLICATE KEY will accomplish this using MYSQLdb, however I'm having trouble getting it working. My code is below
cursor = database.cursor()
cursor.execute("INSERT INTO userfan (user_id, number, round VALUES (%s, %s, %s) ON DUPLICATE KEY UPDATE user_id =%s, number=%s, round=%s", (user_id, number, round))
database.commit()
primary key is user_id
A parenthesis was missiing. You can also use the VALUES(column) in the ON DUPLICATE KEY UPDATE section of the statement:
cursor = database.cursor()
cursor.execute("""
INSERT INTO userfan
(user_id, number, round)
VALUES
(%s, %s, %s)
ON DUPLICATE KEY UPDATE
-- no need to update the PK
number = VALUES(number),
round = VALUES(round) ;
""", (user_id, number, round) # python variables
)
database.commit()
def insertAndUpdateData(lVideoList, no_of_gate):
connection = sqlite3.connect('db.sqlite',
detect_types=sqlite3.PARSE_DECLTYPES |
sqlite3.PARSE_COLNAMES)
cursor = connection.cursor()
success = 200
unsuccess = 500
default_value = 0
lDefaultEntry = None
for i in range(no_of_gate):
gate_id = i+1
for videofilename in lVideoList:
cursor.execute("SELECT * FROM dailyfootfall WHERE csv_name=? AND gate_id=?", [videofilename, gate_id])
lDefaultEntry = cursor.fetchone()
try:
if lDefaultEntry is not None:
print ('Entry found...!!!')
cursor.execute("UPDATE dailyfootfall SET video_download=?, processed=?, send_status=? ,male_footfall=?, send_status_male=?, "
"female_footfall =?,send_status_female=?, outsiders=?, send_status_outsiders=? "
"WHERE csv_name=? AND gate_id=? AND footfall=0", [unsuccess,unsuccess,unsuccess,default_value,unsuccess,
default_value,unsuccess,default_value,unsuccess,videofilename,gate_id])
print("Data_Updated..!!!")
else:
cursor = connection.cursor()
print ('Entry Not found...!!!')
print("videofilename: ", videofilename)
insert_query = ("INSERT or IGNORE INTO dailyfootfall(csv_name, video_download, processed, footfall, send_status, "
"male_footfall, send_status_male, female_footfall, send_status_female, gate_id,outsiders, send_status_outsiders) "
"VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?,?)")
cursor.execute(insert_query,[videofilename, unsuccess, unsuccess, default_value, unsuccess, default_value,
unsuccess, default_value, unsuccess, gate_id, default_value, unsuccess])
print("Data_Inserted..!!")
print("="*20)
except Exception as e:
exc_type, exc_obj, exc_tb = sys.exc_info()
fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
print("Entry found: ",exc_type, fname, exc_tb.tb_lineno)
print("Data Inserted Successfully !")
connection.commit()
cursor.close()
connection.close()
if __name__ == "__main__":
lVideoList = ['2022_01_27_10_00_00-2022_01_25_10_30_00', '2022_01_27_10_30_00-2022_01_25_11_00_00',
'2022_01_27_11_00_00-2022_01_25_11_30_00', '2022_01_27_11_30_00-2022_01_25_12_00_00']
no_of_gate = 3
UpdateData(lVideoList, no_of_gate)
print("default_value inserted!!!!")