Related
I have to make a request to a Brazil ZIPCODES API to get JSON data and insert it on a sqlite database using python. I'm currenctly using pycharm but I need to insert a lot of columns, but somehow the code don't insert the values. Here's the code
import requests
import sqlite3
import json
CEPC = input("Please type the zipcode:")
print("Identifying the ZIP CODE")
Requisicao = requests.get(f"https://viacep.com.br/ws/{CEPC}/json")
if Requisicao.status_code == 200:
data = Requisicao.json()
# Database
con = sqlite3.connect("Banco de dados/CEPS.db")
cur = con.cursor()
cur.execute("DROP TABLE IF EXISTS Requisicao")
cur.execute("CREATE TABLE Requisicao (cep, logradouro, bairro, uf, ddd, siafi,
validation, created json)")
cur.executemany("insert into Requisicao values (?, ?, ?, ?, ?, ?, ?, ?)", (data["cep"],
json.dumps(data)))
con.commit()
con.close()
else:
print(f"Request failed with status code {Requisicao.status_code} ")
The outpout of the zipcode is:
{
"cep": "05565-000",
"logradouro": "Avenida General Asdrúbal da Cunha",
"complemento": "",
"bairro": "Jardim Arpoador",
"localidade": "São Paulo",
"uf": "SP",
"ibge": "3550308",
"gia": "1004",
"ddd": "11",
"siafi": "7107"
}
I need to insert all of these columns: "cep, logadouro, complemento, bairro, localidade, uf, ibge, gia, ddd, siafi".When I try to run the code, It gives me the error:
Traceback (most recent call last):
File "C:\Users\Gui\PycharmProjects\pythonProject\main.py", line 19, in <module>
cur.executemany("insert into Requisicao values (?, ?, ?, ?, ?, ?, ?, ?)", (data["cep"],
json.dumps(data)))
sqlite3.ProgrammingError: Incorrect number of bindings supplied. The current statement
uses 8, and there are 9 supplied
When I try to put the exact same value of columns with the "?", the errors says that "uses 8, and there are 7 supplied.
This code will insert all 10 values from the JSON into the table Requisicao and 0 for both validation and created, though that can be changed.
import requests
import sqlite3
import json
CEPC = input("Please type the zipcode:")
print("Identifying the ZIP CODE")
Requisicao = requests.get(f"https://viacep.com.br/ws/{CEPC}/json")
if Requisicao.status_code == 200:
data = Requisicao.json()
# Database
con = sqlite3.connect("CEPS.db")
cur = con.cursor()
cur.execute("DROP TABLE IF EXISTS Requisicao")
cur.execute("CREATE TABLE Requisicao (cep,logradouro,complemento,bairro,localidade,uf,ibge,gia,ddd,siafi, validation, created)")
cur.execute("insert into Requisicao values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",tuple(data.values())+(0, 0))
con.commit()
con.close()
else:
print(f"Request failed with status code {Requisicao.status_code} ")
I created a function to fill a database with API requests values but since I added new columns to the database, that are modified through a web page, I changed INSERT OR REPLACE by INSERT OR UPDATE(Also tried UPSERT) but the function doesn't run anymore.
Since I'm using Django and VSCode, I have no idea how to print things or see an error (this runs in a separate thread so doesn't crash the webpage).
There might be something wrong with the function but I can't see what so I'm hoping someone can help me out.
def fill_resources():
r = pip._vendor.requests.get(BASE_URL+'/api/resources?&maxResults=500',auth=(USER,PASS))
data0 = json.loads(r.text)
conn = sqlite3.connect('/app/docaret.sqlite3')
c = conn.cursor()
for res in data0['data']:
resId = res['id']
r1 = pip._vendor.requests.get(BASE_URL+'/api/resources/'+str(resId)+'/administrative?&maxResults=500',auth=(USER,PASS))
admin = json.loads(r1.text)
r2 = pip._vendor.requests.get(BASE_URL+'/api/resources/'+str(resId)+'/information?&maxResults=500',auth=(USER,PASS))
info = json.loads(r2.text)
r3 = pip._vendor.requests.get(BASE_URL+'/api/resources/'+str(resId)+'/technical-data?&maxResults=500',auth=(USER,PASS))
tech = json.loads(r3.text)
if res['relationships']['mainManager']['data'] == None:
mainManager = 0
else:
mainManager = res['relationships']['mainManager']['data']['id']
if tech['data']['attributes']['diplomas'] != None:
diplomas = tech['data']['attributes']['diplomas']
else:
diplomas = ''
dipText = ''
for dip in diplomas:
newdip = dip + '#µ§'
dipText += newdip
collab = {
"BoondID": resId,
"lastName": encrypt(res['attributes']['lastName']),
"firstName": encrypt(res['attributes']['firstName']),
"dateOfBirth": encrypt(admin['data']['attributes']['dateOfBirth']),
"placeOfBirth": encrypt(admin['data']['attributes']['placeOfBirth']),
"address": encrypt(info['data']['attributes']['address']),
"postcode": encrypt(info['data']['attributes']['postcode']),
"town": encrypt(info['data']['attributes']['town']),
"country": encrypt(info['data']['attributes']['country']),
"email1": encrypt(res['attributes']['email1']),
"email2": encrypt(info['data']['attributes']['email2']),
"phone1": encrypt(res['attributes']['phone1']),
"phone2": encrypt(res['attributes']['phone2']),
"administrativeComments": encrypt(admin['data']['attributes']['administrativeComments']),
"title": encrypt(res['attributes']['title']),
"diplomas": dipText,
"mainManager": mainManager,
"agency": res['relationships']['agency']['data']['id'],
"healthCareNumber": encrypt(admin['data']['attributes']['healthCareNumber']),
"state": info['data']['attributes']['state']
}
values = (collab['BoondID'],collab['lastName'],collab['firstName'],collab['dateOfBirth'],collab['placeOfBirth'],collab['address'],collab['postcode'],collab['town'],collab['country'],collab['email1'],collab['email2'],collab['phone1'],collab['phone2'],collab['administrativeComments'],collab['title'],collab['mainManager'],collab['agency'],collab['healthCareNumber'],collab['diplomas'],collab['state'])
#collabList.append(collab)
c.execute("INSERT OR UPDATE INTO RESOURCES (BoondID,lastName,firstName,dateOfBirth,placeOfBirth,address,postcode,town,country,email1,email2,phone1,phone2,administrativeComments,title,mainManager,agency,healthCareNumber,diplomas,state) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)", values)
conn.commit()
conn.close()
I get the following error when running the function independently:
Traceback (most recent call last):
File "<ipython-input-12-87021414c9f9>", line 1, in <module>
fill_resources()
File "C:/Users/valen/OneDrive/Bureau/DOCARET/tous les tests/testcontracts.py", line 63, in fill_resources
c.execute("INSERT OR UPDATE INTO RESOURCES (BoondID,lastName,firstName,dateOfBirth,placeOfBirth,address,postcode,town,country,email1,email2,phone1,phone2,administrativeComments,title,mainManager,agency,healthCareNumber,diplomas,state) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)", values)
OperationalError: near "UPDATE": syntax error
SQLite does not support INSERT OR UPDATE INTO....
You can use UPSERT if your version of SQLite is 3.24.0+.
Assuming that BoondID is the PRIMARY KEY of the table or there is a unique constraint defined for it:
sql = """
INSERT INTO RESOURCES (
BoondID, lastName, firstName, dateOfBirth, placeOfBirth, address,
postcode, town, country, email1, email2, phone1, phone2,
administrativeComments, title, mainManager, agency, healthCareNumber,
diplomas, state
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(BoondID) DO UPDATE SET
(lastName, firstName, dateOfBirth, placeOfBirth, address, postcode, town,
country, email1, email2, phone1, phone2, administrativeComments, title,
mainManager, agency, healthCareNumber, diplomas, state) =
(EXCLUDED.lastName, EXCLUDED.firstName, EXCLUDED.dateOfBirth,
EXCLUDED.placeOfBirth, EXCLUDED.address, EXCLUDED.postcode,
EXCLUDED.town, EXCLUDED.country, EXCLUDED.email1, EXCLUDED.email2,
EXCLUDED.phone1, EXCLUDED.phone2, EXCLUDED.administrativeComments,
EXCLUDED.title, EXCLUDED.mainManager, EXCLUDED.agency,
EXCLUDED.healthCareNumber, EXCLUDED.diplomas, EXCLUDED.state);
"""
c.execute(sql, values)
I was trying to insert the CSV file to an already existing table in the SSMS database table. I have a data column in my data. But I keep getting this error when I try to insert data. Please tell me where am I doing it wrong because server connection and extracting data from the database are fine. Below is the code.
with open("combine.csv", encoding="utf8") as f:
csvreader = csv.reader(f)
csvdata = []
for row in csvreader:
csvdata.append(row)
print(csvdata)
for row in csvdata:
# Insert a row of data
print(row)
if len(row)>=8:
data = [row[0],row[1],row[2],row[3],row[4],row[5],row[6],row[7]]
cursor.execute("INSERT INTO BILLING_COPY (DATE, DEPARTMENT_NUMBER, DEPARTMENT_NAME, DIVISION_CODE, DIVISION_NAME, O_T_AMT, R_AMT, U_AMT ) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", data)
Error:
File "", line 7, in
cursor.execute("INSERT INTO BILLING_COPY (DATE, DEPARTMENT_NUMBER, DEPARTMENT_NAME, DIVISION_CODE, DIVISION_NAME, O_T_AMT, R_AMT, U_AMT ) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", data)
DataError: ('22007', '[22007] [Microsoft][ODBC Driver 17 for SQL Server][SQL Server]Conversion failed when converting date and/or time from character string. (241) (SQLExecDirectW)')
File "", line 7, in
cursor.execute("INSERT INTO BILLING_COPY (DATE, DEPARTMENT_NUMBER, DEPARTMENT_NAME, DIVISION_CODE, DIVISION_NAME, O_T_AMT, R_AMT, U_AMT ) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", data)
I think the data type you mentioned in VALUES(?,?,? etc) is not right valid data type, try using it as %d or %s
Here is some example:
mySql_insert_query = """INSERT INTO Laptop (Id, Name, Price, Purchase_date)
VALUES
(10, 'ProductValues SP99', 6459, '2019-12-27') """
cursor = connection.cursor()
cursor.execute(mySql_insert_query)
connection.commit()
My two cents: Better to assign insert query to a variable just like data variable.
I am using:
Flask 0.12.2
Python 3.6.1
To connect to my database, I am using the following code snippet, from Using SQLite 3 with Flask document:
import sqlite3
from flask import g
DATABASE = '/path/to/database.db'
def get_db():
db = getattr(g, '_database', None)
if db is None:
db = g._database = sqlite3.connect(DATABASE)
return db
#app.teardown_appcontext
def close_connection(exception):
db = getattr(g, '_database', None)
if db is not None:
db.close()
and after the following piece of code is executed:
cursor = get_db().cursor()
hashed_password = werkzeug.security.generate_password_hash(request.form['password'], method='pbkdf2:sha512', salt_length=25)
activation_token = werkzeug.security.generate_password_hash(request.form['login'], method='pbkdf2:sha512', salt_length=25)
cursor.execute('INSERT INTO Users (activation_token, login, email, password, name) VALUES (?, ?, ?, ?, ?)' (activation_token, request.form['login'], request.form['email'], hashed_password, request.form['name']))
cursor.close()
close_connection(None)
I am receiving:
File "path/to/my_file.py", line 33, in register
cursor.execute('INSERT INTO Users (activation_token, login, email, password, name) VALUES (?, ?, ?, ?, ?)' (activation_token, request.form['login'], request.form['email'], hashed_password, request.form['name']))
TypeError: 'str' object is not callable
What is the mistake here?
I sense it should be some trivial thing, though I wasn't able to pin it down. What adds to the confusion — SELECT queries in other places of my code are working just fine.
Figured it out myself, while composing the question =).
Edited the title of the question accordingly.
The culprit turned out to be missing comma, between arguments of cursor.execute() (pretty trivial, just like I have anticipated).
So, line:
cursor.execute('INSERT INTO Users (activation_token, login, email, password, name) VALUES (?, ?, ?, ?, ?)' (activation_token, request.form['login'], request.form['email'], hashed_password, request.form['name']))
should be:
cursor.execute('INSERT INTO Users (activation_token, login, email, password, name) VALUES (?, ?, ?, ?, ?)', (activation_token, request.form['login'], request.form['email'], hashed_password, request.form['name']))
I have also learned from this answer, that in order for INSERT to actually happen, connection.commit() should be called after cursor.execute('INSERT ...') (single or multiple).
Thus, the final (revised) code, should look something like this:
db_connection = get_db()
cursor = db_connection.cursor()
hashed_password = werkzeug.security.generate_password_hash(request.form['password'], method='pbkdf2:sha512', salt_length=25)
activation_token = werkzeug.security.generate_password_hash(request.form['login'], method='pbkdf2:sha512', salt_length=25)
cursor.execute('INSERT INTO Users (activation_token, login, email, password, name) VALUES (?, ?, ?, ?, ?)', (activation_token, request.form['login'], request.form['email'], hashed_password, request.form['name']))
db_connection.commit()
cursor.close()
close_connection(None)
Newbie here, trying to import from multiple csv to sql server, the code did run but no data inserted into sql database.
Attached is my code. Maybe the error is lie on the loop.
Please help.
import csv
import pyodbc as p
import os
# Database Connection Info
server = "cld-077\eform"
database = "E-form"
username = "wsmeform"
password = "M1loA1s!"
connStr = (
'DRIVER={ODBC Driver 13 for SQL Server};SERVER=' + server + ';DATABASE=' + database + ';UID=' + username + ';PWD=' + password)
# Open connection to SQL Server Table
conn = p.connect(connStr)
# Get cursor
cursor = conn.cursor()
# Assign path to Excel files
print("Inserting!")
folder_to_import = 'C:/Users/ck.law/Desktop/VBFU_NOV/'
print("path")
l_files_to_import = os.listdir(folder_to_import)
print("inside loop")
for file_to_import in l_files_to_import:
if file_to_import.endswith('.csv'):
csv_files = os.path.join(folder_to_import, file_to_import)
csv_data = csv.reader(csv_files)
for row in csv_data:
if len(row) >= 19:
cursor.execute(
"INSERT INTO VesselBFUData(ShortCode,DocDT,PostDT,DocNo,LineItm,GlCode,ExpType,InvRef,VBaseCurrcy,VBaseAmt,DocCurrcy,DocAmt,VendorCode,Description,InvFilePath,InvCreateDT,InvAppvDT,InvArriDT,PoRef)" " VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
row)
print("Loop!")
cursor.close()
conn.commit()
conn.close()
print("Script has successfully run!")