How to insert the records into postgresql python? - python

I am trying to insert the pandas dataframe into postgresql table. What I am doing is inserting the record of dataframe loop by loop. I am recursively getting errors, Code is shown below:
Code:
import psycopg2
df = pd.read_csv('dataframe.csv')
conn = psycopg2.connect(database = "postgres",
user = "postgres",
password = "12345",
host = "127.0.0.1",
port = "5432")
cur = conn.cursor()
for i in range(0,len(df)):
cur.execute("INSERT INTO stock_market_forecasting_new (date, open, high, low, close) \
VALUES (df['date'][i], df['open'][i], df['high'][i], df['low'][i], df['close'][i])")
conn.commit()
print("Records created successfully")
conn.close()
Error:
UndefinedColumn: column "df" does not exist
LINE 1: ..._new (date, open, high, low, close) VALUES (df['date']...
Edit1:
I am doing like this,
cur.execute("SELECT * from STOCK_MARKET_FORECASTING")
rows = cur.fetchall()
for row in rows:
print(row)
print("Operation done successfully")
conn.close()
Output giving:
('2021-12-07 00:00:00', 1.12837, 1.12846, 1.1279, 1.128)
('2021-12-07 01:00:00', 1.12799, 1.12827, 1.1276, 1.1282)
Output which I want should be with column names:
**Date open high low close**
('2021-12-07 00:00:00', 1.12837, 1.12846, 1.1279, 1.128)
('2021-12-07 01:00:00', 1.12799, 1.12827, 1.1276, 1.1282)

You didn't use a formatted string. It should be f"VALUES ({df['date'][i]}," ect depending on your data type, but that always leads to issues with quotation marks. I recommend inserting with tuples instead as seen in the documentation:
import psycopg2
df = pd.read_csv('dataframe.csv')
conn = psycopg2.connect(database = "postgres",
user = "postgres",
password = "12345",
host = "127.0.0.1",
port = "5432")
cur = conn.cursor()
for i in range(0 ,len(df)):
values = (df['date'][i], df['open'][i], df['high'][i], df['low'][i], df['close'][i])
cur.execute("INSERT INTO stock_market_forecasting_new (date, open, high, low, close) VALUES (%s, %s, %s, %s, %s)",
values)
conn.commit()
print("Records created successfully")
conn.close()
Alternatively, you could see if df.to_sql() (documentation) supports psycopg2 connections:
df.to_sql('stock_market_forecasting_new', con=conn, if_exists='append', index=False)

Related

how to use one database in several python file

I working on a project and I want to use one database in two python file
but, when I run every project they created database for self
if you know please tell me how I can use that
import sqlite3
def connect():
conn = sqlite3.connect("waiters.db")
cur = conn.cursor()
cur.execute(
"CREATE TABLE IF NOT EXISTS salary (id INTEGER PRIMARY KEY , name text, age INTEGER , price INTEGER )"
)
conn.commit()
conn.close()
def insert(name, age, price):
conn = sqlite3.connect("waiters.db")
cur = conn.cursor()
cur.execute(
"INSERT INTO salary VALUES (NULL ,?,?,?)", (name ,age ,price)
)
conn.commit()
conn.close()
def view():
conn = sqlite3.connect("waiters.db")
cur = conn.cursor()
cur.execute(
"SELECT * FROM salary"
)
rows = cur.fetchall()
conn.close()
return rows
def search(name="", age="", price=""):
conn = sqlite3.connect("waiters.db")
cur = conn.cursor()
cur.execute(
"SELECT * FROM salary WHERE name = ? OR age = ? OR price = ?", (name, age, price)
)
rows = cur.fetchall()
conn.close()
return rows
def delete(id):
conn = sqlite3.connect("waiters.db")
cur = conn.cursor()
cur.execute("DELETE FROM salary WHERE id=?", (id,))
conn.commit()
conn.close()
def update(id, name, age, price):
conn = sqlite3.connect("waiters.db")
cur = conn.cursor()
cur.execute(
"UPDATE salary SET name = ?, age = ?, price = ? WHERE id = ?", (name, age, price, id)
)
conn.commit()
conn.close()
def update_pay_money(name, price):
conn = sqlite3.connect("waiters.db")
cur = conn.cursor()
cur.execute(
"UPDATE salary SET price = ? WHERE name = ?", (price, name)
)
conn.commit()
conn.close()
connect()
enter image description here
Giving exact path like /path/to/waiters.db while connecting to your database should solve your problem?
This line should be changed while connecting to database.
conn = sqlite3.connect("/path/to/waiters.db")
as Other mentioned for using a sqllite3 db in multiple files you can use their absolute or relative path, for example if you have 'DBs' & 'section-1' & 'section-2' directories and your python file are in section directories you can access the database file in each section by using somthing like this '"../DBs/waiters.db"' and so on for others... but whatf of you try make multiple tables in a database file in tgat way you don need to have multiple databases and its the standard way,
hope it's help

SQLite exception: no such table, while successfully connecting to DB

When I executing the following function, I get the following response:
Failed to insert data into sqlite table: no such table: users
user_id = uuid.uuid4()
save_record.name = name
save_record.score = score
last_time = datetime.now()
try:
conn = sqlite3.connect('OnlineJeopardy.db')
cursor = conn.cursor()
print("Successfully Connected to OnlineJeopardy DB.")
sqlite_insert_query = """INSERT INTO users
(User_id, Name, Score, Last_time)
VALUES
(?, ?, ?, ?)"""
add_to_db = cursor.execute(sqlite_insert_query, (user_id, name, score, last_time))
conn.commit()
cursor.close()
except sqlite3.Error as error:
print("Failed to insert data into sqlite table: ", error)
finally:
if (conn):
conn.close()
print("The SQLite connection is closed")
When I execute this query in DB Browser, with actual values instead of placeholders, it all goes well.
I've tried swapping placeholders to actual values (as below) within the query in sqlite3 but the outcome was the same.
conn = sqlite3.connect('OnlineJeopardy.db')
cursor = conn.cursor()
print("Successfully Connected to OnlineJeopardy DB.")
sqlite_insert_query = """INSERT INTO users
(User_id, Name, Score, Last_time)
VALUES
('ID-1337', 'Adam', 20, '2020-06-12 23:18:58')"""
add_to_db = cursor.execute(sqlite_insert_query)
conn.commit()
cursor.close()

Write Python dataframe to Oracle

I have a dataframe testdata like this:
Here are the variables' types in Python:
detectorid:int64
starttime:str
volume:float64
speed:float64
occupancy:float64
Now I want to creat a datatable in oracle and insert this dataframe into it, here is what I tried:
import pandas as pd
import cx_Oracle
host = "192.168.1.100"
port = "1521"
sid = "orcl"
dsn = cx_Oracle.makedsn(host, port, sid)
conn = cx_Oracle.connect("scott", "tiger", dsn)
cursor = conn.cursor()
#creat datatable:
sql_creat = "create table portland(detectorid number(32), starttime varchar(32), volume number(32), speed number(32), occupancy number(32))"
cursor.execute(sql_creat)
query = "insert into portland (detectorid,starttime,volume,speed,occupancy) VALUES (%d,'%s',%f,%f,%f)"
#insert by rows:
for i in range(len(testdata)):
detectorid= testdata.ix[i,0]
starttime= testdata.ix[i,1]
volume= testdata.ix[i,2]
speed= testdata.ix[i,3]
occupancy= testdata.ix[i,4]
cursor.execute(query % (detectorid,starttime,volume,speed,occupancy))
conn.commit()
cursor.close()
conn.close()
However it gives me DatabaseError: ORA-00984:column not allowed here. I think there are something wrong about the columns' types in my sql statement but I don't know how to solve it. Could somebody give me some instructions? Thank you for your attention!
#!/usr/local/bin/python3
import cx_Oracle
import os
conn = cx_Oracle.connect("user", "xxx", "localhost:1512/ORCLPDB1", encoding="UTF-8")
cursor = conn.cursor()
#creat datatable:
sql_creat = "create table portland(detectorid number(32), starttime varchar(32), volume number(32), speed number(32), occupancy number(32))"
#cursor.execute(sql_creat)
query = "insert into portland (detectorid,starttime,volume,speed,occupancy) VALUES (%d,'%s',%f,%f,%f)"
detectorid = 1345
starttime = '2011-09-15 00:00:00'
volume = 0
speed = 0
occupancy= 0
cursor.execute(query % (detectorid,starttime,volume,speed,occupancy))
conn.commit()
cursor.close()
conn.close()

importing from excel to mysql

I am trying to import data from excel to MySQl below is my code , problem here is that it only writes the last row from my excel sheet to MySQl db and i want it to import all the rows from my excel sheet.
import pymysql
import xlrd
book = xlrd.open_workbook('C:\SqlExcel\Backup.xlsx')
sheet = book.sheet_by_index(0)
# Connect to the database
connection = pymysql.connect(host='localhost',
user='root',
password='',
db='test')
cursor = connection.cursor()
query = """INSERT INTO report_table (FirstName, LastName) VALUES (%s, %s)"""
for r in range(1, sheet.nrows):
fname = sheet.cell(r,1).value
lname = sheet.cell(r,2).value
values = (fname, lname)
cursor.execute(query, values)
connection.commit()
cursor.close()
connection.close()
You code is currently only storing the last pair, and writing that to the database. You need to call fname and lname inside the loop and write each pair seperately to the database.
You can ammend your code to this:
import pymysql
import xlrd
book = xlrd.open_workbook('C:\SqlExcel\Backup.xlsx')
sheet = book.sheet_by_index(0)
# Connect to the database
connection = pymysql.connect(host='localhost',
user='root',
password='',
db='test',
autocommit=True)
cursor = connection.cursor()
query = """INSERT INTO report_table (FirstName, LastName) VALUES (%s, %s)"""
# loop over each row
for r in range(1, sheet.nrows):
# extract each cell
fname = sheet.cell(r,1).value
lname = sheet.cell(r,2).value
# extract cells into pair
values = fname, lname
# write pair to db
cursor.execute(query, values)
# close everything
cursor.close()
connection.close()
Note: You can set autocommit=True in the connect phase. PyMySQL disables autocommit by default. This means you dont have to call cursor.commit() after your query.
Your variable values have to be inside the for instruction like this :
import pymysql
import xlrd
book = xlrd.open_workbook('C:\SqlExcel\Backup.xlsx')
sheet = book.sheet_by_index(0)
# Connect to the database
connection = pymysql.connect(host='localhost',
user='root',
password='',
db='test')
cursor = connection.cursor()
query = """INSERT INTO report_table (FirstName, LastName) VALUES (%s, %s)"""
for r in range(1, sheet.nrows):
fname = sheet.cell(r,1).value
lname = sheet.cell(r,2).value
values = (fname, lname)
cursor.execute(query, values)
connection.commit()
cursor.close()
connection.close()
Sorry, I don't know much about databases, so nor about pymysql. But assumed all the rest is correct I guess it could work like:
...
cursor = connection.cursor()
query = """INSERT INTO report_table (FirstName, LastName) VALUES (%s, %s)"""
for r in range(1, sheet.nrows):
fname = sheet.cell(r,1).value
lname = sheet.cell(r,2).value
values = (fname, lname)
cursor.execute(query, values)
connection.commit()
cursor.close()
connection.close()
Is this something you will do on a regular basis? I see the script you're writing but I am not sure if this is something you need to run over and over again or if you are just importing the data into MySQL once.
If this is a one shot deal, you can try this.
Open the spreadsheet and SELECT ALL then COPY all your data. Paste it into a text document and save the text document (let's say the text document will be in c:\temp\exceldata.txt). You can then load it all into the table with one command:
LOAD DATA INFILE 'c:/temp/exceldata.txt'
INTO TABLE report_table
FIELDS TERMINATED BY '\t'
LINES TERMINATED BY '\r\n'
IGNORE 1 LINES;
I am making a few assumptions here:
The spreadsheet has only two columns and they are in the same order as the fields in your table.
You do NOT need to clear out the table before the load. If you do, issue the command TRUNCATE TABLE report_table; before the load.
Note, I chose a tab delimited format because I prefer it. You could save the file as a .CSV file and adjust the command as follows:
LOAD DATA INFILE 'c:/temp/exceldata.txt'
INTO TABLE report_table
FIELDS TERMINATED BY ','
OPTIONALLY ENCLOSED BY '"'
LINES TERMINATED BY '\r\n'
IGNORE 1 LINES;
The "optionally enclosed by" is there because Excel will put quotes around text data with a comma in it.
If you need to do this on a regular basis, you can still use the CSV method by writing an excel script that saves the file to a .CSV copy whenever the spreadsheet is saved. I have done that too.
I have never written python but this is how I do it in PHP.
HTH
This code worked for me after taking help from the above suggestion the error was of indentation now its working :)
import pymysql
import xlrd
book = xlrd.open_workbook('C:\SqlExcel\Backup.xlsx')
sheet = book.sheet_by_index(0)
# Connect to the database
connection = pymysql.connect(host='localhost',
user='root',
password='',
db='test',
autocommit=True)
cursor = connection.cursor()
query = """INSERT INTO report_table (FirstName, LastName) VALUES (%s, %s)"""
for r in range(1, sheet.nrows):
fname = sheet.cell(r,1).value
lname = sheet.cell(r,2).value
values = (fname, lname)
cursor.execute(query, values)
cursor.close()
connection.close()

Problem with inserting into MySQL database from Python

I am having trouble inserting a record into a MySQL database from python. This is what I am doing.
def testMain2():
conn = MySQLdb.connect(charset='utf8', host="localhost", user="root", passwd="root", db="epf")
cursor = conn.cursor()
tableName = "test_table"
columnsDef = "(export_date BIGINT, storefront_id INT, genre_id INT, album_id INT, album_rank INT)"
exStr = """CREATE TABLE %s %s""" % (tableName, columnsDef)
cursor.execute(exStr)
#Escape the record
values = ["1305104402172", "12", "34", "56", "78"]
values = [conn.literal(aField) for aField in values]
stringList = "(%s)" % (", ".join(values))
columns = "(export_date, storefront_id, genre_id, album_id, album_rank)"
insertStmt = """INSERT INTO %s %s VALUES %s""" % (tableName, columns, stringList)
cursor.execute(insertStmt)
cursor.close()
conn.close()
The table is created however nothing is in the table. I can run the INSERT statement successfully in Terminal with the same credentials.
Any suggestions on what I may be doing wrong?
You haven't committed the transaction.
conn.commit()
(The MySQLdb library sets autocommit to False when connecting to MySQL. This means that you need to manually call commit or your changes will never make it into the database.)

Categories