I'm trying to create a python script that will copy a glob result into postgres. This is what I have:
import psycopg2, glob
sitesizetemp = glob.glob('C:/Data/Sheltered BLPUs/CSVs/sitesize*.csv')
conn = psycopg2.connect("dbname=postgres user=postgres")
cur = conn.cursor()
cur.execute("COPY sitesize FROM" sitesizetemp "DELIMITER ',' CSV;")
conn.commit()
cur.close()
conn.close()
I'm pretty new to all of this so any help is very welcome! thanks
import psycopg2, glob
sitesizetemp = glob.glob('C:/Data/Sheltered BLPUs/CSVs/sitesize*.csv')
conn = psycopg2.connect("dbname=postgres user=postgres")
cur = conn.cursor()
for f in sitesizetemp:
cur.execute("COPY sitesize FROM '%s' DELIMITER ',' CSV;" % f)
conn.commit()
cur.close()
conn.close()
Related
# Module Imports
import mariadb
import sys
import csv
from pathlib import Path
def connect_to_mariaDB(databse, user, passwd):
# Connect to MariaDB Platform
try: conn = mariadb.connect(
user=user,
password=passwd,
host="localhost",
port=3306,
database=databse
)
except mariadb.Error as e:
print(f"Error connecting to MariaDB Platform: {e}")
sys.exit(1)
return conn
def check_if_table_exists_and_overwrite(conn, tableName, database, overwrite):
cur = conn.cursor()
cur.execute(f"SELECT table_name FROM information_schema.tables WHERE table_schema = '{database}';")
for(table_name) in cur:
if table_name[0] == tableName:
if overwrite == "YES":
print("table exists - DROP TABLE")
cur.execute(f"DROP TABLE {tableName}")
return True
else:
return False
return True
def import_file_into_db_table_(
filename, database, user, passwd, tableName,
create_table_statement = "", overwrite = False):
conn = connect_to_mariaDB(database, user, passwd)
cur = conn.cursor()
if conn != None:
print(f"Connection successful to database {database}")
if check_if_table_exists_and_overwrite(conn, tableName, database, overwrite):
cur.execute(create_table_statement)
print("table is created")
path = f"{Path().absolute()}\\{filename}".replace("\\","/")
print(path)
load_data_statement = f"""LOAD DATA INFILE '{path}'
INTO TABLE {tableName}
FIELDS TERMINATED BY ';'
OPTIONALLY ENCLOSED BY '\"'
LINES TERMINATED BY '\\n'
IGNORE 1 LINES
"""
print(load_data_statement)
cur.execute(load_data_statement)
print("load data into table - successful")
else:
print("table exists - no permission to overwrite")
cur.execute("SELECT * FROM student_mat;")
for da in cur:
print(da)
# variables
filename = "student-mat.csv"
database = "dbs2021"
tableName = "student_mat"
# load the create_table_statement
create_table_statement = ""
path = f"{Path().absolute()}\\create_table_statement.txt"
with open(path, newline='') as file:
spamreader = csv.reader(file, delimiter='\n', quotechar='|')
for row in spamreader:
create_table_statement += row[0]
parameters_length = len(sys.argv)
if parameters_length == 3:
user, passwd = sys.argv[1], sys.argv[2]
import_file_into_db_table_(filename, database, user, passwd, tableName, create_table_statement, "YES")
elif parameters_length == 4:
user, passwd, overwrite = sys.argv[1], sys.argv[2], sys.argv[3]
import_file_into_db_table_(filename, database, user, passwd, tableName, create_table_statement, overwrite)
else:
print("wrong parameters\nTry -user -passwd or additional -overwrite")
The code checks if there is a table with the same name in the db and then potentially drops it, creates a new table and loads the data of the csv file into the table.
When executing the code it seems like everything is working but when going in the mariadb command prompt the created table is empty even though when outputting the table in the code it is filled.
By default MariaDB Connector/Python doesn't use autocommit mode.
You need either set autocommit=True when establishing the connection or you have to commit your changes with conn.commit().
I'm new to python. I'm trying to request an URL (thanks to an id stored in a postgresql data base) which sends me zip folders with several files inside.
import psycopg2
import requests
url = "https://myurl/"
conn = psycopg2.connect(user="XXX", password="XXX", database="XXX", host="localhost", port="5432")
print("Successfully connected!")
cur = conn.cursor()
sql ="select id from public.base"
cur.execute(sql)
row = [item[0] for item in cur.fetchall()]
for d in row:
requests.post(url+d)
The requests.post(url+d) is working, i have a 200 response.
But I don't know how to do the following steps, that is to say to upload in my workspace these zip folders...
You could use the zipfile & io library for that, specifying your download location within a extractall, like so :)
from psycopg2 import (
connect,
OpertionalError,
)
from zipfile import (
BadZipFile,
ZipFile,
)
from io import BytesIO
import requests
def download_zip(url):
response = requests.get(url)
if response.ok:
try:
z = ZipFile(BytesIO(response.content))
z.extractall("/path/to/destination_directory")
except BadZipFile as ex:
print('Error: {}'.format(ex))
print('Download succeeded: {}'.format(url))
else:
print('Connection failed: {}'.format(url))
def main():
conn = connect(
user='XXX',
password='XXX',
database='XXX',
host='localhost',
port='5432',
)
try:
cur = conn.cursor()
cur.execute('select id from public.base')
except OperationalError:
exit(0)
row = [item[0] for item in cur.fetchall()]
for id in row:
download_zip('https://myurl/{}'.format(id))
print('Download completed')
if __name__ == '__main__':
main()
I have a table in SQLite 3 as follows and I am planning on using it to store a variety of files: txt, pdf, images and zip files.
CREATE TABLE zip (filename TEXT PRIMARYKEY NOT NULL, zipfile BLOB NOT NULL);
To store and retrieve I am experimenting with the following python code
#!env/bin/python
import sqlite3 as lite
import os
import sys
def insertfile(_filename):
try:
con = lite.connect('histogram.db', detect_types=lite.PARSE_DECLTYPES)
con.row_factory = lite.Row
cur = con.cursor()
cur.execute('PRAGMA foreign_keys=ON;')
_f = open(_filename,'rb')
_split = os.path.split(_filename)
_file = _split[1]
_blob = _f.read()
cur.execute('INSERT INTO zip (filename,zipfile) VALUES (?,?)', (_file,lite.Binary(_blob)))
_f.close()
con.commit()
cur.close()
con.close()
except Exception as ex:
print ex
def getfile(_filename):
try:
con = lite.connect('histogram.db', detect_types=lite.PARSE_DECLTYPES)
con.row_factory = lite.Row
cur = con.cursor()
cur.execute('PRAGMA foreign_keys=ON;')
cur.execute('SELECT zipfile from zip where filename = ?', (_filename,))
_files = cur.fetchall()
if len(_files) > 0:
_file = open('Test/'+ _filename,'wb')
_file.write(_files[0]['zipfile'])
_file.close()
cur.close()
con.close()
except Exception as ex:
print ex
if __name__ == '__main__':
print 'works'
insertfile(sys.argv[1])
getfile(os.path.split(sys.argv[1])[1])
When I test this on files like .txt, .py, .pdf etc., it works fine.
With Zip files, there is no error while storing into the table but an error while retrieving the file:
Could not decode to UTF-8 column 'zipfile' with text 'PK '
There seems to be some encoding or decoding issue.
I basically tried using the code from one of the questions
Insert binary file in SQLite database with Python
.
It worked originally for the pdf, png, jpg files. But I was still getting the error for Zip files. When I commented out the insertion and just ran the retrieval code it worked. Now the code below works.
def insertfile(_filename):
try:
con = lite.connect('histogram.db', detect_types=lite.PARSE_DECLTYPES)
con.row_factory = lite.Row
cur = con.cursor()
cur.execute('PRAGMA foreign_keys=ON;')
_f = open(_filename,'rb')
_split = os.path.split(_filename)
_file = _split[1]
_blob = _f.read()
cur.execute('INSERT INTO zip (filename,zipfile) VALUES (?,?)', (_file,lite.Binary(_blob)))
_f.close()
con.commit()
cur.close()
con.close()
except Exception as ex:
print ex
def getfile(_filename):
try:
con = lite.connect('histogram.db', detect_types=lite.PARSE_DECLTYPES)
con.row_factory = lite.Row
cur = con.cursor()
cur.execute('PRAGMA foreign_keys=ON;')
cur.execute('SELECT zipfile from zip where filename = ?', (_filename,))
_files = cur.fetchall()
if len(_files) > 0:
_file = open('Downloads/'+ _filename,'wb')
_file.write(_files[0]['zipfile'])
_file.close()
cur.close()
con.close()
except Exception as ex:
print ex
if __name__ == '__main__':
print 'works'
insertfile(sys.argv[1])
getfile(os.path.split(sys.argv[1])[1])
conn = sqlite3.connect('./mydb.db')
c = conn.cursor()
with open('./mydb_tmp.sql', 'w') as f:
for row in c.execute('SELECT * FROM FLOWS'):
print >>f, row
c.execute('DELETE FROM FLOWS;')
conn.close()
After that, all rows are still in mydb
You need to commit the command.
conn = sqlite3.connect('./mydb.db')
c = conn.cursor()
with open('./mydb_tmp.sql', 'w') as f:
for row in c.execute('SELECT * FROM FLOWS'):
print >>f, row
c.execute('DELETE FROM FLOWS;')
# Flush your commands to the db with conn.commit().
conn.commit()
conn.close()
Check the documentation : https://docs.python.org/2/library/sqlite3.html
I used this code to insert an image into mysql database and retrieve back the image.
This code works perfectly with no errors , but the problem is even after inserting an image into img table , when I execute the command select * from img ; in the mysql command line it shows no records.
Table created in database;
create table img(images blob not null);
import mysql.connector
import sys
from PIL import Image
import base64
import cStringIO
import PIL.Image
db = mysql.connector.connect(user='root', password='abhi',
host='localhost',
database='cbir')
#image = Image.open('C:\Users\Abhi\Desktop\cbir-p\images.jpg')
with open("C:\Users\Abhi\Desktop\cbir-p\images.jpg", "rb") as image_file:
encoded_string = base64.b64encode(image_file.read())
#blob_value = open('C:\Users\Abhi\Desktop\cbir-p\images.jpg', 'rb').read()
sql = 'INSERT INTO img(images) VALUES(%s)'
args = (encoded_string, )
cursor=db.cursor()
cursor.execute(sql,args)
sql1='select * from img'
cursor.execute(sql1)
data=cursor.fetchall()
#print type(data[0][0])
data1=base64.b64decode(data[0][0])
file_like=cStringIO.StringIO(data1)
img=PIL.Image.open(file_like)
img.show()
db.close()
import mysql.connector
import sys
from PIL import Image
import base64
import cStringIO
import PIL.Image
db = mysql.connector.connect(user='root', password='abhi',
host='localhost',
database='cbir')
image = Image.open('C:\Users\Abhi\Desktop\cbir-p\images.jpg')
blob_value = open('C:\Users\Abhi\Desktop\cbir-p\images.jpg', 'rb').read()
sql = 'INSERT INTO img(images) VALUES(%s)'
args = (blob_value, )
cursor=db.cursor()
cursor.execute(sql,args)
sql1='select * from img'
db.commit()
cursor.execute(sql1)
data=cursor.fetchall()
print type(data[0][0])
file_like=cStringIO.StringIO(data[0][0])
img=PIL.Image.open(file_like)
img.show()
db.close()
This code works fine
import mysql.connector
import base64
import io
import PIL.Image
with open('lemonyellow_logo.jpg', 'rb') as f:
photo = f.read()
encodestring = base64.b64encode(photo)
db= mysql.connector.connect(user="root",password="lyncup",host="localhost",database="demo")
mycursor=db.cursor()
sql = "insert into sample values(%s)"
mycursor.execute(sql,(encodestring,))
db.commit()
sql1="select * from sample"
mycursor.execute(sql1)
data = mycursor.fetchall()
data1=base64.b64decode(data[0][0])
file_like=io.BytesIO(data1)
img=PIL.Image.open(file_like)
img.show()
db.close()
import mysql.connector
import pyautogui
import time
import base64
def byte_string(image_name):
with open(image_name, 'rb') as image_file:
encoded_string = base64.b64encode(image_file.read())
return encoded_string
def insertBLOB(image_id, image_name, image):
print("Inserting BLOB into my_images table")
try:
connection = mysql.connector.connect(host="localhost",
user="root",
password="Rushi#2001",
database = 'my_images',
port = 3306)
cursor = connection.cursor()
sql_insert_blob_query = " INSERT INTO image_data (image_id, image_name, image) VALUES (%s,%s,%s)"
binary_image = byte_string(image)
print(len(binary_image))
# Convert data into tuple format
insert_blob_tuple = (image_id, image_name, binary_image)
result = cursor.execute(sql_insert_blob_query, insert_blob_tuple)
connection.commit()
print("Image and file inserted successfully as a BLOB into my_images table", result)
except mysql.connector.Error as error:
print("Failed inserting BLOB data into MySQL table {}".format(error))
finally:
if connection.is_connected():
cursor.close()
connection.close()
print("MySQL connection is closed")
insertBLOB(1,'img0',"C:\\Users\\hulag\\OneDrive\\Desktop\\python_sql\\images\\img0.png")