I am getting below error
"Failed inserting BLOB data into MySQL table 2006 (HY000): MySQL
server has gone away"
def insertBLOB(request):
id =(1)
name=("Eric")
photo = ("/home/gauri/Pictures/w3schools.jpg")
biodata = ("/home/gauri/Downloads/mnum.npy")
try:
connection = mysql.connector.connect(host="127.0.0.1", user="root", passwd="",
port=3307, db="data2")
cursor = connection.cursor()
sql_insert_blob_query = "INSERT INTO python_employee(id, name, photo, biodata)
VALUES (%s,%s,%s,%s)"
with open(photo, 'rb') as file:
binaryData = file.read()
with open(biodata, 'rb') as file:
binaryData1 = file.read()
insert_blob_tuple = (id, name, binaryData, binaryData1,)
result = cursor.execute(sql_insert_blob_query,insert_blob_tuple)
connection.commit()
print("Image and file inserted successfully as a BLOB into python_employee
table",result)
except mysql.connector.Error as error:
print("Failed inserting BLOB data into MySQL table {}".format(error))
return HttpResponse("data")
Add the following line into my.cnf file should solve your problem.
max_allowed_packet=64M
Here is the explanation.
Related
I want to write some basic code to do querys on read only mode on sqlite databases
These are daily db files so it is important after closing the connections not to leave other files in the server like db-shm or db-wal files associated
I have been reading the documentation and it seems that even though I try to close the connections explicitly, these are not closed so these files stay there
import sqlite3
import pandas as pd
def _connect(f):
con = None
try:
con = sqlite3.connect("file:"+f+"?mode=ro", uri=True)
except sqlite3.Error as er:
print('SQLite error in the db connection: %s' % (' '.join(er.args)))
return con
def _disconnect(con):
try:
con.close()
except sqlite3.Error as er:
print('SQLite error in the db disconnection: %s' % (' '.join(er.args)))
return con
def fl_query(file):
'''Returns information about a db file
file : string
absolute path to the db file
returns
-------
list
'''
cn = _connect(file)
cur = cn.cursor()
query = """SELECT ....etc"""
cur.execute(query)
info = [(row[0],row[1],row[2],row[3],row[4],row[5]) for row in cur.fetchall()]
cur.close()
_disconnect(cn)
return info
folder = 'path to the db file'
file = folder+'6100000_2022-09-18.db'
info = fl_query(file)
I have read about how to close cleanly the databases but so far nothing works and db-shm and db-wal stay there everytime I open a file. Remark: it is a server with thousand of files so it is important not to create more files
I realized that I was unable to modify the PRAGMA Journal because I was opening the database in read only mode. In read-write mode it performs the modification of the PRAGMA journal to DELETE mode
import sqlite3
import pandas as pd
def _connect(f):
con = None
try:
con = sqlite3.connect("file:"+f+"?mode=rw", uri=True)
except sqlite3.Error as er:
print('SQLite error in the db connection: %s' % (' '.join(er.args)))
return con
def _disconnect(con):
try:
con.close()
except sqlite3.Error as er:
print('SQLite error in the db disconnection: %s' % (' '.join(er.args)))
return con
def fl_query(file):
'''Returns information about a db file
file : string
absolute path to the db file
returns
-------
list
'''
cn = _connect(file)
cur = cn.cursor()
cur.execute("PRAGMA journal_mode=DELETE")
query = """SELECT ....etc"""
cur.execute(query)
info = [(row[0],row[1],row[2],row[3],row[4],row[5]) for row in cur.fetchall()]
cur.close()
_disconnect(cn)
My python code looks like below where I am unloading data from Redshift to Amazon S3 bucket. I am trying to get row count from Redshift and S3 bucket to ensure that all the data is loaded. Additionally, I would also like to get last uploaded date from S3 bucket so that I know when last unload was performed. Kindly suggest the code with explanation.
Thanks in advance for your time and efforts!
import csv
import redshift_connector
import sys
CSV_FILE="Tables.csv"
CSV_DELIMITER=';'
S3_DEST_PATH="s3://..../"
DB_HOST="MY HOST"
DB_PORT=1234
DB_DB="MYDB"
DB_USER="MY_READ"
DB_PASSWORD="MY_PSWD"
IM_ROLE="arn:aws:iam::/redshift-role/unload data","arn:aws::iam::/write in bucket"
def get_tables(path):
tables=[]
with open (path, 'r') as file:
csv_reader = csv.reader (file,delimiter=CSV_DELIMITER)
header = next(csv_reader)
if header != None:
for row in csv_reader:
tables.append(row)
return tables
def unload(conn, tables, s3_path):
cur = conn.cursor()
for table in tables:
print(f">{table[0]}.{table[1]}")
try:
query= f'''unload('select * from {table[0]}.{table[1]}' to '{s3_path}/{table[1]}/'
iam_role '{IAM_ROLE}'
CSV
PARALLEL FALSE
CLEANPATH;'''
print(f"loading in progress")
cur.execute(query)
print(f"Done.")
except Esception as e:
print("Failed to load")
print(str(e))
sys.exit(1)
cur.close()
def main():
try:
conn = redshift_connector.connect(
host=DB_HOST,
port=DB_PORT,
database= DB_DB,
user= DB_USER,
password=DB_PASSWORD
)
tables = get_tables(CSV_FILE)
unload(conn,tables,S3_DEST_PATH)
conn.close()
except Exception as e:
print(e)
sys.exit(1)
Update code based on SO User's comment
tables=['schema1.tablename','schema2.table2']
conn=redshift_connector.connect(
host='my_host',
port= "my_port",
database='my_db'
user="user"
password='password')
cur=conn.cursor()
cur.execute ('select count(*) from {',' .join("'"+y+"'" for y in tables)}')
results=cur.fetchall()
print("The table {} contained".format(tables[0]),*result[0],"rows"+"\n" ) #Printing row counts along with table names
cur.close()
conn.close()
2nd Update:
tables=['schema1.tablename','schema2.table2']
conn=redshift_connector.connect(
host='my_host',
port= "my_port",
database='my_db'
user="user"
password='password')
cur=conn.cursor()
for table in tables:
cur.execute(f'select count(*) from {table};')
results=cur.fetchone()
for row in result:
print("The table {} contained".format(tables[0]),result[0],"rows"+"\n" ) #Printing row counts along with table names
The simple query to get number of rows is
query = "select count(*) from {table_name}"
For Redshift, all you need to do is
cur.execute(query)
row_count = cur.fetchall()
Using boto3, you can use a similar SQL query to fetch S3 row count as well, as elucidated in this answer.
Edit:
Corrected your updated approach a little:
cur=conn.cursor()
for table in tables:
cur.execute(f'select count(*) from {table};')
result=cur.fetchone()
count = result[0] if result else 0
print(f"The table {table} contained {count} rows.\n" )
I want to send my variable to MySQL database. But I have a warning. This code was successful to upload.
import mysql.connector
try:
connection = mysql.connector.connect(host='localhost',
database='mydb',
user='root',
password='')
cursor = connection.cursor()
username = 1111111 #example
isian = "tryfile.pkl" #example, i want send string data
print("Before updating a record ")
sql_select_query = """select * from signature where id = %s"""
cursor.execute(sql_select_query %(username))
record = cursor.fetchone()
print(record)
# Update single record now
sql_update_query = """Update signature set signature = "%s" where id = %s"""
cursor.execute(sql_update_query %(isian,username))
connection.commit()
print("Record Updated successfully ")
print("After updating record ")
cursor.execute(sql_select_query)
record = cursor.fetchone()
print(record)
except mysql.connector.Error as error:
print("Failed to update table record: {}".format(error))
finally:
if connection.is_connected():
connection.close()
print("MySQL connection is closed")
Warning
Failed to update table record: 1064 (42000): You have an error in your SQL syntax; check the manual that corresponds to your MariaDB server version for the right syntax to use near '%s' at line 1
I don't understand with error, because I am using MySQL not MariaDB
What's happen to my code?
MariaDB is the engine used in modern MySQL databases.
The problem is your second call to sql_select_query, because you forgot to add the % (username), so nothing was substituted. HOWEVER, you should not be doing the substitutions yourself. You need to let the database connector do it. It's an easy change:
print("Before updating a record ")
sql_select_query = """select * from signature where id = ?"""
cursor.execute(sql_select_query, (username,))
record = cursor.fetchone()
print(record)
# Update single record now
sql_update_query = """Update signature set signature = ? where id = ?"""
cursor.execute(sql_update_query, (isian,username))
connection.commit()
print("Record Updated successfully ")
print("After updating record ")
cursor.execute(sql_select_query, (username,))
record = cursor.fetchone()
print(record)
i'm taking data from textfile which contains some duplicate data.And i'm trying to insert them into database without duplicating.i'm in trouble where inserting duplicate data.it should not be inserted again.data are not static values.
text_file = open(min_file, "r")
#doc = text_file.readlines()
for line in text_file:
field = line.split(";")
print(field)
try:
connection = mysql.connector.connect(host='localhost',
database='testing',
user='root',
password='root')
if connection.is_connected():
db_Info = connection.get_server_info()
print("Connected to MySQL Server version ", db_Info)
cursor = connection.cursor()
cursor.execute("select database();")
record = cursor.fetchone()
print("You're connected to database: ", record)
mycursor = connection.cursor()
#before inserting
mycursor.execute("Select * from ftp")
myresult = mycursor.fetchall()
for i in myresult:
print(i)
sql ="Insert into ftp(a,b,c,d) \
select * from( Select VALUES(%s,%s,%s,%s) as temp \
where not exists \
(Select a from ftp where a = %s) LIMIT 1"
mycursor.execute(sql,field)
print(mycursor.rowcount, "record inserted.")
connection.commit()
except Error as e:
print("Error while connecting to MySQL", e)
finally:
if connection.is_connected():
cursor.close()
connection.close()
print("MySQL connection is closed")
One option is to add Unique constraint and let the DB validate uniqueness, this will throw exception which you can catch and skip.
I have a problem with creating SQL query for Oracle database using Python.
I want to bind string variable and it does not work, could you tell me what am I doing wrong?
This is my code:
import cx_Oracle
dokList = []
def LoadDatabase():
conn = None
cursor = None
try:
conn = cx_Oracle.connect("login", "password", "localhost")
cursor = conn.cursor()
query = "SELECT * FROM DOCUMENT WHERE DOC = :param"
for doknumber in dokList:
cursor.execute(query, {'doknr':doknumber})
print(cursor.rowcount)
except cx_Oracle.DatabaseError as err:
print(err)
finally:
if cursor:
cursor.close()
if conn:
conn.close()
def CheckData():
with open('changedNamed.txt') as f:
lines = f.readlines()
for line in lines:
dokList.append(line)
CheckData()
LoadDatabase()
The output of cursor.rowcount is 0 but it should be number greater than 0.
You're using a dictionary ({'doknr' : doknumber}) for your parameter, so it's a named parameter - the :param needs to match the key name. Try this:
query = "SELECT * FROM DOCUMENT WHERE DOC = :doknr"
for doknumber in dokList:
cursor.execute(query, {'doknr':doknumber})
print(cursor.rowcount)
For future troubleshooting, to check whether your parameter is getting passed properly, you can also try changing your query to "select :param from dual".