We Keep Coding

Excel crashed while exporting the data from Oracle using python

Below code is for extracting the Oracle data into xlsx format using Python,
Code run successfully but when i opened the xlsx file giving error:
"file format or file extension is not valid. Verify that the file has not been corrupted and that the file extension matches the format of the file."
What should i change in below code to get the data without excel crash.
import xlsxwriter
from xlsxwriter import Workbook
import cx_Oracle
import datetime
from datetime import date
dsn_tns = cx_Oracle.makedsn('HOST', 'port', sid='sid')
conn = cx_Oracle.connect(user=r'username', password='password', dsn=dsn_tns)
cursor = conn.cursor()
xlsx_file = open("path.xlsx", "w")
writer = xlsxwriter.Workbook("path.xlsx")
worksheet = writer.add_worksheet()
sql ='''
SELECT *
FROM ( SELECT STRGUID,ACTIVITYUSERID,ACTIVITYSESSIONID,ACTIVITYCODE,SERVERNAME,APPNAME,STARTTIME,ENDTIME,STRDESCRIPTION,
(To_Date('12/30/1899', 'MM/DD/YYYY HH24:MI:SS')+ STARTTIME)Decoded_Date
FROM tablename
)SUB
WHERE SUB.Decoded_Date between to_date('26-APR-2020', 'DD-MON-YYYY')
and to_date('26-JUN-2020', 'DD-MON-YYYY')
'''
cursor.execute(sql)
for r, row in enumerate(cursor.fetchall()):
for c, col in enumerate(row):
worksheet.write(r, c, col)
cursor.close()
conn.close()
xlsx_file.close()

I tried below code and working successfully for exporting the data from oracle into xlsx format using python:
import xlsxwriter
from xlsxwriter import Workbook
import cx_Oracle
import datetime
from datetime import date
import os
import logging
import sys
import getopt
import traceback
## Define Function
def writeToExcel(cur_sor):
workbook = xlsxwriter.Workbook('path.xlsx')
worksheet = workbook.add_worksheet("DATA") #Add a New Worksheet Name -
for row, row1 in enumerate(cur_sor.fetchall()):
for col, col1 in enumerate(row1):
worksheet.write(row, col, col1)
workbook.close()
def setSqlCommand():
sqlCommand = '''
SELECT *
FROM
( SELECT LROWNUM,DTIMESTAMP,LSCENARIO,LYEAR,LPERIOD,
LENTITY,LPARENT,LVALUE,LACCOUNT,LICP,LCUSTOM1,
LCUSTOM2,STRUSERNAME,STRSERVERNAME,
LACTIVITY,DDATAVALUE,BNODATA,
To_Date('12/30/1899','MM/DD/YYYY') +
DTIMESTAMP as Decoded_Date
FROM tablename
) SUB
WHERE SUB.Decoded_Date
<= to_date('28-JUN-2020', 'DD-MON-YYYY')
'''
return sqlCommand
# Function to Execute Sql commands over TNS
def runSqlTNS (sqlCommand, username, password , hostName, portNumber, sID):
dsn_tns = cx_Oracle.makedsn('HOST', 'PORT', sid='SERVICEID')
db = cx_Oracle.connect(user=r'USERNAME', password='PASSWORD', dsn=dsn_tns)
cursor = db.cursor()
cursor.execute(sqlCommand)
return cursor
def main(argv=None):
#Setup Logging Information
logging.basicConfig(filename='PATH/myapp.log', level=logging.INFO, format='%(asctime)s %(message)s', datefmt='%m/%d/%Y %I:%M:%S %p')
logging.info('Started')
username = 'USERNAME'
password = 'PASSWORD'
#Define dsn entries to create a tns connection
hostName = 'HOST'
portNumber = 'PORT'
sid = 'SERVICEID'
#
try:
sqlCommand = setSqlCommand()
except Exception as e:
logging.info('Function - sqlCommand - In Exception')
logging.info(traceback.print_exc())
try:
c = runSqlTNS(sqlCommand, username, password , hostName, portNumber, sid)
except Exception as e:
logging.info('Function - runSql In Exception')
logging.info(traceback.print_exc())
try:
writeToExcel(c) # Send the Cursor to writetoExcel Function
c.close()
except Exception as e:
logging.info('Function - writeToExcel In Exception')
logging.info(traceback.print_exc())
if __name__ == "__main__":
main(sys.argv)

How to read a CSV file from s3 and write the content in RDS database table using python lambda function?

I have a CSV file Employee.csv in the S3 bucket with all info about employee: name, age, salary, designation.
I have to write a python lambda function to read this file and write in RDS db such as it should create a table as Employee, with columns name, age, salary, designation and rows will have the data.
The Employee.csv is just for example, actually it can be any csv file with any number of columns in it.

from __future__ import print_function
import boto3
import logging
import os
import sys
import uuid
import pymysql
import csv
import rds_config
rds_host = rds_config.rds_host
name = rds_config.db_username
password = rds_config.db_password
db_name = rds_config.db_name
logger = logging.getLogger()
logger.setLevel(logging.INFO)
try:
conn = pymysql.connect(rds_host, user=name, passwd=password, db=db_name, connect_timeout=5)
except Exception as e:
logger.error("ERROR: Unexpected error: Could not connect to MySql instance.")
logger.error(e)
sys.exit()
logger.info("SUCCESS: Connection to RDS mysql instance succeeded")
s3_client = boto3.client('s3')
def handler(event, context):
bucket = event['Records'][0]['s3']['bucket']['name']
key = event['Records'][0]['s3']['object']['key']
download_path = '/tmp/{}{}'.format(uuid.uuid4(), key)
s3_client.download_file(bucket, key,download_path)
csv_data = csv.reader(file( download_path))
with conn.cursor() as cur:
for idx, row in enumerate(csv_data):
logger.info(row)
try:
cur.execute('INSERT INTO target_table(name, age, salary, designation)' \
'VALUES("%s", "%s", "%s", "%s")'
, row)
except Exception as e:
logger.error(e)
if idx % 100 == 0:
conn.commit()
conn.commit()
return 'File loaded into RDS:' + str(download_path)

Here is the code which is working for me now:
s3 = boto3.resource('s3')
file_object=event['Records'][0]
key=str(file_object['s3']['object']['key'])
obj = s3.Object(bucket, key)
content_lines=obj.get()['Body'].read().decode('utf-8').splitlines(True)
tableName= key.strip('folder/').strip('.csv')
with conn.cursor() as cur:
try:
cur.execute('TRUNCATE TABLE '+tableName)
except Exception as e:
print("ERROR: Unexpected error:Table does not exit.")
sys.exit()
header=True
for row in csv.reader(content_lines):
if(header):
numberOfColumns=len(row)
columnNames= str(row).replace('[','').replace(']','').replace("'",'')
print("columnNames:"+columnNames)
values='%s'
numberOfValues=len(values)
numberOfValues=1
while numberOfValues< numberOfColumns:
values=values+",%s"
numberOfValues+=1
print("INSERT into "+tableName+"("+columnNames+") VALUES("+values+")")
header=False
else:
try:
cur.execute('INSERT into '+tableName+'('+columnNames+') VALUES('+values+')', row)
except Exception as e:
raise e
conn.commit()

Storing and retrieving zip files in SQLite gives "Could not decode to UTF-8"

I have a table in SQLite 3 as follows and I am planning on using it to store a variety of files: txt, pdf, images and zip files.
CREATE TABLE zip (filename TEXT PRIMARYKEY NOT NULL, zipfile BLOB NOT NULL);
To store and retrieve I am experimenting with the following python code
#!env/bin/python
import sqlite3 as lite
import os
import sys
def insertfile(_filename):
try:
con = lite.connect('histogram.db', detect_types=lite.PARSE_DECLTYPES)
con.row_factory = lite.Row
cur = con.cursor()
cur.execute('PRAGMA foreign_keys=ON;')
_f = open(_filename,'rb')
_split = os.path.split(_filename)
_file = _split[1]
_blob = _f.read()
cur.execute('INSERT INTO zip (filename,zipfile) VALUES (?,?)', (_file,lite.Binary(_blob)))
_f.close()
con.commit()
cur.close()
con.close()
except Exception as ex:
print ex
def getfile(_filename):
try:
con = lite.connect('histogram.db', detect_types=lite.PARSE_DECLTYPES)
con.row_factory = lite.Row
cur = con.cursor()
cur.execute('PRAGMA foreign_keys=ON;')
cur.execute('SELECT zipfile from zip where filename = ?', (_filename,))
_files = cur.fetchall()
if len(_files) > 0:
_file = open('Test/'+ _filename,'wb')
_file.write(_files[0]['zipfile'])
_file.close()
cur.close()
con.close()
except Exception as ex:
print ex
if __name__ == '__main__':
print 'works'
insertfile(sys.argv[1])
getfile(os.path.split(sys.argv[1])[1])
When I test this on files like .txt, .py, .pdf etc., it works fine.
With Zip files, there is no error while storing into the table but an error while retrieving the file:
Could not decode to UTF-8 column 'zipfile' with text 'PK '
There seems to be some encoding or decoding issue.

I basically tried using the code from one of the questions
Insert binary file in SQLite database with Python
.
It worked originally for the pdf, png, jpg files. But I was still getting the error for Zip files. When I commented out the insertion and just ran the retrieval code it worked. Now the code below works.
def insertfile(_filename):
try:
con = lite.connect('histogram.db', detect_types=lite.PARSE_DECLTYPES)
con.row_factory = lite.Row
cur = con.cursor()
cur.execute('PRAGMA foreign_keys=ON;')
_f = open(_filename,'rb')
_split = os.path.split(_filename)
_file = _split[1]
_blob = _f.read()
cur.execute('INSERT INTO zip (filename,zipfile) VALUES (?,?)', (_file,lite.Binary(_blob)))
_f.close()
con.commit()
cur.close()
con.close()
except Exception as ex:
print ex
def getfile(_filename):
try:
con = lite.connect('histogram.db', detect_types=lite.PARSE_DECLTYPES)
con.row_factory = lite.Row
cur = con.cursor()
cur.execute('PRAGMA foreign_keys=ON;')
cur.execute('SELECT zipfile from zip where filename = ?', (_filename,))
_files = cur.fetchall()
if len(_files) > 0:
_file = open('Downloads/'+ _filename,'wb')
_file.write(_files[0]['zipfile'])
_file.close()
cur.close()
con.close()
except Exception as ex:
print ex
if __name__ == '__main__':
print 'works'
insertfile(sys.argv[1])
getfile(os.path.split(sys.argv[1])[1])

Call mysql stored procedure in python and write the result into a csv file

im new to python and Im trying to define a function in python that calls a stored procedure in python using mysql.connector and write the result into a csv file.
i have defined a function to call the stored procedure and print the result. But couldn't figure out how to write my result into a csv file instead of printing. Tried some methods found in web but didn't take me anywhere
so far my codes are:
def call_accessionsByaffliction():
try:
conn = mysql.connector.connect( host="localhost", user="root",
password="1232580", database="db")
cur = conn.cursor()
cur.callproc('AccessionsBycauseOfAffliction_year')
for result in cur.stored_results():
print(result.fetchall())
except Error as e:
print(e)
finally:
cur.close()
conn.close()
and
if __name__ == '__main__':
call_accessionsByaffliction()
Any help? Much appreciated

import csv
def cs_writer(string_chunk):
# define desired output file name
file_name = "output.csv"
try:
#open file in write mode and hold object
out_file = open(file_name, "wt")
# create csv write object
writer = csv.writer(out_file, delimiter=',', quotechar='"', quoting=csv.QUOTE_ALL)
for line in string_chunk:
writer.writerow(line)
except Exception as e:
print(e)
finally:
# close file
out_file.close()
def call_accessionsByaffliction():
try:
conn = mysql.connector.connect( host="localhost", user="root", password="1232580", database="db")
cur = conn.cursor()
# execute stored procedure
cur.callproc('AccessionsBycauseOfAffliction_year')
# fetch all results or use fetchone()
for res in cur.stored_results():
result = res.fetchall()
print(result)
return result
except Error as e:
print(e)
finally:
cur.close()
conn.close()
if __name__ == '__main__':
cs_writer(call_accessionsByaffliction())
I hope this answers your question.

If you're working with data, I strongly recommend looking into Pandas. Also, for forums like this, it's best to leave out __main__ checks. Here's an untested code which does the things you're trying to do in Pandas.
import pandas
connection_string = "'mysql://root:1232580#localhost/db'"
df = pandas.read_sql_query('EXEC AccessionsBycauseOfAffliction_year', connection_string)
df.to_csv('AccessionsBycauseOfAffliction_year.csv')

if you are able to find SP, for calling you can use following code
Please give more formation that you want code only for storing result in csv??
from mysql.connector import MySQLConnection, Error
from python_mysql_dbconfig import read_db_config
def call_find_by_isbn():
try:
db_config = read_db_config()
conn = MySQLConnection(**db_config)
cursor = conn.cursor()
args = ['1236400967773', 0]
result_args = cursor.callproc('AccessionsBycauseOfAffliction_year', args)
print(result_args[1])
except Error as e:
print(e)
finally:
cursor.close()
conn.close()
if __name__ == '__main__':
call_find_by_isbn()

Inserting and retrieving images into mysql through python

I used this code to insert an image into mysql database and retrieve back the image.
This code works perfectly with no errors , but the problem is even after inserting an image into img table , when I execute the command select * from img ; in the mysql command line it shows no records.
Table created in database;
create table img(images blob not null);
import mysql.connector
import sys
from PIL import Image
import base64
import cStringIO
import PIL.Image
db = mysql.connector.connect(user='root', password='abhi',
host='localhost',
database='cbir')
#image = Image.open('C:\Users\Abhi\Desktop\cbir-p\images.jpg')
with open("C:\Users\Abhi\Desktop\cbir-p\images.jpg", "rb") as image_file:
encoded_string = base64.b64encode(image_file.read())
#blob_value = open('C:\Users\Abhi\Desktop\cbir-p\images.jpg', 'rb').read()
sql = 'INSERT INTO img(images) VALUES(%s)'
args = (encoded_string, )
cursor=db.cursor()
cursor.execute(sql,args)
sql1='select * from img'
cursor.execute(sql1)
data=cursor.fetchall()
#print type(data[0][0])
data1=base64.b64decode(data[0][0])
file_like=cStringIO.StringIO(data1)
img=PIL.Image.open(file_like)
img.show()
db.close()

import mysql.connector
import sys
from PIL import Image
import base64
import cStringIO
import PIL.Image
db = mysql.connector.connect(user='root', password='abhi',
host='localhost',
database='cbir')
image = Image.open('C:\Users\Abhi\Desktop\cbir-p\images.jpg')
blob_value = open('C:\Users\Abhi\Desktop\cbir-p\images.jpg', 'rb').read()
sql = 'INSERT INTO img(images) VALUES(%s)'
args = (blob_value, )
cursor=db.cursor()
cursor.execute(sql,args)
sql1='select * from img'
db.commit()
cursor.execute(sql1)
data=cursor.fetchall()
print type(data[0][0])
file_like=cStringIO.StringIO(data[0][0])
img=PIL.Image.open(file_like)
img.show()
db.close()
This code works fine

import mysql.connector
import base64
import io
import PIL.Image
with open('lemonyellow_logo.jpg', 'rb') as f:
photo = f.read()
encodestring = base64.b64encode(photo)
db= mysql.connector.connect(user="root",password="lyncup",host="localhost",database="demo")
mycursor=db.cursor()
sql = "insert into sample values(%s)"
mycursor.execute(sql,(encodestring,))
db.commit()
sql1="select * from sample"
mycursor.execute(sql1)
data = mycursor.fetchall()
data1=base64.b64decode(data[0][0])
file_like=io.BytesIO(data1)
img=PIL.Image.open(file_like)
img.show()
db.close()

import mysql.connector
import pyautogui
import time
import base64
def byte_string(image_name):
with open(image_name, 'rb') as image_file:
encoded_string = base64.b64encode(image_file.read())
return encoded_string
def insertBLOB(image_id, image_name, image):
print("Inserting BLOB into my_images table")
try:
connection = mysql.connector.connect(host="localhost",
user="root",
password="Rushi#2001",
database = 'my_images',
port = 3306)
cursor = connection.cursor()
sql_insert_blob_query = " INSERT INTO image_data (image_id, image_name, image) VALUES (%s,%s,%s)"
binary_image = byte_string(image)
print(len(binary_image))
# Convert data into tuple format
insert_blob_tuple = (image_id, image_name, binary_image)
result = cursor.execute(sql_insert_blob_query, insert_blob_tuple)
connection.commit()
print("Image and file inserted successfully as a BLOB into my_images table", result)
except mysql.connector.Error as error:
print("Failed inserting BLOB data into MySQL table {}".format(error))
finally:
if connection.is_connected():
cursor.close()
connection.close()
print("MySQL connection is closed")
insertBLOB(1,'img0',"C:\\Users\\hulag\\OneDrive\\Desktop\\python_sql\\images\\img0.png")