I am trying to extract images from a location to (mysql) ID location, it creates the folders according to the mysql table ID but no images are copied.( extracting images from one location and copying them on the ID location)
here is my script.
import datetime
import sys
import os
import MySQLdb
db_driver = "mysql"
host = "localhost"
db = "customer_1"
user = "root"
passwd = "H0t"
db = MySQLdb.connect(host='localhost', user='root', passwd='H0t', db='customer_1')
cursor = db.cursor()
cam_name = sys.argv[1]
topLevel = "/var/wwwdev/cam_images/%s_anpr_vega" % cam_name
sql = """select id,image FROM %s_anpr_vega WHERE image IS NOT NULL LIMIT 20000""" % cam_name
print sql
cursor.execute(sql)
retval = cursor.fetchall()
for values in retval:
(id, image) = values
id_string = "%s" % id
path_string = ""
for i in range(len(id_string)):
path_string = "%s/%s" % (path_string, id_string[i])
imdir = "%s%s" % (topLevel, path_string)
try:
os.makedirs(imdir)
except:
pass
image_path = "%s.jpg" % imdir
print image_path
fp = open(image_path, "w")
fp.write(image)
fp.close();
sql = """UPDATE %s_anpr_vega SET image=NULL WHERE id=%s""" % (cam_name, id)
print sql
cursor.execute(sql)
sql = """OPTIMIZE TABLE %s_anpr_vega """ % cam_name
print sql
#cursor.execute(sql)
Are you sure this isn't just creating image files named ".jpg" in each folder?
It looks to me like you forgot the filename portion of your image_path. You should see that in the prints coming out of that code.
If you're using Windows Explorer or the like you might not notice the dot-named JPEG files.
Related
I have written a little python script to get files in a directory, get a hash and then write them to a table.
The first part, getting the files and calculating the hash was easy. But now I added the function (write_record) to store the filename, log date and hash to a database. But I am struggling how to call it form the get_files function an write a record for each file in the directory
from datetime import datetime
from os import scandir
import os
import hashlib
import psycopg2
BLOCKSIZE = 65536
hasher = hashlib.sha256()
basepath = '.'
def convert_date(timestamp):
d = datetime.utcfromtimestamp(timestamp)
formated_date = d.strftime('%d%m%Y%H%M%S')
return formated_date
def get_hash(entry):
with open(entry, 'rb') as afile:
buf = afile.read(BLOCKSIZE)
while len(buf) > 0:
hasher.update(buf)
buf = afile.read(BLOCKSIZE)
# print(hasher.hexdigest())
def get_files():
dir_entries = scandir('.')
for entry in dir_entries:
if entry.is_file():
info = entry.stat()
print(' %s %s %s' % (entry.name, convert_date(info.st_mtime),hasher.hexdigest()))
log_filename = entry.name
log_hashvalue = hasher.hexdigest()
log_date = convert_date(info.st_mtime)
return log_filename,log_hashvalue,log_date
# write_record()
def write_record():
log_filename,log_hashvalue,log_date = get_files()
try:
print(log_filename,log_hashvalue,log_date)
connection = psycopg2.connect(user="postgres",password="xxxxxxxx",host="xxx.xxx.xxx.xxx",port="5432",database="evidence_logging")
cursor = connection.cursor()
postgres_insert_query = """ INSERT INTO logfiles (log_name,log_date,log_hashvalue) VALUES (%s,%s,%s)"""
record_to_insert = (log_filename,log_date,log_hashvalue)
print(postgres_insert_query, record_to_insert)
cursor.execute(postgres_insert_query, record_to_insert)
connection.commit()
count = cursor.rowcount
print (count, "Record inserted successfully into logfiles table")
except (Exception, psycopg2.Error) as error :
if(connection):
print("Failed to insert record into logfiles table", error)
finally:
#closing database connection.
if(connection):
cursor.close()
connection.close()
print("PostgreSQL connection is closed")
write_record()
Thanks in advance
Regards
Georg
In your code you are calling write_record() method this will insert only one file beacause get_files() method will return the first file not all the files.
first you need to call get_files() method instead of returning in this method you should call write_record() method with the values you are returning from get_files().
And do not close the connection after insertion of every record close the connection after insertion of all the records.
try this
from datetime import datetime
from os import scandir
import os
import hashlib
import psycopg2
BLOCKSIZE = 65536
hasher = hashlib.sha256()
basepath = '.'
connection = None
def convert_date(timestamp):
d = datetime.utcfromtimestamp(timestamp)
formated_date = d.strftime('%d%m%Y%H%M%S')
return formated_date
def get_hash(entry):
with open(entry, 'rb') as afile:
buf = afile.read(BLOCKSIZE)
while len(buf) > 0:
hasher.update(buf)
buf = afile.read(BLOCKSIZE)
# print(hasher.hexdigest())
def get_files():
dir_entries = scandir('.')
for entry in dir_entries:
if entry.is_file():
info = entry.stat()
print(' %s %s %s' % (entry.name, convert_date(info.st_mtime),hasher.hexdigest()))
log_filename = entry.name
log_hashvalue = hasher.hexdigest()
log_date = convert_date(info.st_mtime)
write_record(log_filename,log_hashvalue,log_date)
#close the connection after writing all records
close_connection()
def write_record(log_filename,log_hashvalue,log_date):
try:
print(log_filename,log_hashvalue,log_date)
connection = psycopg2.connect(user="postgres",password="xxxxxxxx",host="xxx.xxx.xxx.xxx",port="5432",database="evidence_logging")
cursor = connection.cursor()
postgres_insert_query = """ INSERT INTO logfiles (log_name,log_date,log_hashvalue) VALUES (%s,%s,%s)"""
record_to_insert = (log_filename,log_date,log_hashvalue)
print(postgres_insert_query, record_to_insert)
cursor.execute(postgres_insert_query, record_to_insert)
connection.commit()
count = cursor.rowcount
print (count, "Record inserted successfully into logfiles table")
except (Exception, psycopg2.Error) as error :
if(connection):
print("Failed to insert record into logfiles table", error)
finally:
cursor.close()
def close_connection():
if(connection):
connection.close()
print("PostgreSQL connection is closed")
get_files()
I am exploring pytest to automate test activities. I'd like to pass var values store in one .py file, which is outside 'TestCases' folder. JSON body is:
req_json= "{ \r\n\"ReqParamList\": { \r\n \"StdValue\": \"age#name\" \r\n } \r\n}"
resp = requests.request("POST", cURL, data=req_json, headers=headers, params=eid)
Age and name are fetched from DB and stored in as var in a separate py file
import mysql.connector
from mysql.connector import errors
try:
cnx = mysql.connector.connect(
host="hostname",
port=port,
database="dbname",
user="uname",
connection_timeout=5,
password="pw")
# Fname
FnameCur = cnx.cursor()
RandomFname = ("select fname from firstnames limit 1")
FnameCur .execute(RandomFname)
for row in FnameCur :
fname = (row[0])
print("fname is:", fname)
# Lname
LnameCur = cnx.cursor()
RandomLname = ("select Lname from lasttnames where Lname in (select "
"Lname from (select Lname from lasttnames ORDER BY RAND("
") limit 1)t)")
LnameCur.execute(RandomLname )
for row in LnameCur:
Lname = (row[0])
print ("Lname is:",Lname)
except Error as e:
print("Error", e)
finally:
if (cnx.is_connected()):
fnameCur .close()
LnameCur.close()
Could anyone please help me with this?
i want to add the result of console in my database when i put :
# Do ALPR processing of selected image
results = alpr.recognize_ndarray(speed_image)
for i, plate in enumerate(results['results']):
best_candidate = plate['candidates'][0]
print('Plate #{}: {:7s} ({:.2f}%)'.format(i, best_candidate['plate'].upper(), best_candidate['confidence']))
# update speed_cam.db to indicate image processing has been done
sql_cmd = '''UPDATE speed SET status=" i need to add ressult of print here " WHERE idx="{}"'''.format(row_index)
db_conn.execute(sql_cmd)
db_conn.commit()
#!/usr/bin/env python
import numpy as np
import cv2
from openalpr import Alpr
import sqlite3
import sys
import time
import os
# WINDOW_NAME = 'openalpr'
DB_FILE = '/home/pi/speed-camera/data/speed_cam.db'
SPEED_DIR = '/home/pi/speed-camera' # path to speed-camera folder
alpr = Alpr('tw', 'tx2.conf', '/usr/local/share/openalpr/runtime_data')
if not alpr.is_loaded():
print('Error loading OpenALPR')
sys.exit(1)
alpr.set_top_n(3)
#alpr.set_default_region('new')
# Connect to speed_cam.db
try:
db_conn = sqlite3.connect(DB_FILE)
except sqlite3.Error as err_msg:
logging.error("Failed: sqlite3 Connect to DB %s", DB_FILE)
logging.error("Error Msg: %s", err_msg)
sys.exit(1)
# setup cursor for processing db query rows
db_conn.row_factory = sqlite3.Row
cursor = db_conn.cursor()
while True:
# run sql query to select unprocessed images from speed_cam.db
cursor.execute("SELECT idx, image_path FROM speed WHERE status=''")
while True:
row = cursor.fetchone()
if row is None:
break
row_index = (row["idx"])
row_path = (row["image_path"])
# create full path to image file to process
image_path = os.path.join(SPEED_DIR, row_path)
speed_image = cv2.imread(image_path)
# This may have to be tweaked since image is from a file.
print('Processing %s' % image_path)
# Do ALPR processing of selected image
results = alpr.recognize_ndarray(speed_image)
for i, plate in enumerate(results['results']):
best_candidate = plate['candidates'][0]
# Could add to a database table eg speed_cam.db plate table image_path, plate columns
print('Plate #{}: {:7s} ({:.2f}%)'.format(i, best_candidate['plate'].upper(),
best_candidate['confidence']))
# update speed_cam.db to indicate image processing has been done
sql_cmd = '''UPDATE speed SET status="done" WHERE idx="{}"'''.format(row_index)
db_conn.execute(sql_cmd)
db_conn.commit()
print('Waiting 30 seconds')
time.sleep(30)
db_conn.close()
alpr.unload()
I have a table in SQLite 3 as follows and I am planning on using it to store a variety of files: txt, pdf, images and zip files.
CREATE TABLE zip (filename TEXT PRIMARYKEY NOT NULL, zipfile BLOB NOT NULL);
To store and retrieve I am experimenting with the following python code
#!env/bin/python
import sqlite3 as lite
import os
import sys
def insertfile(_filename):
try:
con = lite.connect('histogram.db', detect_types=lite.PARSE_DECLTYPES)
con.row_factory = lite.Row
cur = con.cursor()
cur.execute('PRAGMA foreign_keys=ON;')
_f = open(_filename,'rb')
_split = os.path.split(_filename)
_file = _split[1]
_blob = _f.read()
cur.execute('INSERT INTO zip (filename,zipfile) VALUES (?,?)', (_file,lite.Binary(_blob)))
_f.close()
con.commit()
cur.close()
con.close()
except Exception as ex:
print ex
def getfile(_filename):
try:
con = lite.connect('histogram.db', detect_types=lite.PARSE_DECLTYPES)
con.row_factory = lite.Row
cur = con.cursor()
cur.execute('PRAGMA foreign_keys=ON;')
cur.execute('SELECT zipfile from zip where filename = ?', (_filename,))
_files = cur.fetchall()
if len(_files) > 0:
_file = open('Test/'+ _filename,'wb')
_file.write(_files[0]['zipfile'])
_file.close()
cur.close()
con.close()
except Exception as ex:
print ex
if __name__ == '__main__':
print 'works'
insertfile(sys.argv[1])
getfile(os.path.split(sys.argv[1])[1])
When I test this on files like .txt, .py, .pdf etc., it works fine.
With Zip files, there is no error while storing into the table but an error while retrieving the file:
Could not decode to UTF-8 column 'zipfile' with text 'PK '
There seems to be some encoding or decoding issue.
I basically tried using the code from one of the questions
Insert binary file in SQLite database with Python
.
It worked originally for the pdf, png, jpg files. But I was still getting the error for Zip files. When I commented out the insertion and just ran the retrieval code it worked. Now the code below works.
def insertfile(_filename):
try:
con = lite.connect('histogram.db', detect_types=lite.PARSE_DECLTYPES)
con.row_factory = lite.Row
cur = con.cursor()
cur.execute('PRAGMA foreign_keys=ON;')
_f = open(_filename,'rb')
_split = os.path.split(_filename)
_file = _split[1]
_blob = _f.read()
cur.execute('INSERT INTO zip (filename,zipfile) VALUES (?,?)', (_file,lite.Binary(_blob)))
_f.close()
con.commit()
cur.close()
con.close()
except Exception as ex:
print ex
def getfile(_filename):
try:
con = lite.connect('histogram.db', detect_types=lite.PARSE_DECLTYPES)
con.row_factory = lite.Row
cur = con.cursor()
cur.execute('PRAGMA foreign_keys=ON;')
cur.execute('SELECT zipfile from zip where filename = ?', (_filename,))
_files = cur.fetchall()
if len(_files) > 0:
_file = open('Downloads/'+ _filename,'wb')
_file.write(_files[0]['zipfile'])
_file.close()
cur.close()
con.close()
except Exception as ex:
print ex
if __name__ == '__main__':
print 'works'
insertfile(sys.argv[1])
getfile(os.path.split(sys.argv[1])[1])
I used this code to insert an image into mysql database and retrieve back the image.
This code works perfectly with no errors , but the problem is even after inserting an image into img table , when I execute the command select * from img ; in the mysql command line it shows no records.
Table created in database;
create table img(images blob not null);
import mysql.connector
import sys
from PIL import Image
import base64
import cStringIO
import PIL.Image
db = mysql.connector.connect(user='root', password='abhi',
host='localhost',
database='cbir')
#image = Image.open('C:\Users\Abhi\Desktop\cbir-p\images.jpg')
with open("C:\Users\Abhi\Desktop\cbir-p\images.jpg", "rb") as image_file:
encoded_string = base64.b64encode(image_file.read())
#blob_value = open('C:\Users\Abhi\Desktop\cbir-p\images.jpg', 'rb').read()
sql = 'INSERT INTO img(images) VALUES(%s)'
args = (encoded_string, )
cursor=db.cursor()
cursor.execute(sql,args)
sql1='select * from img'
cursor.execute(sql1)
data=cursor.fetchall()
#print type(data[0][0])
data1=base64.b64decode(data[0][0])
file_like=cStringIO.StringIO(data1)
img=PIL.Image.open(file_like)
img.show()
db.close()
import mysql.connector
import sys
from PIL import Image
import base64
import cStringIO
import PIL.Image
db = mysql.connector.connect(user='root', password='abhi',
host='localhost',
database='cbir')
image = Image.open('C:\Users\Abhi\Desktop\cbir-p\images.jpg')
blob_value = open('C:\Users\Abhi\Desktop\cbir-p\images.jpg', 'rb').read()
sql = 'INSERT INTO img(images) VALUES(%s)'
args = (blob_value, )
cursor=db.cursor()
cursor.execute(sql,args)
sql1='select * from img'
db.commit()
cursor.execute(sql1)
data=cursor.fetchall()
print type(data[0][0])
file_like=cStringIO.StringIO(data[0][0])
img=PIL.Image.open(file_like)
img.show()
db.close()
This code works fine
import mysql.connector
import base64
import io
import PIL.Image
with open('lemonyellow_logo.jpg', 'rb') as f:
photo = f.read()
encodestring = base64.b64encode(photo)
db= mysql.connector.connect(user="root",password="lyncup",host="localhost",database="demo")
mycursor=db.cursor()
sql = "insert into sample values(%s)"
mycursor.execute(sql,(encodestring,))
db.commit()
sql1="select * from sample"
mycursor.execute(sql1)
data = mycursor.fetchall()
data1=base64.b64decode(data[0][0])
file_like=io.BytesIO(data1)
img=PIL.Image.open(file_like)
img.show()
db.close()
import mysql.connector
import pyautogui
import time
import base64
def byte_string(image_name):
with open(image_name, 'rb') as image_file:
encoded_string = base64.b64encode(image_file.read())
return encoded_string
def insertBLOB(image_id, image_name, image):
print("Inserting BLOB into my_images table")
try:
connection = mysql.connector.connect(host="localhost",
user="root",
password="Rushi#2001",
database = 'my_images',
port = 3306)
cursor = connection.cursor()
sql_insert_blob_query = " INSERT INTO image_data (image_id, image_name, image) VALUES (%s,%s,%s)"
binary_image = byte_string(image)
print(len(binary_image))
# Convert data into tuple format
insert_blob_tuple = (image_id, image_name, binary_image)
result = cursor.execute(sql_insert_blob_query, insert_blob_tuple)
connection.commit()
print("Image and file inserted successfully as a BLOB into my_images table", result)
except mysql.connector.Error as error:
print("Failed inserting BLOB data into MySQL table {}".format(error))
finally:
if connection.is_connected():
cursor.close()
connection.close()
print("MySQL connection is closed")
insertBLOB(1,'img0',"C:\\Users\\hulag\\OneDrive\\Desktop\\python_sql\\images\\img0.png")