recently I try to find the right way to read LSUN dataset which is in the form of lmdb. However, I do not find any useful information. I want to know how to read image data from lmdb and what the advantage is in that way. Thank you!
Finally, I use the following code to extract LUSN images from lmbd file.
import os
import lmdb
from PIL import Image
import tempfile
def _export_mdb_images(db_path, out_dir=None, flat=True, limit=-1, size=256):
out_dir = out_dir
env = lmdb.open(
db_path, map_size=1099511627776,
max_readers=1000, readonly=True
)
count = 0
with env.begin(write=False) as txn:
cursor = txn.cursor()
for key, val in cursor:
key = str(key, 'utf-8')
# decide image out directory
if not flat:
image_out_dir = os.path.join(out_dir, '/'.join(key[:6]))
else:
image_out_dir = out_dir
# create the directory if an image out directory doesn't exist
if not os.path.exists(image_out_dir):
os.makedirs(image_out_dir)
with tempfile.NamedTemporaryFile('wb') as temp:
temp.write(val)
temp.flush()
temp.seek(0)
image_out_path = os.path.join(image_out_dir, key + '.jpg')
Image.open(temp.name).resize((size, size)).save(image_out_path)
count += 1
if count == limit:
break
if count % 1000 == 0:
print('Finished', count, 'images')
print("start")
db_path = "path to lmbd"
out_dir = os.path.join(db_path, "data")
_export_mdb_images(db_path, out_dir)
Related
My code was working just fine before adding the hash function. I was getting the list of all folders and files in my directory in the Pretty Table. Once I added the hash function, I got maybe 5 of the files in that directory with hashes in the table. I am not sure where I have gone wrong. Please forgive me, I am new to this. We are not learning to code from scratch, but have to modify existing codes to function the way we need it to.
# Python Standard Libaries
import os #file system methode
import hashlib #hashing function
import sys #system methods
import time #time conversions
# Python 3rd Party Libraries
from prettytable import PrettyTable # pip install prettytable
# Local Functions
def GetFileMetaData(fileName):
#obtain file system metadata
try:
metaData = os.stat(fileName) # Use the stat method to obtain meta data
fileSize = metaData.st_size # Extract fileSize and MAC Times
timeLastAccess = metaData.st_atime
timeLastModified = metaData.st_mtime
timeCreated = metaData.st_ctime
macTimeList = [timeLastModified, timeCreated, timeLastAccess] # Group the MAC Times in a List
return True, None, fileSize, macTimeList
except Exception as err:
return False, str(err), None, None
# Psuedo Constants
# Start of the Script
tbl = PrettyTable(['FilePath','FileSize','UTC-Modified', 'UTC-Accessed', 'UTC-Created', 'SHA-256 HASH'])
#file check
while True:
targetFolder = input("Enter Target Folder: ")
if os.path.isdir(targetFolder):
break
else:
print("\nInvalid Folder ... Please Try Again")
print("Walking: ", targetFolder, "\n")
print()
for currentRoot, dirList, fileList in os.walk(targetFolder):
for nextFile in fileList:
fullPath = os.path.join(currentRoot, nextFile)
absPath = os.path.abspath(fullPath)
fileSize = os.path.getsize(absPath)
success, errInfo, fileSize, macList = GetFileMetaData(absPath)
if success:
#convert to readable Greenich Time
modTime = time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime(macList[0]))
accTime = time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime(macList[1]))
creTime = time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime(macList[2]))
#hashing function
with open(absPath, 'rb') as target:
fileContents = target.read()
sha256Obj = hashlib.sha256()
sha256Obj.update(fileContents)
hexDigest = sha256Obj.hexdigest()
tbl.add_row( [ absPath, fileSize,modTime, accTime, creTime, hexDigest] )
tbl.align = "l" # align the columns left justified
# display the table
print (tbl.get_string(sortby="FileSize", reversesort=True))
print("\nScript-End\n")
I am saving my video frames with below code but I want to add save_path to file name as well:
cv2.imwrite(f"{save_path}/{idx}.png", frame)
How could I add save_path to file name?
I need it as:
save_path/idxsavepath
since we lack the debugging info, here is a piece of code you should have written to avoid path problems. first, check that the directory is found and only then continue.
import os
import cv2
import numpy as np
def main():
# before continuing, must check if folder exists
save_dir = r'D:\pics'
valid_path = os.path.exists(save_dir)
print('is {} a valid path ? {}'.format(save_dir, valid_path))
if not valid_path:
print('folder {} does not exist - create it and try again'.format(save_dir))
else:
idx = 0
fake_frame = np.zeros(shape=(480, 640, 3))
image_full_path = '{}/{}.png'.format(save_dir, idx)
cv2.imwrite(image_full_path, fake_frame)
print('image saved successfully on {}'.format(image_full_path))
return
if __name__ == '__main__':
main()
I want to resize images in the folder and store them into database with filesystem using postgres as database and jupyter notebook. I am using resize function for resizing my images and then saving then into database but resize function seems not working and unable to understand my mistake.
subject= input("Enter Subject Name:")
cursor.execute("DROP TABLE IF EXISTS %s"%(subject))
cursor.execute( """CREATE TABLE %s (ID SERIAL PRIMARY KEY, PHOTO BYTEA NOT NULL)"""%(subject))
conn.commit()
userfilepath=input("enter file path:")
dirs = os.listdir( userfilepath )
def resize():
for item in dirs:
if os.path.isfile(userfilepath+item):
im = Image.open(userfilepath+item)
f, e = os.userfilepath.splitext(userfilepath+item)
imResize = im.resize((200,200), Image.ANTIALIAS)
imResize.save(f + ' resized.jpg', 'JPEG', quality=90)
import cv2
import os, sys
from PIL import Image
import io
import glob
img_dir = userfilepath # Enter Directory of all images
data_path = os.path.join(img_dir,'*g')
files = glob.glob(data_path)
data = []
for f1 in files:
# img = cv2.imread(f1)
# data.append(img)
with open(f1,"rb") as file:
resize()
BinaryData=file.read()
cursor.execute("INSERT INTO {tab} (photo)
VALUES({})".format(psycopg2.Binary(BinaryData, ) , tab=subject ) )
conn.commit()
#Insert_blob(img_dir)
Feel free to change the variables at the top since you seem to want to get them from user input. I've chosen to hardcode them for the purpose of this example.
Have a look at the code below
import os
userfilepath = "files"
dirs = os.listdir("files")
def do_open(_file):
print(f"Opening {_file}")
def do_resize(_file):
print(f"resizing: {_file}")
def resize():
for item in dirs:
f = userfilepath + item
print(f)
if os.path.isfile(f):
im = do_open(userfilepath + item)
resize()
# output
# filesf1.txt
# filesf2.txt
def resize_well():
for item in dirs:
f = os.path.join(userfilepath, item)
print(f)
if os.path.isfile(f):
im = do_open(f)
resize_well()
# output
# files\f1.txt
# Opening files\f1.txt
# files\f2.txt
# Opening files\f2.txt
In resize_well() os.path.join() creates a proper path, where as using string concatenation misses out the / or \ delimiter on linux and windows respectively.
Your code isn't passing the if statement because userfilepath+item doesn't exist, but probably userfilepath/item does.
after having concatenated 10 strips of the same image, I want to convert them into reflectance and therefore divide them by 10,000. Nevertheless I have two types of files in my folders, except I want to apply my code only to my.img file and not to the.hdr...
Do you know how I can proceed to make this selection with os.listdir?
my code is as follows :
import os
import spectral as sp
import spectral.io.envi as envi
src_directory = "/d/afavro/Bureau/3_stack/"
dossier = os.listdir (src_directory)
print(dossier)
for fichier in dossier:
print (fichier)
ssrc_directory = "/d/afavro/Bureau/3_stack/" + fichier
rasters = os.listdir (ssrc_directory)
print(rasters)
OUTPUT_FOLDER = "/d/afavro/Bureau/4_reflectance/" + 'reflectance_' + fichier
print(OUTPUT_FOLDER)
if not os.path.exists(OUTPUT_FOLDER):
os.makedirs(OUTPUT_FOLDER)
for image in rasters:
print (image)
img = sp.open_image(image)
print("%s opened successfully" %os.path.basename(image))
im_HS = img[:,:,:]/10000
header = envi.read_envi_header('/d/afavro/Bureau/3_stack/'+ image)
#Save index image
sp.envi.save_image(OUTPUT_FOLDER + '/reflectance_' + image, im_HS, metadate = header, force = True, interleave = 'bsq')
I think that making a yew loop would be a good idea but I don't know how to do it...
Ideas ?
Find the extension of the file using os.path.splitext
for f in os.listdir('<path>'):
name, ext = os.path.splitext(f)
if ext == '.img':
#do stuff
Why don't you use glob?
from glob import glob
for f in glob('/your/path/*.img'):
pass # add your code here
Hello You can use use Pathlib as an object oriented Path management library
and do something like
from Pathlib2 import Path
pattern_1 = "type1"
pattern_2 = "type2"
list_pattern_1_files = list(Path(<YOUR_PATH>).glob(f'**/*.{pattern_1}'))
list_pattern_2_files = list(Path(<YOUR_PATH>).glob(f'**/*.{pattern_2}'))
Preface: this is required for a class, I know ECB should not be used.
I am trying to encrypt images using AES and then display the images
Steps needed:
Read the image,
Convert to byte object,
Pad the bytes,
Encrypt the bytes,
Convert back to image object,
Save as image file
This is my code right now:
from PIL import Image
from Crypto.Cipher import AES
from Crypto import Random
img = Image.open("photo.jpg")
img.tobytes()
key = '0123456789abcdef'
mode = AES.MODE_ECB
encryptor = AES.new(key, mode)
img.frombytes("RGB")
At this point I am stuck. I am getting a "not enough image data" error on the line "img.frombytes("RGB"), and am also stuck at the part to pad the bytes
So I needed a way to transfer files in the form of images (don't ask me why), if you just want to transfer text you can maybe create a txt file.
This is probably not exactly the best solution to the question as you probably want a way to hide data inside an existing image but I would like anyway to share the code in case it will help someone sometime somewhere.
Basically this will create an image with a size dependent on the file size and will put a sequence of 3 bytes in one pixel (RGB)
So I wrote a small folder2ImageEncoder.py
(it will encrypt all the data that is located in a folder named "files" by default)
from PIL import Image
from pathlib import Path
encryptedImagesFolder = 'encryptedImagesFolder'
Path(f"./{encryptedImagesFolder}").mkdir(parents=True, exist_ok=True)
newLine = b'\new\n\rL'
def encode_data_to_image(data: bytes, imagePath: str):
data += b'FINISH_OF_DATA'
data = str(
{
'path': imagePath,
'data': data
}
)
data = data.encode()
n = int((len(data)/3)**0.5) + 1
print(n, len(data))
img = Image.new('RGB', (n, n))
# data = img.getdata()
encryptedPixelsList = []
pixel = []
if len(data) % 3 != 0:
data += (3 - (len(data) % 3)) * b'\x00'
for i, Byte in enumerate(data):
if i % 3 == 2:
pixel.append(Byte)
encryptedPixelsList.append(tuple(pixel))
pixel = []
else:
pixel.append(Byte)
for _ in range(len(encryptedPixelsList), n**2):
encryptedPixelsList.append((0, 0, 0))
img.putdata(encryptedPixelsList)
imagePath = imagePath.replace('\\', '_')
img.save(f'./{encryptedImagesFolder}/{imagePath}.png')
# img.show()
def encode_folder(folder: Path):
for file in folder.iterdir():
if not file.is_dir():
with open(str(file), 'rb') as rb:
data = rb.readlines()
encode_data_to_image(
data=newLine.join(data),
imagePath=str(file))
else:
encode_folder(folder=file)
if __name__ == '__main__':
# ./files is the name of the folder you want to encrypt
encode_folder(folder=Path(r'./files'))
and a Image2FilesDecoder.py
this will iterate through the encrypted images folder and will retrieve its former form (run this in another folder with the encrypted images folder so it won't override the original files folder)
from PIL import Image
from pathlib import Path
newLine = b'\new\n\rL'
def decode_encrypted_images(folder: Path):
for pic in folder.iterdir():
img = Image.open(str(pic))
data = img.getdata()
totalData = []
for pixel in data:
totalData.extend(list(pixel))
decryptedData = bytes(totalData)
try:
decryptedData = eval(
decryptedData[:decryptedData.rfind(b'}')+1].decode())
except:
decryptedData.replace(b'\\', '')
decryptedData = eval(
decryptedData[:decryptedData.rfind(b'}')+1].decode())
decryptedData['data'] = decryptedData['data'][:-14]
filePathObj = Path(decryptedData['path'])
Path(filePathObj.parent).mkdir(
parents=True, exist_ok=True)
writeBytes = decryptedData['data'].split(newLine)
with open(str(filePathObj), 'wb') as wb:
wb.writelines(writeBytes)
if __name__ == '__main__':
decode_encrypted_images(folder=Path(
r".\encryptedImagesFolder"))