Related
I'm doing the basic lasagne example:
https://github.com/Lasagne/Lasagne/blob/master/examples/mnist.py
which I've modified slightly by combining it with another similar example.
I'm trying to run the CNN model, where I've added some extra input to the CNN def, but it shouldn't be any different. Also changed the default value of 28 for the input layer in the example to 60 (for height and width), classes is being used later on in the code, but the code "hangs" on at that last network line, meaning the code is still running, but nothing is happening.
Output when running the code. The input_var is defined as such in the mainloop:
input_var = T.tensor4('input_var')
rest of the code:
def build_cnn(classes, height, width, input_var=None):
print("Input layer, with height: {}, width: {} and input var: {}".format(height, width, input_var))
network = lasagne.layers.InputLayer(shape = (None, 1, height, width),
input_var=input_var)
print("Convolutional layer with 32 kernels of size 5x5")
network = lasagne.layers.Conv2DLayer(network,
num_filters = 32,
filter_size = (5, 5),
nonlinearity = lasagne.nonlinearities.rectify,
W = lasagne.init.HeNormal(gain = 'relu'))
EDIT:
Okay, based on what I've tried so far it seems to be my own dataset that is the problem. I've reshaped my dataset to match the MNIST dataset where for ex. X_train has the shape [images, channel, height, width]. Where channel = 1 and height, width = 60. The code for retrieving these is:
def load_images():
dataset_path = os.path.abspath("C:/Users/laende/Dropbox/Skole UiS/4. semester/Master/Master/data/test_database")
[bilder, label, names] = read_images1(dataset_path, (28, 28))
label = np.array(label)
bilder = bilder / np.float32(256)
bilder = bilder[:, newaxis, :, :]
X_train1, X_test1, Y_train1, Y_test1 = train_test_split(bilder, label, test_size = 0.2)
list_of_labels = list(xrange(max(label) + 1))
classes = len(list_of_labels)
return X_train1, X_test1, Y_train1, Y_test1, classes
where read_images1 is:
def read_images1(path, sz = None, channel = None):
c = 0
X = []
y = []
folder_names = []
for dirname, dirnames, filenames, in os.walk(path):
for subdirname in dirnames:
subject_path = os.path.join(dirname, subdirname)
folder_names.append(subdirname)
for filename in os.listdir(subject_path):
try:
im = cv2.imread(os.path.join(subject_path, filename), cv2.IMREAD_GRAYSCALE)
if (sz is not None):
im = cv2.resize(im, sz)
X.append(np.asarray(im, dtype = np.uint8))
y.append(c)
except IOError, (errno, strerror):
print "I/O error ({0]): {1}".format(errno, strerror)
except:
print "unexpected error:", sys.exc_info()[0]
raise
c = c + 1
return [X, y, folder_names]
The code in main which is ran:
def main(model='mlp', num_epochs=100):
# Load the dataset
print("Loading data...")
mnist = 1
if mnist == 1:
classes = 10
X_train, y_train, X_val, y_val, X_test, y_test = load_dataset()
dataset = {
'train': {'X': X_train, 'y': y_train},
'test': {'X': X_test, 'y': y_test}}
shape = dataset['train']['X'][0].shape
else:
X_train, X_test, y_train, y_test, classes = load_images()
dataset = {
'train': {'X': X_train, 'y': y_train},
'test': {'X': X_test, 'y': y_test}}
shape = dataset['train']['X'][0].shape
input_var = T.tensor4('inputs')
target_var = T.ivector('targets')
print("Building model and compiling functions...")
if model == 'mlp':
network = build_mlp(height=int(shape[1]),
width=int(shape[2]),
channel=int(shape[0]),
classes=int(classes),
input_var=input_var)
if mnist = 1 (in main) the code runs fine, if I try with my own dataset it gets stuck at in build_mlp (similar to original problem with cnn):
def build_mlp(classes, channel, height, width, input_var=None):
neurons = int(height * width)
network = lasagne.layers.InputLayer(shape = (None, channel, height, width),
input_var=input_var)
network = lasagne.layers.DropoutLayer(network, p = 0.2)
#Code gets stuck on this point, running forever, doing nothing.
#No error messages received either.
network = lasagne.layers.DenseLayer(
network,
num_units = neurons,
nonlinearity = lasagne.nonlinearities.rectify,
W = lasagne.init.GlorotUniform())
EDIT 2:
Afters struggling with this for a while I found out that the the image resizing done in read_images1() caused the problem:
def read_images1(path, sz = None, channel = None):
c = 0
X = []
y = []
folder_names = []
for dirname, dirnames, filenames, in os.walk(path):
for subdirname in dirnames:
subject_path = os.path.join(dirname, subdirname)
folder_names.append(subdirname)
for filename in os.listdir(subject_path):
try:
im = cv2.imread(os.path.join(subject_path, filename), cv2.IMREAD_GRAYSCALE)
#This part caused the problems.
if (sz is not None):
im = cv2.resize(im, sz)
X.append(np.asarray(im, dtype = np.uint8))
y.append(c)
except IOError, (errno, strerror):
print "I/O error ({0]): {1}".format(errno, strerror)
except:
print "unexpected error:", sys.exc_info()[0]
raise
c = c + 1
return [X, y, folder_names]
If I didn't pass any resizing and used the default image sizes that were in the folders, the neural network was able to compile. Does anyone have any idea why? I updated the read_images1() to this:
def read_images1(path, sz = None, na = False):
"""
:param path: sti til mappe med underliggende mapper tilhørende personer.
:param sz: Størrelse på bildefilene
:return: returnerer liste av bilder, labels og navn
"""
c = 0
X = []
y = []
folder_names = []
for dirname, dirnames, filenames, in os.walk(path):
for subdirname in dirnames:
subject_path = os.path.join(dirname, subdirname)
folder_names.append(subdirname)
for filename in os.listdir(subject_path):
try:
im = cv2.imread(os.path.join(subject_path, filename), cv2.IMREAD_GRAYSCALE)
if (sz is not None):
im = cv2.resize(im, dsize=sz, interpolation = cv2.INTER_LANCZOS4)
if (na == True):
im = im[newaxis, :, :]
X.append(np.asarray(im, dtype = np.uint8))
y.append(c)
except IOError, (errno, strerror):
print "I/O error ({0]): {1}".format(errno, strerror)
except:
print "unexpected error:", sys.exc_info()[0]
raise
c = c + 1
return [X, y, folder_names]
If I run the program with sz = None and na = True, then it works. If any size is given to the sz parameter the code gets stuck trying to compile the neural network again.
Okay, I think I can see a few issues here, not sure which one you're running into...
In read_images1(), X is a python list of numpy arrays. Where does it get converted to a single numpy array? Try adding X = numpy.asarray(X). You also need to reshape it to (n_images, n_channels, width, height) where I assume n_channels=1 for grayscale. The network expects 4D input not 3D.
This code list_of_labels = list(xrange(max(label) + 1)); classes = len(list_of_labels) assumes the labels are sequential numbers from 0 to N. Are they?
Your build_mlp(classes, height, width, input_var=None) is pretty different from the original example build_mlp(input_var=None). The original example clearly works, so whatever is wrong has to do with the differences. One of the big differences is that you keep assigning to the same variable, like this network = lasagne.layers.DenseLayer(network, ...) where the original has different variables for each layer l_hid1 = lasagne.layers.DenseLayer(l_in_drop, ...)
Also, if it hangs during build_mlp() then clearly the problem is not in how you read images. Try using the original version of build_mlp() with your images. Try running it by itself. Skip the image reading, just call build_mlp() with constant parameters.
hoping somebody can provide guidance as to how to compute pairwise hamming distance of a bunch of hashes and then cluster them. I don't care so much as to performance as from looking at what I am doing and what I want to do its going to be slow no matter what and its not something that will be repeatedly run over and over.
So...in a nutshell I had mistakenly erased 1000's of photos off a drive and had no backups (I know...bad practise). Using various tools I was able to recover a very high % of them from the drive but was left with hundreds of 1000's of photos. Due to the techniques used for recovering some of the photos (such as file carving) some of the images are corrupt to various degrees, others were identical copies, and yet others were essentially identical visually but byte for byte were different.
What I am looking at doing to help the situation is the following:
check each image and identify if image file is structurally corrupt or not (done)
generate perceptual hashes (fingerprint) for each image so that images can be compared for similarity and clustered (fingerprinting part is done)
calculate the pairwise distance of the fingerprints
cluster the pairwise distance so that similar images can be viewed together to aide manual cleanup
In the script attached you will notice a couple of places I calculate hashes, I will explain as to not cause confusion...
for images that are supported by PIL I generate three hashes, 1st for original image, 2nd is rotated 90 degrees, and 3rd is rotated 180 degrees. This was done so that when the pairwise calculations are done I can account for images that just vary in orientation.
for raw images not supported by PIL I instead favour hashes that are generated from the extracted embedded preview image. I did this instead of using the raw image because in the case of a corrupt raw image file there was a high chance of probability that the preview image was intact due to its smaller size and thus would be better for identifying if the image is similar to others
other place hashes are generated is during a last ditch effort to identify corrupt raw images. I would compare hashes of the extracted/converted raw image to that of the extracted embedded preview image and if the similarity does not meet a defined threshold it is assumed that there is probably corruption of the raw file as a whole.
What I need guidance on is how to accomplish the following:
take the three hashes I have for each image and calculate hamming pairwise distances
for each image comparison keep only the hamming distance that is most similar
feed the results into scipy hierarchical clustering so that I can group similar images
I am just learning Python so that is part of the my challenge... I think from what I have got from google I can do this by first getting the pairwise distances using scipy.spatial.distance.pdist, then process this to keep the most similar distance for each image comparison, then feed this to a scipy clustering function. But I cannot figure how to organize this and provide things in the proper format etc. Can anyone provide some guidance on this?
Here is my current script for reference in case anyone else finds it interesting that I will need to alter for storing some sort of dictionary of hashes or maybe some sort of on disk storage.
from PIL import Image
from PIL import ImageFile
import os, sys, imagehash, pyexiv2, rawpy, re
from tempfile import NamedTemporaryFile
from subprocess import check_call, call
# allow PIL to load truncated images (so that perceptual hashes can be created for truncated/damaged images still)
ImageFile.LOAD_TRUNCATED_IMAGES = True
# image files this script will handle
# PIL supported image formats
stdimageext = ('.jpg','.jpeg', '.bmp', '.png', '.gif', '.tif', '.tiff')
# libraw/ufraw supported formats
rawimageext = ('.nef', '.dng', '.tif', '.tiff')
devnull = open(os.devnull, 'w')
corruptRegex = re.compile(r'_\[.+\]\..{3,4}$')
for root, dirs, files in os.walk(sys.argv[1]):
for filename in files:
ext = os.path.splitext(filename.lower())[1]
filepath = os.path.join(root, filename)
if ext in (stdimageext + rawimageext):
hashes = [None] * 4
print(filename)
# reset corrupt string
corrupt_str = None
if ext in (stdimageext):
metadata = pyexiv2.ImageMetadata(filepath)
metadata.read()
rotate = 0
try:
im = Image.open(filepath)
except:
None
else:
for x in range(3):
hashes[x] = imagehash.dhash(im.rotate(90 * (x + 1)),32)
# use jpeginfo against all jpg images as its pretty accurate
if ext in ('.jpg','.jpeg'):
rc = 0
rc = call(["jpeginfo", "--check", filepath], stdout=devnull, stderr=devnull)
if rc == 1:
corrupt_str = 'JpegInfo'
if corrupt_str is None:
try:
im = Image.open(filepath)
im.verify()
except:
e = sys.exc_info()[0]
corrupt_str = 'PIL_Verify'
else:
try:
im = Image.open(filepath)
im.load()
except:
e = sys.exc_info()[0]
corrupt_str = 'PIL_Load'
# raw image processing
else:
# extract largest embedded preview image first
metadata_orig = pyexiv2.ImageMetadata(filepath)
metadata_orig.read()
if len(metadata_orig.previews) > 0:
preview = metadata_orig.previews[-1]
# save preview to temp file
temp_preview = NamedTemporaryFile()
preview.write_to_file(temp_preview.name)
os.rename(temp_preview.name + preview.extension, temp_preview.name)
rotate = 0
try:
im = Image.open(temp_preview.name)
except:
None
else:
for x in range(4):
hashes[x] = imagehash.dhash(im.rotate(90 * (x + 1)),32)
# close temp file
temp_preview.close()
# try to load raw using libraw via rawpy first,
# generally if libraw can't load it then ufraw extraction would also fail
try:
with rawpy.imread(filepath) as im:
None
except:
e = sys.exc_info()[0]
corrupt_str = 'Libraw_Load'
else:
# as a final last ditch effort compare perceptual hashes of extracted
# raw and embedded preview to detect possible internal corruption
if len(metadata_orig.previews) > 0:
# extract and convert raw to jpeg image using ufraw
temp_raw = NamedTemporaryFile(suffix='.jpg')
try:
check_call(['ufraw-batch', '--wb=camera', '--rotate=camera', '--out-type=jpg', '--compression=95', '--noexif', '--lensfun=none', '--output=' + temp_raw.name, '--overwrite', '--silent', filepath],stdout=devnull, stderr=devnull)
except:
e = sys.exc_info()[0]
corrupt_str = 'Ufraw-conv'
else:
rhash = imagehash.dhash(Image.open(temp_raw.name),32)
# compare preview with raw image and compute the most similar hamming distance (best)
hamdiff = .0
for h in range(4):
# calculate hamming distance to compare similarity
hamdiff = max((256 - sum(bool(ord(ch1) - ord(ch2)) for ch1, ch2 in zip(str(hashes[h]), str(rhash))))/256,hamdiff)
if hamdiff < .7: # raw file is probably corrupt
corrupt_str = 'hash' + str(round(hamdiff*100,2))
# close temp files
temp_raw.close()
print(hamdiff)
print(rhash)
print(hashes[0])
print(hashes[1])
print(hashes[2])
print(hashes[3])
# prefix file if corruption was detected ensuring that existing files already prefixed are re prefixed
mo = corruptRegex.search(filename)
if corrupt_str is not None:
if mo is not None:
os.rename(filepath,os.path.join(root, re.sub(corruptRegex, '_[' + corrupt_str + ']', filename) + ext))
else:
os.rename(filepath,os.path.join(root, os.path.splitext(filename)[0] + '_[' + corrupt_str + ']' + ext))
else:
if mo is not None:
os.rename(filepath,os.path.join(root, re.sub(corruptRegex, '', filename) + ext))
EDITED
Just want to provide an update with what I came up with in the end that seems to work quite nicely for my intended purpose and maybe it will prove useful for other users in a similar situation. The script can still use some polishing but otherwise all the meat is there. As I am green with respect using Python if anyone see something that can improved greatly please let me know.
The script does the following:
attempts to detect image corruption in terms of file structure using various methods. For raw image formats (NEF, DNG, TIF) sometimes I found that a corrupt image could still load fine so I decided to hash both the preview image and an extracted .jpg of the raw image and compare the hashes and if they were not similar enough I assume that image is corrupted in some form.
create perceptual hashes for each image that could be loaded. Three are created for the base file (original, original rotated 90, original rotated 180). In addition, for raw images an additional 3 hashes were created for the extracted preview image, this was done so that in cases where the raw image was corrupted we would still have hashes based on the full image (assuming the preview is fine).
for images that are identified as corrupt they are renamed with a suffix that indicates that are corrupt and what determined it.
Pairwise hamming distances were computed by comparing hashes against all file pairs and stored in a numpy array.
square form of pairwise distances are fed to fastcluster for clustering
output from fastcluster is used to generate a dendrogram to visualize clusters of similar images
I save the numpy array to disk so that I can later rerun the fastcluster/dendrogram part without recomputing the hashes for each file which is slow. This is something I have to alter the script to allow yet....
from PIL import Image
from PIL import ImageFile
import os, sys, imagehash, pyexiv2, rawpy, re
from tempfile import NamedTemporaryFile
from subprocess import check_call, call
import numpy as np
from scipy.cluster.hierarchy import dendrogram
from scipy.spatial.distance import squareform
import fastcluster
import matplotlib.pyplot as plt
# allow PIL to load truncated images (so that perceptual hashes can be created for truncated/damaged images still)
ImageFile.LOAD_TRUNCATED_IMAGES = True
# image files this script will handle
# PIL supported image formats
stdimageext = ('.jpg','.jpeg', '.bmp', '.png', '.gif', '.tif', '.tiff')
# libraw/ufraw supported formats
rawimageext = ('.nef', '.dng', '.tif', '.tiff')
devnull = open(os.devnull, 'w')
corruptRegex = re.compile(r'_\[.+\]\..{3,4}$')
hashes = []
filelist = []
for root, _, files in os.walk(sys.argv[1]):
for filename in files:
ext = os.path.splitext(filename.lower())[1]
relpath = os.path.relpath(root, sys.argv[1])
filepath = os.path.join(root, filename)
if ext in (stdimageext + rawimageext):
hashes_tmp = []
rhash = []
# reset corrupt string
corrupt_str = None
if ext in (stdimageext):
try:
im=Image.open(filepath)
for x in range(3):
hashes_tmp.append(str(imagehash.dhash(im.rotate(90 * x, expand=1),32)))
except:
None
# use jpeginfo against all jpg images as its pretty accurate
if ext in ('.jpg','.jpeg'):
rc = 0
rc = call(["jpeginfo", "--check", filepath], stdout=devnull, stderr=devnull)
if rc == 1:
corrupt_str = 'JpegInfo'
if corrupt_str is None:
try:
im = Image.open(filepath)
im.verify()
except:
e = sys.exc_info()[0]
corrupt_str = 'PIL_Verify'
else:
try:
im = Image.open(filepath)
im.load()
except:
e = sys.exc_info()[0]
corrupt_str = 'PIL_Load'
# raw image processing
if ext in (rawimageext):
# extract largest embedded preview image first
metadata_orig = pyexiv2.ImageMetadata(filepath)
metadata_orig.read()
if len(metadata_orig.previews) > 0:
preview = metadata_orig.previews[-1]
# save preview to temp file
temp_preview = NamedTemporaryFile()
preview.write_to_file(temp_preview.name)
os.rename(temp_preview.name + preview.extension, temp_preview.name)
try:
im = Image.open(temp_preview.name)
for x in range(3):
hashes_tmp.append(str(imagehash.dhash(im.rotate(90 * x,expand=1),32)))
except:
None
# try to load raw using libraw via rawpy first,
# generally if libraw can't load it then ufraw extraction would also fail
try:
im = rawpy.imread(filepath)
except:
e = sys.exc_info()[0]
corrupt_str = 'Libraw_Load'
else:
# as a final last ditch effort compare perceptual hashes of extracted
# raw and embedded preview to detect possible internal corruption
# extract and convert raw to jpeg image using ufraw
temp_raw = NamedTemporaryFile(suffix='.jpg')
try:
check_call(['ufraw-batch', '--wb=camera', '--rotate=camera', '--out-type=jpg', '--compression=95', '--noexif', '--lensfun=none', '--output=' + temp_raw.name, '--overwrite', '--silent', filepath],stdout=devnull, stderr=devnull)
except:
e = sys.exc_info()[0]
corrupt_str = 'Ufraw-conv'
else:
try:
im = Image.open(temp_raw.name)
for x in range(3):
rhash.append(str(imagehash.dhash(im.rotate(90 * x,expand=1),32)))
except:
None
# compare preview with raw image and compute the most similar hamming distance (best)
if len(hashes_tmp) > 0 and len(rhash) > 0:
hamdiff = 1
for rh in rhash:
# calculate hamming distance to compare similarity
hamdiff = min(hamdiff,(sum(bool(ord(ch1) - ord(ch2)) for ch1, ch2 in zip(hashes_tmp[0], rh))/len(hashes_tmp[0])))
if hamdiff > .3: # raw file is probably corrupt
corrupt_str = 'hash' + str(round(hamdiff*100,2))
hashes_tmp = hashes_tmp + rhash
# prefix file if corruption was detected ensuring that existing files already prefixed are re prefixed
mo = corruptRegex.search(filename)
newfilename = None
if corrupt_str is not None:
if mo is not None:
newfilename = re.sub(corruptRegex, '_[' + corrupt_str + ']', filename) + ext
else:
newfilename = os.path.splitext(filename)[0] + '_[' + corrupt_str + ']' + ext
else:
if mo is not None:
newfilename = re.sub(corruptRegex, '', filename) + ext
if newfilename is not None:
os.rename(filepath,os.path.join(root, newfilename))
if len(hashes_tmp) > 0:
hashes.append(hashes_tmp)
if newfilename is not None:
filelist.append(os.path.join(relpath, newfilename))
else:
filelist.append(os.path.join(relpath, filename))
print(len(filelist))
print(len(hashes))
a = np.empty(shape=(len(filelist),len(filelist)))
for hash_idx1, hash in enumerate(hashes):
a[hash_idx1,hash_idx1] = 0
hash_idx2 = hash_idx1 + 1
while hash_idx2 < len(hashes):
ham_dist = 1
for h1 in hash:
for h2 in hashes[hash_idx2]:
ham_dist = min(ham_dist, (sum(bool(ord(ch1) - ord(ch2)) for ch1, ch2 in zip(h1, h2)))/len(h1))
a[hash_idx1,hash_idx2] = ham_dist
a[hash_idx2,hash_idx1] = ham_dist
hash_idx2 = hash_idx2 + 1
print(a)
X = squareform(a)
print(X)
linkage = fastcluster.single(X)
clustdict = {i:[i] for i in range(len(linkage)+1)}
fig = plt.figure(figsize=(25,25))
plt.title('test title')
plt.xlabel('perpetual hash hamming distance')
plt.axvline(x=.15,c='red',linestyle='--')
dg = dendrogram(linkage, labels=filelist, orientation='right', show_leaf_counts=True)
ax = fig.gca()
ax.set_xlim(-.01,ax.get_xlim()[1])
plt.show
plt.savefig('foo1.pdf', bbox_inches='tight', dpi=100)
with open('numpyarray.npy','wb') as f:
np.save(f,a)
It took awhile...but I figured things out eventually and got a script that does a pretty good job of identifying if an image is corrupt and then uses perceptual hashes to try and group similar images together.
from PIL import Image, ImageFile
import os, sys, imagehash, pyexiv2, rawpy, re
from tempfile import NamedTemporaryFile
from subprocess import Popen, PIPE
import shlex
import numpy as np
from scipy.cluster.hierarchy import dendrogram, fcluster
from scipy.spatial.distance import squareform
import fastcluster
#import matplotlib.pyplot as plt
import math
import string
from wand.image import Image as wImage
import wand.exceptions
from io import BytesIO
from datetime import datetime
#import fd_table_status
def redirect_stdout():
print("Redirecting stdout and stderr")
sys.stdout.flush() # <--- important when redirecting to files
sys.stderr.flush()
newstdout = os.dup(1)
newstderr = os.dup(2)
devnull = os.open(os.devnull, os.O_WRONLY)
devnull2 = os.open(os.devnull, os.O_WRONLY)
os.dup2(devnull, 1)
os.dup2(devnull2,2)
os.close(devnull)
os.close(devnull2)
sys.stdout = os.fdopen(newstdout, 'w')
sys.stderr = os.fdopen(newstderr, 'w')
redirect_stdout()
def ct(linkage_matrix,flist,score):
cluster_id = []
for fidx, file_ in enumerate(flist):
link_ = np.where(linkage_matrix[:,:2] == fidx)[0]
if len(link_) == 1:
link = link_[0]
if linkage_matrix[link][2] <= score:
fcluster_idx = str(link).zfill(len(str(len(linkage_matrix))))
while True:
match = np.where(linkage_matrix[:,:2] == link+1+len(linkage_matrix))[0]
if len(match) == 1:
link = match[0]
link_d = linkage_matrix[link]
if link_d[2] <= score:
fcluster_idx = str(match[0]).zfill(len(str(len(linkage_matrix)))) + fcluster_idx
else:
break
else:
break
else:
fcluster_idx = None
cluster_id.append(fcluster_idx)
return cluster_id
def get_exitcode_stdout_stderr(cmd):
"""
Execute the external command and get its exitcode, stdout and stderr.
"""
args = shlex.split(cmd)
proc = Popen(args, stdout=PIPE, stderr=PIPE, close_fds=True)
out, err = proc.communicate()
exitcode = proc.returncode
del proc
return exitcode, out, err
if os.path.isdir(sys.argv[1]):
start_time = datetime.now()
# allow PIL to load truncated images (so that perceptual hashes can be created for truncated/damaged images still)
ImageFile.LOAD_TRUNCATED_IMAGES = True
# image files this script will handle
# PIL supported image formats
stdimageext = ('.jpg','.jpeg', '.bmp', '.png', '.gif', '.tif', '.tiff')
# libraw/ufraw supported formats
rawimageext = ('.nef', '.dng', '.tif', '.tiff')
corruptRegex = re.compile(r'_\[.+\]\..{3,4}$')
groupRegex = re.compile(r'^\[\d+\]_')
ufrawRegex = re.compile(r'Corrupt data near|Unexpected end of file|has the wrong dimensions!|Cannot open file|Cannot decode file|requests a nonexistent image!')
for subdirs,dirs,files in os.walk(sys.argv[1]):
files.clear()
dirs.clear()
for root,_,files in os.walk(subdirs):
print('\n******** Processing files in ' + root)
hashes = []
w_hash = []
w_hash_idx = []
filelist = []
files_ = []
cnt = 0
for f in files:
#cnt = cnt + 1
#if cnt < 10:
files_.append(f)
continue
cnt = 0
for f_idx, fname in enumerate(files_):
e=None
ext = os.path.splitext(fname.lower())[1]
filepath = os.path.join(root, fname)
imformat = ''
hashes_tmp = []
# reset corrupt string
corrupt_str = None
if ext in (stdimageext + rawimageext):
print(str(int(round(((f_idx+1)/len(files_))*100))) + '%' + ' : ' + fname + '....', end='', flush=True)
try:
with wImage(filename=filepath) as im:
imformat = '.' + im.format.lower()
ext = imformat if imformat is not '' else ext
with im.convert('jpeg') as converted:
jpeg_bin = converted.make_blob()
with Image.open(BytesIO(jpeg_bin)) as im2:
hash_image = []
for x in range(3):
print('.',end='',flush=True)
hash_i = str(imagehash.dhash(im2.rotate(90 * x, expand=1),32))
if ''.join(set(hash_i)) != '0':
hash_image.append(hash_i)
if hash_image:
hash_image.append(1)
hashes_tmp.append(hash_image)
except:
e = sys.exc_info()[0]
errcode = str([k for k, v in wand.exceptions.TYPE_MAP.items() if v == e][0]).zfill(3)
if int(errcode[-2:]) in (15,25,30,35,40,50,55):
corrupt_str = 'magick'
finally:
try:
im.close()
except:
pass
try:
im2.close()
except:
pass
if ext in (stdimageext):
try:
with Image.open(filepath) as im:
hash_image = []
for x in range(3):
print('.',end='',flush=True)
hash_i = str(imagehash.dhash(im.rotate(90 * x, expand=1),32))
if ''.join(set(hash_i)) != '0':
hash_image.append(hash_i)
if hash_image:
hash_image.append(2)
hashes_tmp.append(hash_image)
except:
pass
finally:
try:
im.close()
except:
pass
# use jpeginfo against all jpg images as its pretty accurate
if ext in ('.jpg','.jpeg'):
#rc = 0
print('.',end='',flush=True)
cmd = 'jpeginfo --check "' + filepath + '"'
exitcode, out, err = get_exitcode_stdout_stderr(cmd)
#rc = call(["jpeginfo", "--check", filepath], stdout=DEVNULL, stderr=DEVNULL, close_fds=True)
if exitcode == 1:
corrupt_str = 'JpegInfo' if corrupt_str == None else corrupt_str
#del rc
if corrupt_str is None:
try:
with Image.open(filepath) as im:
print('.',end='',flush=True)
im.verify()
except:
e = sys.exc_info()[0]
corrupt_str = 'PIL_Verify' if corrupt_str == None else corrupt_str
else:
try:
with Image.open(filepath) as im:
print('.',end='',flush=True)
temp = im.copy()
im.load()
except:
e = sys.exc_info()[0]
corrupt_str = 'PIL_Load' if corrupt_str == None else corrupt_str
finally:
try:
temp.close()
except:
pass
try:
im.close()
except:
pass
finally:
try:
im.close()
except:
pass
try:
temp.close()
except:
pass
# raw image processing
if ext in (rawimageext):
print('.',end='',flush=True)
# try to load raw using libraw via rawpy first,
# generally if libraw can't load it then ufraw extraction would also fail
if corrupt_str == None:
try:
with rawpy.imread(filepath) as raw:
rgb = raw.postprocess(use_camera_wb=True)
temp_raw = NamedTemporaryFile(suffix='.jpg')
Image.fromarray(rgb).save(temp_raw.name)
with Image.open(temp_raw.name) as im:
hash_image = []
for x in range(3):
print('.',end='',flush=True)
hash_i = str(imagehash.dhash(im.rotate(90 * x, expand=1),32))
if ''.join(set(hash_i)) != '0':
hash_image.append(hash_i)
if hash_image:
hash_image.append(3)
hashes_tmp.append(hash_image)
except(rawpy.LibRawFatalError):
e = sys.exc_info()[1]
corrupt_str = 'Libraw_FE'
except(rawpy.LibRawNonFatalError):
e = sys.exc_info()[1]
corrupt_str = 'Libraw_NFE'
except:
#print(sys.exc_info())
corrupt_str = 'Libraw'
finally:
try:
im.close()
except:
pass
try:
temp_raw.close()
except:
pass
try:
raw.close()
except:
pass
if corrupt_str == None:
# as a final last ditch effort compare perceptual hashes of extracted
# raw and embedded preview to detect possible internal corruption
# extract and convert raw to jpeg image using ufraw
temp_raw = NamedTemporaryFile(suffix='.jpg')
#rc = 0
cmd = 'ufraw-batch --wb=camera --rotate=camera --out-type=jpg --compression=95 --noexif --lensfun=none --auto-crop --output=' + temp_raw.name + ' --overwrite "' + filepath + '"'
print('.',end='',flush=True)
exitcode, out, err = get_exitcode_stdout_stderr(cmd)
if exitcode == 1 or ufrawRegex.search(str(err)) is not None:
corrupt_str = 'Ufraw' if corrupt_str is None else corrupt_str
tmpfilesize = os.stat(temp_raw.name).st_size
if tmpfilesize > 0:
try:
with Image.open(temp_raw.name) as im:
hash_image = []
for x in range(3):
print('.',end='',flush=True)
hash_i = str(imagehash.dhash(im.rotate(90 * x, expand=1),32))
if ''.join(set(hash_i)) != '0':
hash_image.append(hash_i)
if hash_image:
hash_image.append(4)
hashes_tmp.append(hash_image)
except:
pass
finally:
try:
im.close()
except:
pass
try:
temp_raw.close()
except:
pass
# attempt to extract preview images
imfile = filepath
try:
with pyexiv2.ImageMetadata(imfile) as metadata_orig:
metadata_orig.read()
#for i,p in enumerate(metadata_orig.previews):
if metadata_orig.previews:
preview = metadata_orig.previews[-1]
# save preview to temp file
temp_preview = NamedTemporaryFile()
preview.write_to_file(temp_preview.name)
os.rename(temp_preview.name + preview.extension, temp_preview.name)
try:
with Image.open(temp_preview.name) as im:
hash_image = []
for x in range(3):
print('.',end='',flush=True)
hash_i = str(imagehash.dhash(im.rotate(90 * x, expand=1),32))
if ''.join(set(hash_i)) != '0':
hash_image.append(hash_i)
if hash_image:
hash_image.append(5)
hashes_tmp.append(hash_image)
except:
pass
finally:
try:
temp_preview.close()
except:
pass
try:
im.close()
except:
pass
except:
pass
finally:
try:
metadata_orig.close()
except:
pass
# compare hashes for all images that were found or extracted and find most dissimilar hamming distance (worst)
if len(hashes_tmp) > 1:
#print('checking_hashes')
print('.',end='',flush=True)
scores = []
for h_idx, hash in enumerate(hashes_tmp):
i = h_idx + 1
while i < len(hashes_tmp):
ham_dist = 1
for h1 in hash[:-1]:
for h2 in hashes_tmp[i][:-1]:
ham_dist = min(ham_dist, (sum(bool(ord(ch1) - ord(ch2)) for ch1, ch2 in zip(h1, h2)))/len(h1))
if (hash[-1] == 5 and hashes_tmp[i][-1] != 5) or (hash[-1] != 5 and hashes_tmp[i][-1] == 5):
scores.append([ham_dist,hash[-1],hashes_tmp[i][-1]])
i = i + 1
if scores:
worst = sorted(scores, key = lambda x: x[0])[-1]
if worst[0] > 0.3:
worst1 = str(worst[1])
worst2 = str(worst[2])
corrupt_str = 'hash' + str(round(worst[0]*100,2)) + '_' + worst1 + '-' + worst2 if corrupt_str == None else corrupt_str
# prefix file if corruption was detected ensuring that existing files already prefixed are re prefixed
mo = corruptRegex.search(fname)
newfilename = None
if corrupt_str is not None:
print('Corrupt: ' + corrupt_str)
if mo is not None:
newfilename = re.sub(corruptRegex, '_[' + corrupt_str + ']', fname) + ext
else:
newfilename = os.path.splitext(fname)[0] + '_[' + corrupt_str + ']' + ext
else:
print('OK!')
if mo is not None:
newfilename = re.sub(corruptRegex, '', fname) + ext
# remove group index from name if present, this will be assigned in the next step if needed
newfilename = newfilename if newfilename is not None else fname
mo = groupRegex.search(newfilename)
if mo is not None:
newfilename = re.sub(groupRegex, '', newfilename)
if hashes_tmp:
# set function unduplicates flattened list
hashes.append(set([item for sublist in hashes_tmp for item in sublist[:-1]]))
filelist.append([root,fname,newfilename, len(hashes_tmp)])
print('******** Grouping similar images... ************')
if len(hashes) > 1:
scores = []
for h_idx, hash in enumerate(hashes):
i = h_idx + 1
while i < len(hashes):
ham_dist = 1
for h1 in hash:
for h2 in hashes[i]:
ham_dist = min(ham_dist, (sum(bool(ord(ch1) - ord(ch2)) for ch1, ch2 in zip(h1, h2)))/len(h1))
scores.append(ham_dist)
i = i + 1
X = np.array(scores)
linkage = fastcluster.single(X)
w_hash_idx = [el_idx for el_idx, el in enumerate(filelist) if el[3] > 0]
w_hash = [filelist[i] for i in w_hash_idx]
test=ct(linkage,[el[2] for el in w_hash],.2)
for i, prfx in enumerate(test):
curfilename = w_hash[i][2]
mo = groupRegex.search(curfilename)
newfilename = None
if prfx is not None:
if mo is not None:
newfilename = re.sub(groupRegex, '[' + prfx + ']_', curfilename)
else:
newfilename = '[' + prfx + ']_' + curfilename
else:
if mo is not None:
newfilename = re.sub(groupRegex, '', curfilename)
# if newfilename is not None:
filelist[w_hash_idx[i]][2] = newfilename if newfilename is not None else curfilename
#fig = plt.figure(figsize=(25,25))
#plt.title(root)
#plt.xlabel('perpetual hash hamming distance')
#plt.axvline(x=.15,c='red',linestyle='--')
#dg = dendrogram(linkage, labels=[el[2] for el in w_hash], orientation='right', show_leaf_counts=True)
#ax = fig.gca()
#ax.set_xlim(-.02,ax.get_xlim()[1])
#plt.show
#plt.savefig(os.path.join(root,'dendrogram.pdf'), bbox_inches='tight', dpi=100)
w_hash.clear()
w_hash_idx.clear()
print('******** Renameing file if applicable... ************')
for fr in filelist:
if fr[1] != fr[2]:
#print(fr[1] + ' -- ' + fr[2])
path = fr[0]
os.rename(os.path.join(path,fr[1]),os.path.join(path,fr[2]))
filelist.clear()
duration = datetime.now() - start_time
days = divmod(duration.total_seconds(), 86400) # Get days (without [0]!)
hours = divmod(days[1], 3600) # Use remainder of days to calc hours
minutes = divmod(hours[1], 60) # Use remainder of hours to calc minutes
seconds = divmod(minutes[1], 1) # Use remainder of minutes to calc seconds
print("Time to complete: %d days, %d:%d:%d" % (days[0], hours[0], minutes[0], seconds[0]))
I am trying to compare one image with all images of another file , get the difference percentage and print file name of the least difference percentage .... if i try to append the output differences to a list ... i get an error saying " float values cannot be iterated".... this is what i have done so far ....
from itertools import izip
import os
import numpy as np
import cv2
from matplotlib import pyplot as plt
from PIL import Image
import math
res = 0
def take_and_save_picture(im_save):
'''Take a picture and save it
Args:
im_save: filepath where the image should be stored
'''
camera_port = 0
ramp_frames = 30
cap = cv2.VideoCapture(camera_port)
def get_image():
retval, im = cap.read()
return im
for i in xrange(ramp_frames):
temp = get_image()
print("Taking image...")
# Take the actual image we want to keep
camera_capture = get_image()
#im_save_tmp = im_save + '.jpg'
im_save_tmp = im_save
# A nice feature of the imwrite method is that it will automatically choose the
# correct format based on the file extension you provide. Convenient!
cv2.imwrite(im_save_tmp, camera_capture)
# You'll want to release the camera, otherwise you won't be able to create a new
# capture object until your script exits
# del(cap)
img1 = cv2.imread(im_save_tmp, 0)
edges = cv2.Canny(img1, 100, 200)
cv2.imwrite(im_save, edges)
cv2.waitKey(0)
cv2.destroyAllWindows()
def re(path1,path2):
#path1 = raw_input("Enter the path1:")
#path2 = raw_input("Enter the path2:")
i2= Image.open(path2)
listing = os.listdir(path1)
for file in listing:
i1 = Image.open(path1 + file)
assert i1.mode == i2.mode, "Different kinds of images."
assert i1.size == i2.size, "Different sizes."
pairs = izip(i1.getdata(), i2.getdata())
if len(i1.getbands()) == 1:
# for gray-scale jpegs
dif = sum(abs(p1-p2) for p1,p2 in pairs)
else:
dif = sum(abs(c1-c2) for p1,p2 in pairs for c1,c2 in zip(p1,p2))
ncomponents = i1.size[0] * i1.size[1] * 3
res = (dif / 255.0 * 100) / ncomponents
print "Difference (percentage):", res
def main():
capture_img = "/Users/Me/Documents/python programs/New/pro.png"
#img_to_compare = "/Users/Me/Documents/python programs/compare/img2.jpg"
take_and_save_picture(capture_img)
path1 = "/Users/Me/Documents/python programs/New/numbers1/"
path2 = "/Users/Me/Documents/python programs/New/pro.png"
re(path1,path2)
if __name__ == '__main__':
main()
the output is the difference
Difference (percentage): 2.52484809028
Difference (percentage): 2.64822048611
Difference (percentage): 2.64822048611
Difference (percentage): 3.55436197917
the values that i get in "res" have to be stored in a list and the minimum value should be found and printed.... please give me some code ... totally new to python ... thank you ...
You're code must be like this:
#######
list_dif = []
def re(path1,path2):
#path1 = raw_input("Enter the path1:")
#path2 = raw_input("Enter the path2:")
i2= Image.open(path2)
listing = os.listdir(path1)
for file in listing:
i1 = Image.open(path1 + file)
assert i1.mode == i2.mode, "Different kinds of images."
assert i1.size == i2.size, "Different sizes."
pairs = izip(i1.getdata(), i2.getdata())
if len(i1.getbands()) == 1:
# for gray-scale jpegs
dif = sum(abs(p1-p2) for p1,p2 in pairs)
else:
dif = sum(abs(c1-c2) for p1,p2 in pairs for c1,c2 in zip(p1,p2))
ncomponents = i1.size[0] * i1.size[1] * 3
#######
for n in range(ncomponents):
res = (dif / 255.0 * 100) / (ncomponents + 1)
list_dif.append(res)
print "Difference (percentage):", list_dif
Something like this?
def re(path1,path2):
#path1 = raw_input("Enter the path1:")
#path2 = raw_input("Enter the path2:")
i2= Image.open(path2)
listing = os.listdir(path1)
res = []
for file in listing:
i1 = Image.open(path1 + file)
assert i1.mode == i2.mode, "Different kinds of images."
assert i1.size == i2.size, "Different sizes."
pairs = izip(i1.getdata(), i2.getdata())
if len(i1.getbands()) == 1:
# for gray-scale jpegs
dif = sum(abs(p1-p2) for p1,p2 in pairs)
else:
dif = sum(abs(c1-c2) for p1,p2 in pairs for c1,c2 in zip(p1,p2))
ncomponents = i1.size[0] * i1.size[1] * 3
res.append((dif / 255.0 * 100) / ncomponents)
print "Difference (percentage):", res
minimum = min(res) # Find minimum value in res
print(minimum)
I am currently writing a Bag Of Words analyser using OpenCV and Python. I have been able to extract the info from the images, learn but the problem is the training part. As a baseline, I am using this and translating it into Python, but I get to the learn part and when it should train the classifier it fails saying that it has an unsupported response type (I assume labels is not in the correct format). I have tried a bit of everything but I cannot get it to work. Any ideas?
import cv2
import os
import numpy as np
dictSize = 1000
retries = 1
flags = cv2.KMEANS_PP_CENTERS
tc = (cv2.TERM_CRITERIA_MAX_ITER, 10, 0.001)
matcher = cv2.DescriptorMatcher_create("FlannBased")
extractor = cv2.DescriptorExtractor_create("SURF")
detector = cv2.FeatureDetector_create("SURF")
bowTrainer = cv2.BOWKMeansTrainer(dictSize,tc,retries,flags)
bowDE = cv2.BOWImgDescriptorExtractor(extractor,matcher)
def extractTrainingVocabulary(path):
global bowTrainer
global extractor
lst=os.listdir(path)
for i in range(0,len(lst)):
if lst[i][0] != ".":
fullPath = path + lst[i]
print "Processing Image " + fullPath
img = cv2.imread(fullPath)
if not (len(img) == 0):
keypoints = detector.detect(img)
if (len(keypoints) == 0):
print "Warning! Could not find any keypoints in image " + fullPath
else:
# Returns 2 vars. The underscore is used to discard the first one
_,features = extractor.compute(img,keypoints)
bowTrainer.add(features)
else:
print "Could not read image " + fullPath
def extractBOWDescriptor(path, descriptors, labels):
global bowTrainer
global extractor
lst=os.listdir(path)
for i in range(0,len(lst)):
if lst[i][0] != ".":
fullPath = path + lst[i]
print "Processing Image " + fullPath
img = cv2.imread(fullPath)
if not (len(img) == 0):
keypoints = detector.detect(img)
if (len(keypoints) == 0):
print "Warning! Could not find any keypoints in image " + fullPath
else:
bowDescriptor = bowDE.compute(img,keypoints)
# descriptors.append(bowDescriptor)
#np.vstack((descriptors,bowDescriptor))
descriptors = np.vstack((descriptors,bowDescriptor))
#labels.append(lst[i][:-4])
labels = np.vstack((labels,float(lst[i][:-4])))
else:
print "Could not read image " + fullPath
return labels, descriptors
def main():
global bowDE
# LEARN
print "Creating Dict..."
extractTrainingVocabulary("./testImages/")
descriptors = bowTrainer.getDescriptors()
print "Clustering " + str(len(descriptors)) + " features. This might take a while..."
dictionary = bowTrainer.cluster()
print "Done clustering"
# EXTRACT
size1 = 0,dictSize
trainingData = np.zeros(size1,dtype=np.float32)
size2 = 0,1
labels = np.zeros(size2,dtype=np.float32)
bowDE.setVocabulary(dictionary)
labels,trainingData = extractBOWDescriptor("./evalImages/",trainingData,labels)
print(trainingData)
print(labels)
print "Training classifier"
size3 = len(trainingData),len(trainingData[0])
responseData = np.zeros(size3,dtype=np.float32)
classifier = cv2.NormalBayesClassifier()
classifier.train(trainingData,labels)
main()
EDIT
as per #berak's suggestion, I changed the following:
labels = np.vstack((labels,float(lst[i][:-4]))) -> labels = np.vstack((labels,int(lst[i][:-4])))
labels = np.zeros(size2,dtype=np.float32) -> labels = np.zeros(size2,dtype=np.int32)
Unfortunately still fails. Now I get the following:
error: (-5) There is only a single class in function cvPreprocessCategoricalResponses
I am using this code to combine multiple images of different sizes into one large image.
#!/usr/bin/env python
"""
Pack multiple images of different sizes into one image.
Based on S W's recipe:
http://code.activestate.com/recipes/442299/
Licensed under the PSF License
"""
import argparse
import glob
import Image
import ImageChops
try: import timing # Optional, http://stackoverflow.com/a/1557906/724176
except: None
def tuple_arg(s):
try:
if ',' in s:
w, h = map(int, s.split(','))
elif ':' in s:
w, h = map(int, s.split(':'))
elif 'x' in s:
w, h = map(int, s.split('x'))
return w, h
except:
raise argparse.ArgumentTypeError("Value must be w,h or w:h or wxh")
class PackNode(object):
"""
Creates an area which can recursively pack other areas of smaller sizes into itself.
"""
def __init__(self, area):
#if tuple contains two elements, assume they are width and height, and origin is (0,0)
if len(area) == 2:
area = (0,0,area[0],area[1])
self.area = area
def __repr__(self):
return "<%s %s>" % (self.__class__.__name__, str(self.area))
def get_width(self):
return self.area[2] - self.area[0]
width = property(fget=get_width)
def get_height(self):
return self.area[3] - self.area[1]
height = property(fget=get_height)
def insert(self, area):
if hasattr(self, 'child'):
a = self.child[0].insert(area)
if a is None:
return self.child[1].insert(area)
return a
area = PackNode(area)
if area.width <= self.width and area.height <= self.height:
self.child = [None,None]
self.child[0] = PackNode((self.area[0]+area.width, self.area[1], self.area[2], self.area[1] + area.height))
self.child[1] = PackNode((self.area[0], self.area[1]+area.height, self.area[2], self.area[3]))
return PackNode((self.area[0], self.area[1], self.area[0]+area.width, self.area[1]+area.height))
if __name__ == "__main__":
parser = argparse.ArgumentParser(
description='Pack multiple images of different sizes into one image.',
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument('-o', '--outfile', default='output.png',
help='Output image file')
parser.add_argument('-s', '--size', type=tuple_arg, metavar='pixels',
help="Size (width,height tuple) of the image we're packing into",
default="1024,1024")
parser.add_argument('-l', '--largest_first', action='store_true',
help='Pack largest images first')
parser.add_argument('-t', '--tempfiles', action='store_true',
help='Save temporary files to show filling')
args = parser.parse_args()
print args
format = 'RGBA'
#get a list of PNG files in the current directory
names = glob.glob("*.png")
if args.outfile in names:
names.remove(args.outfile) # don't include any pre-existing output
#create a list of PIL Image objects, sorted by size
print "Create a list of PIL Image objects, sorted by size"
images = sorted([(i.size[0]*i.size[1], name, i) for name,i in ((x,Image.open(x).convert(format)) for x in names)], reverse=args.largest_first)
print "Create tree"
tree = PackNode(args.size)
image = Image.new(format, args.size)
#insert each image into the PackNode area
for i, (area, name, img) in enumerate(images):
print name, img.size
uv = tree.insert(img.size)
if uv is None: raise ValueError('Pack size ' + str(args.size) + ' too small, cannot insert ' + str(img.size) + ' image.')
image.paste(img, uv.area)
if args.tempfiles:
image.save("temp" + str(i).zfill(4) + ".png")
image.save(args.outfile)
image.show()
# End of file
But it keeps giving me the following output with error:
Namespace(largest_first=False, outfile='output.png', size=(1024, 1024), tempfiles=False)
Create a list of PIL Image objects, sorted by size
Create tree
creativemarketcanvas-f.png (580, 386)
nature_trees_simple_set_1.png (8102, 3378)
Traceback (most recent call last):
File "/home/nabila/Desktop/trying combining images/recipe.py", line 98, in <module>
if uv is None: raise ValueError('Pack size ' + str(args.size) + ' too small, cannot insert ' + str(img.size) + ' image.')
ValueError: Pack size (1024, 1024) too small, cannot insert (8102, 3378) image.
I have tried using different images of various sizes but it still keeps giving errors like the one above.
What am I doing wrong?