I was using multi class U-Net segmentation where I am encountering value error while training by data model. My multi class model is divided into 4 classes.
Code for training model:
from simple_multi_unet_model import multi_unet_model #Uses softmax
from tensorflow.keras.utils import normalize
import os
import glob
import cv2
import numpy as np
from matplotlib import pyplot as plt
from sklearn.model_selection import train_test_split
import tensorflow as tf
#Resizing images, if needed
SIZE_X = 128
SIZE_Y = 128
n_classes=4 #Number of classes for segmentation
#Capture training image info as a list
train_images = []
directory_path = '/home/Documents/Multiclass/images/'
list_of_files = sorted( filter( os.path.isfile, glob.glob(directory_path + '*.png', recursive=True) ) )
for img_path in list_of_files:
img = cv2.imread(img_path, 0)
img = cv2.resize(img, (SIZE_Y, SIZE_X))
train_images.append(img)
#Convert list to array for machine learning processing
train_images = np.array(train_images)
#Capture mask/label info as a list
train_masks = []
labels_path = '/home/Documents/Multiclass/labels/'
list_of_labels = sorted( filter( os.path.isfile, glob.glob(labels_path + '*.png', recursive=True) ) )
for mask_path in list_of_labels:
mask = cv2.imread(mask_path, 0)
mask = cv2.resize(mask, (SIZE_Y, SIZE_X), interpolation = cv2.INTER_NEAREST) #Otherwise ground truth changes due to interpolation
train_masks.append(mask)
#Convert list to array for machine learning processing
train_masks = np.array(train_masks)
###############################################
#Encode labels... but multi dim array so need to flatten, encode and reshape
from sklearn.preprocessing import LabelEncoder
labelencoder = LabelEncoder()
n, h, w = train_masks.shape
train_masks_reshaped = train_masks.reshape(-1,1)
train_masks_reshaped_encoded = labelencoder.fit_transform(train_masks_reshaped)
train_masks_encoded_original_shape = train_masks_reshaped_encoded.reshape(n, h, w)
np.unique(train_masks_encoded_original_shape)
#################################################
train_images = np.expand_dims(train_images, axis=3)
train_images = normalize(train_images, axis=1)
train_masks_input = np.expand_dims(train_masks_encoded_original_shape, axis=3)
#Create a subset of data for quick testing
#Picking 10% for testing and remaining for training
from sklearn.model_selection import train_test_split
x_train, X_test, y_train, y_test = train_test_split(train_images, train_masks_input, test_size = 0.10, random_state = 0)
print("Class values in the dataset are ... ", np.unique(y_train)) # 0 is the background/few unlabeled
from tensorflow.keras.utils import to_categorical
train_masks_cat = to_categorical(y_train, num_classes=n_classes)
y_train_cat = train_masks_cat.reshape((y_train.shape[0], y_train.shape[1], y_train.shape[2], n_classes))
test_masks_cat = to_categorical(y_test, num_classes=n_classes)
y_test_cat = test_masks_cat.reshape((y_test.shape[0], y_test.shape[1], y_test.shape[2], n_classes))
###############################################################
from sklearn.utils import class_weight
class_weights = class_weight.compute_class_weight('balanced',
np.unique(train_masks_reshaped_encoded),
train_masks_reshaped_encoded)
print("Class weights are...:", class_weights)
IMG_HEIGHT = x_train.shape[1]
IMG_WIDTH = x_train.shape[2]
IMG_CHANNELS = x_train.shape[3]
def get_model():
return multi_unet_model(n_classes=n_classes, IMG_HEIGHT=IMG_HEIGHT, IMG_WIDTH=IMG_WIDTH, IMG_CHANNELS=IMG_CHANNELS)
model = get_model()
model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'])
model.summary()
#If starting with pre-trained weights.
#model.load_weights('???.hdf5')
history = model.fit(x_train, y_train_cat,
batch_size = 16,
verbose=1,
epochs=100,
validation_data=(X_test, y_test_cat),
class_weight=class_weights,
shuffle=False)
I used following approach to define class weights:
from sklearn.utils import class_weight
class_weights = class_weight.compute_class_weight('balanced',
np.unique(train_masks_reshaped_encoded),
train_masks_reshaped_encoded)
print("Class weights are...:", class_weights)
The result is class_weights : 0.276965 ,13.5112 ,5.80929,6.97915.
I am encountering ValueError when I train my model. How can I possibly resolve it? Please suggest a better approach of using class weights if you think my approach is not viable.
File "/home/anaconda3/lib/python3.8/site-packages/tensorflow/python/keras/engine/data_adapter.py", line 1185, in _configure_dataset_and_inferred_steps
if class_weight:
ValueError: The truth value of an array with more than one element is ambiguous. Use a.any() or a.all()
I had the same problem but solved it with this!!!
You have to zip it together as a dictionary
Try The below code:
from sklearn.utils import class_weight
class_weights = dict(zip(np.unique(train_masks_reshaped_encoded), class_weight.compute_class_weight('balanced', np.unique(train_masks_reshaped_encoded),
train_masks_reshaped_encoded)))
My understanding of this error message is that numpy does not know whether to evaluate an array as True if any element is true, or as True only if all elements are true. Hence, a ValueError is returned because the boolean evaluation is ambiguous in this regard.
Therefore, when evaluating an array, you should use a.any() or a.all(), as indicated in the error message attached.
The error is likely to be occurring from somewhere else in your code (not shared?) when you try to evaluate the class weights in a boolean context.
my probleme is that my ANN predicts only about 2 times out of 10 the right digit but when the ANN was fitted it tells me about 98% accuarcy. I'm quit a starter with ANNS and I don't know if I'm missing something abvious or why it is like it is.
For testing I use a tabel with 81 digits (allways in a row from 1 to 9).
I would be grateful if someone can help me :)
Training the ANN
import tensorflow as tf
import tensorflow_datasets as tfds
from tensorflow.keras.models import save_model
# See available datasets
print(tfds.list_builders())
# Construct a tf.data.Dataset
dataset = tfds.load(name="mnist", split=tfds.Split.TRAIN)
# Build your input pipeline
dataset = dataset.shuffle(1024).batch(32).prefetch(tf.data.experimental.AUTOTUNE)
for features in dataset.take(1):
image, label = features["image"], features["label"]
# tfds works in both Eager and Graph modes
# See available datasets
print(tfds.list_builders())
# Construct a tf.data.Dataset
dataset = tfds.load(name="mnist", split=tfds.Split.TRAIN)
# Build your input pipeline
dataset = dataset.shuffle(1024).batch(32).prefetch(tf.data.experimental.AUTOTUNE)
for features in dataset.take(1):
image, label = features["image"], features["label"]
mnist = tf.keras.datasets.mnist
# Aufteilung in Training- und Testset
(x_train, y_train),(x_test, y_test) = mnist.load_data()
x_train, x_test = x_train / 255.0, x_test / 255.0
print(x_test[0])
# TF Bilderkennungsmodell
model = tf.keras.models.Sequential([
tf.keras.layers.Flatten(input_shape=(28, 28)),
tf.keras.layers.Dense(512, activation=tf.nn.relu),
tf.keras.layers.Dropout(0.2),
tf.keras.layers.Dense(10, activation=tf.nn.softmax)
])
# Crossentropy für die 10 Zahlen Klassen
model.compile(optimizer='adam',
loss='sparse_categorical_crossentropy',
metrics=['accuracy'])
# Modellfitting und Evaluation
model.fit(x_train, y_train, epochs=50)
model.evaluate(x_test, y_test)
filepath = './saved_model2'
save_model(model, filepath)
For testing with my table:
import matplotlib.pyplot as plt
import numpy as np
from tensorflow.keras.models import load_model
import cv2
import PIL
# Model configuration
img_width, img_height = 28, 28
batch_size = 250
no_epochs = 25
no_classes = 10
validation_split = 0.2
verbosity = 1
filepath = './saved_model2'
model = load_model(filepath, compile=True)
img = PIL.Image.open("Dataset/raster.jpg")
bilder = []
i= 0
for x in range(0,224,28):
img1 = img.crop([0,x,28,x+28])
img2 = img.crop([28,x,56,x+28])
img3 = img.crop([56,x,84,x+28])
img4 = img.crop([84,x,112,x+28])
img5 = img.crop([112,x,140,x+28])
img6 = img.crop([140,x,168,x+28])
img7 = img.crop([168,x,196,x+28])
img8 = img.crop([196,x,224,x+28])
img9 = img.crop([224,x,252,x+28])
img1.save("Dataset/eigeneBilder/"+str(i+1)+".jpg")
img2.save("Dataset/eigeneBilder/"+str(i+2)+".jpg")
img3.save("Dataset/eigeneBilder/"+str(i+3)+".jpg")
img4.save("Dataset/eigeneBilder/"+str(i+4)+".jpg")
img5.save("Dataset/eigeneBilder/"+str(i+5)+".jpg")
img6.save("Dataset/eigeneBilder/"+str(i+6)+".jpg")
img7.save("Dataset/eigeneBilder/"+str(i+7)+".jpg")
img8.save("Dataset/eigeneBilder/"+str(i+8)+".jpg")
img9.save("Dataset/eigeneBilder/"+str(i+9)+".jpg")
i += 9
print(bilder)
samples_to_predict = []
for i in range(1,81,1):
# Generate a plot
sample = cv2.imread("Dataset/eigeneBilder/"+str(i)+".jpg")
img = cv2.bitwise_not(sample)
sample = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
reshaped_image = sample.reshape((img_width, img_height,1))
plt.imshow(reshaped_image)
reshaped_image/255
# Add sample to array for prediction
samples_to_predict.append(reshaped_image)
samples_to_predict = np.array(samples_to_predict)
print(samples_to_predict[0])
predictions = model.predict(samples_to_predict)
classes = np.argmax(predictions, axis=1)
print(samples_to_predict)
print(classes)
One thing you can do is add validation_data in model.fit like this :
model.fit(x_train, y_train, epochs=50, validation_data=(x_test,y_test))
You can also add some 2Dconv layers with pooling before the flatten one. Or add more neurones.
Let me know if it helps.
I am building a model to detect solar panel defects. I am using 255*255 images to train the model and I want to use the confusion matrix to improve my model.
The matrix gives me incorrectly classified images, but I need to find out exact filenames for the false positive and false negative images
How can I achieve this goal?
I have provided my code below:
import numpy as np
import os
import time
import keras
from keras.applications.resnet50 import preprocess_input
from keras.preprocessing import image
from keras.layers import GlobalAveragePooling2D, Dense, Dropout
#from keras.layers import GlobalAveragePooling2D, Dense, Dropout,Activation,Flatten
#from keras.layers import Input
from keras.models import Model
from keras.utils import np_utils
from sklearn.utils import shuffle
from sklearn.model_selection import train_test_split
from keras.layers import Input
from keras import models
from keras import layers
from keras import optimizers
from keras_applications.resnet import ResNet101
from keras.optimizers import SGD, Adagrad, Adadelta, RMSprop, Adam
from keras.callbacks import LearningRateScheduler
from keras.models import load_model
from keras import regularizers
from keras.preprocessing.image import ImageDataGenerator, array_to_img, img_to_array, load_img
###########################################################################################################################
## Model Initials
IMAGE_SIZE = (255, 255)
BATCH_SIZE = 24
NUM_EPOCHS = 1
WEIGHTS_FINAL = 'defectdetection.hdf5'
MODEL_FINAL = 'defectdetection.h5'
BEST_WEIGHT ='1defectdetection.hdf5'
##############################################################################################
## Loading dataset for the training process
## Define data path
# Loading the training data
img_path = 'C:/Users/TeamSoloMid/SolarCellsImages/dataset/Sample0001.jpg'
img = image.load_img(img_path, target_size=(255, 255))
x = image.img_to_array(img)
print (x.shape)
x = np.expand_dims(x, axis=0)
print (x.shape)
x = preprocess_input(x)
print('Input image shape:', x.shape)
PATH = os.getcwd()
data_path = 'C:/Users/TeamSoloMid/SolarCellsImages'
data_dir_list = os.listdir(data_path)
img_data_list=[]
for dataset in data_dir_list:
img_list=os.listdir(data_path+'/'+ dataset)
print ('Loaded the images of dataset-'+'{}\n'.format(dataset))
for img in img_list:
img_path = data_path + '/'+ dataset + '/'+ img
img = image.load_img(img_path, target_size=(255,255))
x = image.img_to_array(img)
x = np.expand_dims(x, axis=0)
x = preprocess_input(x)
#print('Input image shape:', x.shape)
img_data_list.append(x)
img_data = np.array(img_data_list)
print (img_data.shape)
img_data=np.rollaxis(img_data,1,0)
print (img_data.shape)
img_data=img_data[0]
print (img_data.shape)
#t=time.time()
# Define the number of classes
num_classes = 2
num_of_samples = img_data.shape[0]
labels = np.ones((num_of_samples,),dtype='int64')
labels[0:1603]=0
labels[1604:3225]=1
names = ['Defect', 'Almost']
Y = np_utils.to_categorical(labels, num_classes)
#Shuffle the dataset
x,y = shuffle(img_data,Y, random_state=2)
# Split the dataset
TestPcnt = 0.2
X_train, X_test, y_train, y_test = train_test_split(x, y,
test_size=TestPcnt,
random_state=2)
epoch=NUM_EPOCHS
###############################################################################################
# Fine tune the resnet 101
image_input = Input(shape=(255, 255, 3))
model = ResNet101(include_top=False,
input_tensor=image_input,
weights='imagenet',
backend=keras.backend,
layers=keras.layers,
models=keras.models,
utils=keras.utils)
# Freeze all the layers
for layer in model.layers[:-3]:
layer.trainable = False
#model.summary()
last_layer = model.output
# add a global spatial average pooling layer
x = GlobalAveragePooling2D()(last_layer)
x = Dense(256, activation='relu',name='fc-1')(x)
x = Dropout(0.5)(x)
out = Dense(num_classes, activation='softmax',name='output_layer')(x)
# this is the model we will train
net_model = Model(inputs=model.input, outputs=out)
net_model.summary()
for layer in net_model.layers[:-5]:
layer.trainable = False
net_model.summary()
for layer in net_model.layers:
print(layer, layer.trainable)
#my_opti= optimizers.Adam(lr=0.00002)
#my_opti= optimizers.Adam(lr=0.00001)
################################################################################################
#Define learning Rate
learning_rate = 0.00002
decay_rate = learning_rate / epoch
momentum = 0.9
sgd = SGD(lr=learning_rate, momentum=momentum,
decay=decay_rate,
nesterov=False)
##############################################################################
## we will keep the weights of the epoch that scores highest in terms of accuracy on the test set.
from keras.callbacks import ModelCheckpoint
checkpointer = ModelCheckpoint(filepath=BEST_WEIGHT,
monitor = 'val_acc',
verbose=1,
save_best_only=True,
mode = 'max')
###################################################################
callback_list = [checkpointer]
net_model.compile(loss='categorical_crossentropy',
optimizer=sgd,
metrics=['accuracy'])
t=time.time()
hist = net_model.fit(X_train, y_train, batch_size=BATCH_SIZE,
epochs=NUM_EPOCHS, verbose=1,
callbacks = [checkpointer],
validation_data=(X_test, y_test))
print('Training time: %s' % (time.time()-1))
(loss, accuracy) = net_model.evaluate(X_test, y_test,
batch_size=BATCH_SIZE,
verbose=1)
print("[INFO] loss={:.4f}, accuracy: {:.4f}%".format(loss,accuracy * 100))
############################################################################################
## Saving The weights of the model after training
net_model.save_weights(WEIGHTS_FINAL)
print('1. Weights Saved')
net_model.save_weights(BEST_WEIGHT)
print('2. Best Weights Saved')
##############################################################################
## Saving The Complete model after training
net_model.save(MODEL_FINAL)
print('3. Model Saved')
############################################################################################
import matplotlib.pyplot as plt
# visualizing losses and accuracy
train_loss=hist.history['loss']
val_loss=hist.history['val_loss']
train_acc=hist.history['acc']
val_acc=hist.history['val_acc']
xc=range(NUM_EPOCHS)
plt.figure(1,figsize=(7,5))
plt.plot(xc,train_loss)
plt.plot(xc,val_loss)
plt.xlabel('num of Epochs')
plt.ylabel('loss')
plt.title('train_loss vs val_loss')
plt.grid(True)
plt.legend(['train','val'])
#print plt.style.available # use bmh, classic,ggplot for big pictures
plt.style.use(['classic'])
plt.figure(2,figsize=(7,5))
plt.plot(xc,train_acc)
plt.plot(xc,val_acc)
plt.xlabel('num of Epochs')
plt.ylabel('accuracy')
plt.title('train_acc vs val_acc')
plt.grid(True)
plt.legend(['train','val'],loc=4)
#print plt.style.available # use bmh, classic,ggplot for big pictures
plt.style.use(['classic'])
############################################################################
from sklearn.metrics import confusion_matrix, classification_report
import itertools
from sklearn.utils.multiclass import unique_labels
from sklearn import metrics
import seaborn as sns
import pandas as pd
from sklearn.datasets import load_files
from sklearn.svm import LinearSVC
from sklearn import svm
LABELS= ['Defect', 'Almost']
# Print confusion matrix for training data
y_pred_train = net_model.predict(X_train)
def show_confusion_matrix(validations, predictions):
matrix = metrics.confusion_matrix(validations, predictions)
plt.figure(figsize=(10, 10))
sns.heatmap(matrix,
cmap='coolwarm',
linecolor='white',
linewidths=1,
xticklabels=LABELS,
yticklabels=LABELS,
annot=True,
fmt='d')
plt.title('Confusion Matrix')
plt.ylabel('True Label')
plt.xlabel('Predicted Label')
plt.show()
y_pred_test = net_model.predict(X_test)
# Take the class with the highest probability from the test predictions
max_y_pred_test = np.argmax(y_pred_test, axis=1)
max_y_test = np.argmax(y_test, axis=1)
show_confusion_matrix(max_y_test, max_y_pred_test)
print(classification_report(max_y_test, max_y_pred_test))
I would calculate a checksum (i.e. md5) and use that as a dictionary key that would keep the image path as value, i.e.
import hashlib
...
image_paths = {}
...
for dataset in data_dir_list:
img_list=os.listdir(data_path+'/'+ dataset)
print ('Loaded the images of dataset-'+'{}\n'.format(dataset))
for img in img_list:
img_path = data_path + '/'+ dataset + '/'+ img
img = image.load_img(img_path, target_size=(255,255))
x = image.img_to_array(img)
x = np.expand_dims(x, axis=0)
x = preprocess_input(x)
#print('Input image shape:', x.shape)
img_data_list.append(x)
# in this example i am only hashing the first 5 pixels of the image. You would probably want to use all of the pixels of the image.
image_hash = hashlib.md5(str(x[0]) + str(x[1]) + str(x[2]) + str(x[3]) + str(x[4])).hexdigest()
image_paths[image_hash] = img_path
...
when you want to decode the image path you simply calculate the hash again and look the path up in the dictionary
image_hash = hashlib.md5(str(x[0]) + str(x[1]) + str(x[2]) + str(x[3]) + str(x[4])).hexdigest()
image_path = image_paths[image_hash]
While that is not the most flexible approach, I believe that it will still help you achieve your goal.
One note, hashing might be pretty expensive if you have a lot of images, but if your images do not change, you only need to hash them once and save somewhere. In the consequent attempts, you would only need to load the data instead of hashing everything again
Thanks for your answering, I have tried your solution, and there is an IndexError like:
File "C:/Users/TeamSoloMid/Solar cells Defect Detection.py", line 102, in <module>
image_hash = hashlib.md5(str(x[0])+str(x[1])+str(x[2])+str(x[3])+str(x[4])).hexdigest()
IndexError: index 1 is out of bounds for axis 0 with size 1
Below is the code I added your solution in
import numpy as np
import os
import time
import keras
from keras.applications.resnet50 import preprocess_input
from keras.preprocessing import image
from keras.layers import GlobalAveragePooling2D, Dense, Dropout
#from keras.layers import GlobalAveragePooling2D, Dense, Dropout,Activation,Flatten
#from keras.layers import Input
from keras.models import Model
from keras.utils import np_utils
from sklearn.utils import shuffle
from sklearn.model_selection import train_test_split
from keras.layers import Input
from keras import models
from keras import layers
from keras import optimizers
from keras_applications.resnet import ResNet101
from keras.optimizers import SGD, Adagrad, Adadelta, RMSprop, Adam
from keras.callbacks import LearningRateScheduler
from keras.models import load_model
from keras import regularizers
from keras.preprocessing.image import ImageDataGenerator, array_to_img, img_to_array, load_img
import hashlib
########################################################################################
## Model Initials
IMAGE_SIZE = (255, 255)
BATCH_SIZE = 24
NUM_EPOCHS = 1
WEIGHTS_FINAL = 'defectdetection.hdf5'
MODEL_FINAL = 'defectdetection.h5'
BEST_WEIGHT ='defectdetection.hdf5'
#########################################################################################
## Loading dataset for the training process
## Define data path
# Loading the training data
img_path = 'C:/Users/TeamSoloMid/SolarCellsImages/dataset/Sample0001.jpg'
img = image.load_img(img_path, target_size=(255, 255))
x = image.img_to_array(img)
print (x.shape)
x = np.expand_dims(x, axis=0)
print (x.shape)
x = preprocess_input(x)
print('Input image shape:', x.shape)
PATH = os.getcwd()
data_path = 'C:/Users/TeamSoloMid/SolarCellsImages'
data_dir_list = os.listdir(data_path)
image_paths = {}
img_data_list=[]
for dataset in data_dir_list:
img_list=os.listdir(data_path+'/'+ dataset)
print ('Loaded the images of dataset-'+'{}\n'.format(dataset))
for img in img_list:
img_path = data_path + '/'+ dataset + '/'+ img
img = image.load_img(img_path, target_size=(255,255))
x = image.img_to_array(img)
x = np.expand_dims(x, axis=0)
x = preprocess_input(x)
#print('Input image shape:', x.shape)
img_data_list.append(x)
image_hash = hashlib.md5(str(x[0])+str(x[1])+str(x[2])+str(x[3])+str(x[4])).hexdigest()
image_paths[image_hash] = img_path
img_data = np.array(img_data_list)
print (img_data.shape)
img_data=np.rollaxis(img_data,1,0)
print (img_data.shape)
img_data=img_data[0]
print (img_data.shape)
#t=time.time()
# Define the number of classes
num_classes = 2
num_of_samples = img_data.shape[0]
labels = np.ones((num_of_samples,),dtype='int64')
labels[0:1603]=0
labels[1604:3225]=1
#labels[3226:4847]=2
names = ['Defect', 'Almost']
# convert class labels to on-hot encoding
Y = np_utils.to_categorical(labels, num_classes)
#Shuffle the dataset
x,y = shuffle(img_data,Y, random_state=2)
# Split the dataset
TestPcnt = 0.2
X_train, X_test, y_train, y_test = train_test_split(x, y,
test_size=TestPcnt,
random_state=2)
epoch=NUM_EPOCHS
########################################################################################
# Fine tune the resnet 101
image_input = Input(shape=(255, 255, 3))
model = ResNet101(include_top=False,
input_tensor=image_input,
weights='imagenet',
backend=keras.backend,
layers=keras.layers,
models=keras.models,
utils=keras.utils)
# Freeze all the layers
for layer in model.layers[:-3]:
layer.trainable = False
#model.summary()
last_layer = model.output
# add a global spatial average pooling layer
x = GlobalAveragePooling2D()(last_layer)
x = Dense(256, activation='relu',name='fc-1')(x)
x = Dropout(0.5)(x)
out = Dense(num_classes, activation='softmax',name='output_layer')(x)
# this is the model we will train
net_model = Model(inputs=model.input, outputs=out)
net_model.summary()
for layer in net_model.layers[:-5]:
layer.trainable = False
net_model.summary()
for layer in net_model.layers:
print(layer, layer.trainable)
#my_opti= optimizers.Adam(lr=0.00002)
#my_opti= optimizers.Adam(lr=0.00001)
#########################################################################################
Define learning Rate
learning_rate = 0.00002
decay_rate = learning_rate / epoch
momentum = 0.9
sgd = SGD(lr=learning_rate, momentum=momentum,
decay=decay_rate,
nesterov=False)
##############################################################################
## we will keep the weights of the epoch that scores highest in terms of accuracy on the test set.
from keras.callbacks import ModelCheckpoint
checkpointer = ModelCheckpoint(filepath=BEST_WEIGHT,
monitor = 'val_acc',
verbose=1,
save_best_only=True,
mode = 'max')
###################################################################
callback_list = [checkpointer]
net_model.compile(loss='categorical_crossentropy',
optimizer=sgd,
metrics=['accuracy'])
t=time.time()
hist = net_model.fit(X_train, y_train, batch_size=BATCH_SIZE,
epochs=NUM_EPOCHS, verbose=1,
callbacks = [checkpointer],
validation_data=(X_test, y_test))
print('Training time: %s' % (time.time()-1))
(loss, accuracy) = net_model.evaluate(X_test, y_test,
batch_size=BATCH_SIZE,
verbose=1)
print("[INFO] loss={:.4f}, accuracy: {:.4f}%".format(loss,accuracy * 100))
############################################################################################
## Saving The weights of the model after training
net_model.save_weights(WEIGHTS_FINAL)
print('1. Weights Saved')
net_model.save_weights(BEST_WEIGHT)
print('2. Best Weights Saved')
##############################################################################
## Saving The Complete model after training
net_model.save(MODEL_FINAL)
print('3. Model Saved')
############################################################################################
import matplotlib.pyplot as plt
# visualizing losses and accuracy
train_loss=hist.history['loss']
val_loss=hist.history['val_loss']
train_acc=hist.history['acc']
val_acc=hist.history['val_acc']
xc=range(NUM_EPOCHS)
plt.figure(1,figsize=(7,5))
plt.plot(xc,train_loss)
plt.plot(xc,val_loss)
plt.xlabel('num of Epochs')
plt.ylabel('loss')
plt.title('train_loss vs val_loss')
plt.grid(True)
plt.legend(['train','val'])
#print plt.style.available # use bmh, classic,ggplot for big pictures
plt.style.use(['classic'])
plt.figure(2,figsize=(7,5))
plt.plot(xc,train_acc)
plt.plot(xc,val_acc)
plt.xlabel('num of Epochs')
plt.ylabel('accuracy')
plt.title('train_acc vs val_acc')
plt.grid(True)
plt.legend(['train','val'],loc=4)
#print plt.style.available # use bmh, classic,ggplot for big pictures
plt.style.use(['classic'])
############################################################################
from sklearn.metrics import confusion_matrix, classification_report
import itertools
from sklearn.utils.multiclass import unique_labels
from sklearn import metrics
import seaborn as sns
import pandas as pd
from sklearn.datasets import load_files
from sklearn.svm import LinearSVC
from sklearn import svm
LABELS= ['Defect', 'Almost']
# Print confusion matrix for training data
y_pred_train = net_model.predict(X_train)
def show_confusion_matrix(validations, predictions):
matrix = metrics.confusion_matrix(validations, predictions)
plt.figure(figsize=(10, 10))
sns.heatmap(matrix,
cmap='coolwarm',
linecolor='white',
linewidths=1,
xticklabels=LABELS,
yticklabels=LABELS,
annot=True,
fmt='d')
plt.title('Confusion Matrix')
plt.ylabel('True Label')
plt.xlabel('Predicted Label')
plt.show()
y_pred_test = net_model.predict(X_test)
# Take the class with the highest probability from the test predictions
max_y_pred_test = np.argmax(y_pred_test, axis=1)
max_y_test = np.argmax(y_test, axis=1)
show_confusion_matrix(max_y_test, max_y_pred_test)
print(classification_report(max_y_test, max_y_pred_test))
image_hash = hashlib.md5(str(x[0]) + str(x[1]) + str(x[2]) + str(x[3]) + str(x[4])).hexdigest()
image_path = image_paths[image_hash]
I am working with fashion MNIST dataset and I want to build an Artificial Neural Network using custom layers created by my own:
Here is my preprocessing phase:
#ANN with fashion mnist data_set
import tensorflow as tf
import numpy as np
from tensorflow.keras.datasets import fashion_mnist
from tensorflow.keras.layers import Layer
from tensorflow.keras import Model
#Loading dataset and preprocessing:
(X_train,Y_train),(X_test,Y_test) = fashion_mnist.load_data()
unique_labels = set(Y_train)
X_train = X_train/255 #because max value of pixel is 255
X_test = X_test/255 #because max value of pixel is 255
X_train = X_train.reshape(-1,28*28)#flatten image from 28*28 to N*(28*28), second dimension - all the pixels of image
X_test = X_test.reshape(-1,28*28)
Y_train = Y_train.reshape(Y_train.shape[0],1)
Y_test = Y_test.reshape(Y_test.shape[0],1)
dataset_train = tf.data.Dataset.from_tensor_slices((X_train, Y_train))
dataset_train = dataset.shuffle(buffer_size=1024).batch(64)
dataset_test = tf.data.Dataset.from_tensor_slices((X_test, Y_test))
dataset_test = dataset.shuffle(buffer_size=1024).batch(64)
Here you can see, that I have preprocessed data so that they have 2-D shape(which suits better for ANN).
I know, that I could use CNN for the following task, but my goal is to practice with ANN.
And at the end I am getting the following error:
InvalidArgumentError Traceback (most recent call last)
<ipython-input-104-24b493b0dcc5> in <module>()
102
103 for epoch in range(EPOCHS):
--> 104 for X,y in dataset_train:
105 training(X,y)
106
4 frames
/usr/local/lib/python3.6/dist-packages/six.py in raise_from(value, from_value)
InvalidArgumentError: Cannot batch tensors with different shapes in component 0. First element had shape [256,2] and element 36 had shape [32,2]. [Op:IteratorGetNextSync]
But if I don't use batch(64) everything works perfeclty fine.
How can I fix this problem?
You might be using some previously cached tf.Dataset. Because you have train_dataset and test_dataset, not dataset. I tested the following code and it will return a batch of size 64.
#ANN with fashion mnist data_set
import tensorflow as tf
import numpy as np
from tensorflow.keras.datasets import fashion_mnist
from tensorflow.keras.layers import Layer
from tensorflow.keras import Model
#Loading dataset and preprocessing:
(X_train,Y_train),(X_test,Y_test) = fashion_mnist.load_data()
unique_labels = set(Y_train)
X_train = X_train/255 #because max value of pixel is 255
X_test = X_test/255 #because max value of pixel is 255
X_train = X_train.reshape(-1,28*28)#flatten image from 28*28 to N*(28*28), second dimension - all the pixels of image
X_test = X_test.reshape(-1,28*28)
Y_train = Y_train.reshape(Y_train.shape[0],1)
Y_test = Y_test.reshape(Y_test.shape[0],1)
dataset_train = tf.data.Dataset.from_tensor_slices((X_train, Y_train))
dataset_train = dataset_train.shuffle(buffer_size=1024).batch(64)
dataset_test = tf.data.Dataset.from_tensor_slices((X_test, Y_test))
dataset_test = dataset_test.shuffle(buffer_size=1024).batch(64)
# Checking if we are actually getting a batch of data of size 64
# Creating an iterator
iterator = dataset_test.make_one_shot_iterator()
# Getting one batch of data
bi,bl = iterator.get_next()
with tf.Session() as sess:
e_bi = sess.run(bi)
print(e_bi.shape)
There's a useful method to batch that kind of dataset:
tf.data.Dataset.padded_batch
you can also get help from TF official documents:
tf.data.Dataset.padded_batch