We Keep Coding

Opencv readNetFromTensorflow fails with Unknown enumeration DT_VARIANT

I trained a keras model and saved it as PB as in the below script.
import sys
import os
import tensorflow.compat.v1 as tf
from keras import backend as K
from tensorflow.python.keras.backend import set_session
from tensorflow.keras.preprocessing.image import ImageDataGenerator
from tensorflow.keras import optimizers
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dropout, Flatten, Dense, Activation
from keras.layers.convolutional import Convolution2D, MaxPooling2D
from keras import callbacks
import time
def freeze_session(session, keep_var_names=None, output_names=None, clear_devices=True):
"""
Freezes the state of a session into a pruned computation graph.
Creates a new computation graph where variable nodes are replaced by
constants taking their current value in the session. The new graph will be
pruned so subgraphs that are not necessary to compute the requested
outputs are removed.
#param session The TensorFlow session to be frozen.
#param keep_var_names A list of variable names that should not be frozen,
or None to freeze all the variables in the graph.
#param output_names Names of the relevant graph outputs.
#param clear_devices Remove the device directives from the graph for better portability.
#return The frozen graph definition.
"""
graph = session.graph
with graph.as_default():
freeze_var_names = list(set(v.op.name for v in tf.global_variables()).difference(keep_var_names or []))
output_names = output_names or []
output_names += [v.op.name for v in tf.global_variables()]
input_graph_def = graph.as_graph_def()
if clear_devices:
for node in input_graph_def.node:
node.device = ''
frozen_graph = tf.graph_util.convert_variables_to_constants(
session, input_graph_def, output_names, freeze_var_names)
return frozen_graph
start = time.time()
tf.compat.v1.disable_eager_execution()
DEV = False
argvs = sys.argv
argc = len(argvs)
if argc > 1 and (argvs[1] == "--development" or argvs[1] == "-d"):
DEV = True
epochs = 2
train_data_path = 'data/train'
validation_data_path = 'data/test'
"""
Parameters
"""
img_width, img_height = 150, 150
batch_size = 32
samples_per_epoch = 24
validation_steps = 24
nb_filters1 = 32
nb_filters2 = 64
conv1_size = 3
conv2_size = 2
pool_size = 2
classes_num = 2
lr = 0.0004
model = Sequential()
model.add(Convolution2D(nb_filters1, conv1_size, conv1_size, padding ="same", input_shape=(img_width, img_height, 3)))
model.add(Activation("relu"))
model.add(MaxPooling2D(pool_size=(pool_size, pool_size)))
model.add(Convolution2D(nb_filters2, conv2_size, conv2_size, padding ="same"))
model.add(Activation("relu"))
model.add(MaxPooling2D(pool_size=(pool_size, pool_size)))
model.add(Flatten())
model.add(Dense(256))
model.add(Activation("relu"))
model.add(Dropout(0.5))
model.add(Dense(classes_num, activation='softmax'))
model.compile(loss='categorical_crossentropy',
optimizer=optimizers.RMSprop(learning_rate=lr),
metrics=['accuracy'])
train_datagen = ImageDataGenerator(
rescale=1. / 255,
shear_range=0.2,
zoom_range=0.2,
horizontal_flip=True)
test_datagen = ImageDataGenerator(rescale=1. / 255)
train_generator = train_datagen.flow_from_directory(
train_data_path,
target_size=(img_height, img_width),
batch_size=batch_size,
class_mode='categorical')
validation_generator = test_datagen.flow_from_directory(
validation_data_path,
target_size=(img_height, img_width),
batch_size=batch_size,
class_mode='categorical')
"""
Tensorboard log
"""
log_dir = './tf-log/'
tb_cb = callbacks.TensorBoard(log_dir=log_dir, histogram_freq=0)
cbks = [tb_cb]
model.fit(
train_generator,
epochs=epochs,
validation_data=validation_generator,
callbacks=cbks)
target_dir = './models/'
if not os.path.exists(target_dir):
os.mkdir(target_dir)
model.save('./mymodels/model')
pred_node_names = [None]
pred = [None]
for i in range(1):
pred_node_names[i] = "output_node"+str(i)
pred[i] = tf.identity(model.outputs[i], name=pred_node_names[i])
frozen_graph = freeze_session(tf.keras.backend.get_session(), output_names=[out.op.name for out in model.outputs])
tf.train.write_graph(frozen_graph, './', 'xor.pbtxt', as_text=True)
tf.train.write_graph(frozen_graph, './', 'xor.pb', as_text=False)
#Calculate execution time
end = time.time()
dur = end-start
if dur<60:
print("Execution Time:",dur,"seconds")
elif dur>60 and dur<3600:
dur=dur/60
print("Execution Time:",dur,"minutes")
else:
dur=dur/(60*60)
print("Execution Time:",dur,"hours")
Post that I want to load the model in opencv to perform the predictions but readNetFromTensorflow() fails with the following error
[libprotobuf ERROR D:\a\opencv-python\opencv-python\opencv\3rdparty\protobuf\src\google\protobuf\text_format.cc:292] Error parsing text-format opencv_tensorflow.GraphDef: 700:9: Unknown enumeration value of "DT_VARIANT" for field "type".
Traceback (most recent call last):
File "C:\Users\admin\Downloads\Image-Classification-by-Keras-and-Tensorflow-master\Using Tensorflow\classify_opencv.py", line 4, in <module>
tensorflowNet = cv2.dnn.readNetFromTensorflow('xor.pb','xor.pbtxt')
cv2.error: OpenCV(4.5.4-dev) D:\a\opencv-python\opencv-python\opencv\modules\dnn\src\tensorflow\tf_io.cpp:54: error: (-2:Unspecified error) FAILED: ReadProtoFromTextFile(param_file, param). Failed to parse GraphDef file: xor.pbtxt in function 'cv::dnn::ReadTFNetParamsFromTextFileOrDie
I went through the .pbtxt file, it indeed contains DT_VARIANT type which I guess is not defined in 3rd party protobuf.
How do I solve this error?

How to train imbalanced data with Keras?

I am trying to experiment on ISIC 2019 data as a newbie. Firstly, I downloaded the training data and divided the data into 3 parts as train, test, and validation data, and every dataset folder contains 2 subfolders which are benign and malignant. In short, I just moved all the categories into benign folders except the melenoma category and melanoma images are inside malignant folders. After the division, I get imbalanced data. In the training dataset for benign data, I get 16596 images and for malignant data, I get 3629 images. I tried to train my data and I couldn't get a good result for malignant and my precision value was about 0.18 for malignant. I used ResNet50 to train my model and I would like to ask how can I train my model without data augmentation and oversampling? I am also trying decayed learning metrics at the moment and it seems it won't give a good result too.
import os
import tensorflow as tf
import math
import numpy as np
import matplotlib.pyplot as plt
from tensorflow import keras
from keras.applications.resnet50 import ResNet50, preprocess_input
from keras.layers import Dense, GlobalMaxPooling2D
from keras.models import Model
from keras.optimizers import Adam
from sklearn.metrics import roc_curve
from tensorflow.keras.preprocessing.image import ImageDataGenerator
train_examples = 20225
test_examples = 2551
validation_examples = 2555
img_height = img_width = 224
channel = 3
batch_size = 32
base_model = ResNet50(weights = 'imagenet' , include_top = False, input_shape = (img_height, img_width, channel))
x = base_model.output
x = GlobalMaxPooling2D()(x)
x = Dense(1, activation= 'sigmoid')(x)
model = Model(
inputs = base_model.input,
outputs = x)
model.summary()
train_datagen = ImageDataGenerator(
rotation_range = 20,
width_shift_range=0.10,
height_shift_range=0.10,
zoom_range = 0.10,
horizontal_flip = True,
preprocessing_function = preprocess_input,
fill_mode='nearest'
)
validation_datagen = ImageDataGenerator(
preprocessing_function = preprocess_input,
)
test_datagen = ImageDataGenerator(
preprocessing_function = preprocess_input,
)
train_gen = train_datagen.flow_from_directory(
"dataset/train/",
target_size = (img_height, img_width),
batch_size = batch_size,
color_mode = "rgb",
class_mode = "binary",
shuffle = True,
seed = 123,
)
validation_gen = validation_datagen.flow_from_directory(
"dataset/validation/",
target_size = (img_height, img_width),
batch_size = batch_size,
color_mode = "rgb",
class_mode = "binary",
shuffle = True,
seed = 123,
)
test_gen = test_datagen.flow_from_directory(
"dataset/test/",
target_size =(img_height, img_width),
batch_size = batch_size,
color_mode = "rgb",
class_mode = "binary",
shuffle = True,
seed = 123,
)
METRICS = [
keras.metrics.Precision(name = "precision"),
keras.metrics.Recall(name = "recall"),
keras.metrics.AUC(name = "auc"),
]
model.compile(
optimizer = Adam(lr = 3e-4),
loss = [keras.losses.BinaryCrossentropy(from_logits = False)],
metrics = METRICS,
)
history = model.fit(train_gen,
epochs=50,
verbose=1,
validation_data=validation_gen,
callbacks=[keras.callbacks.ModelCheckpoint("isic_binary_model")],
)

Why I am getting error while using ResNet50 for transfer learning?

So I am trying to use ResNet50 for transfer learning with my keras model(i.e Trinity), but the problem is I am getting an error when I try to execute Trinity.fit().
Exact error "ValueError: Input 0 of layer sequential_8 is incompatible with the layer: expected ndim=4, found ndim=2. Full shape received: [None, 2048]"
What is the reason for this error? How can I solve it?
Here is the python code:
1) from keras.models import Sequential
from keras.layers import Conv2D,MaxPooling2D
from keras.layers import Activation, Dense, Flatten, Dropout
from keras.preprocessing.image import ImageDataGenerator
import os
2) import keras
import wandb
from wandb.keras import WandbCallback
from keras.models import Sequential
from keras.layers import Dense, Flatten
from keras.applications.resnet50 import ResNet50, decode_predictions,preprocess_input
import matplotlib.pyplot as plt
3) training_dir = '../input/fruits/fruits-360/Training/'
validation_dir = '../input/fruits/fruits-360/Test/'
test_dir = '../input/fruits/fruits-360/test-multiple_fruits/'
4) image_size = 224
data_generator = ImageDataGenerator(preprocessing_function=preprocess_input)
train_generator = data_generator.flow_from_directory(
'../input/fruits/fruits-360/Training/',
target_size=(image_size, image_size),
batch_size=24,
class_mode='categorical')
validation_generator = data_generator.flow_from_directory(
'../input/fruits/fruits-360/Test/',
target_size=(image_size, image_size),
batch_size=24,
class_mode='categorical')
#OUTPUT
#[Found 67692 images belonging to 131 classes.
#Found 22688 images belonging to 131 classes.]
5) resnet_model = ResNet50(weights="imagenet")
6) x_train_preprocessed = train_generator
x_test_preprocessed = validation_generator
7) last_layer = resnet_model.get_layer("avg_pool")
resnet_layers = keras.Model(inputs=resnet_model.inputs, outputs=last_layer.output)
resnet_layers.summary()
8) x_train_features = resnet_layers.predict(x_train_preprocessed)
x_test_features = resnet_layers.predict(x_test_preprocessed)
9) Trinity = Sequential()
Trinity.add(Conv2D(filters = 16, kernel_size = 2,input_shape=(224,224,3),padding='same'))
Trinity.add(Activation('relu'))
Trinity.add(MaxPooling2D(pool_size=2))
Trinity.add(Conv2D(filters = 32,kernel_size = 2,activation= 'relu',padding='same'))
Trinity.add(MaxPooling2D(pool_size=2))
Trinity.add(Conv2D(filters = 64,kernel_size = 2,activation= 'relu',padding='same'))
Trinity.add(MaxPooling2D(pool_size=2))
Trinity.add(Conv2D(filters = 128,kernel_size = 2,activation= 'relu',padding='same'))
Trinity.add(MaxPooling2D(pool_size=2))
Trinity.add(Dropout(0.3))
Trinity.add(Flatten())
Trinity.add(Dense(132))
Trinity.add(Activation('relu'))
Trinity.add(Dropout(0.4))
Trinity.add(Dense(131,activation = 'softmax'))
Trinity.compile(loss='categorical_crossentropy',optimizer='rmsprop',metrics=['accuracy'])
10) Trinity.fit(x_train_features,epochs=50,validation_data=x_test_features)
#OUTPUT OF 7)
#OUTPUT OF 10)

you should add include_top=False to the instance of resnet, then add your model to the top of resnet. that way it should work. Here is an example:
base_model = ResNet50V2(include_top=False, weights="imagenet", input_shape=(224,224,3), pooling="avg")
base_model.summary()
model2 = Sequential()
model2.add(base_model)
model2.add(Dense(64, activation="relu"))
model2.add(Dropout(0.2))
model2.add(Dense(64, activation="relu"))
model2.add(Dropout(0.2))
model2.add(Dense(32, activation="relu"))
model2.add(Dropout(0.2))
model2.add(Dense(32, activation="relu"))
model2.add(Dropout(0.2))
model2.add(Dense(1, activation="sigmoid"))
base_model.trainable = False
model2.summary()
model2.compile(optimizer=Adam(), loss="binary_crossentropy", metrics=["accuracy"])
and here is another example from keras website: https://keras.io/examples/vision/image_classification_efficientnet_fine_tuning/

How do i interpret results from Keras predict_generator?

I am performing binary classification of data. I am using predict_generator to obtain the classification results. The input to the predict_generator are 44 examples, 22 positive, 22 negative. The following is the output obtained:
[9.98187363e-01 1.81267178e-03]
[5.02341951e-04 9.99497652e-01]
[8.41189444e-01 1.58810586e-01]
[7.26610771e-04 9.99273360e-01]
[9.96826649e-01 3.17337317e-03]
[8.83008718e-01 1.16991334e-01]
[3.84130690e-04 9.99615788e-01]
[8.65039527e-01 1.34960532e-01]
[1.78014021e-03 9.98219788e-01]
[9.96107757e-01 3.89222591e-03]
[6.16264821e-04 9.99383688e-01]
[2.98170745e-03 9.97018337e-01]
[9.92357790e-01 7.64221745e-03]
[9.93237853e-01 6.76209433e-03]
[9.98248339e-01 1.75163767e-03]
[1.17816392e-03 9.98821795e-01]
[9.84322488e-01 1.56775210e-02]
[3.11790430e-03 9.96882081e-01]
[4.62388212e-04 9.99537587e-01]
[1.42699364e-03 9.98572946e-01]
[9.43281949e-01 5.67180961e-02]
[9.98008907e-01 1.99115812e-03]
[4.12312744e-04 9.99587715e-01]
[9.29474115e-01 7.05258474e-02]
[3.37766513e-04 9.99662280e-01]
[1.75693433e-03 9.98243093e-01]
[9.92154586e-04 9.99007881e-01]
[1.87152205e-03 9.98128474e-01]
[9.20654461e-02 9.07934546e-01]
[9.95722532e-01 4.27750358e-03]
[9.96877313e-01 3.12273414e-03]
[9.87601459e-01 1.23985587e-02]
[1.11398198e-01 8.88601840e-01]
[1.48968585e-02 9.85103130e-01]
[6.73048152e-03 9.93269503e-01]
[1.65761902e-03 9.98342395e-01]
[9.94634032e-01 5.36595425e-03]
[5.00697970e-01 4.99302000e-01]
[1.65578525e-03 9.98344183e-01]
[9.68859911e-01 3.11401486e-02]
CODE:
from keras.applications import Xception
from keras.models import Model
from keras.layers import Dense, Input, Dropout
from keras.optimizers import Nadam
from keras.callbacks import EarlyStopping, ModelCheckpoint
from keras.preprocessing.image import ImageDataGenerator
img_height = 299
img_width = 299
no_of_frames = 15
channels = 3
no_of_epochs = 50
batch_size_value = 60
cnn_base = Xception(input_shape=(img_width, img_height, channels),
weights="imagenet", include_top=False, pooling='avg')
cnn_base.trainable = False
hidden_layer_1 = Dense(activation="relu", units=1024)(cnn_base.output)
drop_layer=Dropout(0.2)(hidden_layer_1)
hidden_layer_2 = Dense(activation="relu", units=512)(drop_layer)
outputs = Dense(2, activation="softmax")(hidden_layer_2)
model = Model(cnn_base.input, outputs)
nadam_optimizer = Nadam(lr=0.0001, beta_1=0.9, beta_2=0.999,
epsilon=1e-08, schedule_decay=0.004)
model.compile(loss="categorical_crossentropy",
optimizer=nadam_optimizer, metrics=["accuracy"])
# for data augmentation
train_datagen = ImageDataGenerator( zoom_range=.1, rotation_range=8,
width_shift_range=.2, height_shift_range=.2)
train_generator = train_datagen.flow_from_directory(
'/home/Train', # this is the target directory
target_size=(img_width, img_height),
batch_size=batch_size_value,
class_mode="categorical")
validation_generator = ImageDataGenerator().flow_from_directory(
'/home/Val',
target_size=(img_width, img_height),
batch_size=batch_size_value,
class_mode="categorical")
history = model.fit_generator(
train_generator,
validation_data=validation_generator,
verbose=1,
epochs=no_of_epochs,
steps_per_epoch=17,
validation_steps=7)
Test_generator = ImageDataGenerator().flow_from_directory(
'/home/Test',
target_size=(img_width, img_height),
batch_size=batch_size_value,
class_mode="categorical")
Prob_val = model.predict_generator(test_set)
print((Prob_val))
I assume they are probabilities, but the column contains only 40 entries. How do they correspond with the 44 inputs example ?

Can't train model for hand gesture (ASL) by using CNN

I've successfully distinguish dog and cat by using CNN, now I am trying to train model for (ASL) American Sign Language, I made some changes but not worked and now I've no idea what to change in code and which way and also I google for this but unfortunately didn't worked, this is my FYP- (Final Year Project) and I am stuck, please help me.
I changed loss = binary_crossentropy to loss = sparse_categorical_crossentropy and but still showing label error.
1 class Data_preprocessing:
'Data preprocessing before goes to ML'
# Train by data list initilization
training_data = []
def __init__(self, datadir, categories, img_size):
Data_preprocessing.img_size = img_size
Data_preprocessing.datadir = datadir
Data_preprocessing.categories = categories
def Create_training_data(self):
for category in Data_preprocessing.categories:
# path to cats or dogs dir
path = os.path.join(Data_preprocessing.datadir, category)
class_num = Data_preprocessing.categories.index(category)
# After having the directory for images
# Started to read image by using OpenCv and directly convert it to GRAYSCALE
for img in os.listdir(path):
try:
img_array = cv2.imread(os.path.join(path, img), cv2.IMREAD_GRAYSCALE)
new_array = cv2.resize(img_array, (Data_preprocessing.img_size, Data_preprocessing.img_size))
Data_preprocessing.training_data.append([new_array, class_num])
except Exception as e:
pass
self.Saving_processed_data()
def Saving_processed_data(self):
random.shuffle(Data_preprocessing.training_data)
x = []
y = []
for features, label in Data_preprocessing.training_data:
x.append(features)
y.append(label)
x = np.array(x).reshape(-1, Data_preprocessing.img_size, Data_preprocessing.img_size, 1)
# Saving data by using "pickle"
pickle_out = open("x.pickle", "wb")
pickle.dump(x, pickle_out)
pickle_out.close()
pickle_out = open("y.pickle", "wb")
pickle.dump(y, pickle_out)
pickle_out.close()
categories = ["Dog","Cat"]
categories = ["A","B","C","D","del","E","F","G","H","I","J","K","L","M","N","nothing","O","P","Q","R","S","space","T","U","V","W","X","Y","Z"]
data_preprocessing = Data_preprocessing("ASLDS\\ASLDS",categories, 50)
data_preprocessing.Create_training_data()
2 class Learning_model:
def __init__(self):
pass
def TrainModel(self):
self.x = pickle.load(open("x.pickle", "rb"))
self.y = pickle.load(open("y.pickle", "rb"))
self.x = self.x/255.0
self.model = Sequential()
self.model.add(Conv2D(64, (3,3), input_shape = self.x.shape[1:]))
self.model.add(Activation("relu"))
self.model.add(MaxPooling2D(pool_size=(2,2)))
self.model.add(Conv2D(64, (3,3)))
self.model.add(Activation("relu"))
self.model.add(MaxPooling2D(pool_size=(2,2)))
self.model.add(Flatten())
self.model.add(Dense(64))
self.model.add(Dense(1))
self.model.add(Activation('sigmoid'))
self.model.compile(optimizer='adam',
loss='sparse_categorical_crossentropy',
metrics=['accuracy'])
self.model.fit(self.x, self.y, batch_size = 32, epochs=10, validation_split = 0.1)
self.model.save("64x3-CNN-ASL.model")
trained_model = Learning_model()
trained_model.TrainModel()
I am expecting that if I input an image of any alphabet so it is supposed to show me corresponding name of that alphabet.

You should change loss to categorical cross entropy. Even I built a similar CNN with Keras.
This CNN is built to recognize 3 different types of images, but you can change input_shape to detect any number of categories.
# Importing the Keras libraries and packages
from keras.models import Sequential
from keras.layers import Conv2D
from keras.layers import MaxPooling2D
from keras.layers import Flatten
from keras.layers import Dense
# Initialising the CNN
classifier = Sequential()
classifier.add(Conv2D(32, (3, 3), input_shape = (64, 64, 3), activation = 'relu'))
classifier.add(MaxPooling2D(pool_size = (2, 2)))
classifier.add(Conv2D(32, (3, 3), activation = 'relu'))
classifier.add(MaxPooling2D(pool_size = (2, 2)))
classifier.add(Flatten())
classifier.add(Dense(units = 128, activation = 'relu'))
classifier.add(Dense(units = 3, activation = 'softmax')) # output layer
# Compiling the CNN
classifier.compile(optimizer = 'adam', loss = 'categorical_crossentropy', metrics = ['accuracy'])
# Using the CNN on the images
from keras.preprocessing.image import ImageDataGenerator
train_datagen = ImageDataGenerator(rescale = 1./255,
shear_range = 0.2,
zoom_range = 0.2,
horizontal_flip = True)
test_datagen = ImageDataGenerator(rescale = 1./255)
training_set = train_datagen.flow_from_directory('dataset/training_set',
target_size = (64, 64),
batch_size = 32,
class_mode = 'categorical')
test_set = test_datagen.flow_from_directory('dataset/test_set',
target_size = (64, 64),
batch_size = 32,
class_mode = 'categorical')
classifier.fit_generator(training_set,
steps_per_epoch = (8000/32),
epochs = 25,
validation_data = test_set,
validation_steps = (2000/32))
# Fetching Predictions
import numpy as np
from skimage.io import imread
from skimage.transform import resize
class_labels = {v: k for k, v in training_set.class_indices.items()}
img = imread('dataset/single_prediction/random.jpg')
img = resize(img,(64,64))
img = np.expand_dims(img,axis=0)
if(np.max(img)>1):
img = img/255.0
prediction = classifier.predict_classes(img)
print ("\n\n")
print (class_labels[prediction[0]])