We Keep Coding

Opencv readNetFromTensorflow fails with Unknown enumeration DT_VARIANT

I trained a keras model and saved it as PB as in the below script.
import sys
import os
import tensorflow.compat.v1 as tf
from keras import backend as K
from tensorflow.python.keras.backend import set_session
from tensorflow.keras.preprocessing.image import ImageDataGenerator
from tensorflow.keras import optimizers
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dropout, Flatten, Dense, Activation
from keras.layers.convolutional import Convolution2D, MaxPooling2D
from keras import callbacks
import time
def freeze_session(session, keep_var_names=None, output_names=None, clear_devices=True):
"""
Freezes the state of a session into a pruned computation graph.
Creates a new computation graph where variable nodes are replaced by
constants taking their current value in the session. The new graph will be
pruned so subgraphs that are not necessary to compute the requested
outputs are removed.
#param session The TensorFlow session to be frozen.
#param keep_var_names A list of variable names that should not be frozen,
or None to freeze all the variables in the graph.
#param output_names Names of the relevant graph outputs.
#param clear_devices Remove the device directives from the graph for better portability.
#return The frozen graph definition.
"""
graph = session.graph
with graph.as_default():
freeze_var_names = list(set(v.op.name for v in tf.global_variables()).difference(keep_var_names or []))
output_names = output_names or []
output_names += [v.op.name for v in tf.global_variables()]
input_graph_def = graph.as_graph_def()
if clear_devices:
for node in input_graph_def.node:
node.device = ''
frozen_graph = tf.graph_util.convert_variables_to_constants(
session, input_graph_def, output_names, freeze_var_names)
return frozen_graph
start = time.time()
tf.compat.v1.disable_eager_execution()
DEV = False
argvs = sys.argv
argc = len(argvs)
if argc > 1 and (argvs[1] == "--development" or argvs[1] == "-d"):
DEV = True
epochs = 2
train_data_path = 'data/train'
validation_data_path = 'data/test'
"""
Parameters
"""
img_width, img_height = 150, 150
batch_size = 32
samples_per_epoch = 24
validation_steps = 24
nb_filters1 = 32
nb_filters2 = 64
conv1_size = 3
conv2_size = 2
pool_size = 2
classes_num = 2
lr = 0.0004
model = Sequential()
model.add(Convolution2D(nb_filters1, conv1_size, conv1_size, padding ="same", input_shape=(img_width, img_height, 3)))
model.add(Activation("relu"))
model.add(MaxPooling2D(pool_size=(pool_size, pool_size)))
model.add(Convolution2D(nb_filters2, conv2_size, conv2_size, padding ="same"))
model.add(Activation("relu"))
model.add(MaxPooling2D(pool_size=(pool_size, pool_size)))
model.add(Flatten())
model.add(Dense(256))
model.add(Activation("relu"))
model.add(Dropout(0.5))
model.add(Dense(classes_num, activation='softmax'))
model.compile(loss='categorical_crossentropy',
optimizer=optimizers.RMSprop(learning_rate=lr),
metrics=['accuracy'])
train_datagen = ImageDataGenerator(
rescale=1. / 255,
shear_range=0.2,
zoom_range=0.2,
horizontal_flip=True)
test_datagen = ImageDataGenerator(rescale=1. / 255)
train_generator = train_datagen.flow_from_directory(
train_data_path,
target_size=(img_height, img_width),
batch_size=batch_size,
class_mode='categorical')
validation_generator = test_datagen.flow_from_directory(
validation_data_path,
target_size=(img_height, img_width),
batch_size=batch_size,
class_mode='categorical')
"""
Tensorboard log
"""
log_dir = './tf-log/'
tb_cb = callbacks.TensorBoard(log_dir=log_dir, histogram_freq=0)
cbks = [tb_cb]
model.fit(
train_generator,
epochs=epochs,
validation_data=validation_generator,
callbacks=cbks)
target_dir = './models/'
if not os.path.exists(target_dir):
os.mkdir(target_dir)
model.save('./mymodels/model')
pred_node_names = [None]
pred = [None]
for i in range(1):
pred_node_names[i] = "output_node"+str(i)
pred[i] = tf.identity(model.outputs[i], name=pred_node_names[i])
frozen_graph = freeze_session(tf.keras.backend.get_session(), output_names=[out.op.name for out in model.outputs])
tf.train.write_graph(frozen_graph, './', 'xor.pbtxt', as_text=True)
tf.train.write_graph(frozen_graph, './', 'xor.pb', as_text=False)
#Calculate execution time
end = time.time()
dur = end-start
if dur<60:
print("Execution Time:",dur,"seconds")
elif dur>60 and dur<3600:
dur=dur/60
print("Execution Time:",dur,"minutes")
else:
dur=dur/(60*60)
print("Execution Time:",dur,"hours")
Post that I want to load the model in opencv to perform the predictions but readNetFromTensorflow() fails with the following error
[libprotobuf ERROR D:\a\opencv-python\opencv-python\opencv\3rdparty\protobuf\src\google\protobuf\text_format.cc:292] Error parsing text-format opencv_tensorflow.GraphDef: 700:9: Unknown enumeration value of "DT_VARIANT" for field "type".
Traceback (most recent call last):
File "C:\Users\admin\Downloads\Image-Classification-by-Keras-and-Tensorflow-master\Using Tensorflow\classify_opencv.py", line 4, in <module>
tensorflowNet = cv2.dnn.readNetFromTensorflow('xor.pb','xor.pbtxt')
cv2.error: OpenCV(4.5.4-dev) D:\a\opencv-python\opencv-python\opencv\modules\dnn\src\tensorflow\tf_io.cpp:54: error: (-2:Unspecified error) FAILED: ReadProtoFromTextFile(param_file, param). Failed to parse GraphDef file: xor.pbtxt in function 'cv::dnn::ReadTFNetParamsFromTextFileOrDie
I went through the .pbtxt file, it indeed contains DT_VARIANT type which I guess is not defined in 3rd party protobuf.
How do I solve this error?

How do I solve this problem during transfer learning ResNet-50?

I am training Resnet-50 to classify 9 classes. I am using following code, transfer learning, to train the model.
Train and test loss and accuracy seem to be fine but when I am testing network against new images I see lots of mistakes.
I feel like that the model is not learning well, I was wondering if you please let me know what is wrong in my approach? How do I solve this problem?
NUM_CLASSES = 9
CHANNELS = 3
IMAGE_RESIZE = 224
RESNET50_POOLING_AVERAGE = 'avg'
DENSE_LAYER_ACTIVATION = 'softmax'
OBJECTIVE_FUNCTION = 'categorical_crossentropy'
LOSS_METRICS = ['accuracy']
NUM_EPOCHS = 100
EARLY_STOP_PATIENCE = 3
STEPS_PER_EPOCH_TRAINING = 10
STEPS_PER_EPOCH_VALIDATION = 10
BATCH_SIZE_TRAINING = 100
BATCH_SIZE_VALIDATION = 100
BATCH_SIZE_TESTING = 1
from tensorflow.python.keras.applications import ResNet50
from tensorflow.python.keras.models import Sequential
from tensorflow.python.keras.layers import Dense
resnet_weights_path = '/path/resnet50_weights_tf_dim_ordering_tf_kernels_notop.h5'
model = Sequential()
model.add(ResNet50(include_top = False, pooling = RESNET50_POOLING_AVERAGE, weights = resnet_weights_path))
model.add(Dense(NUM_CLASSES, activation = DENSE_LAYER_ACTIVATION))
model.layers[0].trainable = False
from tensorflow.python.keras import optimizers
sgd = optimizers.SGD(lr = 0.01, decay = 1e-6, momentum = 0.9, nesterov = True)
model.compile(optimizer = sgd, loss = OBJECTIVE_FUNCTION, metrics = LOSS_METRICS)
model.summary()
from tensorflow.python.keras import optimizers
sgd = optimizers.SGD(lr = 0.01, decay = 1e-6, momentum = 0.9, nesterov = True)
model.compile(optimizer = sgd, loss = OBJECTIVE_FUNCTION, metrics = LOSS_METRICS)
from keras.applications.resnet50 import preprocess_input
from keras.preprocessing.image import ImageDataGenerator
image_size = IMAGE_RESIZE
data_generator = ImageDataGenerator(preprocessing_function=preprocess_input)
train_generator = data_generator.flow_from_directory(
'/path train folder/train',
target_size=(image_size, image_size),
batch_size=BATCH_SIZE_TRAINING,
class_mode='categorical')
validation_generator = data_generator.flow_from_directory(
'/path test folder/test',
target_size=(image_size, image_size),
batch_size=BATCH_SIZE_VALIDATION,
class_mode='categorical')
(BATCH_SIZE_TRAINING, len(train_generator), BATCH_SIZE_VALIDATION, len(validation_generator))
from tensorflow.python.keras.callbacks import EarlyStopping, ModelCheckpoint
cb_early_stopper = EarlyStopping(monitor = 'val_loss', patience = EARLY_STOP_PATIENCE)
cb_checkpointer = ModelCheckpoint(filepath = '/path/best.hdf5', monitor = 'val_loss', save_best_only = True, mode = 'auto')
fit_history = model.fit_generator(
train_generator,
steps_per_epoch=STEPS_PER_EPOCH_TRAINING,
epochs = NUM_EPOCHS,
validation_data=validation_generator,
validation_steps=STEPS_PER_EPOCH_VALIDATION,
callbacks=[cb_checkpointer, cb_early_stopper]
)
model.load_weights("/path/best.hdf5")
model.save('transfer_resnet.h5')
print(fit_history.history.keys())

This can have many reasons.
For one, it is possible that your data set is too small or not varied enough.
What you can try is to add a few more Dense Layers in the top section.

Failed to train with tf.keras.applications.MobileNetV2

Environment:
TF2.0
Python 3.5
ubuntu 16.04
Problem:
I try to use the pre-trained mobilenet_V2 but accuracy doesn't increase:
base_model = tf.keras.applications.MobileNetV2(input_shape=IMG_SHAPE,
include_top=False,
weights='imagenet')
The script is copied from the tutorial of the tensorflow 2.0(https://www.tensorflow.org/tutorials/images/transfer_learning?hl=zh-cn)
The only change I made is the dataset which feed into the network. The original code makes binary classification between dogs and cats, and everything works. However, the accuracy never increases while using multi-classes datasets like: "mnist", "tf_flowers". Please note that, I used the correct loss function and metrics.
Naive model and results:
Keras.mobilenetv2:
Here is the code:
from __future__ import absolute_import, division, print_function, unicode_literals
import os
import numpy as np
import tensorflow as tf
from tensorflow.keras.layers import Input, Dense, Flatten, Conv2D, GlobalAveragePooling2D
from tensorflow.keras import Model
keras = tf.keras
import tensorflow_datasets as tfds
# tfds.disable_progress_bar()
IMG_SIZE = 224
IMG_SHAPE = (IMG_SIZE, IMG_SIZE, 3)
def format_example(image, label):
if image.shape[-1] == 1:
image = tf.concat([image, image, image], 2)
image = tf.cast(image, tf.float32)
image = (image/127.5) - 1
image = tf.image.resize(image, (IMG_SIZE, IMG_SIZE))
return image, label
##----functional model----##
class TinyModel():
def __init__(self, num_classes, hiddens=32, input_shape=IMG_SHAPE):
import tensorflow as tf
self.num_classes = num_classes
self.input_shape = input_shape
self.hiddens = hiddens
def build(self):
inputs = Input(shape=self.input_shape)
x = Conv2D(16, 3, activation="relu", strides=2)(inputs)
x = Conv2D(32, 3, activation="relu", strides=2)(x)
x = Conv2D(32, 3, activation="relu", strides=2)(x)
x = Conv2D(16, 3, activation="relu")(x)
x = Flatten()(x)
x = Dense(self.hiddens, activation="relu")(x)
outputs = Dense(self.num_classes, activation="softmax")(x)
model = Model(inputs=inputs, outputs=outputs, name='my_model')
return model
def assemble_model(num_classes, model_name='MobileNetV2'):
import tensorflow as tf
base_model = tf.keras.applications.MobileNetV2(input_shape=IMG_SHAPE,
weights='imagenet',
include_top=False)
model = tf.keras.Sequential([
base_model,
GlobalAveragePooling2D(),
Dense(num_classes, activation='softmax')
])
model.trainable = True
return model
## ---- dataset preparation -----##
SPLIT_WEIGHTS = (8, 1, 1)
splits = tfds.Split.TRAIN.subsplit(weighted=SPLIT_WEIGHTS)
(raw_train, raw_validation, raw_test), metadata = tfds.load(
'tf_flowers', split=list(splits),
with_info=True, as_supervised=True)
get_label_name = metadata.features['label'].int2str
train = raw_train.map(format_example)
validation = raw_validation.map(format_example)
test = raw_test.map(format_example)
BATCH_SIZE = 32
SHUFFLE_BUFFER_SIZE = 1000
train_ds = train.shuffle(SHUFFLE_BUFFER_SIZE).batch(BATCH_SIZE)
validation_ds = validation.batch(BATCH_SIZE)
test_ds = test.batch(BATCH_SIZE)
IMG_SHAPE = (IMG_SIZE, IMG_SIZE, 3)
## ----- model config ---- ##
# Create an instance of the model
model = TinyModel(num_classes=5).build() # model 1
# model = assemble_model(num_classes=5) # model 2
model.summary()
## ----- training config -----##
loss_object = tf.keras.losses.SparseCategoricalCrossentropy()
optimizer = tf.keras.optimizers.Adam()
train_loss = tf.keras.metrics.Mean(name='train_loss')
train_accuracy = tf.keras.metrics.SparseCategoricalAccuracy(name='train_accuracy')
test_loss = tf.keras.metrics.Mean(name='test_loss')
test_accuracy = tf.keras.metrics.SparseCategoricalAccuracy(name='test_accuracy')
## ----- training loop -----##
#tf.function
def train_step(images, labels):
with tf.GradientTape() as tape:
predictions = model(images)
loss = loss_object(labels, predictions)
gradients = tape.gradient(loss, model.trainable_variables)
optimizer.apply_gradients(zip(gradients, model.trainable_variables))
train_loss(loss)
train_accuracy(labels, predictions)
#tf.function
def test_step(images, labels):
predictions = model(images)
t_loss = loss_object(labels, predictions)
test_loss(t_loss)
test_accuracy(labels, predictions)
EPOCHS = 5
for epoch in range(EPOCHS):
# Reset the metrics at the start of the next epoch
train_loss.reset_states()
train_accuracy.reset_states()
test_loss.reset_states()
test_accuracy.reset_states()
for images, labels in train_ds:
train_step(images, labels)
for test_images, test_labels in test_ds:
test_step(test_images, test_labels)
template = 'Epoch {}, Loss: {}, Accuracy: {}, Test Loss: {}, Test Accuracy: {}'
print(template.format(epoch+1,
train_loss.result(),
train_accuracy.result()*100,
test_loss.result(),
test_accuracy.result()*100))
----------------------SOLVED-----------------------
Solution：add the argument "training=True" when training the keras.application.. For example
model = tf.keras.applications.MobileNetV2(input_shape=IMG_SHAPE,weights="imagenet",include_top=False)
pred = model(inputs, training=True)
The reason might be caused by "batchnorm" layer. Those model which has BN layers works well in keras training loop, "model.fit()", and nothing to takecare. However, they cannot learn anything by costume training loop if you forget to set training=True in model()

The problem is that you set all your parameters to be non-trainable, check this on hte summary of the model, you will see something like this
Change this line, (or just delete it)
base_model.trainable = False
To
base_model.trainable = True
And everything will work fine

Trying the CNN code in python using Keras and Tensorflow

I will show you the predefined code below.
from __future__ import division
import os, keras
os.environ["KERAS_BACKEND"] = "theano"
os.environ["THEANO_FLAGS"] = "device=gpu%d"%(1)
import numpy as np
import theano as th
import theano.tensor as T
from keras.utils import np_utils
import keras.models as models
from keras.layers.core import Reshape,Dense,Dropout,Activation
from keras.optimizers import adam
from scipy.io import loadmat, savemat
import os.path
from keras import backend as K
# Model training function
def train(In_train, Out_train, In_test, Out_test,
nb_epoch, batch_size,dr,
num_hidden_layers, nodes_per_layer,
loss_fn,n_BS,n_beams):
in_shp = list(In_train.shape[1:])
AP_models = []
for idx in range(0, n_BS*n_beams-2, n_beams):
idx_str = str(idx / n_beams + 1)
model = models.Sequential()
model.add(Dense(nodes_per_layer, activation='relu', init='he_normal',
name="dense" + idx_str + "1", input_shape=in_shp))
model.add(Dropout(dr))
for h in range(num_hidden_layers):
model.add(Dense(nodes_per_layer, activation='relu',
init='he_normal', name="dense" + idx_str + "h" + str(h)))
model.add(Dropout(dr))
model.add(Dense(n_beams, activation='relu', init='he_normal',
name="dense" + idx_str + "o"))
model.compile(loss=loss_fn, optimizer='adam')
model.summary()
# perform training ...
earlyStoppingCallback = \
keras.callbacks.EarlyStopping(monitor='val_loss',
patience=5,
verbose=0,
mode='auto')
filepath = 'DLCB_code_output/Results_mmWave_ML'+str(idx)
history = model.fit(In_train,
Out_train[:, idx:idx + n_beams],
batch_size=batch_size,
nb_epoch=nb_epoch,
verbose=2,
validation_data=(In_test, Out_test[:,idx:idx + n_beams]),
callbacks = [
keras.callbacks.ModelCheckpoint(filepath, monitor='val_loss', verbose=0, save_best_only=True, mode='auto'),
keras.callbacks.EarlyStopping(monitor='val_loss', patience=5, verbose=0, mode='auto')
])
# we re-load the best weights once training is finished
model.load_weights(filepath)
AP_models.append(model)
return AP_models
# Reading input and output sets generated from MATLAB
In_set_file=loadmat('DLCB_dataset/DLCB_input.mat')
Out_set_file=loadmat('DLCB_dataset/DLCB_output.mat')
In_set=In_set_file['DL_input']
Out_set=Out_set_file['DL_output']
# Parameter initialization
num_user_tot=In_set.shape[0]
n_DL_size=[.001,.05,.1,.15,.2,.25,.3,.35,.4,.45,.5,.55,.6,.65,.7,.75,.8]
count=0
num_tot_TX=4
num_beams=128
for DL_size_ratio in n_DL_size:
print (DL_size_ratio)
count=count+1
DL_size=int(num_user_tot*DL_size_ratio)
np.random.seed(2016)
n_examples = DL_size
num_train = int(DL_size * 0.8)
num_test = int(num_user_tot*.2)
train_index = np.random.choice(range(0,num_user_tot), size=num_train, replace=False)
rem_index = set(range(0,num_user_tot))-set(train_index)
test_index= list(set(np.random.choice(list(rem_index), size=num_test, replace=False)))
In_train = In_set[train_index]
In_test = In_set[test_index]
Out_train = Out_set[train_index]
Out_test = Out_set[test_index]
# Learning model parameters
nb_epoch = 10
batch_size = 100
dr = 0.05 # dropout rate
num_hidden_layers=4
nodes_per_layer=In_train.shape[1]
loss_fn='mean_squared_error'
# Model training
AP_models = train(In_train, Out_train, In_test, Out_test,
nb_epoch, batch_size,dr,
num_hidden_layers, nodes_per_layer,
loss_fn,num_tot_TX,num_beams)
# Model running/testing
DL_Result={}
for id in range(0,num_tot_TX,1):
beams_predicted=AP_models[id].predict( In_test, batch_size=10, verbose=0)
DL_Result['TX'+str(id+1)+'Pred_Beams']=beams_predicted
DL_Result['TX'+str(id+1)+'Opt_Beams']=Out_test[:,id*num_beams:(id+1)*num_beams]
DL_Result['user_index']=test_index
savemat('DLCB_code_output/DL_Result'+str(count),DL_Result)
UnboundLocalError: local variable 'batch_index' referenced before assignment
As you can see above, there is an error message about 'batch_index'.
I searched through the internet, and I think there is a problem in the local/global variables.
Adversely, I don't use the variable 'batch_index' in the code.
I don't know how I should deal with this problem.

How can we fix following error " InternalError: Unable to get element as bytes"?

I am a student, and I am trying to implement text classification using elmo on keras. I imported elmo layer from tensorflow-hub.
def ELMoEmbedding(x):
return embed(inputs={ "tokens": tf.squeeze(tf.cast(x, tf.string)), "sequence_len": tf.constant(100*[max_len])}, signature="tokens", as_dict=True)["elmo"]
url = "https://tfhub.dev/google/elmo/2"
embed = hub.Module(url)
Model :
inpt = Input(shape=(max_len,), dtype = tf.string)
emb_layer = Lambda(ELMoEmbedding, output_shape=(max_len,1024))(inpt)
bdlstm1 = Bidirectional(LSTM(1024))
drp = Dropout(0.5)(bdlstm1)
dns1 = Dense(2, activation='relu')(drp)
dns2 = Dense(no_labels, activation='softmax')(dns1)
model = Model(inpt, dns2)
model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
model.fit(x, y, batch_size = 100, epochs=10)
x and y are int32 numpy arrays
x = [[0,0,1,2,3],[0,0,4,5,6]]
y = [[0,1],[1,0]]
On implementing above code I am getting following error
c_api.TF_GetCode(self.status.status))
InternalError: Unable to get element as bytes.
Updated code
import tensorflow as tf
import tensorflow_hub as hub
from tensorflow.keras.layers import Input, Bidirectional, Lambda, Dropout, Dense, LSTM
from tensorflow.keras.models import Model
def ELMoEmbedding(x):
return embed(inputs={ "tokens": tf.squeeze(tf.cast(x, tf.string)), "sequence_len": tf.constant(100*[max_len])}, signature="tokens", as_dict=True)["elmo"]
url = "https://tfhub.dev/google/elmo/2"
embed = hub.Module(url)
inpt = Input(shape=(max_len,))
emb_layer = Lambda(ELMoEmbedding, output_shape=(max_len,1024))(inpt)
bdlstm1 = Bidirectional(LSTM(1024))
drp = Dropout(0.5)(bdlstm1)
dns1 = Dense(2, activation='softmax')(drp)
model = Model(inpt, dns1)
model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
model.fit(x, y, batch_size = 100, epochs=10)
New Error
TypeError: Failed to convert object of type <class 'tensorflow.python.keras.layers.wrappers.Bidirectional'> to Tensor. Contents: <tensorflow.python.keras.layers.wrappers.Bidirectional object at 0x7fc724d042d0>. Consider casting elements to a supported type.