We Keep Coding

Tensorflow data pipeline stops working during training

I made a Tensorflow pipeline for loading numpy arrays (video data shape (40,160,160,3)). However, it stops working after loading the first x batches.
The problem is solved when removing num_parallel_calls=AUTOTUNE. However, if I do this, the training becomes significantly slower (ETA/epoch ~30 min -> ETA/epoch ~ 4 hours) . Is there a way to load the numpy arrays in parallel (or apply num_parallel_calls=AUTOTUNE) without any problems?
def get_label(file_path):
import os
parts = tf.strings.split(file_path, os.path.sep)
return parts[-2]
def process_video(file_path):
label = get_label(file_path)
video = np.load(file_path, allow_pickle=True)
return np.float32(video/255), np.float32(label)
def set_shape(video, label):
video.set_shape((40,160,160, 3))
label.set_shape([])
return video, label
## Data pipeline
AUTOTUNE = tf.data.experimental.AUTOTUNE
train_ds = tf.data.Dataset.list_files("path/train/*/*",shuffle=True)
train_ds = train_ds.map(lambda item: tf.numpy_function(
process_video, [item], (tf.float32, tf.float32)) ,num_parallel_calls=AUTOTUNE)
train_ds = train_ds.map(set_shape)
train_ds = train_ds.batch(8)
train_ds = train_ds.prefetch(AUTOTUNE)
## Model
def create_LRCN_model():
model = Sequential()
model.add(TimeDistributed(Conv2D(64, (3, 3), padding='same',activation = 'relu'),
input_shape = (40, 160, 160, 3)))
model.add(TimeDistributed(MaxPooling2D((4, 4))))
model.add(TimeDistributed(Dropout(0.25)))
model.add(TimeDistributed(Conv2D(64, (3, 3), padding='same',activation = 'relu')))
model.add(TimeDistributed(MaxPooling2D((4, 4))))
model.add(TimeDistributed(Dropout(0.25)))
model.add(TimeDistributed(Conv2D(64, (3, 3), padding='same',activation = 'relu')))
model.add(TimeDistributed(MaxPooling2D((2, 2))))
model.add(TimeDistributed(Dropout(0.25)))
model.add(TimeDistributed(Conv2D(32, (3, 3), padding='same',activation = 'relu')))
model.add(TimeDistributed(MaxPooling2D((2, 2))))
#model.add(TimeDistributed(Dropout(0.25)))
model.add(TimeDistributed(Flatten()))
model.add(LSTM(32))
model.add(Dense(1, activation = 'sigmoid'))
model.summary()
return model
LRCN_model = create_LRCN_model()
early_stopping_callback = EarlyStopping(monitor = 'val_loss', patience = 15, mode = 'min', restore_best_weights = True)
LRCN_model.compile(loss='binary_crossentropy', optimizer = 'Adam', metrics = ["accuracy"])
LRCN_model_training_history = LRCN_model.fit(train_ds, validation_data= val_ds, epochs = 70, callbacks = [early_stopping_callback]) #class_weight= class_weights,

How to solve Input to reshape is a tensor with

I want to solve this below error.
InvalidArgumentError: Input to reshape is a tensor with 737280 values, but the requested shape requires a multiple of 184832
so I see reference.
reference
Python / Tensorflow - Input to reshape is a tensor with 92416 values, but the requested shape requires a multiple of 2304
However, looking at the answers to this problem, I do not know where to fix it.
So ,I would like to know how to check the size of the input image.
Thank you for your time.
my model:
# For multi_model
activationFunction='elu'
def build_multi2(main_input_shape, output_dim):
inputA = Input(shape=main_input_shape)
ch1_model = create_convolution_layers(inputA)
inputB = Input(shape=main_input_shape)
ch2_model = create_convolution_layers(inputB)
inputC = Input(shape=main_input_shape)
ch3_model = create_convolution_layers(inputC)
inputD = Input(shape=main_input_shape)
ch4_model = create_convolution_layers(inputD)
conv = concatenate([ch1_model, ch2_model, ch3_model, ch4_model])
conv = Flatten()(conv)
dense = Dense(512)(conv)
dense = LeakyReLU(alpha=0.1)(dense)
dense = Dropout(0.5)(dense)
output = Dense(N_class, activation='softmax')(dense)
return Model(inputs=[inputA, inputB, inputC, inputD], outputs=[output])
def create_convolution_layers(input_img):
model = Conv2D(32, (3, 3), padding='same', input_shape=main_input_shape)(input_img)
model = LeakyReLU(alpha=0.1)(model)
model = MaxPooling2D((2, 2),padding='same')(model)
model = Dropout(0.25)(model)
model = Conv2D(64, (3, 3), padding='same')(model)
model = LeakyReLU(alpha=0.1)(model)
model = MaxPooling2D(pool_size=(2, 2),padding='same')(model)
model = Dropout(0.25)(model)
model = Conv2D(128, (3, 3), padding='same')(model)
model = LeakyReLU(alpha=0.1)(model)
model = MaxPooling2D(pool_size=(2, 2),padding='same')(model)
model = Dropout(0.4)(model)
return model
my model call
# For model declaration
N_class = 20
main_input_shape = (150,150, 3)
output_dim = N_class
# opt = tf.keras.optimizers.RMSprop(lr=0.001)
opt = tf.keras.optimizers.Adam()
clf = build_multi2(main_input_shape, output_dim)
clf.compile(optimizer=opt, loss=['categorical_crossentropy'], metrics=['accuracy'])
clf.summary()
my image size: 96×96 pixel
my tensorflow. ImageDataGenerator
train_imgen = ImageDataGenerator(rescale = 1./255,
# shear_range = 0.2,
# zoom_range = 0.2,
# rotation_range=5.,
horizontal_flip = False)
'''

You have specified your input shape as (150, 150, 3) and your image shape is (96, 96, 3), these are incompatible.
You can either resize your images to (150, 150, 3) or change your input shape to be the same as your image shape.

Can't train model for hand gesture (ASL) by using CNN

I've successfully distinguish dog and cat by using CNN, now I am trying to train model for (ASL) American Sign Language, I made some changes but not worked and now I've no idea what to change in code and which way and also I google for this but unfortunately didn't worked, this is my FYP- (Final Year Project) and I am stuck, please help me.
I changed loss = binary_crossentropy to loss = sparse_categorical_crossentropy and but still showing label error.
1 class Data_preprocessing:
'Data preprocessing before goes to ML'
# Train by data list initilization
training_data = []
def __init__(self, datadir, categories, img_size):
Data_preprocessing.img_size = img_size
Data_preprocessing.datadir = datadir
Data_preprocessing.categories = categories
def Create_training_data(self):
for category in Data_preprocessing.categories:
# path to cats or dogs dir
path = os.path.join(Data_preprocessing.datadir, category)
class_num = Data_preprocessing.categories.index(category)
# After having the directory for images
# Started to read image by using OpenCv and directly convert it to GRAYSCALE
for img in os.listdir(path):
try:
img_array = cv2.imread(os.path.join(path, img), cv2.IMREAD_GRAYSCALE)
new_array = cv2.resize(img_array, (Data_preprocessing.img_size, Data_preprocessing.img_size))
Data_preprocessing.training_data.append([new_array, class_num])
except Exception as e:
pass
self.Saving_processed_data()
def Saving_processed_data(self):
random.shuffle(Data_preprocessing.training_data)
x = []
y = []
for features, label in Data_preprocessing.training_data:
x.append(features)
y.append(label)
x = np.array(x).reshape(-1, Data_preprocessing.img_size, Data_preprocessing.img_size, 1)
# Saving data by using "pickle"
pickle_out = open("x.pickle", "wb")
pickle.dump(x, pickle_out)
pickle_out.close()
pickle_out = open("y.pickle", "wb")
pickle.dump(y, pickle_out)
pickle_out.close()
categories = ["Dog","Cat"]
categories = ["A","B","C","D","del","E","F","G","H","I","J","K","L","M","N","nothing","O","P","Q","R","S","space","T","U","V","W","X","Y","Z"]
data_preprocessing = Data_preprocessing("ASLDS\\ASLDS",categories, 50)
data_preprocessing.Create_training_data()
2 class Learning_model:
def __init__(self):
pass
def TrainModel(self):
self.x = pickle.load(open("x.pickle", "rb"))
self.y = pickle.load(open("y.pickle", "rb"))
self.x = self.x/255.0
self.model = Sequential()
self.model.add(Conv2D(64, (3,3), input_shape = self.x.shape[1:]))
self.model.add(Activation("relu"))
self.model.add(MaxPooling2D(pool_size=(2,2)))
self.model.add(Conv2D(64, (3,3)))
self.model.add(Activation("relu"))
self.model.add(MaxPooling2D(pool_size=(2,2)))
self.model.add(Flatten())
self.model.add(Dense(64))
self.model.add(Dense(1))
self.model.add(Activation('sigmoid'))
self.model.compile(optimizer='adam',
loss='sparse_categorical_crossentropy',
metrics=['accuracy'])
self.model.fit(self.x, self.y, batch_size = 32, epochs=10, validation_split = 0.1)
self.model.save("64x3-CNN-ASL.model")
trained_model = Learning_model()
trained_model.TrainModel()
I am expecting that if I input an image of any alphabet so it is supposed to show me corresponding name of that alphabet.

You should change loss to categorical cross entropy. Even I built a similar CNN with Keras.
This CNN is built to recognize 3 different types of images, but you can change input_shape to detect any number of categories.
# Importing the Keras libraries and packages
from keras.models import Sequential
from keras.layers import Conv2D
from keras.layers import MaxPooling2D
from keras.layers import Flatten
from keras.layers import Dense
# Initialising the CNN
classifier = Sequential()
classifier.add(Conv2D(32, (3, 3), input_shape = (64, 64, 3), activation = 'relu'))
classifier.add(MaxPooling2D(pool_size = (2, 2)))
classifier.add(Conv2D(32, (3, 3), activation = 'relu'))
classifier.add(MaxPooling2D(pool_size = (2, 2)))
classifier.add(Flatten())
classifier.add(Dense(units = 128, activation = 'relu'))
classifier.add(Dense(units = 3, activation = 'softmax')) # output layer
# Compiling the CNN
classifier.compile(optimizer = 'adam', loss = 'categorical_crossentropy', metrics = ['accuracy'])
# Using the CNN on the images
from keras.preprocessing.image import ImageDataGenerator
train_datagen = ImageDataGenerator(rescale = 1./255,
shear_range = 0.2,
zoom_range = 0.2,
horizontal_flip = True)
test_datagen = ImageDataGenerator(rescale = 1./255)
training_set = train_datagen.flow_from_directory('dataset/training_set',
target_size = (64, 64),
batch_size = 32,
class_mode = 'categorical')
test_set = test_datagen.flow_from_directory('dataset/test_set',
target_size = (64, 64),
batch_size = 32,
class_mode = 'categorical')
classifier.fit_generator(training_set,
steps_per_epoch = (8000/32),
epochs = 25,
validation_data = test_set,
validation_steps = (2000/32))
# Fetching Predictions
import numpy as np
from skimage.io import imread
from skimage.transform import resize
class_labels = {v: k for k, v in training_set.class_indices.items()}
img = imread('dataset/single_prediction/random.jpg')
img = resize(img,(64,64))
img = np.expand_dims(img,axis=0)
if(np.max(img)>1):
img = img/255.0
prediction = classifier.predict_classes(img)
print ("\n\n")
print (class_labels[prediction[0]])

Keras Graph disconnects when combining a LSTM with a CNN

The idea is to train a CNN on a cosine similarity matrix of the hidden states of two bilstms.
I try to get the following code working, but it is failing giving the error message:
Graph disconnected: cannot obtain value for tensor
Tensor("bidirectional_4/concat:0", shape=(?, ?, 100), dtype=float32)
at layer "input_11". The following previous layers were accessed without issue: []
The code to train the model is the following:
def train_model(self, sentences_pair, is_similar,
embedding_meta_data_skt, embedding_meta_data_tib ,
model_save_directory='./'):
tokenizer_skt = embedding_meta_data_skt['tokenizer']
tokenizer_tib = embedding_meta_data_tib['tokenizer']
embedding_matrix_skt = embedding_meta_data_skt['embedding_matrix']
embedding_matrix_tib = embedding_meta_data_tib['embedding_matrix']
train_data_x1, train_data_x2, train_labels, leaks_train, \
val_data_x1, val_data_x2, val_labels, leaks_val = create_train_dev_set(tokenizer_skt, sentences_pair,
is_similar, self.max_sequence_length,
self.validation_split_ratio)
nb_words_skt = len(tokenizer_skt.word_index) + 1
nb_words_tib = len(tokenizer_tib.word_index) + 1
# Creating word embedding layer
embedding_layer_skt = Embedding(nb_words_skt, self.embedding_dim, weights=[embedding_matrix_skt],
input_length=self.max_sequence_length, trainable=False)
embedding_layer_tib = Embedding(nb_words_tib, self.embedding_dim, weights=[embedding_matrix_tib],
input_length=self.max_sequence_length, trainable=False)
# Creating LSTM Encoder
lstm_layer = Bidirectional(LSTM(self.number_lstm_units, dropout=self.rate_drop_lstm, recurrent_dropout=self.rate_drop_lstm,return_sequences=True))
# Creating LSTM Encoder layer for First Sentence
sequence_1_input = Input(shape=(self.max_sequence_length,), dtype='int32')
embedded_sequences_1 = embedding_layer_skt(sequence_1_input)
skt_lstm = lstm_layer(embedded_sequences_1)
# Creating LSTM Encoder layer for Second Sentence
sequence_2_input = Input(shape=(self.max_sequence_length,), dtype='int32')
embedded_sequences_2 = embedding_layer_tib(sequence_2_input)
tib_lstm = lstm_layer(embedded_sequences_2)
A_input = keras.Input(tensor=skt_lstm)
B_input = keras.Input(tensor=tib_lstm)
dist_output = keras.layers.Lambda(pairwise_cosine_sim)([skt_lstm,tib_lstm,A_input,B_input])
dist_output = Reshape((40,40,1))(dist_output)
input_shape = (40,40,1)
cnn_model = Conv2D(128, (2, 2), input_shape=input_shape)(dist_output)
cnn_model = BatchNormalization(axis=-1)(cnn_model)
cnn_model = Activation('relu')(cnn_model)
cnn_model = Conv2D(164, (2, 2))(cnn_model)
cnn_model = BatchNormalization(axis=-1)(cnn_model)
cnn_model = Activation('relu')(cnn_model)
cnn_model = Conv2D(192,(3, 3))(cnn_model)
cnn_model = BatchNormalization(axis=-1)(cnn_model)
cnn_model = Activation('relu')(cnn_model)
cnn_model = Conv2D(192, (3, 3))(cnn_model)
cnn_model = BatchNormalization(axis=-1)(cnn_model)
cnn_model = Activation('relu')(cnn_model)
cnn_model = Conv2D(128, (3, 3))(cnn_model)
cnn_model = BatchNormalization(axis=-1)(cnn_model)
cnn_model = Activation('relu')(cnn_model)
cnn_model = MaxPooling2D(pool_size=(2,2))(cnn_model)
cnn_model = Dropout(0.40)(cnn_model)
cnn_model = Flatten()(cnn_model)
# Fully connected layer
cnn_model = Dense(256)(cnn_model)
cnn_model = BatchNormalization()(cnn_model)
cnn_model = Activation('relu')(cnn_model)
cnn_model = Dropout(0.5)(cnn_model)
cnn_model = Dense(num_classes)(cnn_model)
preds = Dense(1, activation='sigmoid')(cnn_model)
model = Model(inputs=[sequence_1_input, sequence_2_input], outputs=preds)
model.compile(loss=keras.losses.binary_crossentropy,
optimizer=keras.optimizers.Adam(lr=learning_rate),
metrics=['accuracy'])
#model.compile(loss='binary_crossentropy', optimizer='nadam', metrics=['acc'])
filepath="skt-tib-bs" + str(batch_size) + "-" + "{epoch:02d}-{val_acc:.2f}.hdf5"
checkpoint = ModelCheckpoint('skt-tib.h5', monitor='val_acc')
callbacks_list = [checkpoint]
model.fit([train_data_x1, train_data_x2, leaks_train], train_labels,validation_data=([val_data_x1, val_data_x2, leaks_val], val_labels),
batch_size=batch_size,
epochs=epochs,
verbose=1,
class_weight = class_weight,
callbacks = callbacks_list)
score = model.evaluate(x_test, y_test, verbose=1)
print('Test loss:', score[0])
print('Test accuracy:', score[1])
model.save(file_name)
The definition of the function calculating the pairwise cosine similarity is the following:
def l2_norm(x, axis=None):
square_sum = K.sum(K.square(x), axis=axis, keepdims=True)
norm = K.sqrt(K.maximum(square_sum, K.epsilon()))
return norm
def pairwise_cosine_sim(A_B):
A,B,A_tensor,B_tensor = A_B
A_mag = l2_norm(A, axis=2)
B_mag = l2_norm(B, axis=2)
num = K.batch_dot(A_tensor, K.permute_dimensions(B_tensor, (0,2,1)))
den = (A_mag * K.permute_dimensions(B_mag, (0,2,1)))
dist_mat = num / den
return dist_mat
I Have been trying for a couple of hours to fix it, but it seems to be no good. Somewhere the input and outputs are not connected, but I just can't figure out where the problem lies. Any suggestions on this?

Either remove A_input and B_input entirely as they are not input layers in the first place and use skt_lstm and tib_lstm directly instead of them, or if you would like to keep them pass them as the inputs of the model as well when you are defining the Model since they are actually input layers:
model = Model(inputs=[sequence_1_input, sequence_2_input, A_input, B_input], outputs=preds)
However, you don't need to pass any corresponding arrays for them when calling fit method as they will be fed using their corresponding tensors skt_lstm and tib_lstm (i.e. they will act as wrappers around these tensors).

TensorFlow Experiment/Estimator with Constant Initializers

I'm having some trouble using the Estimator API with constant_initializer. Originally, I was trying to load model weights from a .npy file, but the evaluation loss didn't seem to move at all.
I've made a smaller example that seems to have the same problem. When I replace the constant_initializer with any other random initializer, it seems to work. Can anybody explain what is going on?
Here is the main part of the code:
# Big thanks to https://medium.com/onfido-tech/higher-level-apis-in-tensorflow-67bfb602e6c0
import os
import tensorflow as tf
from tensorflow.contrib.learn import ModeKeys
from tensorflow.contrib.learn import learn_runner
from fcn import fcn32_vgg
from fcn import loss as fcn_loss
import voc_dataset
from voc_to_tfrecord import load_voc_dataset
from test_model import SimpleNet, WeightInitializerHook
FLAGS = tf.app.flags.FLAGS
tf.app.flags.DEFINE_string(
flag_name='weights_dir', default_value='...',
docstring='Top-level directory where the input data will be looked for.')
tf.app.flags.DEFINE_string(
flag_name='model_dir', default_value='...',
docstring='Output directory for model and training stats.')
tf.app.flags.DEFINE_string(
flag_name='data_dir', default_value='...',
docstring='Directory containing the "voc_segmentation_{train|val}.tfrecord" files.')
def run_experiment(argv=None):
# Define model parameters
params = tf.contrib.training.HParams(
learning_rate=0.002,
n_classes=22,
train_steps=100,
eval_steps=1,
min_eval_frequency=10,
eval_delay_secs=0
)
# Set the run_config and the directory to save the model and stats
run_config = tf.contrib.learn.RunConfig()
run_config = run_config.replace(model_dir=FLAGS.model_dir)
run_config = run_config.replace(tf_random_seed=42)
learn_runner.run(
experiment_fn=experiment_fn,
run_config=run_config, # RunConfig
schedule="train_and_evaluate", # What to run
hparams=params # HParams
)
def experiment_fn(run_config, params):
# You can change a subset of the run_config properties as
run_config = run_config.replace(
save_checkpoints_steps=params.min_eval_frequency)
estimator = tf.estimator.Estimator(
model_fn=model_fn, # First-class function
params=params, # HParams
config=run_config # RunConfig
)
# Setup data loaders
train_input_fn, train_input_hook = voc_dataset.get_inputs(
batch_size=64,
tfrecords_path=os.path.join(FLAGS.data_dir,'voc_segmentation_train.tfrecords'),
name_scope='train_data',
shuffle_and_repeat=True)
eval_input_fn, eval_input_hook = voc_dataset.get_inputs(
batch_size=64,
tfrecords_path=os.path.join(FLAGS.data_dir, 'voc_segmentation_val.tfrecords'),
name_scope='eval_data',
shuffle_and_repeat=False)
# Define the experiment
experiment = tf.contrib.learn.Experiment(
estimator=estimator, # Estimator
train_input_fn=train_input_fn,
eval_input_fn=eval_input_fn,
train_steps=params.train_steps,
eval_steps=params.eval_steps,
min_eval_frequency=params.min_eval_frequency, # Eval frequency
train_monitors=[train_input_hook], # Hooks for training
eval_hooks=[eval_input_hook], # Hooks for evaluation
eval_delay_secs=params.eval_delay_secs,
)
return experiment
def model_fn(features, labels, mode, params):
is_training = mode == ModeKeys.TRAIN
net = SimpleNet()
net.build(features, is_training=is_training)
logits = net.logits
predictions = net.predictions
loss = None
train_op = None
eval_metric_ops = {}
if mode != ModeKeys.INFER:
loss = fcn_loss.loss(logits, labels, params.n_classes)
if mode == ModeKeys.TRAIN:
train_op = get_train_op_fn(loss, params)
tf.summary.image('INPUT' + str(is_training), features, max_outputs=64)
tf.summary.image('OUTPUT' + str(is_training), tf.expand_dims(tf.argmax(predictions, -1) / 22, -1), max_outputs=64)
tf.summary.image('LABELS' + str(is_training), tf.expand_dims(tf.argmax(labels, -1) / 22, -1), max_outputs=64)
return tf.estimator.EstimatorSpec(
mode=mode,
predictions={'result': predictions},
loss=loss,
train_op=train_op,
# eval_metric_ops=eval_metric_ops
)
def get_train_op_fn(loss, params):
return tf.contrib.layers.optimize_loss(
loss=loss,
global_step=tf.train.get_global_step(),
optimizer=tf.train.AdamOptimizer,
learning_rate=params.learning_rate,
name='optimize_loss',
summaries=['loss']
)
# Run script ##############################################
if __name__ == "__main__":
tf.app.run(
main=train_manual
)
and here is the architecture:
class SimpleNet:
def __init__(self, vgg16_npy_path=None):
pass
def build(self, rgb, is_training=False, debug=False):
k_init = None
if is_training:
k_init = tf.constant_initializer(0.1)
self.conv_1 = tf.layers.conv2d(rgb, 5, (5, 5), activation=tf.nn.elu, padding='same', name='conv1', kernel_initializer=k_init)
self.conv_2 = tf.layers.conv2d(self.conv_1, 10, (5, 5), activation=tf.nn.elu, padding='same', name='conv2', kernel_initializer=k_init)
self.conv_3 = tf.layers.conv2d(self.conv_2, 15, (5, 5), activation=tf.nn.elu, padding='same', name='conv3', kernel_initializer=k_init)
self.conv_4 = tf.layers.conv2d(self.conv_3, 20, (5, 5), activation=tf.nn.elu, padding='same', name='conv4', kernel_initializer=k_init)
self.logits = tf.layers.conv2d(self.conv_4, 22, (5, 5), activation=None, padding='same', name='logits', kernel_initializer=k_init)
with tf.name_scope('softmax'):
self.predictions = tf.nn.softmax(self.logits)
If I set the is_training flag to False, then the evaluation loss seems to drop. Otherwise, it is completely flat. Any ideas on why this might be?