We Keep Coding

Python : ValueError: Shapes (3, 1) and (512, 3) are incompatible during the training of a Transformer model

I'm trying to build a model from a "classic" dataset.
This dataset has 2 columns :
The "text" column which contains some text extracted from pdf documents
the "label" column : it is the label associated to the extracted text.
I'm trying to build NLP model using the "transformers" library but when
the code execute the training part, the following error is raising :
**
Une exception s'est produite : ValueError
in user code:
File "C:\Users\lione\anaconda3\lib\site-packages\keras\engine\training.py", line 1021, in train_function *
return step_function(self, iterator)
File "C:\Users\lione\anaconda3\lib\site-packages\keras\engine\training.py", line 1010, in step_function **
outputs = model.distribute_strategy.run(run_step, args=(data,))
File "C:\Users\lione\anaconda3\lib\site-packages\keras\engine\training.py", line 1000, in run_step **
outputs = model.train_step(data)
File "C:\Users\lione\anaconda3\lib\site-packages\transformers\modeling_tf_utils.py", line 909, in train_step
loss = self.compiled_loss(y, y_pred, sample_weight, regularization_losses=self.losses)
File "C:\Users\lione\anaconda3\lib\site-packages\keras\engine\compile_utils.py", line 201, in __call__
loss_value = loss_obj(y_t, y_p, sample_weight=sw)
File "C:\Users\lione\anaconda3\lib\site-packages\keras\losses.py", line 141, in __call__
losses = call_fn(y_true, y_pred)
File "C:\Users\lione\anaconda3\lib\site-packages\keras\losses.py", line 245, in call **
return ag_fn(y_true, y_pred, **self._fn_kwargs)
File "C:\Users\lione\anaconda3\lib\site-packages\keras\losses.py", line 1790, in categorical_crossentropy
y_true, y_pred, from_logits=from_logits, axis=axis)
File "C:\Users\lione\anaconda3\lib\site-packages\keras\backend.py", line 5083, in categorical_crossentropy
target.shape.assert_is_compatible_with(output.shape)
ValueError: Shapes (3, 1) and (512, 3) are incompatible
File"C:\Users\lione\AppData\Local\Temp
\__autograph_generated_fileacj4
qa7l.py", line 15, in tf__train_function
retval_ = ag__.converted_call(ag__.ld(step_function),
(ag__.ld(self), ag__.ld(iterator)), None, fscope)
During handling of the above exception, another exception
occurred:
File"C:\Users\lione\OneDrive\Bureau\Lionel\LDK_360\Projets
\Novasecur\Tests\Transformers.py", line 92, in <module>
model.fit(dataset,epochs=3)
My code :
from transformers import AutoTokenizer
from transformers import DataCollatorWithPadding
from transformers import create_optimizer
from transformers import TFAutoModelForSequenceClassification
import numpy as np
import os
import pandas as pd
import tensorflow as tf
import io
import nltk
from tensorflow.keras.utils import to_categorical
df = pd.read_excel('transformers.xlsx')
print(df.head())
tokenizer = AutoTokenizer.from_pretrained("distilbert-base-uncased")
X = list(df['text'])
train_encodings = tokenizer(X, truncation=True, padding=True)
y =list(df['label'])
y = tf.cast(y,tf.int32)
y = to_categorical(y, 3)
y = tf.cast(y,tf.int32)
dataset = tf.data.Dataset.from_tensor_slices((dict(train_encodings),y))
model = TFAutoModelForSequenceClassification.from_pretrained("distilbert-base-
uncased", num_labels=3)
batch_size = 16
num_epochs = 5
batches_per_epoch = len(dataset) // batch_size
total_train_steps = int(batches_per_epoch * num_epochs)
optimizer, schedule = create_optimizer(init_lr=2e-5, num_warmup_steps=0,
num_train_steps=total_train_steps)
#
model.compile(optimizer=optimizer,
loss='categorical_crossentropy',
metrics=['accuracy'])
# Train the model
model.fit(dataset,epochs=3)
You can execute this code with the transformers.xlsx file you can download here :
https://docs.google.com/spreadsheets/d/16tPffyOXd6i1R2ohNUO3BhcSkinNtOIn/edit?usp=sharing&ouid=117798974480569779583&rtpof=true&sd=true
So far, I tried to change the loss function to sparse_categorical_crossentropy
but without success...
Can you help me to fix this error/bug ?
Regards,
Lionel

Trying to load a saved Tensorflow ELMO model but get "TypeError: 'str' object is not callable" when loading

I'm trying to load a saved Tensorflow ELMO model in a different function than I trained it, because I want to do multiple predictions with the model without having to train it every time. My (simplified) code is as follows:
(builder.py)
from word_classifier import train_word_classifier, predict_labels
def builder(lines):
train_word_classifier()
for lst in lines:
print('PRED_LABELS: ', predict_labels(lst))
(word_classifier.py)
import pandas as pd
import numpy as np
import tensorflow as tf
import tensorflow_hub as hub
from tensorflow.python.keras import backend as K
from tensorflow.keras.preprocessing.sequence import pad_sequences
from tensorflow.keras.models import Model, Sequential, model_from_json
from tensorflow.keras.layers import LSTM, Embedding, Dense, TimeDistributed, Dropout, Bidirectional, Lambda, add, Input
def train_word_classifier():
"""
Input data preparation excluded for readability
"""
sess = tf.compat.v1.Session()
K.set_session(sess)
elmo_model = hub.Module("https://tfhub.dev/google/elmo/3", trainable=True)
init = tf.compat.v1.global_variables_initializer()
sess.run(init)
input_text = Input(shape=(MAX_LEN,), dtype=tf.string)
def elmo_embedding(inData):
return \
elmo_model(inputs={"tokens": tf.squeeze(tf.cast(inData, tf.string)),
"sequence_len": tf.constant(BATCH_SIZE * [MAX_LEN])},
signature="tokens", as_dict=True)["elmo"]
embedding = Lambda(lambda text, : elmo_embedding(text), output_shape=(MAX_LEN, 1024))(input_text, )
x = Bidirectional(LSTM(units=LSTM_UNITS, return_sequences=LSTM_RETURN_SEQ,
recurrent_dropout=LSTM_RO_DROPOUT, dropout=LSTM_DROPOUT))(embedding)
x_rnn = Bidirectional(LSTM(units=LSTM_UNITS, return_sequences=LSTM_RETURN_SEQ,
recurrent_dropout=LSTM_RO_DROPOUT, dropout=LSTM_DROPOUT))(x)
x = add([x, x_rnn]) # residual connection to the first biLSTM
out = TimeDistributed(Dense(n_tags, activation="softmax"))(x)
model = Model(input_text, out)
model.compile(optimizer="adam", loss="sparse_categorical_crossentropy", metrics=["accuracy"])
line_count_training_data = count_lines_in_file(CLASSIFIER_SENTENCE_FILE, 10)
size_train, size_test = get_count_for_batch_train_test_data(line_count_training_data)
print(size_train)
mode_dict = {
"train": size_train,
"test": size_test,
}
x_tr, x_val = x_tr[:mode_dict["train"] * BATCH_SIZE], x_tr[-mode_dict["test"] * BATCH_SIZE:]
y_tr, y_val = y_tr[:mode_dict["train"] * BATCH_SIZE], y_tr[-mode_dict["test"] * BATCH_SIZE:]
y_tr = y_tr.reshape(y_tr.shape[0], y_tr.shape[1], 1)
y_val = y_val.reshape(y_val.shape[0], y_val.shape[1], 1)
history = model.fit(np.array(x_tr),
y_tr,
validation_data=(np.array(x_val), y_val),
batch_size=BATCH_SIZE,
epochs=NUM_EPOCHS,
verbose=VERBOSE_VALUE)
model_json = model.to_json()
with open("resources/SavedModel/word_classifier/model.json", "w") as json_file:
json_file.write(model_json)
# serialize weights to HDF5
model.save_weights("resources/SavedModel/word_classifier/model.h5")
def predict_labels(input_data_list):
with open('resources/SavedModel/word_classifier/model.json', 'r') as json_file:
loaded_model_json = json_file.read()
def elmo_embedding(inData):
return \
elmo_model(inputs={"tokens": tf.squeeze(tf.cast(inData, tf.string)),
"sequence_len": tf.constant(BATCH_SIZE * [MAX_LEN])},
signature="tokens", as_dict=True)["elmo"]
loaded_model = tf.keras.models.model_from_json(loaded_model_json, custom_objects={'elmo_embedding': elmo_embedding})
# load weights into new model
loaded_model.load_weights("resources/SavedModel/word_classifier/model.h5")
print("Loaded model from disk")
In the end, after training the model, this gives me the error "TypeError: 'str' object is not callable" with the following traceback:
Traceback (most recent call last):
File "usc_coordinator.py", line 62, in <module>
run_usc_coordinator(fIn, fOut, mode)
File "usc_coordinator.py", line 32, in run_usc_coordinator
user_story_builder(fast_mode, file_in)
File "/home/ubuntu/PA/PA_AI4US/PythonVersion/src/builder.py", line 45, in builder
print('PRED_LABELS: ', predict_labels(lst))
File "/home/ubuntu/PA/PA_AI4US/PythonVersion/src/word_classifier.py", line 161, in predict_labels
loaded_model = tf.keras.models.model_from_json(loaded_model_json, custom_objects={'elmo_embedding': elmo_embedding})
File "/home/ubuntu/.local/lib/python3.8/site-packages/tensorflow/python/keras/saving/model_config.py", line 122, in model_from_json
return deserialize(config, custom_objects=custom_objects)
File "/home/ubuntu/.local/lib/python3.8/site-packages/tensorflow/python/keras/layers/serialization.py", line 171, in deserialize
return generic_utils.deserialize_keras_object(
File "/home/ubuntu/.local/lib/python3.8/site-packages/tensorflow/python/keras/utils/generic_utils.py", line 354, in deserialize_keras_object
return cls.from_config(
File "/home/ubuntu/.local/lib/python3.8/site-packages/tensorflow/python/keras/engine/functional.py", line 616, in from_config
input_tensors, output_tensors, created_layers = reconstruct_from_config(
File "/home/ubuntu/.local/lib/python3.8/site-packages/tensorflow/python/keras/engine/functional.py", line 1214, in reconstruct_from_config
process_node(layer, node_data)
File "/home/ubuntu/.local/lib/python3.8/site-packages/tensorflow/python/keras/engine/functional.py", line 1162, in process_node
output_tensors = layer(input_tensors, **kwargs)
File "/home/ubuntu/.local/lib/python3.8/site-packages/tensorflow/python/keras/engine/base_layer_v1.py", line 776, in __call__
outputs = call_fn(cast_inputs, *args, **kwargs)
File "/home/ubuntu/.local/lib/python3.8/site-packages/tensorflow/python/keras/layers/core.py", line 903, in call
result = self.function(inputs, **kwargs)
File "/home/ubuntu/PA/PA_AI4US/PythonVersion/src/word_classifier.py", line 101, in <lambda>
embedding = Lambda(lambda text, : elmo_embedding(text), output_shape=(MAX_LEN, 1024))(input_text, )
TypeError: 'str' object is not callable
My versions are:
Python 3.8.10
Keras 2.3.0
Tensorflow 2.3.1
Tensorflow-hub 0.10.0
My guess is that the error is caused by the variable input_text that is set as a dtype tf.string. However, I don't how what to do about that without breaking the training sequence.
I hope that somebody can help!

It is a bug in tensorflow v2.3.1:
Loading a model with a Lambda layer causes a 'str' object is not callable exception #46659
https://github.com/tensorflow/tensorflow/issues/46659

You should change the string to something else like if I have to calculate 2+2, this would be the wrong code:
x = "2"
print(x + 2)
This shows:
Traceback (most recent call):
File "main.py", line 2, in <module>
print(x + 2)
TypeError: can only concatenate str (not "int") to str
So what do we do?
Convert it to something else
x = "2"
y = int(x)
print(y + 2)
Output: 4
int = integer,
float = float.
Edit: You can not convert some things to something else, like
x = "This is an example"
y = int(x)
This would show:
Traceback (most recent call last):
File "main.py", line 2, in <module>
y = int(x)
ValueError: invalid literal for int() with base10: 'This is an example'
#For a string to array
x = "1-2"
y = x.split("-")
print(y)
y would be ["1", "2"]
Or if you do not know short forms you can use full forms
Line #32,
'str' is not callable
fast_mode and file_In would be strings
And same for other lines

Perceptual loss for image sequences in Keras

I want to implement perceptual loss for sequential image data of the shape [batch_size, sequence_length, height, width, channels]
The predictions of my model also have the same shape as the input.
My problem is, that I'm not able to properly feed in my predictions to the VGG16 in order to calculate the loss.
Firstly, I build my vgg as follows:
def build_vgg_model(self, weights="imagenet"):
# Input image to extract features from
img = Input(shape=(self.img_rows, self.img_cols, 3))
# Mean center and rescale by variance as in PyTorch
processed = Lambda(lambda x: (x - self.mean) / self.std)(img)
# If inference only, just return empty model
if self.inference_only:
model = Model(inputs=img, outputs=[img for _ in range(len(self.vgg_layers))])
model.trainable = False
model.compile(loss='mse', optimizer='adam')
return model
# Get the vgg network from Keras applications
if weights in ['imagenet', None]:
vgg = VGG16(weights=weights, include_top=False)
# model = Model(inputs=base_model.input, outputs=base_model.get_layer('block4_pool').output)
else:
vgg = VGG16(weights=None, include_top=False)
vgg.load_weights(weights, by_name=True)
# Output the first three pooling layers
vgg.outputs = [vgg.layers[i].output for i in self.vgg_layers]
# Create model and compile
model = Model(inputs=img, outputs=vgg(processed))
model.trainable = False
return model
Then I define my loss function:
def total_loss(self, mask):
def loss(y_true, y_pred):
# Compute predicted image with non-hole pixels set to ground truth
y_comp = mask * y_true + (1 - mask) * y_pred
# Compute the vgg features.
if self.vgg_device:
with tf.device(self.vgg_device):
vgg_out = self.vgg(y_pred)
vgg_gt = self.vgg(y_true)
vgg_comp = self.vgg(y_comp)
else:
vgg_out = self.vgg(y_pred)
vgg_gt = self.vgg(y_true)
vgg_comp = self.vgg(y_comp)
# Compute loss components
l1 = self.loss_valid(mask, y_true, y_pred)
l2 = self.loss_hole(mask, y_true, y_pred)
l3 = self.loss_perceptual(vgg_out, vgg_gt, vgg_comp)
# Return loss function
return l1 + 6 * l2 + 0.05 * l3
return loss
def loss_perceptual(self, vgg_out, vgg_gt, vgg_comp):
loss = 0
for o, c, g in zip(vgg_out, vgg_comp, vgg_gt):
loss += self.l1(o, g) + self.l1(c, g)
return loss
If I now run my code, I get the following error:
Traceback (most recent call last):
File "convlstm_main.py", line 51, in <module>
model = ConvLSTM(64, 64, 3, 5)
File "/mnt/workspace/lm78463/RecurrentDiFoRem/src/model/conv_lstm.py", line 40, in __init__
self.model = self.compile()
File "/mnt/workspace/lm78463/RecurrentDiFoRem/src/model/conv_lstm.py", line 116, in compile
self.model.compile(loss=self.loss_total, optimizer='adadelta')
File "/usr/local/lib/python3.6/dist-packages/keras/engine/training.py", line 229, in compile
self.total_loss = self._prepare_total_loss(masks)
File "/usr/local/lib/python3.6/dist-packages/keras/engine/training.py", line 692, in _prepare_total_loss
y_true, y_pred, sample_weight=sample_weight)
File "/usr/local/lib/python3.6/dist-packages/keras/losses.py", line 71, in __call__
losses = self.call(y_true, y_pred)
File "/usr/local/lib/python3.6/dist-packages/keras/losses.py", line 132, in call
return self.fn(y_true, y_pred, **self._fn_kwargs)
File "/mnt/workspace/lm78463/RecurrentDiFoRem/src/model/conv_lstm.py", line 180, in loss_total
vgg_gts = self.vgg(y_trues)
File "/usr/local/lib/python3.6/dist-packages/keras/engine/base_layer.py", line 489, in __call__
output = self.call(inputs, **kwargs)
File "/usr/local/lib/python3.6/dist-packages/keras/engine/network.py", line 583, in call
output_tensors, _, _ = self.run_internal_graph(inputs, masks)
File "/usr/local/lib/python3.6/dist-packages/keras/engine/network.py", line 740, in run_internal_graph
layer.call(computed_tensor, **kwargs))
File "/usr/local/lib/python3.6/dist-packages/keras/engine/network.py", line 583, in call
output_tensors, _, _ = self.run_internal_graph(inputs, masks)
File "/usr/local/lib/python3.6/dist-packages/keras/engine/network.py", line 740, in run_internal_graph
layer.call(computed_tensor, **kwargs))
File "/usr/local/lib/python3.6/dist-packages/keras/layers/convolutional.py", line 171, in call
dilation_rate=self.dilation_rate)
File "/usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py", line 3717, in conv2d
**kwargs)
File "/usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/nn_ops.py", line 917, in convolution
name=name)
File "/usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/nn_ops.py", line 979, in convolution_internal
strides = _get_sequence(strides, n, channel_index, "strides")
File "/usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/nn_ops.py", line 74, in _get_sequence
name, n, n + 2, current_n))
ValueError: strides should be of length 1, 3 or 5 but was 2

Unable to use MSE of VGG features in loss function

I'm using keras (tf.keras) in tensorflow 2.0.0
I've a network, whose input is an image and output is also an image. I want to use a combination of MSE, MSE in VGG feature space and some other losses, which depend on intermediate layer output. I'm defining a custom loss function. I'm able to build the model, compile with the custom loss. But when I train using fit_generator, I'm getting a SymbolicException saying Inputs to eager execution function cannot be Keras symbolic tensors
Full Code:
Train File:
def __init__(self, gray_images: bool, verbose: bool = True):
super().__init__(gray_images, verbose)
self.model = None
self.vgg_feature_extractor = VggFeaturesExtractor(model_name='vgg16', layers=[3, 6, 10])
def build_model():
image_input = Input(shape=(None, None, num_input_channels))
out1 = self.build_out1_model(image_input, num_filters, depth_t)
out2 = self.build_out2_model(image_input, num_filters, depth_n, use_bnorm)
enhanced_image = ... # Some function of image_input, out1 and out2
self.model = Model(inputs=image_input, outputs=enhanced_image)
self.model.add_loss(loss_weights[1] * self.loss2(out2))
self.model.compile(optimizer='adam', loss=self.vgg_loss)
def vgg_loss(self, gt_image, est_image):
gt_features = self.vgg_feature_extractor.extract_features(gt_image)
est_features = self.vgg_feature_extractor.extract_features(est_image)
loss = tf.reduce_mean(tf.square(gt_features[0] - est_features[0])) + \
tf.reduce_mean(tf.square(gt_features[1] - est_features[1])) + \
tf.reduce_mean(tf.square(gt_features[2] - est_features[2]))
return loss
VggFeatures.py:
class VggFeaturesExtractor:
def __init__(self, model_name: str, layers: List[int]):
self.model_name = model_name
self.layers = layers
if model_name == 'vgg16':
from tensorflow.keras.applications.vgg16 import VGG16, preprocess_input
vgg_model = VGG16(include_top=False)
self.preprocess_input = preprocess_input
elif model_name == 'vgg19':
from tensorflow.keras.applications.vgg19 import VGG19, preprocess_input
vgg_model = VGG19(include_top=False)
self.preprocess_input = preprocess_input
else:
raise RuntimeError(f'Unknown Model: {model_name}')
outputs = []
for layer_num in layers:
outputs.append(vgg_model.layers[layer_num].output)
self.feature_extractor = keras.Model(inputs=vgg_model.input, outputs=outputs)
def extract_features(self, images: numpy.ndarray):
preprocessed_images = self.preprocess_input(images)
features = self.feature_extractor(preprocessed_images)
return features
Stack trace:
Epoch 1/1000
Traceback (most recent call last):
File "/media/nagabhushan/Data02/SoftwareFiles/Anaconda/anaconda3/envs/.../lib/python3.7/site-packages/tensorflow_core/python/eager/execute.py", line 61, in quick_execute
num_outputs)
TypeError: An op outside of the function building code is being passed
a "Graph" tensor. It is possible to have Graph tensors
leak out of the function building context by including a
tf.init_scope in your function building code.
For example, the following function will fail:
#tf.function
def has_init_scope():
my_constant = tf.constant(1.)
with tf.init_scope():
added = my_constant * 2
The graph tensor has name: StridedSliceGrad:0
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/media/nagabhushan/Data02/SNB/IISc/Research/.../Workspace/Ideas/01_Supervised/src/N09.py", line 363, in <module>
main()
File "/media/nagabhushan/Data02/SNB/IISc/Research/.../Workspace/Ideas/01_Supervised/src/N09.py", line 343, in main
args.save_interval)
File "/media/nagabhushan/Data02/SNB/IISc/Research/.../Workspace/Ideas/01_Supervised/src/N09.py", line 92, in train_model
verbose=self.verbose)
File "/media/nagabhushan/Data02/SoftwareFiles/Anaconda/anaconda3/envs/.../lib/python3.7/site-packages/tensorflow_core/python/keras/engine/training.py", line 1297, in fit_generator
steps_name='steps_per_epoch')
File "/media/nagabhushan/Data02/SoftwareFiles/Anaconda/anaconda3/envs/.../lib/python3.7/site-packages/tensorflow_core/python/keras/engine/training_generator.py", line 265, in model_iteration
batch_outs = batch_function(*batch_data)
File "/media/nagabhushan/Data02/SoftwareFiles/Anaconda/anaconda3/envs/.../lib/python3.7/site-packages/tensorflow_core/python/keras/engine/training.py", line 973, in train_on_batch
class_weight=class_weight, reset_metrics=reset_metrics)
File "/media/nagabhushan/Data02/SoftwareFiles/Anaconda/anaconda3/envs/.../lib/python3.7/site-packages/tensorflow_core/python/keras/engine/training_v2_utils.py", line 264, in train_on_batch
output_loss_metrics=model._output_loss_metrics)
File "/media/nagabhushan/Data02/SoftwareFiles/Anaconda/anaconda3/envs/.../lib/python3.7/site-packages/tensorflow_core/python/keras/engine/training_eager.py", line 311, in train_on_batch
output_loss_metrics=output_loss_metrics))
File "/media/nagabhushan/Data02/SoftwareFiles/Anaconda/anaconda3/envs/.../lib/python3.7/site-packages/tensorflow_core/python/keras/engine/training_eager.py", line 268, in _process_single_batch
grads = tape.gradient(scaled_total_loss, trainable_weights)
File "/media/nagabhushan/Data02/SoftwareFiles/Anaconda/anaconda3/envs/.../lib/python3.7/site-packages/tensorflow_core/python/eager/backprop.py", line 1014, in gradient
unconnected_gradients=unconnected_gradients)
File "/media/nagabhushan/Data02/SoftwareFiles/Anaconda/anaconda3/envs/.../lib/python3.7/site-packages/tensorflow_core/python/eager/imperative_grad.py", line 76, in imperative_grad
compat.as_str(unconnected_gradients.value))
File "/media/nagabhushan/Data02/SoftwareFiles/Anaconda/anaconda3/envs/.../lib/python3.7/site-packages/tensorflow_core/python/eager/function.py", line 911, in _backward_function_wrapper
processed_args, remapped_captures)
File "/media/nagabhushan/Data02/SoftwareFiles/Anaconda/anaconda3/envs/.../lib/python3.7/site-packages/tensorflow_core/python/eager/function.py", line 1224, in _call_flat
ctx, args, cancellation_manager=cancellation_manager)
File "/media/nagabhushan/Data02/SoftwareFiles/Anaconda/anaconda3/envs/.../lib/python3.7/site-packages/tensorflow_core/python/eager/function.py", line 511, in call
ctx=ctx)
File "/media/nagabhushan/Data02/SoftwareFiles/Anaconda/anaconda3/envs/.../lib/python3.7/site-packages/tensorflow_core/python/eager/execute.py", line 75, in quick_execute
"tensors, but found {}".format(keras_symbolic_tensors))
tensorflow.python.eager.core._SymbolicException: Inputs to eager execution function cannot be Keras symbolic tensors, but found [<tf.Tensor 'StridedSliceGrad:0' shape=(16, 64, 64, 3) dtype=float32>]
Process finished with exit code 1
Note:
1. If I replace self.model.compile(optimizer='adam', loss=self.vgg_loss) with self.model.compile(optimizer='adam', loss='mse'), code works fine, which implies the other part of code is working correctly.
2. Almost every question I found on SO regarding VGG loss advises to append VGG network to the main network, set trainable=False for VGG network and then train with MSE loss. But I can't do that, since I have many components in my loss function.

I was able to fix this issue by disabling eager execution. In tensorflow 2.0, eager execution is enabled by default.
tf.compat.v1.disable_eager_execution()
I didn't understand how this was able to fix the issue though. If anybody stumbles on a similar problem, you can try disabling eager execution.

Tensorflow model.fit() using a Dataset generator

I am using the Dataset API to generate training data and sort it into batches for a NN.
Here is a minimum working example of my code:
import tensorflow as tf
import numpy as np
import random
def my_generator():
while True:
x = np.random.rand(4, 20)
y = random.randint(0, 11)
label = tf.one_hot(y, depth=12)
yield x.reshape(4, 20, 1), label
def my_input_fn():
dataset = tf.data.Dataset.from_generator(lambda: my_generator(),
output_types=(tf.float64, tf.int32))
dataset = dataset.batch(32)
iterator = dataset.make_one_shot_iterator()
batch_features, batch_labels = iterator.get_next()
return batch_features, batch_labels
if __name__ == "__main__":
tf.enable_eager_execution()
model = tf.keras.Sequential([tf.keras.layers.Flatten(input_shape=(4, 20, 1)),
tf.keras.layers.Dense(128, activation=tf.nn.relu),
tf.keras.layers.Dense(12, activation=tf.nn.softmax)])
model.compile(optimizer='adam',
loss='categorical_crossentropy',
metrics=['accuracy'])
data_generator = my_input_fn()
model.fit(data_generator)
The code fails using TensorFlow 1.13.1 at the model.fit() call with the following error:
Traceback (most recent call last):
File "scripts/min_working_example.py", line 37, in <module>
model.fit(data_generator)
File "~/.local/lib/python3.6/site-packages/tensorflow/python/keras/engine/training.py", line 880, in fit
validation_steps=validation_steps)
File "~/.local/lib/python3.6/site-packages/tensorflow/python/keras/engine/training_arrays.py", line 310, in model_iteration
ins_batch = slice_arrays(ins[:-1], batch_ids) + [ins[-1]]
File "~/.local/lib/python3.6/site-packages/tensorflow/python/keras/utils/generic_utils.py", line 526, in slice_arrays
return [None if x is None else x[start] for x in arrays]
File "~/.local/lib/python3.6/site-packages/tensorflow/python/keras/utils/generic_utils.py", line 526, in <listcomp>
return [None if x is None else x[start] for x in arrays]
File "~/.local/lib/python3.6/site-packages/tensorflow/python/ops/array_ops.py", line 654, in _slice_helper
name=name)
File "~/.local/lib/python3.6/site-packages/tensorflow/python/ops/array_ops.py", line 820, in strided_slice
shrink_axis_mask=shrink_axis_mask)
File "~/.local/lib/python3.6/site-packages/tensorflow/python/ops/gen_array_ops.py", line 9334, in strided_slice
_six.raise_from(_core._status_to_exception(e.code, message), None)
File "<string>", line 3, in raise_from
tensorflow.python.framework.errors_impl.InvalidArgumentError: Attr shrink_axis_mask has value 4294967295 out of range for an int32 [Op:StridedSlice] name: strided_slice/
I tried running the same code on a different machine using TensorFlow 2.0 (after removing the line tf.enable_eager_execution() because it runs eagerly by default) and I got the following error:
Traceback (most recent call last):
File "scripts/min_working_example.py", line 37, in <module>
model.fit(data_generator)
File "~/.local/lib/python3.7/site-packages/tensorflow/python/keras/engine/training.py", line 873, in fit
steps_name='steps_per_epoch')
File "~/.local/lib/python3.7/site-packages/tensorflow/python/keras/engine/training_arrays.py", line 352, in model_iteration
batch_outs = f(ins_batch)
File "~/.local/lib/python3.7/site-packages/tensorflow/python/keras/backend.py", line 3217, in __call__
outputs = self._graph_fn(*converted_inputs)
File "~/.local/lib/python3.7/site-packages/tensorflow/python/eager/function.py", line 558, in __call__
return self._call_flat(args)
File "~/.local/lib/python3.7/site-packages/tensorflow/python/eager/function.py", line 627, in _call_flat
outputs = self._inference_function.call(ctx, args)
File "~/.local/lib/python3.7/site-packages/tensorflow/python/eager/function.py", line 397, in call
(len(args), len(list(self.signature.input_arg))))
ValueError: Arguments and signature arguments do not match: 21 23
I tried changing model.fit() to model.fit_generator() but this fails on both TensorFlow versions too. On TF 1.13.1 I get the following error:
Traceback (most recent call last):
File "scripts/min_working_example.py", line 37, in <module>
model.fit_generator(data_generator)
File "~/.local/lib/python3.6/site-packages/tensorflow/python/keras/engine/training.py", line 1426, in fit_generator
initial_epoch=initial_epoch)
File "~/.local/lib/python3.6/site-packages/tensorflow/python/keras/engine/training_generator.py", line 115, in model_iteration
shuffle=shuffle)
File "~/.local/lib/python3.6/site-packages/tensorflow/python/keras/engine/training_generator.py", line 377, in convert_to_generator_like
num_samples = int(nest.flatten(data)[0].shape[0])
TypeError: __int__ returned non-int (type NoneType)
and on TF 2.0 I get the following error:
Traceback (most recent call last):
File "scripts/min_working_example.py", line 37, in <module>
model.fit_generator(data_generator)
File "~/.local/lib/python3.7/site-packages/tensorflow/python/keras/engine/training.py", line 1515, in fit_generator
steps_name='steps_per_epoch')
File "~/.local/lib/python3.7/site-packages/tensorflow/python/keras/engine/training_generator.py", line 140, in model_iteration
shuffle=shuffle)
File "~/.local/lib/python3.7/site-packages/tensorflow/python/keras/engine/training_generator.py", line 477, in convert_to_generator_like
raise ValueError('You must specify `batch_size`')
ValueError: You must specify `batch_size`
yet batch_size is not a recognized keyword for fit_generator().
I am puzzled by these error messages and I would appreciate if anyone can shed some light on them, or point out what I am doing wrong.

While the origin of the errors is still nebulous, I have found a solution that makes the code work. I'll post it here in case it is useful to anyone in a similar situation.
Basically, I changed the my_input_fn() into a generator and used model.fit_generator() as follows:
import tensorflow as tf
import numpy as np
import random
def my_generator(total_items):
i = 0
while i < total_items:
x = np.random.rand(4, 20)
y = random.randint(0, 11)
label = tf.one_hot(y, depth=12)
yield x.reshape(4, 20, 1), label
i += 1
def my_input_fn(total_items, epochs):
dataset = tf.data.Dataset.from_generator(lambda: my_generator(total_items),
output_types=(tf.float64, tf.int64))
dataset = dataset.repeat(epochs)
dataset = dataset.batch(32)
iterator = dataset.make_one_shot_iterator()
while True:
batch_features, batch_labels = iterator.get_next()
yield batch_features, batch_labels
if __name__ == "__main__":
tf.enable_eager_execution()
model = tf.keras.Sequential([tf.keras.layers.Flatten(input_shape=(4, 20, 1)),
tf.keras.layers.Dense(64, activation=tf.nn.relu),
tf.keras.layers.Dense(12, activation=tf.nn.softmax)])
model.compile(optimizer='adam',
loss='categorical_crossentropy',
metrics=['accuracy'])
total_items = 200
batch_size = 32
epochs = 10
num_batches = int(total_items/batch_size)
train_data_generator = my_input_fn(total_items, epochs)
model.fit_generator(generator=train_data_generator, steps_per_epoch=num_batches, epochs=epochs, verbose=1)
EDIT
As implied by giser_yugang in a comment, it is also possible to do it with my_input_fn() as a function returning the dataset instead of the individual batches.
def my_input_fn(total_items, epochs):
dataset = tf.data.Dataset.from_generator(lambda: my_generator(total_items),
output_types=(tf.float64, tf.int64))
dataset = dataset.repeat(epochs)
dataset = dataset.batch(32)
return dataset
if __name__ == "__main__":
tf.enable_eager_execution()
model = tf.keras.Sequential([tf.keras.layers.Flatten(input_shape=(4, 20, 1)),
tf.keras.layers.Dense(64, activation=tf.nn.relu),
tf.keras.layers.Dense(12, activation=tf.nn.softmax)])
model.compile(optimizer='adam',
loss='categorical_crossentropy',
metrics=['accuracy'])
total_items = 100
batch_size = 32
epochs = 10
num_batches = int(total_items/batch_size)
dataset = my_input_fn(total_items, epochs)
model.fit_generator(dataset, epochs=epochs, steps_per_epoch=num_batches)
There does not appear to be any average performance difference between the approaches.