We Keep Coding

Fixing "OperatorNotAllowedInGraphError" in tensorflow 2.9.1 with custom model

I'm trying to build a custom Transformer model within TensorFlow but I'm stuck. I always receive this error:
OperatorNotAllowedInGraphError: Iterating over a symbolic `tf.Tensor` is not allowed: AutoGraph did convert this function. This might indicate you are trying to use an unsupported feature.
Of course I did some research and I found many solutions, but I'm failing to implement these to my own model since I'm not even sure where exactly this error occurs. I tried to remove all for loops, but the error still remained. I hope someone can help me and point me to the right direction.
I'm using:
python: 3.9
tensorflow: 2.9.1
keras: 2.9.0
import tensorflow as tf
import keras.backend as K # for custom loss function
import tensorflow_probability as tfp
from tensorflow.python.keras.layers import Dense, Dropout, GlobalAveragePooling1D, MultiHeadAttention, Input, Conv1D
from keras.layers import LayerNormalization
import numpy as np
class MultiAttention(tf.keras.Model):
def __init__(self, head_size, num_heads, dropout_attention, dropout1, dropout2, ff_dim):
super(MultiAttention, self).__init__()
# define all layers in init
self.LN1 = LayerNormalization(epsilon=1e-6)
self.MHA = MultiHeadAttention(key_dim=head_size, num_heads=num_heads, dropout=dropout_attention)
self.Dropout1 = Dropout(dropout1)
self.LN2 = LayerNormalization(epsilon=1e-6)
self.CONV1 = Conv1D(filters=ff_dim, kernel_size=1, activation="relu")
self.Dropout2 = Dropout(dropout2)
self.CONV2 = None
def build(self, input_shape):
self.CONV2 = Conv1D(filters=input_shape.shape[-1], kernel_size=1)
def call(self, inputs, training=False):
# Normalization and Attention
x = self.LN1(epsilon=1e-6)(inputs)
x = self.MHA(x, x)
x = self.Dropout1(x)
x_att = x + inputs
# Feed Forward Part
x = self.LN2(x_att)
x = self.Conv1(x)
x = self.Dropout2(x)
x = self.Conv1(x)
return x + x_att
class Transformer(tf.keras.Model): # tf.keras.Model inherits model.fit/predict/evaluate. tf.keras.Layer can't do this
def __init__(self, head_size, num_heads, dropout_attention, dropout1, dropout2, ff_dim,
num_transformer_blocks, mlp_units, mlp_dropout):
super(Transformer, self).__init__()
# define all layers in init
self.head_size = head_size
self.num_heads = num_heads
self.dropout_attention = dropout_attention
self.dropout1 = dropout1
self.dropout2 = dropout2
self.ff_dim = ff_dim
self.num_transformer_blocks = num_transformer_blocks
self.mlp_units = mlp_units
self.GAP = GlobalAveragePooling1D(data_format="channels_first")
self.Dense1 = Dense(mlp_units, activation="relu")
self.Dropout = Dropout(mlp_dropout)
self.Dense2 = Dense(1, activation="softmax")
self.MA = None
def build(self, input_shape):
self.MA = MultiAttention(self.head_size, self.num_heads, self.dropout_attention,
self.dropout1, self.dropout2, self.ff_dim)
# for dim in range(self.mlp_units):
# self.Dense1_list.append(Dense(dim, actiavtion="relu"))
def call(self, input_shape, training=False):
x = Input(input_shape)
for _ in range(self.num_transformer_blocks):
x = self.MA(x)
x = self.GAP(x)
for unit in self.Dense1_list:
x = unit(x)
x = self.Dropout(x)
x = self.Dense1(x)
x = self.Dropout(x)
return self.Dense2(x)
class CustomLoss(tf.keras.losses.Loss):
def __init__(self):
super().__init__()
def call(self, y_true, y_pred):
alpha = K.std(y_pred) / K.std(y_true)
beta = K.sum(y_pred) / K.sum(y_true) # no need to calc mean
r = tfp.stats.correlation(y_true, y_pred, sample_axis=None, event_axis=None)
return K.sqrt(K.square(1 - r) + K.square(1 - alpha) + K.square(1 - beta))
def create_model(head_size, num_heads, dropout_attention, dropout1, dropout2, ff_dim,
num_transformer_blocks, mlp_units, mlp_dropout, lr):
model = Transformer(head_size, num_heads, dropout_attention, dropout1, dropout2, ff_dim,
num_transformer_blocks, mlp_units, mlp_dropout)
model.compile(
loss=CustomLoss(),
optimizer=tf.keras.optimizers.Adam(learning_rate=lr),
)
return model
model = create_model(256,4, 0.10, 0.12, 0.15, 2, 4, 3, 0.10, 0.001)
model.fit(np.zeros((2, 257, 11)),
np.zeros((2, 1)),
epochs=1,
batch_size=512,
)

No attribute 'compile', how can I modify the class, so that it works?

The neuMF class is not a Keras's class and therefore it doesn't provide any compile method.
I would better use keras.Model instead of nn.Blocks.
Unfortunately, I do not really understand what nn.Blocks is and how I could replace it in the class.
How should I modfy my code, so that it works with keras.Model and can use the Keras method?
Here is my code:
from d2l import mxnet as d2l
from mxnet import autograd, gluon, np, npx
from mxnet.gluon import nn
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras import layers
class NeuMF(nn.Block):
def init(self, num_factors, num_users, num_items, nums_hiddens,
kwargs):
super(NeuMF, self).init(kwargs)
self.P = nn.Embedding(num_users, num_factors)
self.Q = nn.Embedding(num_items, num_factors)
self.U = nn.Embedding(num_users, num_factors)
self.V = nn.Embedding(num_items, num_factors)
self.mlp = nn.Sequential()
for num_hiddens in nums_hiddens:
self.mlp.add(nn.Dense(num_hiddens, activation='relu',
use_bias=True))
self.prediction_layer = nn.Dense(1, activation='sigmoid', use_bias=False)
def forward(self, user_id, item_id):
p_mf = self.P(user_id)
q_mf = self.Q(item_id)
gmf = p_mf * q_mf
p_mlp = self.U(user_id)
q_mlp = self.V(item_id)
mlp = self.mlp(np.concatenate([p_mlp, q_mlp], axis=1))
con_res = np.concatenate([gmf, mlp], axis=1)
return self.prediction_layer(con_res)
hidden = [5,5,5]
model = NeuMF(5, num_users, num_items, hidden)
model.compile(
#loss=tf.keras.losses.BinaryCrossentropy(),
loss=tf.keras.losses.MeanSquaredError(),
optimizer=keras.optimizers.Adam(lr=0.001)
)
And I get the following error:
---------------------------------------------------------------------------
AttributeError Traceback (most recent call last)
<ipython-input-21-5979072369bd> in <module>()
2
3 model = NeuMF(5, num_users, num_items, hidden)
----> 4 model.compile(
5 #loss=tf.keras.losses.BinaryCrossentropy(),
6 loss=tf.keras.losses.MeanSquaredError(),
AttributeError: 'NeuMF' object has no attribute 'compile'
Thank you a lot in advance!
Edit:
I replaced nn to layers
class NeuMF(keras.Model):
def __init__(self, num_factors, num_users, num_items, nums_hiddens,
**kwargs):
super(NeuMF, self).__init__(**kwargs)
self.P = layers.Embedding(num_users, num_factors)
self.Q = layers.Embedding(num_items, num_factors)
self.U = layers.Embedding(num_users, num_factors)
self.V = layers.Embedding(num_items, num_factors)
self.mlp = layers.Sequential()
for num_hiddens in nums_hiddens:
self.mlp.add(layers.Dense(num_hiddens, activation='relu',
use_bias=True))
self.prediction_layer = layers.Dense(1, activation='sigmoid', use_bias=False)
def forward(self, user_id, item_id):
p_mf = self.P(user_id)
q_mf = self.Q(item_id)
gmf = p_mf * q_mf
p_mlp = self.U(user_id)
q_mlp = self.V(item_id)
mlp = self.mlp(np.concatenate([p_mlp, q_mlp], axis=1))
con_res = np.concatenate([gmf, mlp], axis=1)
return self.prediction_layer(con_res)
Then I got an new error:
---------------------------------------------------------------------------
ValueError Traceback (most recent call last)
<ipython-input-26-7e09b0f80300> in <module>()
1 hidden = [1,1,1]
2
----> 3 model = NeuMF(1, num_users, num_items, hidden)
4 model.compile(
5 #loss=tf.keras.losses.BinaryCrossentropy(),
1 frames
/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/layers/embeddings.py in __init__(self, input_dim, output_dim, embeddings_initializer, embeddings_regularizer, activity_regularizer, embeddings_constraint, mask_zero, input_length, **kwargs)
102 else:
103 kwargs['input_shape'] = (None,)
--> 104 if input_dim <= 0 or output_dim <= 0:
105 raise ValueError('Both `input_dim` and `output_dim` should be positive, '
106 'found input_dim {} and output_dim {}'.format(
ValueError: The truth value of an array with more than one element is ambiguous. Use a.any() or a.all()

After already quite some discussion in the comments, there are still several problems with your code and clarification needed from your side:
subclasses of keras.Model should implement the __call__ method, but not a forward method.
you can't just build numpy-operations like np.concatenate inside your model, always use keras-layers like tf.keras.layers.Concatenate.
as already commented, the error you've posted most likely comes from num_factors, num_users, num_items not being integer, though I can only guess here, since you did not provide those to us.
also, I can currently only guess what you are trying to achieve, since this is not at all clear from what you posted.
Let us approach the issues in a different way. The following code snippet runs without error and might be a good starting point for you:
import tensorflow as tf
class NeuMF(tf.keras.Model):
def __init__(self, num_factors, num_users, num_items, nums_hiddens,
**kwargs):
super(NeuMF, self).__init__(**kwargs)
self.P = tf.keras.layers.Embedding(num_users, num_factors)
self.Q = tf.keras.layers.Embedding(num_items, num_factors)
self.U = tf.keras.layers.Embedding(num_users, num_factors)
self.V = tf.keras.layers.Embedding(num_items, num_factors)
self.mlp = tf.keras.Sequential()
for num_hiddens in nums_hiddens:
self.mlp.add(
tf.keras.layers.Dense(
num_hiddens,
activation='relu',
use_bias=True
)
)
self.prediction_layer = tf.keras.layers.Dense(1, activation='sigmoid', use_bias=False)
def __call__(self, inputs):
x = self.P(inputs[0])
x1 = self.Q(inputs[1])
x = tf.keras.layers.Multiply()([x,x1])
y = self.U(inputs[0])
y1 = self.V(inputs[1])
y = tf.keras.layers.Concatenate()([y,y1])
y = self.mlp(y)
x = tf.keras.layers.Concatenate()([x,y])
return self.prediction_layer(x)
if __name__ == '__main__':
#replace these with values of your choice:
num_factors = 2
num_users = 3
num_items = 4
nums_hidden = [5,5,5]
model = NeuMF(num_users, num_items, num_items, nums_hidden)
model.compile(
loss = tf.keras.losses.MeanSquaredError(),
optimizer = tf.keras.optimizers.Adam(lr=0.001)
)

Unable to save model architecture (bilstm + attention)

I am working on a multi-label text classification problem. I am trying to add attention mechanism with bilstm model. The attention mechanism code is taken from here. I am not able to save the model architecture and getting an error mentioned below. My tensorflow version -2.2.0
from keras import backend as K
def dot_product(x, kernel):
if K.backend() == 'tensorflow':
return K.squeeze(K.dot(x, K.expand_dims(kernel)), axis=-1)
else:
return K.dot(x, kernel)
class AttentionWithContext(tf.keras.layers.Layer):
"""
# Input shape
3D tensor with shape: `(samples, steps, features)`.
# Output shape
2D tensor with shape: `(samples, features)`.
"""
def __init__(self,
W_regularizer=None, u_regularizer=None, b_regularizer=None,
W_constraint=None, u_constraint=None, b_constraint=None,
bias=True, **kwargs):
self.supports_masking = True
self.init = tf.keras.initializers.get('glorot_uniform')
self.W_regularizer = tf.keras.regularizers.get(W_regularizer)
self.u_regularizer = tf.keras.regularizers.get(u_regularizer)
self.b_regularizer = tf.keras.regularizers.get(b_regularizer)
self.W_constraint = tf.keras.constraints.get(W_constraint)
self.u_constraint = tf.keras.constraints.get(u_constraint)
self.b_constraint = tf.keras.constraints.get(b_constraint)
self.bias = bias
super(AttentionWithContext, self).__init__(**kwargs)
def build(self, input_shape):
assert len(input_shape) == 3
self.W = self.add_weight(shape=(input_shape[-1], input_shape[-1],),
initializer=self.init,
name='{}_W'.format(self.name),
regularizer=self.W_regularizer,
constraint=self.W_constraint)
if self.bias:
self.b = self.add_weight(shape=(input_shape[-1],),
initializer='zero',
name='{}_b'.format(self.name),
regularizer=self.b_regularizer,
constraint=self.b_constraint)
self.u = self.add_weight(shape=(input_shape[-1],),
initializer=self.init,
name='{}_u'.format(self.name),
regularizer=self.u_regularizer,
constraint=self.u_constraint)
super(AttentionWithContext, self).build(input_shape)
def compute_mask(self, input, input_mask=None):
# do not pass the mask to the next layers
return None
def call(self, x, mask=None):
uit = dot_product(x, self.W)
if self.bias:
uit += self.b
uit = K.tanh(uit)
ait = dot_product(uit, self.u)
a = K.exp(ait)
# apply mask after the exp. will be re-normalized next
if mask is not None:
# Cast the mask to floatX to avoid float64 upcasting in theano
a *= K.cast(mask, K.floatx())
# in some cases especially in the early stages of training the sum may be almost zero
# and this results in NaN's. A workaround is to add a very small positive number ε to the sum.
# a /= K.cast(K.sum(a, axis=1, keepdims=True), K.floatx())
a /= K.cast(K.sum(a, axis=1, keepdims=True) + K.epsilon(), K.floatx())
a = K.expand_dims(a)
weighted_input = x * a
return K.sum(weighted_input, axis=1)
def compute_output_shape(self, input_shape):
return input_shape[0], input_shape[-1]
def lstm_with_attention(embedding_matrix,
**kwargs):
STAMP = kwargs['STAMP']
max_seq_length = kwargs['max_seq_length']
EMBEDDING_DIM = kwargs['EMBEDDING_DIM']
nb_words = kwargs['nb_words']
inp = tf.keras.Input(shape=(max_seq_length,))
embedded_seq = tf.keras.layers.Embedding(nb_words,
EMBEDDING_DIM,
weights=[embedding_matrix],
trainable=False)(inp)
x_1_bilstm = tf.keras.layers.Bidirectional(tf.compat.v1.keras.layers.CuDNNLSTM(128, return_sequences=True))(embedded_seq)
x_1_bn = tf.keras.layers.BatchNormalization()(x_1_bilstm)
x_2_bilstm = tf.keras.layers.Bidirectional(tf.compat.v1.keras.layers.CuDNNLSTM(64, return_sequences=True))(x_1_bn)
attention = AttentionWithContext()(x_2_bilstm)
x = tf.keras.layers.Dense(64, activation="relu")(attention)
x = tf.keras.layers.Dense(1, activation="sigmoid")(x)
model = tf.keras.Model(inputs=inp, outputs=x)
optimizer = tf.keras.optimizers.Adam()
model.compile(loss='binary_crossentropy', optimizer=optimizer, metrics=['accuracy'])
model.summary()
with open(STAMP + ".json", "w") as json_file: json_file.write(model.to_json())
return model, attention
Building lstm with attention
embedding_matrix, nb_words = get_embedding('glove',word_index)
model, attention_layer = lstm_with_attention(embedding_matrix,STAMP=STAMP,max_seq_length=max_seq_length,nb_words=nb_words,EMBEDDING_DIM=EMBEDDING_DIM)
Error
---------------------------------------------------------------------------
NotImplementedError Traceback (most recent call last)
<ipython-input-54-4be6d63890f7> in <module>()
20 # # BiGRU CuDNN
21 embedding_matrix, nb_words = get_embedding('glove',word_index)
---> 22 model, attention_layer = lstm_with_attention(embedding_matrix,STAMP=STAMP,max_seq_length=max_seq_length,nb_words=nb_words,EMBEDDING_DIM=EMBEDDING_DIM)
23 # gru_model = make_cudnn_gru_f(max_seq_length,embedding_matrix,loss_func=macro_soft_f1,eval_metric=macro_f1)
24 # model = gru_model()
7 frames
<ipython-input-51-1ae8a90521d0> in lstm_with_attention(embedding_matrix, **kwargs)
115 model.summary()
116
--> 117 with open(STAMP + ".json", "w") as json_file: json_file.write(model.to_json())
118 return model, attention
119
/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/network.py in to_json(self, **kwargs)
1296 A JSON string.
1297 """
-> 1298 model_config = self._updated_config()
1299 return json.dumps(
1300 model_config, default=serialization.get_json_type, **kwargs)
/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/network.py in _updated_config(self)
1274 from tensorflow.python.keras import __version__ as keras_version # pylint: disable=g-import-not-at-top
1275
-> 1276 config = self.get_config()
1277 model_config = {
1278 'class_name': self.__class__.__name__,
/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/network.py in get_config(self)
966 if not self._is_graph_network:
967 raise NotImplementedError
--> 968 return copy.deepcopy(get_network_config(self))
969
970 #classmethod
/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/network.py in get_network_config(network, serialize_layer_fn)
2117 filtered_inbound_nodes.append(node_data)
2118
-> 2119 layer_config = serialize_layer_fn(layer)
2120 layer_config['name'] = layer.name
2121 layer_config['inbound_nodes'] = filtered_inbound_nodes
/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/utils/generic_utils.py in serialize_keras_object(instance)
273 return serialize_keras_class_and_config(
274 name, {_LAYER_UNDEFINED_CONFIG_KEY: True})
--> 275 raise e
276 serialization_config = {}
277 for key, item in config.items():
/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/utils/generic_utils.py in serialize_keras_object(instance)
268 name = get_registered_name(instance.__class__)
269 try:
--> 270 config = instance.get_config()
271 except NotImplementedError as e:
272 if _SKIP_FAILED_SERIALIZATION:
/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/base_layer.py in get_config(self)
634 raise NotImplementedError('Layer %s has arguments in `__init__` and '
635 'therefore must override `get_config`.' %
--> 636 self.__class__.__name__)
637 return config
638
NotImplementedError: Layer AttentionWithContext has arguments in `__init__` and therefore must override `get_config`.

This is a TensorFlow feature to notify you that it does not know how to reconstruct your layers since it does not know how to process your configs. Here is a quote from tensorflow's document:
get_config()
Returns the config of the layer.
A layer config is a Python dictionary (serializable) containing the configuration of a layer. The same layer can be reinstantiated later (without its trained weights) from this configuration.
The config of a layer does not include connectivity information, nor the layer class name. These are handled by Network (one layer of abstraction above).
To solve the problem, all you need is to create a get_config method corresponding to __init__ in your class to instruct TensorFlow how to reinstantiate your layer.
def get_config(self):
config = super().get_config().copy()
config.update({
'W_regularizer': self.W_regularizer,
'u_regularizer': self.u_regularizer,
'b_regularizer': self.b_regularizer,
'W_constraint': self.W_constraint,
'u_constraint': self.u_constraint,
'b_constraint': self.b_constraint,
'bias': self.bias,
})
return config
Then you should be able to save and load it.

How to chain an input layer to tensorflow-hub?

I want to classify text to 2 classes by using this embedding: https://tfhub.dev/google/universal-sentence-encoder-multilingual/3
And I also want to add additional features after the embedding. So I have two inputs:
import tensorflow as tf
import tensorflow_hub as tfh
import tensorflow_datasets as tfds
import tensorflow_text as tft
hp = {
'embedding': 'https://tfhub.dev/google/universal-sentence-encoder-multilingual/3' EMBEDDINGS['senm'],
'units': 64,
'learning_rate': 1e-3,
'dropout': 0.2,
'layers': 2
}
textInput = tf.keras.Input(shape=(1, ), name = 'text', dtype = tf.string)
featuresInput = tf.keras.Input(shape=(36, ), name = 'features')
x = tfh.KerasLayer(hp.get('embedding'), dtype = tf.string, trainable = False)(textInput)
x = tf.keras.layers.concatenate([x, featuresInput])
for index in range(hp.get('layers')):
x = tf.keras.layers.Dense(hp.get('units'), activation = 'relu')(x)
x = tf.keras.layers.Dropout(hp.get('dropout'))(x)
output = tf.keras.layers.Dense(
1,
activation = 'sigmoid',
bias_initializer = tf.keras.initializers.Constant(INITIAL_BIAS) if INITIAL_BIAS else None
)(x)
model = tf.keras.Model(inputs = [textInput, featuresInput], outputs = output)
model.compile(
optimizer = tf.keras.optimizers.Adam(lr = hp.get('learning_rate')),
loss = tf.keras.losses.BinaryCrossentropy(),
metrics = METRICS,
)
And the code fails with error:
---------------------------------------------------------------------------
AssertionError Traceback (most recent call last)
<ipython-input-17-61aed6f885c9> in <module>
10 featuresInput = tf.keras.Input(shape=(36, ), name = 'features')
11
---> 12 x = tfh.KerasLayer(hp.get('embedding'), dtype = tf.string, trainable = False)(textInput)
13 x = tf.keras.layers.concatenate([x, featuresInput])
14
~/.virtualenvs/python3/lib/python3.6/site-packages/tensorflow/python/keras/engine/base_layer.py in __call__(self, *args, **kwargs)
920 not base_layer_utils.is_in_eager_or_tf_function()):
921 with auto_control_deps.AutomaticControlDependencies() as acd:
--> 922 outputs = call_fn(cast_inputs, *args, **kwargs)
923 # Wrap Tensors in `outputs` in `tf.identity` to avoid
924 # circular dependencies.
~/.virtualenvs/python3/lib/python3.6/site-packages/tensorflow/python/autograph/impl/api.py in wrapper(*args, **kwargs)
263 except Exception as e: # pylint:disable=broad-except
264 if hasattr(e, 'ag_error_metadata'):
--> 265 raise e.ag_error_metadata.to_exception(e)
266 else:
267 raise
AssertionError: in user code:
/home/e/.virtualenvs/python3/lib/python3.6/site-packages/tensorflow_hub/keras_layer.py:222 call *
result = f()
/home/e/.virtualenvs/python3/lib/python3.6/site-packages/tensorflow/python/saved_model/load.py:486 _call_attribute **
return instance.__call__(*args, **kwargs)
/home/e/.virtualenvs/python3/lib/python3.6/site-packages/tensorflow/python/eager/def_function.py:580 __call__
result = self._call(*args, **kwds)
/home/e/.virtualenvs/python3/lib/python3.6/site-packages/tensorflow/python/eager/def_function.py:650 _call
return self._concrete_stateful_fn._filtered_call(canon_args, canon_kwds) # pylint: disable=protected-access
/home/e/.virtualenvs/python3/lib/python3.6/site-packages/tensorflow/python/eager/function.py:1665 _filtered_call
self.captured_inputs)
/home/e/.virtualenvs/python3/lib/python3.6/site-packages/tensorflow/python/eager/function.py:1759 _call_flat
"StatefulPartitionedCall": self._get_gradient_function()}):
/usr/lib/python3.6/contextlib.py:81 __enter__
return next(self.gen)
/home/e/.virtualenvs/python3/lib/python3.6/site-packages/tensorflow/python/framework/ops.py:4735 _override_gradient_function
assert not self._gradient_function_map
AssertionError:
BUT it works if I use Sequential
model = tf.keras.Sequential([
hub.KerasLayer(embedding, input_shape=[], dtype = tf.string, trainable = True),
tf.keras.layers.Dense(16, activation = 'relu', input_shape = (train_features.shape[-1],)),
tf.keras.layers.Dropout(0.5),
tf.keras.layers.Dense(1, activation = 'sigmoid', bias_initializer = output_bias),
])
model.compile(optimizer = tf.keras.optimizers.Adam(lr=1e-3), loss = tf.keras.losses.BinaryCrossentropy(), metrics = metrics)
Is there anything I'm doing wrong with the functual api? Can you please help me with the error

I've faced a similar problem. My solution looks like this:
def build_model():
premise = keras.Input(shape=(), dtype=tf.string)
hypothesis = keras.Input(shape=(), dtype=tf.string)
keras_emb = hub.KerasLayer(embed, input_shape=(), output_shape = (512), dtype=tf.string, trainable=True)
prem_emb = keras_emb(premise)
hyp_emb = keras_emb(hypothesis)
emb = layers.Concatenate()([prem_emb, hyp_emb])
dense = layers.Dense(32, activation="relu")(emb)
classifier = layers.Dense(3)(dense)
model = keras.Model(inputs=[premise, hypothesis], outputs=classifier, name="elementary_model")
model.compile(loss=tf.losses.SparseCategoricalCrossentropy(from_logits=True), optimizer="adam", metrics=['accuracy'])
return model
Note: the text input shape should be () (empty tuple)

Keras: custom loss causes "You must feed a value for placeholder tensor"

I'm trying to build a variational autoencoder in Keras following the example in the Keras repository. Here's my setup:
from keras.layers import Dense, Input, merge, concatenate, Dense, LSTM, Lambda, Flatten, Reshape
from keras import backend as K
from keras.models import Model
from keras.losses import mse
import numpy as np
class VAE:
def __init__(self, n_verts=15, n_dims=3, n_layers=3, n_units=128, latent_dim=2):
self.n_verts = n_verts
self.n_dims = n_dims
self.n_layers = n_layers
self.n_units = n_units
self.latent_dim = latent_dim
self.encoder = self.build_encoder()
self.decoder = self.build_decoder()
inputs = Input((self.n_verts, self.n_dims))
outputs = self.decoder(self.encoder(inputs)[2])
self.model = Model(inputs, outputs, name='vae')
self.model.compile(optimizer='adam', loss=self.get_loss)
def build_encoder(self):
i = Input(shape=(self.n_verts, self.n_dims), name='encoder_input')
h = i
h = Flatten()(h)
h = Dense(self.n_units, activation='relu')(h)
for idx in range(1, self.n_layers, 1):
h = Dense(self.n_units // (2*idx), activation='relu')(h)
self.z_mean = Dense(self.latent_dim, name='z_mean')(h)
self.z_log_var = Dense(self.latent_dim, name='z_log_var')(h)
# use reparameterization trick to factor stochastic node out of gradient flow
self.z = Lambda(self.sample, output_shape=(self.latent_dim,), name='z')([self.z_mean, self.z_log_var])
return Model(i, [self.z_mean, self.z_log_var, self.z], name='encoder')
def sample(self, args):
'''
Reparameterization trick by sampling from an isotropic unit Gaussian.
#arg (tensor): mean and log of variance of Q(z|X)
#returns z (tensor): sampled latent vector
'''
z_mean, z_log_var = args
batch = K.shape(z_mean)[0]
dim = K.int_shape(z_mean)[1]
# by default, random_normal has mean = 0 and std = 1.0
epsilon = K.random_normal(shape=(batch, dim))
return z_mean + K.exp(0.5 * z_log_var) * epsilon
def build_decoder(self):
i = Input(shape=(self.latent_dim,), name='z_sampling')
h = i
for idx in range(1, self.n_layers, 1):
h = Dense(self.n_units//(2*(self.n_layers-idx)), activation='relu')(h)
h = Dense(self.n_units, activation='relu')(h)
h = Dense(self.n_verts * self.n_dims, activation='sigmoid')(h)
o = Reshape((self.n_verts, self.n_dims))(h)
return Model(i, o, name='decoder')
def get_loss(self, inputs, outputs):
reconstruction_loss = mse(inputs, outputs)
reconstruction_loss *= self.n_verts * self.n_dims
return reconstruction_loss # this works fine
kl_loss = 1 + self.z_log_var - K.square(self.z_mean) - K.exp(self.z_log_var)
kl_loss = K.sum(kl_loss, axis=-1)
kl_loss *= -0.5
vae_loss = K.mean(reconstruction_loss + kl_loss) # todo: make this balance parameterizable
return vae_loss # this doesn't
def train(self, X, predict='frame', n_epochs=10000):
for idx in range(n_epochs):
i = np.random.randint(0, X.shape[1]-1) # sample idx
frame = np.expand_dims( X[:,i:i+1,:].squeeze(), axis=0) # shape = 1 sample, v verts, d dims
next_frame = np.expand_dims( X[:,i+1:i+2,:].squeeze(), axis=0)
if predict == 'frame': loss = self.model.train_on_batch(frame, frame)
elif predict == 'next_frame': loss = self.model.train_on_batch(frame, next_frame)
if idx % 1000 == 0:
print(' * training idx', idx, 'loss', loss)
X_train = np.random.rand(15, 100, 3)
vae = VAE(n_verts=15, latent_dim=2, n_layers=3, n_units=128)
vae.encoder.summary()
vae.train(X_train, n_epochs=10000, predict='frame')
This works, but if you look at the get_loss function you'll see it's returning a little prematurely. If I comment out return reconstruction_loss so that the loss function returns vae_loss, I get an error:
--------------------------------------------------------------------------- InvalidArgumentError Traceback (most recent call last) <ipython-input-7-57d76ed539a4> in <module>
78 vae = VAE(n_verts=15, latent_dim=2, n_layers=3, n_units=128)
79 vae.encoder.summary()
---> 80 vae.train(X_train, n_epochs=10000, predict='frame')
<ipython-input-7-57d76ed539a4> in train(self, X, predict, n_epochs)
70 frame = np.expand_dims( X[:,i:i+1,:].squeeze(), axis=0) # shape = 1 sample, v verts, d dims
71 next_frame = np.expand_dims( X[:,i+1:i+2,:].squeeze(), axis=0)
---> 72 if predict == 'frame': loss = self.model.train_on_batch(frame, frame)
73 elif predict == 'next_frame': loss = self.model.train_on_batch(frame, next_frame)
74 if idx % 1000 == 0:
~/anaconda/envs/3.5/lib/python3.5/site-packages/keras/engine/training.py in train_on_batch(self, x, y, sample_weight, class_weight) 1215 ins = x + y + sample_weights 1216 self._make_train_function()
-> 1217 outputs = self.train_function(ins) 1218 return unpack_singleton(outputs) 1219
~/anaconda/envs/3.5/lib/python3.5/site-packages/keras/backend/tensorflow_backend.py in __call__(self, inputs) 2713 return self._legacy_call(inputs) 2714
-> 2715 return self._call(inputs) 2716 else: 2717 if py_any(is_tensor(x) for x in inputs):
~/anaconda/envs/3.5/lib/python3.5/site-packages/keras/backend/tensorflow_backend.py in _call(self, inputs) 2673 fetched = self._callable_fn(*array_vals, run_metadata=self.run_metadata) 2674 else:
-> 2675 fetched = self._callable_fn(*array_vals) 2676 return fetched[:len(self.outputs)] 2677
~/anaconda/envs/3.5/lib/python3.5/site-packages/tensorflow/python/client/session.py in __call__(self, *args, **kwargs) 1437 ret = tf_session.TF_SessionRunCallable( 1438 self._session._session, self._handle, args, status,
-> 1439 run_metadata_ptr) 1440 if run_metadata: 1441 proto_data = tf_session.TF_GetBuffer(run_metadata_ptr)
~/anaconda/envs/3.5/lib/python3.5/site-packages/tensorflow/python/framework/errors_impl.py in __exit__(self, type_arg, value_arg, traceback_arg)
526 None, None,
527 compat.as_text(c_api.TF_Message(self.status.status)),
--> 528 c_api.TF_GetCode(self.status.status))
529 # Delete the underlying status object from memory otherwise it stays alive
530 # as there is a reference to status from this from the traceback due to
InvalidArgumentError: You must feed a value for placeholder tensor 'encoder_input_6' with dtype float and shape [?,15,3] [[{{node encoder_input_6}}]]
Does anyone know how to resolve this error? Any suggestions would be hugely appreciated!

Ah, I got this worked out once my variables were scoped properly:
from keras.layers import Dense, Input, merge, concatenate, Dense, LSTM, Lambda, Flatten, Reshape
from keras import backend as K
from keras.models import Model
from keras.losses import mse
import numpy as np
class VAE:
def __init__(self, n_verts=15, n_dims=3, n_layers=3, n_units=128, latent_dim=2):
self.input_shape = (n_verts*n_dims,)
self.n_layers = n_layers
self.n_units = n_units
self.latent_dim = latent_dim
# build the encoder and decoder
inputs = Input(shape=self.input_shape, name='encoder_input')
self.encoder = self.get_encoder(inputs)
self.decoder = self.get_decoder()
# build the VAE
outputs = self.decoder(self.encoder(inputs)[2])
self.model = Model(inputs, outputs, name='vae_mlp')
# add loss and compile
self.model.add_loss(self.get_loss(inputs, outputs))
self.model.compile(optimizer='adam')
def get_encoder(self, inputs):
h = inputs
h = Dense(self.n_units, activation='relu')(h)
for idx in range(1, self.n_layers, 1):
h = Dense(self.n_units // (2*idx), activation='relu')(h)
self.z_mean = Dense(self.latent_dim, name='z_mean')(h)
self.z_log_var = Dense(self.latent_dim, name='z_log_var')(h)
z = Lambda(self.sampling, output_shape=(self.latent_dim,), name='z')([self.z_mean, self.z_log_var])
encoder = Model(inputs, [self.z_mean, self.z_log_var, z], name='encoder')
return encoder
def sampling(self, args):
self.z_mean, self.z_log_var = args
batch = K.shape(self.z_mean)[0]
dim = K.int_shape(self.z_mean)[1]
# by default, random_normal has mean = 0 and std = 1.0
epsilon = K.random_normal(shape=(batch, dim))
return self.z_mean + K.exp(0.5 * self.z_log_var) * epsilon
def get_decoder(self):
latent_inputs = Input(shape=(self.latent_dim,), name='z_sampling')
h = latent_inputs
for idx in range(1, self.n_layers, 1):
h = Dense(self.n_units//(2*(self.n_layers-idx)), activation='relu')(h)
h = Dense(self.n_units, activation='relu')(h)
outputs = Dense(self.input_shape[0], activation='sigmoid')(h)
decoder = Model(latent_inputs, outputs, name='decoder')
return decoder
def get_loss(self, inputs, outputs):
reconstruction_loss = mse(inputs, outputs)
reconstruction_loss *= self.input_shape[0]
kl_loss = 1 + self.z_log_var - K.square(self.z_mean) - K.exp(self.z_log_var)
kl_loss = K.sum(kl_loss, axis=-1)
kl_loss *= -0.5
vae_loss = K.mean(reconstruction_loss + kl_loss)
return vae_loss
# train
x_train = np.random.rand(10000, 45)
vae = VAE(n_verts=15, latent_dim=2, n_layers=3, n_units=128)
vae.model.fit(x_train[:-1000,:],
epochs=100,
batch_size=128,
validation_data=(x_train[-1000:,:], None))