I am working on a multi-label text classification problem. I am trying to add attention mechanism with bilstm model. The attention mechanism code is taken from here. I am not able to save the model architecture and getting an error mentioned below. My tensorflow version -2.2.0
from keras import backend as K
def dot_product(x, kernel):
if K.backend() == 'tensorflow':
return K.squeeze(K.dot(x, K.expand_dims(kernel)), axis=-1)
else:
return K.dot(x, kernel)
class AttentionWithContext(tf.keras.layers.Layer):
"""
# Input shape
3D tensor with shape: `(samples, steps, features)`.
# Output shape
2D tensor with shape: `(samples, features)`.
"""
def __init__(self,
W_regularizer=None, u_regularizer=None, b_regularizer=None,
W_constraint=None, u_constraint=None, b_constraint=None,
bias=True, **kwargs):
self.supports_masking = True
self.init = tf.keras.initializers.get('glorot_uniform')
self.W_regularizer = tf.keras.regularizers.get(W_regularizer)
self.u_regularizer = tf.keras.regularizers.get(u_regularizer)
self.b_regularizer = tf.keras.regularizers.get(b_regularizer)
self.W_constraint = tf.keras.constraints.get(W_constraint)
self.u_constraint = tf.keras.constraints.get(u_constraint)
self.b_constraint = tf.keras.constraints.get(b_constraint)
self.bias = bias
super(AttentionWithContext, self).__init__(**kwargs)
def build(self, input_shape):
assert len(input_shape) == 3
self.W = self.add_weight(shape=(input_shape[-1], input_shape[-1],),
initializer=self.init,
name='{}_W'.format(self.name),
regularizer=self.W_regularizer,
constraint=self.W_constraint)
if self.bias:
self.b = self.add_weight(shape=(input_shape[-1],),
initializer='zero',
name='{}_b'.format(self.name),
regularizer=self.b_regularizer,
constraint=self.b_constraint)
self.u = self.add_weight(shape=(input_shape[-1],),
initializer=self.init,
name='{}_u'.format(self.name),
regularizer=self.u_regularizer,
constraint=self.u_constraint)
super(AttentionWithContext, self).build(input_shape)
def compute_mask(self, input, input_mask=None):
# do not pass the mask to the next layers
return None
def call(self, x, mask=None):
uit = dot_product(x, self.W)
if self.bias:
uit += self.b
uit = K.tanh(uit)
ait = dot_product(uit, self.u)
a = K.exp(ait)
# apply mask after the exp. will be re-normalized next
if mask is not None:
# Cast the mask to floatX to avoid float64 upcasting in theano
a *= K.cast(mask, K.floatx())
# in some cases especially in the early stages of training the sum may be almost zero
# and this results in NaN's. A workaround is to add a very small positive number ε to the sum.
# a /= K.cast(K.sum(a, axis=1, keepdims=True), K.floatx())
a /= K.cast(K.sum(a, axis=1, keepdims=True) + K.epsilon(), K.floatx())
a = K.expand_dims(a)
weighted_input = x * a
return K.sum(weighted_input, axis=1)
def compute_output_shape(self, input_shape):
return input_shape[0], input_shape[-1]
def lstm_with_attention(embedding_matrix,
**kwargs):
STAMP = kwargs['STAMP']
max_seq_length = kwargs['max_seq_length']
EMBEDDING_DIM = kwargs['EMBEDDING_DIM']
nb_words = kwargs['nb_words']
inp = tf.keras.Input(shape=(max_seq_length,))
embedded_seq = tf.keras.layers.Embedding(nb_words,
EMBEDDING_DIM,
weights=[embedding_matrix],
trainable=False)(inp)
x_1_bilstm = tf.keras.layers.Bidirectional(tf.compat.v1.keras.layers.CuDNNLSTM(128, return_sequences=True))(embedded_seq)
x_1_bn = tf.keras.layers.BatchNormalization()(x_1_bilstm)
x_2_bilstm = tf.keras.layers.Bidirectional(tf.compat.v1.keras.layers.CuDNNLSTM(64, return_sequences=True))(x_1_bn)
attention = AttentionWithContext()(x_2_bilstm)
x = tf.keras.layers.Dense(64, activation="relu")(attention)
x = tf.keras.layers.Dense(1, activation="sigmoid")(x)
model = tf.keras.Model(inputs=inp, outputs=x)
optimizer = tf.keras.optimizers.Adam()
model.compile(loss='binary_crossentropy', optimizer=optimizer, metrics=['accuracy'])
model.summary()
with open(STAMP + ".json", "w") as json_file: json_file.write(model.to_json())
return model, attention
Building lstm with attention
embedding_matrix, nb_words = get_embedding('glove',word_index)
model, attention_layer = lstm_with_attention(embedding_matrix,STAMP=STAMP,max_seq_length=max_seq_length,nb_words=nb_words,EMBEDDING_DIM=EMBEDDING_DIM)
Error
---------------------------------------------------------------------------
NotImplementedError Traceback (most recent call last)
<ipython-input-54-4be6d63890f7> in <module>()
20 # # BiGRU CuDNN
21 embedding_matrix, nb_words = get_embedding('glove',word_index)
---> 22 model, attention_layer = lstm_with_attention(embedding_matrix,STAMP=STAMP,max_seq_length=max_seq_length,nb_words=nb_words,EMBEDDING_DIM=EMBEDDING_DIM)
23 # gru_model = make_cudnn_gru_f(max_seq_length,embedding_matrix,loss_func=macro_soft_f1,eval_metric=macro_f1)
24 # model = gru_model()
7 frames
<ipython-input-51-1ae8a90521d0> in lstm_with_attention(embedding_matrix, **kwargs)
115 model.summary()
116
--> 117 with open(STAMP + ".json", "w") as json_file: json_file.write(model.to_json())
118 return model, attention
119
/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/network.py in to_json(self, **kwargs)
1296 A JSON string.
1297 """
-> 1298 model_config = self._updated_config()
1299 return json.dumps(
1300 model_config, default=serialization.get_json_type, **kwargs)
/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/network.py in _updated_config(self)
1274 from tensorflow.python.keras import __version__ as keras_version # pylint: disable=g-import-not-at-top
1275
-> 1276 config = self.get_config()
1277 model_config = {
1278 'class_name': self.__class__.__name__,
/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/network.py in get_config(self)
966 if not self._is_graph_network:
967 raise NotImplementedError
--> 968 return copy.deepcopy(get_network_config(self))
969
970 #classmethod
/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/network.py in get_network_config(network, serialize_layer_fn)
2117 filtered_inbound_nodes.append(node_data)
2118
-> 2119 layer_config = serialize_layer_fn(layer)
2120 layer_config['name'] = layer.name
2121 layer_config['inbound_nodes'] = filtered_inbound_nodes
/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/utils/generic_utils.py in serialize_keras_object(instance)
273 return serialize_keras_class_and_config(
274 name, {_LAYER_UNDEFINED_CONFIG_KEY: True})
--> 275 raise e
276 serialization_config = {}
277 for key, item in config.items():
/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/utils/generic_utils.py in serialize_keras_object(instance)
268 name = get_registered_name(instance.__class__)
269 try:
--> 270 config = instance.get_config()
271 except NotImplementedError as e:
272 if _SKIP_FAILED_SERIALIZATION:
/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/base_layer.py in get_config(self)
634 raise NotImplementedError('Layer %s has arguments in `__init__` and '
635 'therefore must override `get_config`.' %
--> 636 self.__class__.__name__)
637 return config
638
NotImplementedError: Layer AttentionWithContext has arguments in `__init__` and therefore must override `get_config`.
This is a TensorFlow feature to notify you that it does not know how to reconstruct your layers since it does not know how to process your configs. Here is a quote from tensorflow's document:
get_config()
Returns the config of the layer.
A layer config is a Python dictionary (serializable) containing the configuration of a layer. The same layer can be reinstantiated later (without its trained weights) from this configuration.
The config of a layer does not include connectivity information, nor the layer class name. These are handled by Network (one layer of abstraction above).
To solve the problem, all you need is to create a get_config method corresponding to __init__ in your class to instruct TensorFlow how to reinstantiate your layer.
def get_config(self):
config = super().get_config().copy()
config.update({
'W_regularizer': self.W_regularizer,
'u_regularizer': self.u_regularizer,
'b_regularizer': self.b_regularizer,
'W_constraint': self.W_constraint,
'u_constraint': self.u_constraint,
'b_constraint': self.b_constraint,
'bias': self.bias,
})
return config
Then you should be able to save and load it.
Related
I have a code to forecast a time series using an attention mechanism.
Here's what I've got so far, but I'm getting an error.
def dot_product(x, kernel):
if K.backend() == 'tensorflow':
return K.squeeze(K.dot(x, K.expand_dims(kernel)), axis=-1)
else:
return K.dot(x, kernel)
class Attention(Layer):
def __init__(self,
W_regularizer=None, u_regularizer=None, b_regularizer=None,
W_constraint=None, u_constraint=None, b_constraint=None,
bias=True, **kwargs):
self.supports_masking = True
self.init = tf.keras.initializers.get('glorot_uniform')
self.W_regularizer = tf.keras.regularizers.get(W_regularizer)
self.u_regularizer = tf.keras.regularizers.get(u_regularizer)
self.b_regularizer = tf.keras.regularizers.get(b_regularizer)
self.W_constraint = tf.keras.constraints.get(W_constraint)
self.u_constraint = tf.keras.constraints.get(u_constraint)
self.b_constraint = tf.keras.constraints.get(b_constraint)
self.bias = bias
super(Attention, self).__init__(**kwargs)
def build(self, input_shape):
assert len(input_shape) == 3
self.W = self.add_weight(shape=(input_shape[-2], input_shape[-1],),
initializer=self.init,
name='{}_W'.format(self.name),
regularizer=self.W_regularizer,
constraint=self.W_constraint)
if self.bias:
self.b = self.add_weight(shape=(input_shape[-2], input_shape[-1],),
initializer='zero',
name='{}_b'.format(self.name),
regularizer=self.b_regularizer,
constraint=self.b_constraint)
self.u = self.add_weight(shape=(input_shape[-1],),
initializer=self.init,
name='{}_u'.format(self.name),
regularizer=self.u_regularizer,
constraint=self.u_constraint)
super(Attention, self).build(input_shape)
def compute_mask(self, input, input_mask=None):
return None
def call(self, x, mask=None):
uit = dot_product(x, self.W)
if self.bias:
uit += self.b
uit = K.tanh(uit)
ait = dot_product(uit, self.u)
a = K.exp(ait)
if mask is not None:
a *= K.cast(mask, K.floatx())
a /= K.cast(K.sum(a, axis=1, keepdims=True) + K.epsilon(), K.floatx())
a = K.expand_dims(a)
weighted_input = x * a
x = K.sum(weighted_input, axis=1)
return x
n_timesteps= 2
n_features = 1
n_outputs = 1
model_name = 'With_attetion'
path_checkpoint = "model_checkpoint.h5"
# define parameters
verbose, epochs, batch_size = 1, 200, 170
callback = keras.callbacks.EarlyStopping(monitor='loss', patience=3)
modelckpt_callback = keras.callbacks.ModelCheckpoint(monitor="val_loss",filepath=path_checkpoint,
save_weights_only=True,
save_best_only=True,
)
# define model
model = Sequential()
main_input = Input(shape =(n_timesteps,n_features), name = 'main_input')
con1 = Conv1D(filters=128, kernel_size=3, activation='relu', padding='same')(main_input)
bat1 = BatchNormalization()(con1)
max1 = MaxPooling1D(pool_size=3, padding='same')(bat1)
dro1 = Dropout(0.4)(max1)
con2 = Conv1D(filters=64, kernel_size=3, activation='relu', padding='same')(max1)
bat2 = BatchNormalization()(con2)
max2 = MaxPooling1D(pool_size=3, padding='same')(bat2)
dro2 = Dropout(0.2)(max2)
con_f1 = TimeDistributed(Flatten())(dro2)
model.add(RepeatVector(n_outputs))
x1 = LSTM(64, activation='tanh', return_sequences=True)(con_f1)
#model.add(Dropout(0.2))
attention_mul = Attention()(x1)
#attention_mul=tf.reshape(attention_mul, [-1, 1, 64])
attention_mul=tf.expand_dims(attention_mul, axis = 1)
t2 = TimeDistributed(Dense(64, activation='tanh'))(attention_mul)
t1 = TimeDistributed(Dense(1, activation='tanh'))(t2)
model = Model(input = main_input , output = t1)
model.compile(loss="mse", optimizer=keras.optimizers.Adam(lr=3e-6), metrics=[keras.metrics.RootMeanSquaredError()])
# fit network
model.summary()
history = model.fit([x_train], y_train, epochs=epochs, batch_size=250,validation_split=0.2, verbose=verbose,
callbacks=[callback, modelckpt_callback])
Then this is the error I'm getting
C:\Users\HBK\Anaconda3\lib\site-packages\ipykernel_launcher.py:51: UserWarning: Update your `Model` call to the Keras 2 API: `Model(inputs=Tensor("ma..., outputs=Tensor("ti...)`
---------------------------------------------------------------------------
AttributeError Traceback (most recent call last)
<ipython-input-39-a96d7b49ecfc> in <module>
49 # t2 = TimeDistributed(Dense(64, activation='tanh'))(x4)
50 # t1 = TimeDistributed(Dense(1, activation='tanh'))(t2)
---> 51 model = Model(input = main_input , output = t1)
52 #model.compile(loss="mse", optimizer=keras.optimizers.Adam(lr=0.00001), metrics=["accuracy"])
53 model.compile(loss="mse", optimizer=keras.optimizers.Adam(lr=3e-6), metrics=[keras.metrics.RootMeanSquaredError()])
~\Anaconda3\lib\site-packages\keras\legacy\interfaces.py in wrapper(*args, **kwargs)
89 warnings.warn('Update your `' + object_name + '` call to the ' +
90 'Keras 2 API: ' + signature, stacklevel=2)
---> 91 return func(*args, **kwargs)
92 wrapper._original_function = func
93 return wrapper
~\Anaconda3\lib\site-packages\keras\engine\network.py in __init__(self, *args, **kwargs)
92 'inputs' in kwargs and 'outputs' in kwargs):
93 # Graph network
---> 94 self._init_graph_network(*args, **kwargs)
95 else:
96 # Subclassed network
~\Anaconda3\lib\site-packages\keras\engine\network.py in _init_graph_network(self, inputs, outputs, name, **kwargs)
239 # Keep track of the network's nodes and layers.
240 nodes, nodes_by_depth, layers, layers_by_depth = _map_graph_network(
--> 241 self.inputs, self.outputs)
242 self._network_nodes = nodes
243 self._nodes_by_depth = nodes_by_depth
~\Anaconda3\lib\site-packages\keras\engine\network.py in _map_graph_network(inputs, outputs)
1432 layer=layer,
1433 node_index=node_index,
-> 1434 tensor_index=tensor_index)
1435
1436 for node in reversed(nodes_in_decreasing_depth):
~\Anaconda3\lib\site-packages\keras\engine\network.py in build_map(tensor, finished_nodes, nodes_in_progress, layer, node_index, tensor_index)
1419 tensor_index = node.tensor_indices[i]
1420 build_map(x, finished_nodes, nodes_in_progress, layer,
-> 1421 node_index, tensor_index)
1422
1423 finished_nodes.add(node)
~\Anaconda3\lib\site-packages\keras\engine\network.py in build_map(tensor, finished_nodes, nodes_in_progress, layer, node_index, tensor_index)
1419 tensor_index = node.tensor_indices[i]
1420 build_map(x, finished_nodes, nodes_in_progress, layer,
-> 1421 node_index, tensor_index)
1422
1423 finished_nodes.add(node)
~\Anaconda3\lib\site-packages\keras\engine\network.py in build_map(tensor, finished_nodes, nodes_in_progress, layer, node_index, tensor_index)
1391 ValueError: if a cycle is detected.
1392 """
-> 1393 node = layer._inbound_nodes[node_index]
1394
1395 # Prevent cycles.
AttributeError: 'NoneType' object has no attribute '_inbound_nodes'
The error is at this part of the code
model = Model(input = main_input , output = t1)
After the attention mechanism, I got a 2D tensor.
I reshaped it with
attention_mul=tf.expand_dims(attention_mul, axis = 1)
to have a 3D tensor since time distributed layer need 3 dim.
You should embed tf.expand_dims() in a keras Lambda layer.
Convert:
attention_mul=tf.expand_dims(attention_mul, axis = 1)
To:
attention_mul=Lambda(lambda x: tf.expand_dims(x, axis = 1))(attention_mul)
I am trying to concatenate the outputs of two or more models.
Code:
import efficientnet.tfkeras as efn
def build_model(model_name="efficientnet_B0", dim=128,
normalize_features=True, return_feature_model=False,
inp_layer=None):
def get_efficientnet(effnet_model):
effnet_model = int(effnet_model)
EFNS = [efn.EfficientNetB0, efn.EfficientNetB1,
efn.EfficientNetB2, efn.EfficientNetB3,
efn.EfficientNetB4, efn.EfficientNetB5,
efn.EfficientNetB6, efn.EfficientNetB7,]
return EFNS[effnet_model]
model_dict = {
"efficientnet": get_efficientnet,
"resnet50": tf.keras.applications.ResNet50,
"vgg19": tf.keras.applications.VGG19,
"Xception": tf.keras.applications.Xception,
}
# For Image Input
if model_name.find('efficientnet') != -1:
base = model_dict['efficientnet'](model_name[-1])(input_shape=(dim,dim,3),
weights='imagenet',
include_top=False)(inp_layer)
else:
base = model_dict[model_name](input_shape=(dim,dim,3),
weights='imagenet',
include_top=False)(inp_layer)
pooling_layer = keras.layers.GlobalAveragePooling2D()(base)
normalized_feature = keras.layers.BatchNormalization(name="norm_featvec")(pooling_layer)
output = keras.layers.Dense(1,activation='sigmoid')(pooling_layer)
if normalize_features:
feat_output = normalized_feature
else:
feat_output = pooling_layer
if return_feature_model:
featext_model = keras.Model(inputs=[img_inp], outputs=[feat_output])
model = keras.Model(inputs=[img_inp], outputs=[output])
if return_feature_model:
return model, featext_model
else:
return model
def get_complete_model(featvec_model_paths, dim=384):
input_layer = keras.layers.Input(shape=(dim, dim, 3), name="img_input")
models = list()
for model_path in featvec_model_paths:
model_name = os.path.basename(model_path)[5:-5]
_, featvec_model = build_model(model_name=model_name, dim=dim,
return_feature_model=True,
inp_layer=input_layer)
featvec_model.load_weights(model_path)
featvec_model.trainable = False
models.append(featvec_model)
concat_layer = keras.layers.Concatenate()(models)
dense_1 = keras.layers.Dense(4096, activation="selu")(concat_layer)
dense_2 = keras.layers.Dense(2048, activation="selu")(dense_1)
dense_3 = keras.layers.Dense(1024, activation="selu")(dense_2)
output_layer = keras.layers.Dense(1, activation="sigmoid")(dense_3)
model = keras.Model(inputs=[input_layer], outputs=[output_layer])
model.compile(loss=keras.losses.BinaryCrossentropy(),
optimizer=keras.optimizers.Nadam(),
metrics=['AUC'])
return model
featvec_model_paths in get_complete_model are paths to weight files.
Execution Line:
featvec_paths = ['feat_efficientnet_b3_0.h5',
'feat_efficientnet_b3_1.h5',
'feat_efficientnet_b4_0.h5',
'feat_efficientnet_b4_1.h5']
final_model, featvec_model = get_complete_model(featvec_paths, dim=384)
You can run the code by commenting featvec_model.load_weights(model_path).
Error:
TypeError Traceback (most recent call last)
<ipython-input-104-76ecd46bf20d> in <module>
----> 1 final_model, featvec_model = get_complete_model(featvec_paths, dim=384)
<ipython-input-103-d92456535b41> in get_complete_model(featvec_model_paths, dim)
17 #print(models)
18
---> 19 concat_layer = keras.layers.Concatenate()(models)
20 dense_1 = keras.layers.Dense(4096, activation="selu")(concat_layer)
21 dense_2 = keras.layers.Dense(2048, activation="selu")(dense_1)
/opt/conda/lib/python3.7/site-packages/tensorflow/python/keras/engine/base_layer.py in __call__(self, *args, **kwargs)
1006 with ops.name_scope_v2(name_scope):
1007 if not self.built:
-> 1008 self._maybe_build(inputs)
1009
1010 with autocast_variable.enable_auto_cast_variables(
/opt/conda/lib/python3.7/site-packages/tensorflow/python/keras/engine/base_layer.py in _maybe_build(self, inputs)
2708 # operations.
2709 with tf_utils.maybe_init_scope(self):
-> 2710 self.build(input_shapes) # pylint:disable=not-callable
2711 # We must set also ensure that the layer is marked as built, and the build
2712 # shape is stored since user defined build functions may not be calling
/opt/conda/lib/python3.7/site-packages/tensorflow/python/keras/utils/tf_utils.py in wrapper(instance, input_shape)
270 if input_shape is not None:
271 input_shape = convert_shapes(input_shape, to_tuples=True)
--> 272 output_shape = fn(instance, input_shape)
273 # Return shapes from `fn` as TensorShapes.
274 if output_shape is not None:
/opt/conda/lib/python3.7/site-packages/tensorflow/python/keras/layers/merge.py in build(self, input_shape)
490 def build(self, input_shape):
491 # Used purely for shape validation.
--> 492 if not isinstance(input_shape[0], tuple) or len(input_shape) < 2:
493 raise ValueError('A `Concatenate` layer should be called '
494 'on a list of at least 2 inputs')
TypeError: 'NoneType' object is not subscriptable
I am passing a list of models to the concatenation class object. I also tried appending only the last layers of each model:
models.append(featvec_model.layers[-1]) but this also gave the same error.
I just had to do this in get_complete_model():
models = models.append(featvec_model)
When I am defining the Hierarchical Attentional Network, an error is popping up which says "AttributeError: can't set attribute". Please help.
This is the Attention.py file
import keras
import Attention
from keras.engine.topology import Layer, Input
from keras import backend as K
from keras import initializers
#Hierarchical Attention Layer Implementation
'''
Implemented by Arkadipta De (MIT Licensed)
'''
class Hierarchical_Attention(Layer):
def __init__(self, attention_dim):
self.init = initializers.get('normal')
self.supports_masking = True
self.attention_dim = attention_dim
super(Hierarchical_Attention, self).__init__()
def build(self, input_shape):
assert len(input_shape) == 3
self.W = K.variable(self.init((input_shape[-1], self.attention_dim)))
self.b = K.variable(self.init((self.attention_dim, )))
self.u = K.variable(self.init((self.attention_dim, 1)))
self.trainable_weights = [self.W, self.b, self.u]
super(Hierarchical_Attention, self).build(input_shape)
def compute_mask(self, inputs, mask=None):
return mask
def call(self, x, mask=None):
# size of x :[batch_size, sel_len, attention_dim]
# size of u :[batch_size, attention_dim]
# uit = tanh(xW+b)
uit = K.tanh(K.bias_add(K.dot(x, self.W), self.b))
ait = K.dot(uit, self.u)
ait = K.squeeze(ait, -1)
ait = K.exp(ait)
if mask is not None:
# Cast the mask to floatX to avoid float64 upcasting in theano
ait *= K.cast(mask, K.floatx())
ait /= K.cast(K.sum(ait, axis=1, keepdims=True) + K.epsilon(), K.floatx())
ait = K.expand_dims(ait)
weighted_input = x * ait
output = K.sum(weighted_input, axis=1)
return output
def compute_output_shape(self, input_shape):
return (input_shape[0], input_shape[-1])
This is the main file where I'm building the model.
import re
import os
import numpy as np
import pandas as pd
import keras
from keras.engine.topology import Layer, Input
import Attention
from sklearn.model_selection import train_test_split
from keras.models import Model, Input
from keras.layers import Dropout, Dense, LSTM, GRU, Bidirectional, concatenate, Multiply, Subtract
from keras.utils import to_categorical
from keras import backend as K
from keras import initializers
Max_Title_Length = 0
Max_Content_Length = 0
for i in range(0, len(X)):
Max_Title_Length = max(Max_Title_Length, len(X['title'][i]))
Max_Content_Length = max(Max_Content_Length, len(X['text'][i]))
vector_size = 100
input_title = Input(shape = (Max_Title_Length,vector_size,), name = 'input_title')
input_content = Input(shape = (Max_Content_Length,vector_size,), name = 'input_content')
def Classifier(input_title, input_content):
#x = Bidirectional(GRU(units = 100, return_sequences = True, kernel_initializer = keras.initializers.lecun_normal(seed = None), unit_forget_bias = True))(input_title)
x = Bidirectional(GRU(100, return_sequences=True))(input_title)
x_attention = Attention.Hierarchical_Attention(100)(x)
#y = Bidirectional(LSTM(units = 100, return_sequences = True, kernel_initializer = keras.initializers.lecun_normal(seed = None), unit_forget_bias = True))(input_content)
y = Bidirectional(GRU(100, return_sequences=True))(input_content)
y_attention = Attention.Hierarchical_Attention(100)(y)
z = concatenate([x_attention,y_attention])
z = Dense(units = 512, activation = 'relu')(z)
z = Dropout(0.2)(z)
z = Dense(units = 256, activation = 'relu')(z)
z = Dropout(0.2)(z)
z = Dense(units = 128, activation = 'relu')(z)
z = Dropout(0.2)(z)
z = Dense(units = 50, activation = 'relu')(z)
z = Dropout(0.2)(z)
z = Dense(units = 10, activation = 'relu')(z)
z = Dropout(0.2)(z)
output = Dense(units = 2, activation = 'softmax')(z)
model = Model(inputs = [input_title, input_content], outputs = output)
model.summary()
return model
def compile_and_train(model, num_epochs):
model.compile(optimizer= 'adam', loss= 'categorical_crossentropy', metrics=['acc'])
history = model.fit([train_x_title,train_x_content], train_label, batch_size=32, epochs=num_epochs)
return history
Classifier_Model = Classifier(input_title,input_content)
This code is giving me an error which says:
---------------------------------------------------------------------------
AttributeError Traceback (most recent call last)
/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/base_layer.py in __setattr__(self, name, value)
2761 try:
-> 2762 super(tracking.AutoTrackable, self).__setattr__(name, value)
2763 except AttributeError:
AttributeError: can't set attribute
During handling of the above exception, another exception occurred:
AttributeError Traceback (most recent call last)
6 frames
<ipython-input-43-32804502e0b0> in <module>()
32 return history
33
---> 34 Classifier_Model = Classifier(input_title,input_content)
<ipython-input-43-32804502e0b0> in Classifier(input_title, input_content)
7 #x = Bidirectional(GRU(units = 100, return_sequences = True, kernel_initializer = keras.initializers.lecun_normal(seed = None), unit_forget_bias = True))(input_title)
8 x = Bidirectional(GRU(200, return_sequences=True))(input_title)
----> 9 x_attention = Attention.Hierarchical_Attention(100)(x)
10 #y = Bidirectional(LSTM(units = 100, return_sequences = True, kernel_initializer = keras.initializers.lecun_normal(seed = None), unit_forget_bias = True))(input_content)
11 y = Bidirectional(GRU(100, return_sequences=True))(input_content)
/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/base_layer.py in __call__(self, *args, **kwargs)
924 if _in_functional_construction_mode(self, inputs, args, kwargs, input_list):
925 return self._functional_construction_call(inputs, args, kwargs,
--> 926 input_list)
927
928 # Maintains info about the `Layer.call` stack.
/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/base_layer.py in _functional_construction_call(self, inputs, args, kwargs, input_list)
1096 # Build layer if applicable (if the `build` method has been
1097 # overridden).
-> 1098 self._maybe_build(inputs)
1099 cast_inputs = self._maybe_cast_inputs(inputs, input_list)
1100
/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/base_layer.py in _maybe_build(self, inputs)
2641 # operations.
2642 with tf_utils.maybe_init_scope(self):
-> 2643 self.build(input_shapes) # pylint:disable=not-callable
2644 # We must set also ensure that the layer is marked as built, and the build
2645 # shape is stored since user defined build functions may not be calling
/content/Attention.py in build(self, input_shape)
23 self.b = K.variable(self.init((self.attention_dim, )))
24 self.u = K.variable(self.init((self.attention_dim, 1)))
---> 25 self.trainable_weights = [self.W, self.b, self.u]
26 super(Hierarchical_Attention, self).build(input_shape)
27
/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/base_layer.py in __setattr__(self, name, value)
2765 ('Can\'t set the attribute "{}", likely because it conflicts with '
2766 'an existing read-only #property of the object. Please choose a '
-> 2767 'different name.').format(name))
2768 return
2769
AttributeError: Can't set the attribute "trainable_weights", likely because it conflicts with an existing read-only #property of the object. Please choose a different name.
I'm a noob in Neural Networks. Please help.
I ran into the same problem when I was trying to execute the code on Google Colab.
I found some answers on StackOverflow says it's an ongoing issue with tf on Colab.
link here
It remains unsolved for me, but I believe you can try to set self._trainable_weights instead of self.trainable_weights
I used tf2.0 to write a vae model,and after I used the callbacks to save the model weight.
But afeter i use the load_weights, it said trying to load a weight file containing 2 layers into a model with 0 layers.
I used more solutions to solve it, but these failure.
this is my train code
vae = VAE(5529,600,1024)
optimizer = tf.keras.optimizers.Adam(learning_rate=1e-3)
vae.compile(optimizer, loss=tf.keras.losses.MeanSquaredError())
callbacks = []
# save model
cp_callback = tf.keras.callbacks.ModelCheckpoint(filepath='%s/cp.weights.{epoch:03d}-{val_loss:.2f}.hdf5' % save_dir,save_weights_only=True,save_best_only=True,verbose=1)
callbacks.append(cp_callback)
vae.fit(train_dataset,validation_data=valid_dataset, epochs=2, callbacks=callbacks)
this is my VAE Model Code
import tensorflow as tf
class Sampling(tf.keras.layers.Layer):
def call(self, inputs):
z_mean, z_log_var = inputs
batch = tf.shape(z_mean)[0]
dim = tf.shape(z_mean)[1]
epsilon = tf.keras.backend.random_normal(shape=(batch, dim))
return z_mean + tf.exp(0.5 * z_log_var) * epsilon
class Encoder(tf.keras.layers.Layer):
def __init__(self, latent_dim=600, intermediate_dim=1024, name='encoder', **kwargs):
super(Encoder, self).__init__(name, **kwargs)
self.dense_proj = tf.keras.layers.Dense(intermediate_dim, activation='relu')
self.dense_mean = tf.keras.layers.Dense(latent_dim)
self.dense_log_var = tf.keras.layers.Dense(latent_dim)
self.sampling = Sampling()
def call(self, inputs):
h1 = self.dense_proj(inputs)
z_mean = self.dense_mean(h1)
z_log_var = self.dense_log_var(h1)
z = self.sampling((z_mean, z_log_var))
return z_mean, z_log_var, z
class Decoder(tf.keras.layers.Layer):
def __init__(self, original_dim, intermediate_dim=1024, name='decoder', **kwargs):
super(Decoder, self).__init__(name,**kwargs)
self.dense_proj = tf.keras.layers.Dense(intermediate_dim, activation='relu')
self.dense_output = tf.keras.layers.Dense(original_dim, activation='sigmoid')
def call(self, inputs):
h1 = self.dense_proj(inputs)
return self.dense_output(h1)
class VAE(tf.keras.Model):
def __init__(self, original_dim, latent_dim=600, intermediate_dim=1024, name='VAE', **kwargs):
super(VAE, self).__init__(name, **kwargs)
self.original_dim = original_dim
self.encoder = Encoder(latent_dim=latent_dim, intermediate_dim=intermediate_dim)
self.decoder = Decoder(original_dim=original_dim, intermediate_dim=intermediate_dim)
def call(self, inputs):
z_mean, z_log_var, z = self.encoder(inputs)
reconstructed = self.decoder(z)
kl_loss = -0.5 * tf.reduce_sum(
z_log_var-tf.square(z_mean)-tf.exp(z_log_var)+1
)
self.add_loss(kl_loss)
return reconstructed
and i use this way to load it
vae =VAE(original_dim=5529, latent_dim=600, intermediate_dim=1024)
optimizer = tf.keras.optimizers.Adam(learning_rate=1e-3)
vae.compile(optimizer, loss=tf.keras.losses.MeanSquaredError())
vae.load_weights('./cp.weights.002-0.91.hdf5')
then this is my error
ValueError Traceback (most recent call last)
<ipython-input-39-b318cf8b65e4> in <module>
----> 1 vae.load_weights('./cp.weights.002-0.91.hdf5')
~/opt/anaconda3/envs/ehr/lib/python3.7/site-packages/tensorflow_core/python/keras/engine/training.py in load_weights(self, filepath, by_name)
179 raise ValueError('Load weights is not yet supported with TPUStrategy '
180 'with steps_per_run greater than 1.')
--> 181 return super(Model, self).load_weights(filepath, by_name)
182
183 #trackable.no_automatic_dependency_tracking
~/opt/anaconda3/envs/ehr/lib/python3.7/site-packages/tensorflow_core/python/keras/engine/network.py in load_weights(self, filepath, by_name)
1175 saving.load_weights_from_hdf5_group_by_name(f, self.layers)
1176 else:
-> 1177 saving.load_weights_from_hdf5_group(f, self.layers)
1178
1179 def _updated_config(self):
~/opt/anaconda3/envs/ehr/lib/python3.7/site-packages/tensorflow_core/python/keras/saving/hdf5_format.py in load_weights_from_hdf5_group(f, layers)
675 'containing ' + str(len(layer_names)) +
676 ' layers into a model with ' + str(len(filtered_layers)) +
--> 677 ' layers.')
678
679 # We batch weight value assignments in a single backend call
ValueError: You are trying to load a weight file containing 2 layers into a model with 0 layers.
I was able to solve this problem by running the model once.
vae(tf.ones(shape=INPUT_SHAPE))
vae.load_weights(filepath=FILEPATH)
I am trying to create a model similar to the one proposed in this paper: https://ieeexplore.ieee.org/stamp/stamp.jsp?arnumber=8738842
The custom cell code is available at: https://github.com/SungjoonPark/DenoisingRNN/blob/master/dgrud.py
However, I am not able to embed this custom cell into any RNN model and I am assuming it is because the init takes 3 arguments instead of the standard "num_units".
I tried following the example at https://keras.io/layers/recurrent/:
cell = MinimalRNNCell(32)
x = keras.Input((None, 5))
layer = RNN(cell)
y = layer(x)
but I get an error:
TypeError Traceback (most recent call last)
in 2 x = keras.Input((None,
5)) 3 layer = RNN(cell) ----> 4 y = layer(x)
~/.local/lib/python3.5/site-packages/keras/layers/recurrent.py in
call(self, inputs, initial_state, constants, **kwargs) 539 540 if
initial_state is None and constants is None: --> 541 return super(RNN,
self).call(inputs, **kwargs) 542 543 # If any of initial_state or
constants are specified and are Keras
~/.local/lib/python3.5/site-packages/keras/engine/base_layer.py in
call(self, inputs, **kwargs) 487 # Actually call the layer, 488 #
collecting output(s), mask(s), and shape(s). --> 489 output =
self.call(inputs, **kwargs) 490 output_mask =
self.compute_mask(inputs, previous_mask) 491
~/.local/lib/python3.5/site-packages/keras/layers/recurrent.py in
call(self, inputs, mask, training, initial_state, constants) 680
mask=mask, 681 unroll=self.unroll, --> 682 input_length=timesteps) 683
if self.stateful: 684 updates = []
~/.local/lib/python3.5/site-packages/keras/backend/tensorflow_backend.py
in rnn(step_function, inputs, initial_states, go_backwards, mask,
constants, unroll, input_length) 3101 constants=constants, 3102
unroll=unroll, -> 3103 input_length=input_length) 3104 reachable =
tf_utils.get_reachable_from_inputs([learning_phase()], 3105
targets=[last_output])
~/.local/lib/python3.5/site-packages/tensorflow/python/keras/backend.py
in rnn(step_function, inputs, initial_states, go_backwards, mask,
constants, unroll, input_length, time_major, zero_output_for_mask)
3730 # the value is discarded. 3731 output_time_zero, _ =
step_function( -> 3732 input_time_zero, tuple(initial_states) +
tuple(constants)) 3733 output_ta = tuple( 3734
tensor_array_ops.TensorArray(
~/.local/lib/python3.5/site-packages/keras/layers/recurrent.py in
step(inputs, states) 671 else: 672 def step(inputs, states): --> 673
return self.cell.call(inputs, states, **kwargs) 674 675 last_output,
outputs, states = K.rnn(step,
TypeError: call() takes 2 positional arguments but 3 were given
Could you please help me figure out whether it is a init issue, a call issue or I need to define a custom layer for this custom cell?
I tried looking for answers all over the internet and I just can't get any clarity on how embedding a custom cell in a RNN model should be done.
Thank you in advance,
Sam
I was able to recreate your issue while I imported keras directly into the program. See below,
%tensorflow_version 1.x
import keras
from keras import backend as K
import tensorflow as tf
from tensorflow.keras import layers
from tensorflow.keras.layers import RNN
class MinimalRNNCell(keras.layers.Layer):
def __init__(self, units, **kwargs):
self.units = units
self.state_size = units
super(MinimalRNNCell, self).__init__(**kwargs)
def build(self, input_shape):
self.kernel = self.add_weight(shape=(input_shape[-1], self.units),
initializer='uniform',
name='kernel')
self.recurrent_kernel = self.add_weight(
shape=(self.units, self.units),
initializer='uniform',
name='recurrent_kernel')
self.built = True
def call(self, inputs, states):
prev_output = states[0]
h = K.dot(inputs, self.kernel)
output = h + K.dot(prev_output, self.recurrent_kernel)
return output, [output]
# Let's use this cell in a RNN layer:
cell = MinimalRNNCell(32)
x = keras.Input((None, 5))
layer = RNN(cell)
y = layer(x)
Output -
TensorFlow is already loaded. Please restart the runtime to change versions.
---------------------------------------------------------------------------
TypeError Traceback (most recent call last)
<ipython-input-3-0f3bed686a7d> in <module>()
34 x = keras.Input((None, 5))
35 layer = RNN(cell)
---> 36 y = layer(x)
5 frames
/usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py in symbolic_fn_wrapper(*args, **kwargs)
73 if _SYMBOLIC_SCOPE.value:
74 with get_graph().as_default():
---> 75 return func(*args, **kwargs)
76 else:
77 return func(*args, **kwargs)
TypeError: __call__() takes 2 positional arguments but 3 were given
The error vanishes while you import keras from tensorflow import keras. The code runs successfully with tensorflow version 1.x and as well as 2.x. Modify your code as below -
%tensorflow_version 2.x
from keras import backend as K
import tensorflow as tf
from tensorflow.keras import layers
from tensorflow import keras
from tensorflow.keras.layers import RNN
# First, let's define a RNN Cell, as a layer subclass.
class MinimalRNNCell(keras.layers.Layer):
def __init__(self, units, **kwargs):
self.units = units
self.state_size = units
super(MinimalRNNCell, self).__init__(**kwargs)
def build(self, input_shape):
self.kernel = self.add_weight(shape=(input_shape[-1], self.units),
initializer='uniform',
name='kernel')
self.recurrent_kernel = self.add_weight(
shape=(self.units, self.units),
initializer='uniform',
name='recurrent_kernel')
self.built = True
def call(self, inputs, states):
prev_output = states[0]
h = K.dot(inputs, self.kernel)
output = h + K.dot(prev_output, self.recurrent_kernel)
return output, [output]
# Let's use this cell in a RNN layer:
cell = MinimalRNNCell(32)
x = keras.Input((None, 5))
layer = RNN(cell)
y = layer(x)
print("I Ran Successfully")
Output -
I Ran Successfully
Hope this answers your question. Happy Learning.