I want to apply this method to implement Bi-LSTM with attention. The method is discussed here: Bi-LSTM Attention model in Keras
I get the following error:
'module' object is not callable
It can not apply multiply in this line:
sent_representation = merge([lstm, attention], mode='mul')
from keras.layers import merge
import tensorflow as tf
from tensorflow.keras.layers import Concatenate, Dense, Input, LSTM, Embedding, Dropout, Activation, Flatten, Permute, RepeatVector
from tensorflow.keras.layers import Bidirectional
inp =Input(shape=(maxlen,), dtype='float32')
x = Embedding(max_features, embed_size, weights=[emb_matrix])(inp)
lstm = Bidirectional(LSTM(50, return_sequences=True), name="bi_lstm_0")(x)
attention = Dense(1, activation='tanh')(lstm)
attention = Flatten()(attention)
attention = Activation('softmax')(attention)
attention = RepeatVector(max_features*2)(attention)
attention = Permute([2,1])(attention)
sent_representation = merge([lstm, attention], mode='mul')
sent_representation = Lambda(lambda xin: K.sum(xin, axis=1))(sent_representation)
output = Dense(3, activation="softmax")(sent_representation)
In Keras, merge is a module that contains layers that implement various ways of merging outputs of other layers. You need to select a method that you want to use to merge the states.
In this particular case, you want to concatenate the outputs.
Related
Keras model's validation accuracy is not changing, and when I try to run the model it's just outputting same messy numbers. As input I'm giving it flatten 5x4 array of bounding boxes, and the output are keypresses (W,A,S,D).
I'm using it to avoid detected objects for autopilot.
I tried changing activation, layers in model, learning rate and so on, but nothing helped.
Any help would be appreciated.
from tensorflow import keras
import numpy as np
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, Dropout, Activation, Flatten, Conv2D, MaxPooling2D, Input
from tensorflow.keras.callbacks import TensorBoard
data = np.load("training_data/data.npz",allow_pickle=True)
X = data['arr_2']
X = keras.utils.normalize(X,axis=1)
Y = data['arr_1']
# Creating model for object processing
model2 = Sequential()
# Model for object processing
# Inputs
model2.add(Input(shape=(20)))
# Hidden layer
model2.add(Dense(16,activation="relu"))
# Outputs
model2.add(Dense(4,activation="softmax"))
model2.compile(loss='binary_crossentropy',
optimizer="adam",
metrics=['accuracy'])
model2.fit(X,Y,batch_size=256, epochs=20, validation_split=0.1)
I have been trying to create a multi-input model using Keras, but got errors. The idea is to combine the text and corresonding topics to make predictions for sentiments. Here's the code:
import numpy as np
text = np.random.randint(5000, size=(442702, 200), dtype='int32')
topic = np.random.randint(2, size=(442702, 227), dtype='int32')
sentiment = to_categorical(np.random.randint(5, size=442702), dtype='int32')
from keras.models import Sequential
from keras.layers import Dense, Activation, Embedding, Flatten, GlobalMaxPool1D, Dropout, Conv1D
from keras.callbacks import ReduceLROnPlateau, EarlyStopping, ModelCheckpoint
from keras.losses import binary_crossentropy
from keras.optimizers import Adam
text_input = Input(shape=(200,), dtype='int32', name='text')
text_encoded = Embedding(input_dim=5000, output_dim=20, input_length=200)(text_input)
text_encoded = Dropout(0.1)(text_encoded)
text_encoded = Conv1D(300, 3, padding='valid', activation='relu', strides=1)(text_encoded)
text_encoded = GlobalMaxPool1D()(text_encoded)
topic_input = Input(shape=(227,), dtype='int32', name='topic')
concatenated = concatenate([text_encoded, topic_input])
sentiment = Dense(5, activation='softmax')(concatenated)
model = Model(inputs=[text_encoded, topic_input], outputs=sentiment)
# summarize layers
print(model.summary())
# plot graph
plot_model(model)
However, this gives me the below error:
TypeError: Tensors in list passed to 'values' of 'ConcatV2' Op have types [float32, int32] that don't all match.
Now if I change dtype of topic_input from 'int32' to 'float32', I got a different error:
ValueError: Graph disconnected: cannot obtain value for tensor Tensor("text_37:0", shape=(?, 200), dtype=int32) at layer "text". The following previous layers were accessed without issue: []
On the other hand, part of the model works just fine with the sequential API.
model = Sequential()
model.add(Embedding(5000, 20, input_length=200))
model.add(Dropout(0.1))
model.add(Conv1D(300, 3, padding='valid', activation='relu', strides=1))
model.add(GlobalMaxPool1D())
model.add(Dense(227))
model.add(Activation('sigmoid'))
print(model.summary())
Any pointers are highly appreciated.
There are few issues with your Keras functional API implementation,
You should use the Concatenate layer as Concatenate(axis=-1)([text_encoded, topic_input]).
In the concatenate layer you are trying to combine an int32 tensor and a float32 tensor, which is not allowed. What you should do is, from keras.backend import cast and concatenated = Concatenate(axis=-1)([text_encoded, cast(topic_input, 'float32')]).
You got variable conflicts, there are two sentiment variables, one pointing to a to_categorical output and the other the output of the final Dense layer.
Your model inputs cannot be intermediate tensors like text_encoded. They should come from Input layers.
To help with your implementation, here's a working version of your code (I am not sure if this is exactly what you wanted though) in TF 1.13.
from keras.utils import to_categorical
text = np.random.randint(5000, size=(442702, 200), dtype='int32')
topic = np.random.randint(2, size=(442702, 227), dtype='int32')
sentiment1 = to_categorical(np.random.randint(5, size=442702), dtype='int32')
from keras.models import Sequential
from keras.layers import Input, Dense, Activation, Embedding, Flatten, GlobalMaxPool1D, Dropout, Conv1D, Concatenate, Lambda
from keras.callbacks import ReduceLROnPlateau, EarlyStopping, ModelCheckpoint
from keras.losses import binary_crossentropy
from keras.optimizers import Adam
from keras.backend import cast
from keras.models import Model
text_input = Input(shape=(200,), dtype='int32', name='text')
text_encoded = Embedding(input_dim=5000, output_dim=20, input_length=200)(text_input)
text_encoded = Dropout(0.1)(text_encoded)
text_encoded = Conv1D(300, 3, padding='valid', activation='relu', strides=1)(text_encoded)
text_encoded = GlobalMaxPool1D()(text_encoded)
topic_input = Input(shape=(227,), dtype='int32', name='topic')
topic_float = Lambda(lambda x:cast(x, 'float32'), name='Floatconverter')(topic_input)
concatenated = Concatenate(axis=-1)([text_encoded, topic_float])
sentiment = Dense(5, activation='softmax')(concatenated)
model = Model(inputs=[text_input, topic_input], outputs=sentiment)
# summarize layers
print(model.summary())
Hope these help.
I am trying to design a Bi-Directional LSTM model and I want to concatenate features after Max pooling and Average pooling layers.
I have this for my model:
from keras.layers import Dense, Embedding
from keras.layers.recurrent import LSTM
from keras.layers import Bidirectional
from keras.models import Sequential
from keras.layers.core import Dropout
from features import train,embedding_matrix,words
from keras.layers import concatenate,AveragePooling1D,GlobalMaxPooling1D
model=Sequential()
model.add(Embedding(words,300,input_length=train.shape[1],weights=[embedding_matrix]))
model.add(Bidirectional(LSTM(20,activation='tanh',kernel_initializer='glorot_uniform',recurrent_dropout = 0.2, dropout = 0.2,return_sequences=True)))
model.add(concatenate([GlobalMaxPooling1D(),AveragePooling1D()]))
model.add(Dropout(0.2))
model.add(Dense(2, activation='softmax'))
print model.summary()
But I am having:
ValueError: Layer concatenate_1 was called with an input that isn't a symbolic tensor which is because I believe the concatenating layer. As I am not adding the pooling in the model.
Can I add two layers in the same model? or Should I define two separate models and then add pooling layers in each of them?
The trick here is to use a graph model instead of a sequential model.
Before we get started, I assume
your network expects a 2D input tensor of shape (B=batch_size, N=num_of_words), where N is the longest sample length of your training data. (In case you have unequal length samples, you should use keras.preprocessing.sequence.pad_sequences to achieve equal length samples)
your vocabulary size is V (probably is 300 if I understand correctly)
your embedding layer encodes each word to a feature of F dimension, i.e. your embedding layer's weight matrix is VxF.
from keras.layers import Dense, Embedding, Input, Concatenate, Lambda
from keras.layers.recurrent import LSTM
from keras.layers import Bidirectional
from keras.models import Model
from keras.layers.core import Dropout
from keras import backend as BKN
from keras.layers import concatenate,AveragePooling1D,GlobalMaxPooling1D
words = Input( shape=(N,))
f = Embedding(input_dim=V,output_dim=F)( words )
f = Bidirectional(LSTM(20,activation='tanh',
kernel_initializer='glorot_uniform',
recurrent_dropout = 0.2,
dropout = 0.2,return_sequences=True))(f)
gpf = GlobalMaxPooling1D()(f)
gpf = Lambda( lambda t : BKN.expand_dims(t, axis=1) )(gpf)
apf = AveragePooling1D( pool_size=2 )(f)
pf = Concatenate(axis=1)([gpf, apf])
pf = Dropout(0.2)( pf )
pred = Dense(2, activation='softmax')(pf) # <-- make sure this is correct
model = Model( input=words, output=pred )
Finally, I fail to find that keras Embedding layer supports syntax like weights=[embedding_matrix].
I am currently having some issue loading my model, as it includes a lambda layer..
This is my neural network layer which uses lambda layer.
#
# Python scritpt - Keras RCNN model.
#
import keras
from keras.models import Model
from keras.layers import Input, Dense, Dropout, Flatten, Activation
from keras.layers import merge, Conv2D, MaxPooling2D, Input
from keras.layers.normalization import BatchNormalization
from keras.layers.core import Lambda
import numpy as np
from keras.layers import add
from keras import backend as K
# RCL:
# BatchNorm(Relu(conv(L-1) + conv(L)))
#
def make_RCNN(input,number_of_rcl,num_of_filter, filtersize,alpha,pool):
feed_forward = Conv2D(filters=num_of_filter, kernel_size=1, name='init')(input)
for x in xrange(number_of_rcl):
output = RCL(feed_forward,num_of_filter,filtersize,alpha,pool)
feed_forward = output
return feed_forward
def RCL(feed_forward_input,num_of_filter, filtersize, alpha,pool):
conv = Conv2D(filters=num_of_filter, kernel_size=filtersize, padding='same')
recurrent_input = conv(feed_forward_input)
merged = add([feed_forward_input,recurrent_input])
conv_relu = Lambda(lambda x : K.relu(x,alpha=alpha))(merged)
conv_relu_batchnorm = BatchNormalization()(conv_relu)
if pool:
conv_relu_batchnorm_pool = MaxPooling2D()(conv_relu_batchnorm)
return conv_relu_batchnorm_pool
else:
return conv_relu_batchnorm
input = Input(shape=(30,30,3))
output = make_RCNN(input,number_of_rcl=3,num_of_filter=3,filtersize=3,alpha=0.2, pool=True)
model = Model(input = input, output = output)
model.compile(optimizer='rmsprop', loss='binary_crossentropy')
model.summary()
how do i remove the layer, without altering the functionality?...
from keras.models import Sequential
from keras.layers.embeddings import Embedding
from theano import function
model = Sequential()
model.add(Embedding(max_features, 128, input_length = maxlen))
I want to get the outputs from the embedding layers. I read through the source in keras but didnt find any suitable function or attribute. Anyone can help me with this?
You can get the output of any layer, not just an embedding layer, as described here:
from keras import backend as K
get_3rd_layer_output = K.function([model.layers[0].input],
[model.layers[3].output])
layer_output = get_3rd_layer_output([X])[0]
In your case, you would want model.layers[0].output instead of model.layers[3].output.