Following is the code of one simple example for what I want to implement:
Error: raise TypeError("inputs must be a sequence"), TypeError: inputs must be a sequence
How to solve this to make the program can work? Any help will be appreciated.
from keras.models import Sequential
from keras.layers import LSTM, Dense, Flatten
import numpy as np
from keras.engine.topology import Layer
import tensorflow as tf
class MyLayer(Layer):
def __init__(self, **kwargs):
super(MyLayer, self).__init__(**kwargs)
def build(self, input_shape):
super(MyLayer, self).build(input_shape)
def call(self, x):
"Some other tf function will be put at here"
outputs, state = tf.contrib.rnn.static_rnn(tf.contrib.rnn.LSTMBlockCell(32), x, dtype=tf.float32)
return outputs
def compute_output_shape(self, input_shape):
return input_shape
def get_model(timesteps, data_dim):
model = Sequential()
model.add(LSTM(32, return_sequences=True, input_shape=(timesteps, data_dim)))
model.add(LSTM(32, return_sequences=True))
model.add(MyLayer()) # this is my layer
model.add(Flatten())
model.add(Dense(10, activation='softmax'))
return model
def run_demo():
data_dim = 16
timesteps = 8
num_classes = 10
model = get_model(timesteps, data_dim)
model.compile(loss='categorical_crossentropy', optimizer='rmsprop', metrics=['accuracy'])
"""Generate the traning and validation data"""
x_train = np.random.random((1000, timesteps, data_dim))
y_train = np.random.random((1000, num_classes))
x_val = np.random.random((100, timesteps, data_dim))
y_val = np.random.random((100, num_classes))
model.fit(x_train, y_train, batch_size=64, epochs=5, validation_data=(x_val, y_val))
if __name__ == "__main__":
run_demo()
Sorry, I am not too familiar with Recurrent Model.
But I think the problem is input size.
The size into custom layer (?, 8, 32)
but tf.nn.static_rnn require list like
so you need to change the input size then problem will fix.
import numpy as np
import tensorflow as tf
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import LSTM, Flatten, Dense
class MyLayer(tf.keras.layers.Layer):
def __init__(self, **kwargs):
super(MyLayer, self).__init__(**kwargs)
def build(self, input_shape):
self.cell = tf.nn.rnn_cell.BasicRNNCell(32)
super(MyLayer, self).build(input_shape)
def call(self, x):
"Some other tf function will be put at here"
rnn_inputs = tf.unstack(x, axis=1)
outputs, state = tf.nn.static_rnn(self.cell, rnn_inputs, dtype=tf.float32)
for i in range(len(outputs)):
outputs[i] = tf.expand_dims(outputs[i], axis=1)
outputs = tf.concat(outputs, axis=1)
return outputs
def compute_output_shape(self, input_shape):
return input_shape
def get_model(timesteps, data_dim):
model = Sequential()
model.add(LSTM(32, return_sequences=True, input_shape=(timesteps, data_dim)))
model.add(LSTM(32, return_sequences=True))
model.add(MyLayer()) # this is my layer
model.add(Flatten())
model.add(Dense(10, activation='softmax'))
return model
def run_demo():
data_dim = 16
timesteps = 8
num_classes = 10
model = get_model(timesteps, data_dim)
model.compile(loss='categorical_crossentropy', optimizer='rmsprop', metrics=['accuracy'])
"""Generate the traning and validation data"""
x_train = np.random.random((1000, timesteps, data_dim))
y_train = np.random.random((1000, num_classes))
x_val = np.random.random((100, timesteps, data_dim))
y_val = np.random.random((100, num_classes))
model.fit(x_train, y_train, batch_size=64, epochs=5, validation_data=(x_val, y_val))
if __name__ == "__main__":
run_demo()
Related
I am trying to add Attention layer to my model for text clasiffication.
but I get an error after adding the layer and then fitting the model.
here is my code:
model = Sequential()
for i in range(len(kernel_size)):
model.add(Conv1D(filters=nb_filter, kernel_size=kernel_size[i], padding='valid', activation='relu',
input_shape=(data_batch_size, emb_dim)))
model.add(MaxPooling1D(pool_size=pool_size))
model.add(Bidirectional(LSTM(units=lstm_out, return_sequences=True), merge_mode='concat',
input_shape=(X_train.shape[1], X_train.shape[2])))
model.add(Bidirectional(LSTM(units=lstm_out, go_backwards=True)))
# ------------------------------------------------------------------------------------------------------------
# ------------------------------------------------------------------------------------------------------------
model.add(Attention(return_sequences=True))
# ------------------------------------------------------------------------------------------------------------
# ------------------------------------------------------------------------------------------------------------
model.add(Dropout(DropoutP))
model.add(Dense(cat_output, activation='sigmoid'))
model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
Y_tmp = np.zeros([Y_train.shape[0], 2])
Y_tmp[:, 0] = 2 - Y_train
Y_tmp[:, 1] = Y_train - 1
Y_train = Y_tmp
history = model.fit(X_train, Y_train, validation_split=test_size, epochs=nb_epoch, verbose=1,
callbacks=[EarlyStopping(monitor='val_accuracy', patience=0, restore_best_weights=True)])
And This is the Attention class:
class Attention(Layer):
def __init__(self, return_sequences=True):
self.return_sequences = return_sequences
super(Attention, self).__init__()
def build(self, input_shape):
self.W = self.add_weight(name="att_weight", shape=(input_shape[-1], 1), initializer="normal")
self.b = self.add_weight(name="att_bias", shape=(input_shape[1], 1), initializer="zeros")
super(Attention, self).build(input_shape)
def call(self, x):
e = K.tanh(K.dot(x, self.W) + self.b)
a = K.softmax(e, axis=1)
output = x * a
if self.return_sequences:
return output
return K.sum(output, axis=1)
And this is the error: Incompatible shapes: [32,2] vs. [1200,2]
What am I doing wrong?
There is a problem with :
self.b = self.add_weight(name="att_bias", shape=(input_shape[1], 1), initializer="zeros")
which should be :
self.b = self.add_weight(name="att_bias", shape=(1,), initializer="zeros")
In fact, you are redefining the Dense layer. To see for yourself, you can look at the custom Linear layer in layers and models via sub-classing.
The custom attention layer is actually what you want using the Dense layers and is more general (a Bahdanau attention layer).
I am getting a ValueError: logits and labels must have the same shape ((None, 1) vs ()) when doing a model evaluate. I get the model to train but when I evaluate is when I have the problem. I used a tf.expand_dims for logits but wondering if this needs to be applied to the labels as well?
here is my code below.
import tensorflow as tf
import tensorflow_datasets as tfds
dataset, info = tfds.load('imdb_reviews', with_info=True,
as_supervised=True)
train_dataset, test_dataset = dataset['train'], dataset['test']
BUFFER_SIZE = 10000
BATCH_SIZE = 64
train_dataset = train_dataset.shuffle(BUFFER_SIZE).batch(BATCH_SIZE).prefetch(1)
VOCAB_SIZE, EMBED_SIZE, NUM_OOV_BUCKETS = 10000, 128, 1000
encoder = tf.keras.layers.experimental.preprocessing.TextVectorization(
max_tokens=VOCAB_SIZE)
encoder.adapt(train_dataset.map(lambda text, label: text))
class AttentionLayer(tf.keras.layers.Layer):
def __init__(self, **kwargs):
super(AttentionLayer, self).__init__(**kwargs)
self.query_layer = tf.keras.layers.Conv1D(
filters=100,
kernel_size=4,
padding='same'
)
self.value_layer = tf.keras.layers.Conv1D(
filters=100,
kernel_size=4,
padding='same'
)
self.attention_layer = tf.keras.layers.Attention()
def call(self, inputs):
query = self.query_layer(inputs)
value = self.value_layer(inputs)
attention = self.attention_layer([query, value])
return tf.keras.layers.concatenate([query, attention])
attention_layer = AttentionLayer()
model1 = tf.keras.models.Sequential([
tf.keras.Input(shape=(),batch_size=1, dtype=tf.string, name='InputLayer'),
encoder,
tf.keras.layers.Embedding(VOCAB_SIZE + NUM_OOV_BUCKETS, EMBED_SIZE, mask_zero=True, name='Embedding_Layer'),
attention_layer,
tf.keras.layers.Conv1D(filters=32, kernel_size=4, padding = 'same', activation = 'relu', name='Conv1DLayer'),
tf.keras.layers.MaxPooling1D(pool_size=2, name='MaxPoolLayer'),
tf.keras.layers.LSTM(64, dropout = 0.2, name='DropoutLayer'),
tf.keras.layers.Dense(250, activation = 'relu', name='DenseLayer'),
tf.keras.layers.Dense(1, activation='sigmoid', name='Output_Layer')
])
model.compile(loss="binary_crossentropy", optimizer="adam", metrics=["accuracy"])
def preprocess_y(x, y):
return x, tf.expand_dims(y, -1)
history1 = model1.fit(
train_dataset.map(preprocess_y),
batch_size=BATCH_SIZE,
epochs=1)
model1.evaluate(test_dataset)
ValueError: logits and labels must have the same shape ((None, 1) vs ())
I am currently using kerastuner ( 1.0.0) to fine-tune hyperparamteters in a CNN and it keeps displaying the error 'Too many failed attempts to build model.'. I have tried all other availabel solutions including those on this stackoverflow but none of them works. Does my model have too many parameters to tune?
from tensorflow.keras.datasets import fashion_mnist
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras.layers import Dense, Conv2D, Dropout, MaxPooling2D
from tensorflow.keras.initializers import LecunNormal
!pip install keras-tuner==1.0.0
from kerastuner.tuners import RandomSearch
from kerastuner import HyperModel
(x_train, y_train), (x_test, y_test) = fashion_mnist.load_data()
x_train = x_train.reshape(-1, 28, 28, 1)
x_test = x_test.reshape(-1, 28, 28, 1)
class MCDropout(tf.keras.layers.Dropout):
def call(self, inputs):
return super().call(inputs, training=True)
NUM_CLASSES = 10
KERNEL_SIZE = (3, 3)
class CNNHyperModel(HyperModel):
def __init__(self, num_classes, kernel_size):
self.num_classes = num_classes
self.kernel_size = kernel_size
def build_model(self, hp):
model = keras.Sequential()
model.add(Conv2D(ilters=hp.Choice('input_units', values=[32, 16], default=16),
kernel_size=(self.kernel_size),
activation='selu',
kernel_intializer = LecunNormal,
input_shape=x_train.shape[1:]))
model.add(MCDropout(rate=hp.Float('dropout_0',min_value=0.0,max_value=0.2,default=0.1,step=0.1)))
model.add(MaxPooling2D(pool_size=(2, 2)))
for i in range(hp.Choice('n_filters', 1, 4)):
model.add(Conv2D(hp.Choice(f'num_filters_{i}', values=[32, 16, 64], default=32),
kernel_size=(self.kernel_size),
activation='selu',
kernel_intializer = LecunNormal))
model.add(MCDropout(hp.Float(f'dropout_{i}',min_value=0.0,max_value=0.2,default=0.1,step=0.1)))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Flatten())
model.add(Dense(units=hp.Choice('units',values = [128, 256, 512, 1024], defaults = 512),activation='selu'))
model.add(MCDropout(rate=hp.Float('dropout_5',min_value=0.3,max_value=0.7,default=0.4,step=0.1)))
model.add(Dense(self.num_classes, activation='softmax'))
model.compile(optimizer=keras.optimizers.Nadam(hp.Choice('learning_rate',
values=[1e-2, 1e-3, 1e-4])),
loss='sparse_categorical_crossentropy',
metrics=['accuracy'])
return model
hypermodel = CNNHyperModel(num_classes=NUM_CLASSES, kernel_size = KERNEL_SIZE)
SEED = 1
tuner = RandomSearch(
hypermodel,
objective='val_accuracy',
max_trials = 1,
executions_per_trial = 1
)
tuner.search(x=x_train,
y=y_train,
epochs=1,
batch_size=64,
validation_data =(x_test, y_test))```
I RUN THIS CODE AND IT KEEP DISPLAYING TOO MANY FAILED ATTEMPTS TO BUILD MODELS. (BTW I AM WRITING THIS ON GG COLAB.)
IS THERE ANYWAY TO FIX IT?
I'm new to deep learning and tried to tune hyperparameters using kerastuner. Here is my code:
from tensorflow import keras
from tensorflow.keras import layers
from kerastuner.tuners import RandomSearch
from kerastuner import HyperModel
class MyHyperModel(HyperModel):
def __init__(self, input_shape):
self.input_shape = input_shape
def build(self, hp):
model = keras.Sequential()
model.add(layers.Dense(units=hp.Int('units',
min_value=10,
max_value=120,
step=10),
activation='relu',
input_shape=self.input_shape))
model.add(layers.Dense(units=hp.Int('units',min_value=10,max_value=20,step=2),
activation='relu'))
model.add(layers.Dense(1, activation='linear'))
model.compile(
optimizer=keras.optimizers.Adam(
hp.Choice('learning_rate',
values=[1e-2, 1e-3, 1e-4])),
loss='mse',
metrics=['mse'])
return model
input_shape = (X_train.shape[1],)
hypermodel = MyHyperModel(input_shape)
tuner = RandomSearch(
hypermodel,
objective='mse',
max_trials=10,
)
tuner.search(X_train, y_train,
epochs=50,
validation_data=(X_test, y_test),verbose=0)
When I tried to retrieve the best model using
tuner.get_best_models(num_models=1)[0]
I got the following error
ValueError: Shapes (320,) and (1,) are incompatible
I don't have this error when I deleted this hidden layer
model.add(layers.Dense(units=hp.Int('units',min_value=10,max_value=20,step=2),
activation='relu'))
Anyone knows how to fix this? Thanks in advance.
I have a large dataset of n_samples, n_features, n_classes = 346679, 10233, 86. I am trying to build a classifier on this dataset. For this, I am using a Multi-Layer Perceptron which is built using the keras sequential model.
DataGeneratorClass
class DataGeneratorKeras:
def __init__(self, num_rows, n_classes, n_samples, n_features, batch_size=1, shuffle=True):
self.num_rows = num_rows
self.n_samples = n_samples
self.n_features = n_features
self.n_classes = n_classes
self.batch_size = batch_size
self.shuffle = shuffle
self.flag = False
def __get_exploration_order(self, list_ids):
"""
Generates order of exploration
:param list_ids:
:return:
"""
# Find exploration order
indexes = np.arange(len(list_ids))
if self.shuffle:
np.random.shuffle(indexes)
return indexes
def __data_generation(self, list_ids_temp, n_classes):
"""
Generates data of batch_size samples
:param list_ids_temp:
:param n_classes:
:return:
"""
index = list_ids_temp[0]
fv = load_npz("data_file_" + str(index) + ".npz")
labels_complete = load(...) # Load labels
partial_labels = labels_complete[index]
del labels_complete
y = self.sparsify(partial_labels, n_classes)
return fv, y
#staticmethod
def sparsify(y, n_classes):
"""
:return:
"""
label_encoder = np_utils.to_categorical(y, n_classes)
return label_encoder
def generate(self, list_ids):
"""
Generates batches of samples
:param list_ids:
:return:
"""
# Infinite loop
while 1:
# Generate order of exploration of dataset
indexes = self.__get_exploration_order(list_ids)
# Generate batches
imax = int(len(indexes) / self.batch_size)
for i in range(imax):
# Find list of IDs
list_ids_temp = [list_ids[k] for k in indexes[i * self.batch_size:(i + 1) * self.batch_size]]
# Generate data
x, y = self.__data_generation(list_ids_temp, self.n_classes)
yield x.toarray(), y
The Script class
class Script:
def __init__(self, num_rows, batch_size, test_size, n_classes, n_samples, n_features):
self.batch_size = batch_size
self.num_rows = num_rows
self.test_size = test_size
self.n_classes = n_classes
self.n_samples = n_samples
self.n_features = n_features
def main(self):
validation = int(self.test_size * self.num_rows)
train = self.num_rows - validation
params = {
'num_rows': self.num_rows,
'n_samples': self.n_samples,
'n_features': self.n_features,
'n_classes': self.n_classes,
'batch_size': self.batch_size,
'shuffle': True
}
partition = {'train': range(train), 'validation': range(train, self.num_rows)}
# Generators
training_generator = DataGeneratorKeras(**params).generate(partition['train'])
validation_generator = DataGeneratorKeras(**params).generate(partition['validation'])
return training_generator, validation_generator, partition
if __name__ == "__main__":
script = Script(num_rows=347, test_size=0.25, n_classes=86, n_samples=346679, n_features=10233, batch_size=1)
training_generator, validation_generator, partition = script.main()
Building the model
def classifier_base_data(dropout, learning_rate):
model = Sequential()
model.add(Dense(2**13, input_shape=(script.n_features,), activation='relu', name="l_input"))
model.add(BatchNormalization())
model.add(Dropout(dropout))
model.add(Dense(2**12, input_dim=2**13, activation='relu', name="l_hidden_1"))
model.add(BatchNormalization())
model.add(Dropout(dropout))
model.add(Dense(2**11, input_dim=2**12, activation='relu', name="l_hidden_2"))
model.add(BatchNormalization())
model.add(Dropout(dropout))
model.add(Dense(2**10, input_dim=2**11, activation='relu', name="l_hidden_3"))
model.add(BatchNormalization())
model.add(Dropout(dropout))
model.add(Dense(2**9, input_dim=2**10, activation='relu', name="l_hidden_4"))
model.add(BatchNormalization())
model.add(Dropout(dropout))
model.add(Dense(2**8, input_dim=2**9, activation='relu', name="l_hidden_5"))
model.add(BatchNormalization())
model.add(Dropout(dropout))
model.add(Dense(2**7, input_dim=2**8, activation='relu', name="l_hidden_6"))
model.add(BatchNormalization())
model.add(Dropout(dropout))
model.add(Dense(script.n_classes, activation='softmax', name="l_output"))
optimizer = adam(lr=learning_rate)
model.compile(loss='categorical_crossentropy', optimizer=optimizer, metrics=['accuracy'])
print model.summary()
return model
When I run the model using keras fit function, I am able to achieve val_acc and acc above 25%.
history = model.fit(x_train.toarray(), y_train,
batch_size=batch_size,
epochs=epochs,
verbose=1,
validation_data=(x_validation.toarray(), y_validation))
Since the data is large, I am using the DataGenerator by keras following a very well written tutorial keras-datagen-tutorial. When I run the model using the fit_generator, I get 0% val_acc.
model.fit_generator(
generator = training_generator,
steps_per_epoch = len(partition['train']),
epochs = epochs,
validation_data = validation_generator,
validation_steps = len(partition['validation']),
verbose = 1
)
Is there any issue in the DataGenerator written?