Well, I have a problem setting up a network consisting of a CNN + Autoencoder for a classification task. The main idea is to use CNN-generated embedding as the input of an autoencoder for the embedding reconstruction process. Well, I was able to define both architectures, but I couldn't merge them into a single graph.
def autoencoder(cnn_out):
xreal = keras.layers.Input(tensor=cnn_out)
(...)
xhat = keras.layers.Dense(cnn_out.shape[1], activation='sigmoid')(dec)
ae = keras.models.Model(inputs=xreal, outputs=xhat)
loss_mse = mse_loss(xreal, xhat)
ae.add_loss(loss_mse)
return ae
def cnnae_model(input_shape):
h1 = keras.layers.Conv2D(8,strides=(1,1), kernel_size=kernel, kernel_regularizer=r.l2(kl), padding='same')(X)
(...)
h5 = keras.layers.AveragePooling2D(pool_size = (2, 2))(h5)
xreal = keras.layers.Flatten()(h5)
cnn = keras.models.Model(inputs=X, outputs=xreal)
cnn_ae = keras.models.Model(inputs=cnn.input, outputs=autoencoder(cnn.output).output)
return cnn_ae
input_shape = (128, 64, 3)
model = cnnae_siamesa(input_shape)
model.compile(loss=contrastve_loss,bacth_size = 16, optimizer=rms, metrics=[accuracy], callbacks=[reduce_lr])
The following error message appears when I try to compile the model:
ValueError: Graph disconnected: cannot obtain value for tensor Tensor("flatten_11/Identity:0", shape=(None, 2048), dtype=float32) at layer "input_50". The following previous layers were accessed without issue: []
I did some modifications to your code and produced a working version (one without the error you reported). There are a few changes that have to do with how the output layers are called when connecting up the different submodels, but hopefully you can relate it back to your original model. There is some additional information here that might help clarify: https://www.tensorflow.org/guide/keras/functional#using_the_same_graph_of_layers_to_define_multiple_models. I hope this helps. :
import tensorflow as tf
import numpy as np
print(tf.__version__)
tf.keras.backend.clear_session()
# Code with issue:
def autoencoder(cnn_out):
xreal = cnn_out # tf.keras.layers.Input(tensor=cnn_out)
dec = xreal
xhat = tf.keras.layers.Dense(cnn_out.shape[1], activation='sigmoid', name='AE_Dense')(dec)
# ae = tf.keras.models.Model(inputs=xreal, outputs=xhat, name='AE_Model')
# loss_mse = mse_loss(xreal, xhat)
# ae.add_loss(loss_mse)
return xhat # return last layer of model
def cnnae_model(input_shape):
#CNN model start:
X = tf.keras.layers.Input(input_shape, name='CNN_Input')
h1 = tf.keras.layers.Conv2D(8,kernel_size=(2,2), padding='same', name='CNN_Conv2D')(X)
h5 = h1
h5 = tf.keras.layers.AveragePooling2D(pool_size = (2, 2), name='CNN_AvgPooling2D')(h5)
xreal = tf.keras.layers.Flatten(name='CNN_myFlatten')(h5)
cnn = tf.keras.models.Model(inputs=X, outputs=xreal, name='CNN_Model')
#CNN model end:
ae_model = autoencoder(xreal)
cnn_ae = tf.keras.models.Model(inputs=cnn.input, outputs=ae_model, name='cnn_ae_model')
return cnn_ae
input_shape = (128, 64, 3)
model = cnnae_model(input_shape)
print('model.summary():')
print(model.summary())
model.compile(optimizer='rmsprop', loss='mse')
x_train=np.random.random((2,128,64,3))
y_train=np.random.random((2,16384))
print('x_train.shape:')
print(x_train.shape)
print('y_train.shape:')
print(y_train.shape)
model.fit(x_train, y_train, epochs=1)
I saw this code from https://github.com/raducrs/Applications-of-Deep-Learning/blob/master/Image%20captioning%20Flickr8k.ipynb and tried it to run in google colab, however when I run the code below it gave me error. It says
Merge is deprecated
I wonder how I can run this code with keras latest version.
LSTM_CELLS_CAPTION = 256
LSTM_CELLS_MERGED = 1000
image_pre = Sequential()
image_pre.add(Dense(100, input_shape=(IMG_FEATURES_SIZE,), activation='relu', name='fc_image'))
image_pre.add(RepeatVector(MAX_SENTENCE,name='repeat_image'))
caption_model = Sequential()
caption_model.add(Embedding(VOCABULARY_SIZE, EMB_SIZE,
weights=[embedding_matrix],
input_length=MAX_SENTENCE,
trainable=False, name="embedding"))
caption_model.add(LSTM(EMB_SIZE, return_sequences=True, name="lstm_caption"))
caption_model.add(TimeDistributed(Dense(100, name="td_caption")))
combined = Sequential()
combined.add(Merge([image_pre, caption_model], mode='concat', concat_axis=1,name="merge_models"))
combined.add(Bidirectional(LSTM(256,return_sequences=False, name="lstm_merged"),name="bidirectional_lstm"))
combined.add(Dense(VOCABULARY_SIZE,name="fc_merged"))
combined.add(Activation('softmax',name="softmax_combined"))
predictive = Model([image_pre.input, caption_model.input],combined.output)
Merge(mode='concat') is now Concatenate(axis=1).
The following generates a graph correctly on colab.
from tensorflow.python import keras
from keras.layers import *
from keras.models import Model, Sequential
IMG_FEATURES_SIZE = 10
MAX_SENTENCE = 80
VOCABULARY_SIZE = 1000
EMB_SIZE = 100
embedding_matrix = np.zeros((VOCABULARY_SIZE, EMB_SIZE))
LSTM_CELLS_CAPTION = 256
LSTM_CELLS_MERGED = 1000
image_pre = Sequential()
image_pre.add(Dense(100, input_shape=(IMG_FEATURES_SIZE,), activation='relu', name='fc_image'))
image_pre.add(RepeatVector(MAX_SENTENCE,name='repeat_image'))
caption_model = Sequential()
caption_model.add(Embedding(VOCABULARY_SIZE, EMB_SIZE,
weights=[embedding_matrix],
input_length=MAX_SENTENCE,
trainable=False, name="embedding"))
caption_model.add(LSTM(EMB_SIZE, return_sequences=True, name="lstm_caption"))
caption_model.add(TimeDistributed(Dense(100, name="td_caption")))
merge = Concatenate(axis=1,name="merge_models")([image_pre.output, caption_model.output])
lstm = Bidirectional(LSTM(256,return_sequences=False, name="lstm_merged"),name="bidirectional_lstm")(merge)
output = Dense(VOCABULARY_SIZE, name="fc_merged", activation='softmax')(lstm)
predictive = Model([image_pre.input, caption_model.input], output)
predictive.compile('sgd', 'binary_crossentropy')
predictive.summary()
Description:
This is a model with 2 inputs per sample: an image and a caption ( a sequence of words ).
The input graphs merge at the concatenation point (name='merge_models')
The image is processed simply by a Dense layer (you may want to add convolutions to the image branch ); the output of this dense layer is then copied MAX_SENTENCE times in preparation for the merge.
The captions are processed by an LSTM and a Dense layer.
The merge results in MAX_SENTENCE time-steps each with features from both branches.
The combined branch then ends up predicting one class out of VOCABULARY_SIZE.
The model.summary() is a good way to understand the graph.
I want to get the activation values for a given input of a trained LSTM network, specifically the values for the cell, the input gate, the output gate and the forget gate. According to this Keras issue and this Stackoverflow question I'm able to get some activation values with the following code:
(basically I'm trying to classify 1-dimensional timeseries using one label per timeseries, but that doesn't really matter for this general question)
import random
from pprint import pprint
import keras.backend as K
import numpy as np
from keras.layers import Dense
from keras.layers.recurrent import LSTM
from keras.models import Sequential
from keras.utils import to_categorical
def getOutputLayer(layerNumber, model, X):
return K.function([model.layers[0].input],
[model.layers[layerNumber].output])([X])
model = Sequential()
model.add(LSTM(10, batch_input_shape=(1, 1, 1), stateful=True))
model.add(Dense(2, activation='softmax'))
model.compile(
loss='categorical_crossentropy', metrics=['accuracy'], optimizer='adam')
# generate some test data
for i in range(10):
# generate a random timeseries of 100 numbers
X = np.random.rand(10)
X = X.reshape(10, 1, 1)
# generate a random label for the whole timeseries between 0 and 1
y = to_categorical([random.randint(0, 1)] * 10, num_classes=2)
# train the lstm for this one timeseries
model.fit(X, y, epochs=1, batch_size=1, verbose=0)
model.reset_states()
# to keep the output simple use only 5 steps for the input of the timeseries
X_test = np.random.rand(5)
X_test = X_test.reshape(5, 1, 1)
# get the activations for the output lstm layer
pprint(getOutputLayer(0, model, X_test))
Using that I get the following activation values for the LSTM layer:
[array([[-0.04106992, -0.00327154, -0.01524276, 0.0055838 , 0.00969929,
-0.01438944, 0.00211149, -0.04286387, -0.01102304, 0.0113989 ],
[-0.05771339, -0.00425535, -0.02032563, 0.00751972, 0.01377549,
-0.02027745, 0.00268653, -0.06011265, -0.01602218, 0.01571197],
[-0.03069103, -0.00267129, -0.01183739, 0.00434298, 0.00710012,
-0.01082268, 0.00175544, -0.0318702 , -0.00820942, 0.00871707],
[-0.02062054, -0.00209525, -0.00834482, 0.00310852, 0.0045242 ,
-0.00741894, 0.00141046, -0.02104726, -0.0056723 , 0.00611038],
[-0.05246543, -0.0039417 , -0.01877101, 0.00691551, 0.01250046,
-0.01839472, 0.00250443, -0.05472757, -0.01437504, 0.01434854]],
dtype=float32)]
So I get for each input value 10 values, because I specified in the Keras model to use a LSTM with 10 neurons. But which one is a cell, which is is the input gate, which one the output gate, which one the forget gate?
Well, these are the output values, to get and look into the value of each gate look into this issue
I paste the essential part here
for i in range(epochs):
print('Epoch', i, '/', epochs)
model.fit(cos,
expected_output,
batch_size=batch_size,
verbose=1,
nb_epoch=1,
shuffle=False)
for layer in model.layers:
if 'LSTM' in str(layer):
print('states[0] = {}'.format(K.get_value(layer.states[0])))
print('states[1] = {}'.format(K.get_value(layer.states[1])))
print('Input')
print('b_i = {}'.format(K.get_value(layer.b_i)))
print('W_i = {}'.format(K.get_value(layer.W_i)))
print('U_i = {}'.format(K.get_value(layer.U_i)))
print('Forget')
print('b_f = {}'.format(K.get_value(layer.b_f)))
print('W_f = {}'.format(K.get_value(layer.W_f)))
print('U_f = {}'.format(K.get_value(layer.U_f)))
print('Cell')
print('b_c = {}'.format(K.get_value(layer.b_c)))
print('W_c = {}'.format(K.get_value(layer.W_c)))
print('U_c = {}'.format(K.get_value(layer.U_c)))
print('Output')
print('b_o = {}'.format(K.get_value(layer.b_o)))
print('W_o = {}'.format(K.get_value(layer.W_o)))
print('U_o = {}'.format(K.get_value(layer.U_o)))
# output of the first batch value of the batch after the first fit().
first_batch_element = np.expand_dims(cos[0], axis=1) # (1, 1) to (1, 1, 1)
print('output = {}'.format(get_LSTM_output([first_batch_element])[0].flatten()))
model.reset_states()
print('Predicting')
predicted_output = model.predict(cos, batch_size=batch_size)
print('Ploting Results')
plt.subplot(2, 1, 1)
plt.plot(expected_output)
plt.title('Expected')
plt.subplot(2, 1, 2)
plt.plot(predicted_output)
plt.title('Predicted')
plt.show()
I'm trying to use Conv1D for the first time for multiclass classification of time series data and my model keeps throwing this error when I use it.
import numpy as np
import os
import keras
from keras.models import Sequential
from keras.layers import Conv1D, Dense, TimeDistributed, MaxPooling1D, Flatten
# fix random seed for reproducibility
np.random.seed(7)
dataset1 = np.genfromtxt(os.path.join('data', 'norm_cellcycle_384_17.txt'), delimiter=',', dtype=None)
data = dataset1[1:]
# extract columns
genes = data[:,0]
y_all = data[:,1].astype(int)
x_all = data[:,2:-1].astype(float)
# deleted this line when using sparse_categorical_crossentropy
# 384x6
y_all = keras.utils.to_categorical(y_all)
# 5
num_classes = np.unique(y_all).shape[0]
# split entire data into train set and test set
validation_split = 0.2
val_idx = np.random.choice(range(x_all.shape[0]), int(validation_split*x_all.shape[0]), replace=False)
train_idx = [x for x in range(x_all.shape[0]) if x not in val_idx]
x_train = x_all[train_idx]
y_train = y_all[train_idx]
# 308x17x1
x_train = x_train[:, :, np.newaxis]
# 308x1
y_train = y_train[:,np.newaxis]
x_test = x_all[val_idx]
y_test = y_all[val_idx]
# deleted this line when using sparse_categorical_crossentropy
y_test = keras.utils.to_categorical(y_test)
# 76x17x1
x_test = x_test[:, :, np.newaxis]
# 76x1
y_test = y_test[:,np.newaxis]
print(x_train.shape[0],'train samples')
print(x_test.shape[0],'test samples')
# Create Model
# number of filters for 1D conv
nb_filter = 4
filter_length = 5
window = x_train.shape[1]
model = Sequential()
model.add(Conv1D(filters=nb_filter,kernel_size=filter_length,activation="relu", input_shape=(window,1)))
model.add(MaxPooling1D())
model.add(Conv1D(nb_filter=nb_filter, filter_length=filter_length, activation='relu'))
model.add(MaxPooling1D())
model.add(Flatten())
model.add(Dense(num_classes, activation='softmax'))
model.summary()
model.compile(loss='sparse_categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
model.fit(x_train, y_train, epochs=25, batch_size=2, validation_data=(x_test, y_test))
I don't know why I get this error. When I use binary_crossentropy loss and no one hot encoding for y_all, my model works. But it fails when I use one hot encoding for y_all with categorical_crossentropy loss. When I don't use one hot encoding, keras throws an error making me change y_all to one a binary matrix.
I don't even know where the (1,6) are coming from in the array.
ValueError: Error when checking model target: expected dense_1 to have 2 dimensions, but got array with shape (308, 1, 6)
Please help! I've been stuck on this for many hours! Already went through all the related questions but still doesn't make sense.
Update: I now use sparse_categorical_crossentropy because it has integer support. I deleted the to_categorical lines from the above code and I get this new error:
InvalidArgumentError (see above for traceback): Received a label value
of 5 which is outside the valid range of [0, 5). Label values: 2 5
[[Node:
SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits
= SparseSoftmaxCrossEntropyWithLogits[T=DT_FLOAT, Tlabels=DT_INT64, _device="/job:localhost/replica:0/task:0/cpu:0"](Reshape_1, Cast)]]
Requested sample of data:
,Main,Gp,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,c16,c17
YDL179w,1,-0.75808,-0.90319,-0.98935,-0.73995,-0.67193,-0.12777,-0.95307,-1.01656,0.79730,2.11688,1.98537,0.61591,0.56603,-0.13684,-0.52228,-0.05068,0.78823,
YLR079w,1,-0.48845,-0.70828,-0.47688,-0.65814,-0.45374,-0.47302,-0.71214,-1.02839,0.24048,3.11376,1.28952,0.44874,0.04379,-0.31104,-0.30332,-0.34575,0.82285,
YER111c,1,-0.42218,0.23887,1.84427,-0.02083,-0.61105,-0.65827,-0.79992,-0.39857,-0.09166,2.03314,1.58457,0.68744,0.14443,-0.72910,-1.46097,-0.82353,-0.51662,
YBR200w,1,0.09824,0.55258,-0.89641,-1.19111,-1.11744,-0.76133,0.09824,2.16120,1.46126,1.03148,0.67537,-0.33155,-0.60170,-1.39987,-0.42978,-0.15963,0.81045,
YPL209c,2,-0.65282,-0.32055,2.53702,2.00538,0.60982,0.51014,-0.55314,-1.01832,-0.78573,0.01173,0.07818,-0.05473,-0.22087,0.24432,-0.28732,-1.11801,-0.98510,
YJL074c,2,-0.81087,-0.19448,1.72941,0.59002,-0.53069,-0.25051,-0.92294,-0.92294,-0.53069,0.08570,1.87884,1.97223,0.45927,-0.36258,-0.34390,-1.07237,-0.77351,
YNL233w,2,-0.43997,0.66325,2.85098,0.74739,-0.42127,-0.47736,-0.79524,-0.80459,-0.48671,-0.21558,1.25226,1.01852,-0.10339,-0.56151,-0.96353,-0.46801,-0.79524,
YLR313c,2,-0.46611,0.42952,3.01689,1.13856,0.01902,-0.44123,-0.66514,-0.98856,-0.59050,-0.47855,0.84002,0.39220,0.50416,-0.50342,-0.82685,-0.64026,-0.73977,
YGR041w,2,-0.57187,-0.26687,1.10561,-0.38125,-0.68624,-0.26687,-0.87687,-1.18186,-0.80062,0.60999,2.09686,1.82998,1.14374,0.11437,-0.80062,-0.87687,-0.19062,
So I noticed that even though I know there are 5 classes in this dataset as seen by the unique values obtained for y_all, for some reason Keras to_categorical thinks there are 6 classes.
# 384x6
y_all = keras.utils.to_categorical(y_all)
# 5
num_classes = np.unique(y_all).shape[0]
I don't know why that is. Keeping this in mind I changed this line of code and my model began to run:
model.add(Dense(num_classes, activation='softmax'))
to
model.add(Dense(num_classes+1, activation='softmax'))
I still don't know why to_categorical behaves this way. Anyone know?
to_categorical(x) in Keras will encode the given parameter into n number of classes where n = max(x) + 1, i.e. generally speaking from [0 , max(x)].