We Keep Coding

Masking input for ConvLSTM1D

I am doing a binary regression problem using keras.
The input shape is: (None, 2, 94, 3) (channels is the last dimension)
I have the following architecture:
input1 = Input(shape=(time, n_rows, n_channels))
masking = Masking(mask_value=-999)(input1)
convlstm = ConvLSTM1D(filters=16, kernel_size=15,
data_format='channels_last',
activation="tanh")(masking)
dropout = Dropout(0.2)(convlstm)
flatten1 = Flatten()(dropout)
outputs = Dense(n_outputs, activation='sigmoid')(flatten1)
model = Model(inputs=input1, outputs=outputs)
model.compile(loss=keras.losses.BinaryCrossentropy(),
optimizer=tf.keras.optimizers.Adam(learning_rate=0.01))
However when training I get this error: Dimensions must be equal, but are 94 and 80 for '{{node conv_lstm1d/while/SelectV2}} = SelectV2[T=DT_FLOAT](conv_lstm1d/while/Tile, conv_lstm1d/while/mul_5, conv_lstm1d/while/Placeholder_2)' with input shapes: [?,94,16], [?,80,16], [?,80,16].
If I remove the masking layer this error disappears, what is the masking doing that triggers this error? Also the only way I was able to run the above architecture was with a kernel_size of 1.

Seems like the ConvLSTM1D layer needs a mask with the shape (samples, timesteps) according to the docs. The mask you are calculating has the shape (samples, time, rows). Here is one solution to fix your problem but I am not sure if it is the 'correct' way to go:
import tensorflow as tf
input1 = tf.keras.layers.Input(shape=(2, 94, 3))
masking = tf.keras.layers.Masking(mask_value=-999)(input1)
convlstm = tf.keras.layers.ConvLSTM1D(filters=16, kernel_size=15,
data_format='channels_last',
activation="tanh")(inputs = masking, mask = tf.reduce_all(masking._keras_mask, axis=-1))
dropout = tf.keras.layers.Dropout(0.2)(convlstm)
flatten1 = tf.keras.layers.Flatten()(dropout)
outputs = tf.keras.layers.Dense(1, activation='sigmoid')(flatten1)
model = tf.keras.Model(inputs=input1, outputs=outputs)
model.compile(loss=tf.keras.losses.BinaryCrossentropy(),
optimizer=tf.keras.optimizers.Adam(learning_rate=0.01))
This line mask = tf.reduce_all(masking._keras_mask, axis=-1) essentially reduces your mask to (samples, timesteps) by applying an AND operation to the last dimension of the mask. Alternatively, you could just create your own custom mask layer:
import tensorflow as tf
class Reduce(tf.keras.layers.Layer):
def __init__(self):
super(Reduce, self).__init__()
def call(self, inputs):
return tf.reduce_all(tf.reduce_any(tf.not_equal(inputs, -999), axis=-1, keepdims=False), axis=1)
input1 = tf.keras.layers.Input(shape=(2, 94, 3))
reduce_layer = Reduce()
boolean_mask = reduce_layer(input1)
convlstm = tf.keras.layers.ConvLSTM1D(filters=16, kernel_size=15,
data_format='channels_last',
activation="tanh")(inputs = input1, mask = boolean_mask)
dropout = tf.keras.layers.Dropout(0.2)(convlstm)
flatten1 = tf.keras.layers.Flatten()(dropout)
outputs = tf.keras.layers.Dense(1, activation='sigmoid')(flatten1)
model = tf.keras.Model(inputs=input1, outputs=outputs)
model.compile(loss=tf.keras.losses.BinaryCrossentropy(),
optimizer=tf.keras.optimizers.Adam(learning_rate=0.01))
print(model.summary(expand_nested=True))
x = tf.random.normal((50, 2, 94, 3))
y = tf.random.uniform((50, ), maxval=3, dtype=tf.int32)
model.fit(x, y)

Issue retrieiving value error `decode_predictions` expects a batch of predictions

I have the following code trying to perform predictions on part of resnet model. However, I am retrieving error.
def layer_input_shape(Model, layer_index):
input_shape = np.array(Model.layers[layer_index - 1].output_shape)
input_shape = np.ndarray.tolist(np.delete(input_shape, 0))
return input_shape
def resnet50_Model(Model, trainable=True):
input_shape = layer_input_shape(Model, 1)
input = tf.keras.layers.Input(shape=input_shape)
first_layer = Model.layers[0]
first_layer.trainable = trainable
out = first_layer(input)
for i in range(1, 12):
layer_i = Model.layers[i]
layer_i.trainable = trainable
out = layer_i(out)
out = Conv2D(filters=2, kernel_size=2, strides=(2,2), activation='relu')(out)
out = Flatten()(out)
out = Dense(units=2,activation='softmax')(out)
result_model = tf.keras.models.Model(inputs=[input], outputs=out)
return result_model
from tensorflow.keras.applications.resnet50 import preprocess_input, decode_predictions
img='/content/elephant.jpg'
img = image.load_img(img, target_size=(224, 224))
x = image.img_to_array(img)
x = np.expand_dims(x, axis=0)
x = preprocess_input(x)
preds = resnet_skip_model.predict(x)
print('Predicted:', decode_predictions(preds, top=3)[0])
Retrieving below error:
ValueError: `decode_predictions` expects a batch of predictions (i.e. a 2D array of shape (samples,
1000)). Found array with shape: (1, 3)

I added two output dense layer so I can only predict two classes and when I call decode it expects 1000 output last dense layer, therefore changed units from two to 1000
out = Dense(units=1000,activation='softmax')(out)

convert 2D sparse matrix to 3D matrix

I want to convert the 2D sparse matrix to 3D matrix as i need to give it as the input the conv1d layer, which expects 3D tensor.
Here is the input for the conv1d layer.
from scipy.sparse import hstack
other_features_train = hstack((X_train_state_ohe, X_train_teacher_ohe, X_train_grade_ohe, X_train_category_ohe, X_train_subcategory_ohe,X_train_price_norm,X_train_number_norm))
other_features_cv = hstack((X_cv_state_ohe, X_cv_teacher_ohe, X_cv_grade_ohe,X_cv_category_ohe,X_cv_subcategory_ohe,X_cv_price_norm,X_cv_number_norm))
other_features_test = hstack((X_test_state_ohe, X_test_teacher_ohe, X_test_grade_ohe,X_test_category_ohe,X_test_subcategory_ohe,X_test_price_norm,X_test_number_norm))
print(other_features_train.shape)
print(other_features_cv.shape)
print(other_features_test.shape)
shape of the train , cv and test data
(49041, 101)
(24155, 101)
(36052, 101)
This is my model architecture.
tf.keras.backend.clear_session()
vec_size = 300
input_model_1 = Input(shape=(300,),name='essay')
embedding = Embedding(vocab_size_essay, vec_size, weights=[word_vector_matrix], input_length = max_length, trainable=False)(input_model_1)
lstm = LSTM(16)(embedding)
flatten_1 = Flatten()(lstm)
input_model_2 = Input(shape=(101, ),name='other_features')
conv_layer1 = Conv1D(32, 3, strides=1, padding='valid', kernel_initializer='glorot_uniform', activation='relu')(input_model_2)
conv_layer2 = Conv1D(32, 3, strides=1, padding='valid', kernel_initializer='glorot_uniform', activation='relu')(conv_layer1)
conv_layer3 = Conv1D(32, 3, strides=1, padding='valid', kernel_initializer='glorot_uniform', activation='relu')(conv_layer2)
flatten_2 = Flatten()(conv_layer3)
concat_layer = concatenate(inputs=[flatten_1, flatten_2],name='concat')
dense_layer_1 = Dense(units=32, activation='relu', kernel_initializer='he_normal', name='dense_layer_1')(concat_layer)
dropout_1 = Dropout(0.2)(dense_layer_1)
dense_layer_2 = Dense(units=32, activation='relu', kernel_initializer='he_normal', name='dense_layer_2')(dropout_1)
dropout_2 = Dropout(0.2)(dense_layer_2)
dense_layer_3 = Dense(units=32, activation='relu', kernel_initializer='he_normal', name='dense_layer_3')(dropout_2)
output = Dense(units=2, activation='softmax', kernel_initializer='glorot_uniform', name='output')(dense_layer_3)
model_3 = Model(inputs=[input_model_1,input_model_2],outputs=output)
and am getting this error when am trying to give 2d array.
---------------------------------------------------------------------------
ValueError Traceback (most recent call last)
<ipython-input-18-44c8f6f0caa7> in <module>
9
10 input_model_2 = Input(shape=(101, ),name='other_features')
---> 11 conv_layer1 = Conv1D(32, 3, strides=1, padding='valid', kernel_initializer='glorot_uniform', activation='relu')(input_model_2)
12 conv_layer2 = Conv1D(32, 3, strides=1, padding='valid', kernel_initializer='glorot_uniform', activation='relu')(conv_layer1)
13 conv_layer3 = Conv1D(32, 3, strides=1, padding='valid', kernel_initializer='glorot_uniform', activation='relu')(conv_layer2)
~\AppData\Local\Programs\Python\Python37\lib\site-packages\tensorflow_core\python\keras\engine\base_layer.py in __call__(self, inputs, *args, **kwargs)
810 # are casted, not before.
811 input_spec.assert_input_compatibility(self.input_spec, inputs,
--> 812 self.name)
813 graph = backend.get_graph()
814 with graph.as_default(), backend.name_scope(self._name_scope()):
~\AppData\Local\Programs\Python\Python37\lib\site-packages\tensorflow_core\python\keras\engine\input_spec.py in assert_input_compatibility(input_spec, inputs, layer_name)
175 'expected ndim=' + str(spec.ndim) + ', found ndim=' +
176 str(ndim) + '. Full shape received: ' +
--> 177 str(x.shape.as_list()))
178 if spec.max_ndim is not None:
179 ndim = x.shape.ndims
ValueError: Input 0 of layer conv1d is incompatible with the layer: expected ndim=3, found ndim=2. Full shape received: [None, 101]
model_3.summary()
model_3.compile(loss = "binary_crossentropy", optimizer=Adam()
Compile the model
model_3.compile(loss = "binary_crossentropy", optimizer=Adam(), metrics=["accuracy"])
Fit the model
model_3.fit(train_features,y_train_ohe,batch_size=16,epochs=10,validation_data=(cv_features,y_cv_ohe))
train_features = [train_text, other_features_train]
cv_features = [cv_text, other_features_cv]
test_featues = [test_text, other_features_test]
Text Features
train_text = X_train['essay'].tolist()
cv_text = X_cv['essay'].tolist()
test_text = X_test['essay'].tolist()
token = Tokenizer()
token.fit_on_texts(train_text)
vocab_size_essay = len(token.word_index) + 1
print("No. of unique words = ", vocab_size_essay)
encoded_train_text = token.texts_to_sequences(train_text)
encoded_cv_text = token.texts_to_sequences(cv_text)
encoded_test_text = token.texts_to_sequences(test_text)
#print(encoded_test_text[:5])
max_length = 300
train_text = pad_sequences(encoded_train_text, maxlen=max_length, padding='post')
cv_text = pad_sequences(encoded_cv_text, maxlen=max_length, padding='post')
test_text = pad_sequences(encoded_test_text, maxlen=max_length, padding='post')
print("\n")
print(train_text.shape)
print(cv_text.shape)
print(test_text.shape)
shape of text features
No. of unique words = 41468
(49041, 300)
(24155, 300)
(36052, 300)
So, I want the reshape in
(49041,101,1)
(24155,101,1)
(36052,101,1)
Please suggest how to do it.

Solution
The solution here demands clarity on a few concepts as follows. I will explain these concepts
in the following sections.
what keras expects as inputs
what kind of modifications could be done to your keras model to allow sparse input matrices
converting a 2D numpy array to a 3D numpy array
back-and-forth conversion between a sparse and a non-sparse (or, dense) array using
scipy.sparse.coo_matrix for 2D numpy array
sparse.COO for 3D numpy array
Using sparse matrices as input to tf.keras models
One option is to convert your sparse input matrix into the non-sparse (dense) format using
todense() method. This makes the matrix a regular numpy array. See kaggle discussion,
[3] and [4].
Another option is to write your own custom Layers for both sparse and dense inputs by
subclassing tf.keras.layers.Layer class. See this article, [2].
It appears that tensorflow.keras now allows model training with sparse weights. So,
somewhere it has the ability to handle sparsity. You may want to explore the documentation,
[1] for this aspect.
Adding a new-axis to a numpy array
You can add another axis to a numpy array using np.newaxis as follows.
import numpy as np
## Make a 2D array
a2D = np.zeros((10,10))
# Make a few elements non-zero in a2D
aa = a2D.flatten()
aa[[0,13,41,87,98]] = np.random.randint(1,10,size=5)
a2D = aa.reshape(a2D.shape)
# Make 3D array from 2D array by adding another axis
a3D = a2D[:,:,np.newaxis]
#print(a2D)
print('a2D.shape: {}\na3D.shape: {}'.format(a2D.shape, a3D.shape))
Output:
a2D.shape: (10, 10)
a3D.shape: (10, 10, 1)
Having said that, please take a look at the links in the References section.
Sparse Arrays
Since a sparse array has very few non-zero values, a regular numpy array when converted
into a sparse array, stores it in a few sparse-formats:
csr_matrix: row-wise arrays of non-zero values and indices
csc-matrix: column-wise array of nonzero values and indices
coo-matrix: a table with three columns
row
column
non-zero value
Scipy Sparse Matrices expect 2D input-matrix
However, scipy.sparse implementation of the above three types of sparse-matrices, only
considers 2D non-sparse matrix as input.
from scipy.sparse import csr_matrix, coo_matrix
coo_a2D = coo_matrix(a2D)
coo_a2D.shape # output: (10, 10)
# scipy.sparse only accepts 2D input matrices
# the following line will throw an !!! ERROR !!!
coo_a3D = coo_matrix(coo_a2D.todense()[:,:,np.newaxis])
Sparse Matrix from 3D non-sparse input matrix
Yes, you can do this using the sparse library.
It also supports scipy.sparse and numpy arrays. To convert from sparse matrix to
non-sparse (dense) format (this is NOT a Dense Layer in neural networks), use
the todense() method.
## Installation
# pip install -U sparse
import sparse
## Create sparse coo_matrix from a
# 3D numpy array (dense format)
coo_a3D = sparse.COO(a3D)
## Test that
# coo_a3D == coo made from (coo_a2D + newaxis)
print(
(coo_a3D == sparse.COO(coo_a2D.todense()[:,:,np.newaxis])).all()
) # output: True
## Convert to dense (non-sparse) format
# use: coo_a3D.todense()
print((a3D == coo_a3D.todense()).all()) # output: True
Source
PyTorch: torch.sparse 🔥 ⭐
PyTorch library also provides ways to work with sparce tensors.
Documentation torch.sparse: https://pytorch.org/docs/stable/sparse.html#sparse-coo-docs
References
Train sparse TensorFlow models with Keras
How to design deep learning models with sparse inputs in Tensorflow Keras
Neural network for sparse matrices
Training Neural network with scipy sparse matrix?
Documentation of sparse library

You can simply use np.reshape
https://numpy.org/doc/1.18/reference/generated/numpy.reshape.html
other_features_train = other_features_train.reshape(other_features_train.shape[0], other_features_train.shape[1], 1)
other_features_cv = other_features_cv.reshape(other_features_cv.shape[0], other_features_cv.shape[1], 1)
other_features_test = other_features_test.reshape(other_features_test.shape[0], other_features_test.shape[1], 1)
Also, you need to change this line
input_model_2 = Input(shape=(101, 1),name='other_features')
Conv1D expects 3-d data, not 2-d.

How to modify elements of a Keras Tensor object

I am building a Convolution Neural Network in Keras that receives batch of images with dimensions (None, 256, 256, 1) and the output would be batches with size (None, 256, 256, 3). Now after the final layer output I want to add a layer that assigns values to some of the pixels in output layer based on a value condition on inputs. Here is what I tried:
The Function
def SetBoundaries(ins):
xi = ins[0]
xo = ins[1]
bnds = np.where(xi[:, :, :, 0] == 0)
bnds_s, bnds_i, bnds_j = bnds[0], bnds[1], bnds[2]
xo[bnds_s, bnds_i, bnds_j, 0] = 0
xo[bnds_s, bnds_i, bnds_j, 1] = 0
xo[bnds_s, bnds_i, bnds_j, 2] = 0
return xo
Keras model
def conv_res(inputs):
x0 = inputs
...
xc = conv_layer(xc, kernel_size=3, stride=1,
num_filters=3, name="Final_Conv")
# apply assignment function
xc = Lambda(SetBoundaries, name="assign_boundaries")([x0, xc])
return xc
Finally, the model is built using
def build_model(inputs):
xres = int(inputs.shape[1])
yres = int(inputs.shape[2])
cres = int(inputs.shape[3])
inputs = Input((xres, yres, cres))
outputs = UNet.conv_res(inputs)
model = keras.Model(inputs=inputs, outputs=outputs)
return model
However, when I run I get the error:
NotImplementedError: Cannot convert a symbolic Tensor (assign_boundaries/Equal:0) to a numpy array.
Everything works fine without the Lambda function. I understand the issue is assigning value to Tensor object but how can I achieve what I am after?
Thanks

np.where works with NumPy arrays, but the output from your model is a Tensorflow tensor. Try using tf.where, which is the same thing but for tf.Tensors.

I managed to make it work by changing the function to:
def SetBoundaries(ins):
xi = ins[0]
xo = ins[1]
xin = tf.broadcast_to(xi, tf.shape(xo))
mask = K.cast(tf.not_equal(xin, 0), dtype="float32")
xf = layers.Multiply()([mask, xo])
return xf

why I must reshape one image to [n,height,width,channel] in CNN

I try to apply a convolutional layer to a picture of shape [256,256,3]
a have an error when I user the tensor of the image directly
conv1 = conv2d(input,W_conv1) +b_conv1 #<=== error
error message:
ValueError: Shape must be rank 4 but is rank 3 for 'Conv2D' (op: 'Conv2D')
with input shapes: [256,256,3], [3,3,3,1].
but when I reshape the function conv2d work normally
x_image = tf.reshape(input,[-1,256,256,3])
conv1 = conv2d(x_image,W_conv1) +b_conv1
if I must reshape the tensor what the best value to reshape in my case and why?
import tensorflow as tf
import numpy as np
from PIL import Image
def img_to_tensor(img) :
return tf.convert_to_tensor(img, np.float32)
def weight_generater(shape):
return tf.Variable(tf.truncated_normal(shape,stddev=0.1))
def bias_generater(shape):
return tf.Variable(tf.constant(.1,shape=shape))
def conv2d(x,W):
return tf.nn.conv2d(x,W,[1,1,1,1],'SAME')
def pool_max_2x2(x):
return tf.nn.max_pool(x,ksize=[1,2,2,1],strides=[1,1,1,1],padding='SAME')
#read image
img = Image.open("img.tif")
sess = tf.InteractiveSession()
#convetir image to tensor
input = img_to_tensor(img).eval()
#print(input)
# get img dimension
img_dimension = tf.shape(input).eval()
print(img_dimension)
height,width,channel=img_dimension
filter_size = 3
feature_map = 32
x = tf.placeholder(tf.float32,shape=[height*width*channel])
y = tf.placeholder(tf.float32,shape=21)
# generate weigh [kernal size, kernal size,channel,number of filters]
W_conv1 = weight_generater([filter_size,filter_size,channel,1])
#for each filter W has his specific bais
b_conv1 = bias_generater([feature_map])
""" I must reshape the picture
x_image = tf.reshape(input,[-1,256,256,3])
"""
conv1 = conv2d(input,W_conv1) +b_conv1 #<=== error
h_conv1 = tf.nn.relu(conv1)
h_pool1 = pool_max_2x2(h_conv1)
layer1_dimension = tf.shape(h_pool1).eval()
print(layer1_dimension)

The first dimension is the batch size. If you are feeding 1 image at a time you can simply make the first dimension 1 and it doesn't change your data any, just changes the indexing to 4D:
x_image = tf.reshape(input, [1, 256, 256, 3])
If you reshape it with a -1 in the first dimension what you are doing is saying that you will feed in a 4D batch of images (shaped [batch_size, height, width, color_channels], and you are allowing the batch size to be dynamic (which is common to do).

You could also use
im = tf.expand_dims(input, axis=0)
to insert a dimension of 1 into the tensor's shape. im will be a rank 4 tensor. This way you do not have to specify the dimensions of the image.