We Keep Coding

Dimensions must be equal issue

I created an Autoencoder model to minimise a PAPR and BER metrics in the same time i use Categorical Cross entropy for BER and a function 'PAPRLoss' for PAPR reduction i have a problem with the shapes
def paprLoss(y_true, y_pred):
sigR = tf.math.real(y_pred)
sigI = tf.math.imag(y_pred)
sigR = tf.reshape(sigR,(1,(72*btch)))
sigI = tf.reshape(sigI,(1,(72*btch)))
sigRI = tf.concat((sigR, sigI), 0)
yPower = K.sqrt(K.sum(K.square(sigRI), axis=-1))
yMax = K.max(yPower, axis=-1)
yMean = K.mean(yPower, axis=-1)
yPAPR = 10 * tf.experimental.numpy.log10(yMax/yMean)
return yMax
#generating data of size N
N = 1024000
label = np.random.randint(M,size=N)
# creating one hot encoded vectors
data = []
for i in label:
temp = np.zeros(M)
temp[i] = 1
data.append(temp)
data = np.array(data)
n_channel=2
R = k/n_channel
input_signal = Input(shape=(M,))
encoded = Dense(10*M, activation='relu')(input_signal)
encoded1 = Dense(10*M, activation='relu')(encoded)
encoded1 = Dense(n_channel, activation='linear')(encoded1)
encoded2=Lambda(lambda x:x / K.sqrt(K.mean(x**2)))(encoded1)#Average Power constraint
encoded3 = Lambda(lambda x: OFDM_mod(x, Mfft, CP))(encoded2)
SNRdB_train = 55
SNR_Linear = 10**(SNRdB_train/10)
encoded51 = Lambda(lambda x: OFDM_demod(x, Mfft, CP))(encoded3)
decoded0 = Dense(10*M,activation='relu')(encoded51)
decoded = Dense(10*M,activation='relu')(decoded0)
decoded1 = Dense(M, activation='softmax')(decoded)
autoencoder= Model(inputs=input_signal, outputs=[decoded1,encoded3], name="PAPRnet_Encoder")
autoencoder.compile(optimizer='adamax', loss=['categorical_crossentropy',paprLoss],loss_weights=[1.0,0.1], metrics=['accuracy'])
print (autoencoder.summary())
#training phase
history=autoencoder.fit(data, [data,data],
epochs=100,
batch_size=btch*Mfft,
validation_freq=1,)
I got the following error. Shapes of y_true and y_pred are not equal
Dimensions must be equal, but are 512000 and 8000 for '{{node Equal_1}} = Equal[T=DT_INT64, incompatible_shape_error=true](ArgMax_2, ArgMax_3)' with input shapes: [512000], [8000].

Using a multiple matrices to predict a matrix in Python with Keras

I modified the code from here. What I'm trying to do is combine the two matrices to predict the output matrix. The output matrix is built from the two input matrices. The problem seems to be associated to:
self.Combined_dense_1 = tf.keras.layers.Dense(units=32, activation="relu")
self.Combined_dense_2 = tf.keras.layers.Dense(units=16, activation="softmax")
The linked medium tutorial only predicting a single number based on the combined mixed input. I however am trying to predict a whole matrix but don't know how to structure the combined layer (if this is even the problem).
The error: "ValueError: Shape mismatch: The shape of labels (received (40,)) should equal the shape of logits except for the last dimension (received (10, 16))."
The code:
import warnings
import sys
if not sys.warnoptions:
warnings.simplefilter("ignore")
import numpy as np
import os
import random
import tensorflow as tf
from tensorflow import keras
from IPython.display import clear_output
class model(keras.Model):
def __init__(self):
super().__init__()
# The layers to process our image
self.Conv2D_1 = tf.keras.layers.Conv2D(filters=32,
kernel_size=(1, 1),
strides=(1, 1)
)
self.Conv2D_2 = tf.keras.layers.Conv2D(filters=32,
kernel_size=(3, 3),
strides=(1, 1)
)
# our combined layers
self.Combined_dense_1 = tf.keras.layers.Dense(units=32, activation="relu")
self.Combined_dense_2 = tf.keras.layers.Dense(units=16, activation="softmax")
def call(self, input_image_one, input_image_two):
# Image model
I = self.Conv2D_1(input_image_one)
I = self.Conv2D_2(I)
# Flatten I so we can merge our data.
I = tf.keras.layers.Flatten()(I)
N = self.Conv2D_1(input_image_two)
N = self.Conv2D_2(N)
N = tf.keras.layers.Flatten()(N)
# Combined model
x = tf.concat([N, I], 1) # Concatenate through axis #1
x = self.Combined_dense_1(x)
x = self.Combined_dense_2(x)
return x
network = model()
optimizer = tf.keras.optimizers.Adam()
loss_function = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)
def train_step(model, optimizer, loss_function,
images_one_batch, images_two_batch,
labels):
with tf.GradientTape() as tape:
model_output = model(images_one_batch, images_two_batch)
print(model_output)
loss = loss_function(labels, model_output) # our labels vs our predictions
grads = tape.gradient(loss, model.trainable_variables)
optimizer.apply_gradients(zip(grads, model.trainable_variables))
return loss
def train(model, optimizer, loss_function, epochs,
images_one_batch, images_two_batch,
labels):
loss_array = []
for epoch in range(epochs):
loss = train_step(model, optimizer, loss_function, images_one_batch, images_two_batch, labels)
loss_array.append(loss)
if ((epoch + 1) % 20 == 0):
# Calculating accuracy
network_output = network(images_one_batch, images_two_batch)
preds = np.argmax(network_output, axis=1)
acc = 0
for i in range(len(images_one_batch)):
if (preds[i] == labels[i]):
acc += 1
print(" loss:", loss, " Accuracy: ", acc / len(images_one_batch) * 100, "%")
clear_output(wait=True)
NumberofVars = 2;
width= NumberofVars; height = NumberofVars
NumberOfComputationSets = 10
CM_MatrixArr1 = []
CM_MatrixArr2 = []
for j in range(NumberOfComputationSets):
Theta1 = list(np.reshape(np.random.randint(2, size=4), (1,4))[0])
Theta1 = list(np.float_(Theta1))
CM_MatrixArr1.append(Theta1)
Theta2 = list(np.reshape(np.random.randint(2, size=4), (1,4))[0])
Theta2 = list(np.float_(Theta2))
CM_MatrixArr2.append(Theta2)
combinedCM_MatrixArr = []
combinedCM_toIntArr = []
for x,y in zip(CM_MatrixArr1, CM_MatrixArr2):
combinedCM = []
combinedCM_toInt = 0
for a,b in zip(x,y):
LogVal = (a == b)
combinedCM.append(float(LogVal == True))
combinedCM_MatrixArr.append(combinedCM)
combinedCM_MatrixArr = np.array(combinedCM_MatrixArr)
combinedCM_MatrixArr = combinedCM_MatrixArr.reshape(NumberOfComputationSets,2,2)
CM_MatrixArr1 = np.array(CM_MatrixArr1)
CM_MatrixArr1 = CM_MatrixArr1.reshape(NumberOfComputationSets,2,2)
CM_MatrixArr1 = CM_MatrixArr1.reshape(NumberOfComputationSets, 2,2,1)
CM_MatrixArr2 = np.array(CM_MatrixArr2)
CM_MatrixArr2 = CM_MatrixArr2.reshape(NumberOfComputationSets,2,2)
CM_MatrixArr2 = CM_MatrixArr2.reshape(NumberOfComputationSets, 2,2,1)
train(network,optimizer,loss_function,300,CM_MatrixArr1,CM_MatrixArr2,combinedCM_MatrixArr)

tensorflow 2 : loss using hidden layers output

I am trying to implement the OSME MAMC model describe in article https://arxiv.org/abs/1806.05372.
I'm stuck where I have to add a cost that doesn't depend on y_true and y_pred but on hidden layers and y_true.
It can't be right as tensorflow custom loss, for which we need y_true and y_pred.
I wrote the model into class, then tried to use gradient tape to add NPairLoss to Softmax output loss, but gradient is NaN during training.
I think my approach isn't good, but I have no idea how to design / write it.
Here my model :
class OSME_network(tf.keras.Model):
def __init__(self, nbrclass=10, weight="imagenet",input_tensor=(32,32,3)):
super(OSME_network, self).__init__()
self.nbrclass = nbrclass
self.weight = weight
self.input_tensor=input_tensor
self.Resnet_50=ResNet50(include_top=False, weights=self.weight, input_shape=self.input_tensor)
self.Resnet_50.trainable=False
self.split=Lambda(lambda x: tf.split(x,num_or_size_splits=2,axis=-1))
self.s_1=OSME_Layer(ch=1024,ratio=16)
self.s_2=OSME_Layer(ch=1024,ratio=16)
self.fl1=tf.keras.layers.Flatten()
self.fl2=tf.keras.layers.Flatten()
self.d1=tf.keras.layers.Dense(1024, name='fc1')
self.d2=tf.keras.layers.Dense(1024,name='fc2')
self.fc=Concatenate()
self.preds=tf.keras.layers.Dense(self.nbrclass,activation='softmax')
#tf.function
def call(self,x): #set à construire le model sequentiellement
x=self.Resnet_50(x)
x_1,x_2=self.split(x)
xx_1 = self.s_1(x_1)
xx_2 = self.s_2(x_2)
xxx_1 = self.d1(xx_1)
xxx_2 = self.d2(xx_2)
xxxx_1 = self.fl1(xxx_1)
xxxx_2 = self.fl2(xxx_2)
fc = self.fc([xxxx_1,xxxx_2]) #fc1 + fc2
ret=self.preds(fc)
return xxxx_1,xxxx_2,ret
class OSME_Layer(tf.keras.layers.Layer):
def __init__(self,ch,ratio):
super(OSME_Layer,self).__init__()
self.GloAvePool2D=GlobalAveragePooling2D()
self.Dense1=Dense(ch//ratio,activation='relu')
self.Dense2=Dense(ch,activation='sigmoid')
self.Mult=Multiply()
self.ch=ch
def call(self,inputs):
squeeze=self.GloAvePool2D(inputs)
se_shape = (1, 1, self.ch)
se = Reshape(se_shape)(squeeze)
excitation=self.Dense1(se)
excitation=self.Dense2(excitation)
scale=self.Mult([inputs,excitation])
return scale
class NPairLoss():
def __init__(self):
self._inputs = None
self._y=None
#tf.function
def __call__(self,inputs,y):
targets=tf.argmax(y, axis=1)
b, p, _ = inputs.shape
n = b * p
inputs=tf.reshape(inputs, [n, -1])
targets = tf.repeat(targets,repeats=p)
parts = tf.tile(tf.range(p),[b])
prod=tf.linalg.matmul(inputs,inputs,transpose_a=False,transpose_b=True)
same_class_mask = tf.math.equal(tf.broadcast_to(targets,[n, n]),tf.transpose(tf.broadcast_to(targets,(n, n))))
same_atten_mask = tf.math.equal(tf.broadcast_to(parts,[n, n]),tf.transpose(tf.broadcast_to(parts,(n, n))))
s_sasc = same_class_mask & same_atten_mask
s_sadc = (~same_class_mask) & same_atten_mask
s_dasc = same_class_mask & (~same_atten_mask)
s_dadc = (~same_class_mask) & (~same_atten_mask)
loss_sasc = 0
loss_sadc = 0
loss_dasc = 0
for i in range(n):
#loss_sasc
pos = prod[i][s_sasc[i]]
neg = prod[i][s_sadc[i] | s_dasc[i] | s_dadc[i]]
n_pos=tf.shape(pos)[0]
n_neg=tf.shape(neg)[0]
pos = tf.transpose(tf.broadcast_to(pos,[n_neg,n_pos]))
neg = tf.broadcast_to(neg,[n_pos,n_neg])
exp=tf.clip_by_value(tf.math.exp(neg - pos),clip_value_min=0,clip_value_max=9e6) # need to clip value, else inf
loss_sasc += tf.reduce_sum(tf.math.log(1 + tf.reduce_sum(exp,axis=1)))
#loss_sadc
pos = prod[i][s_sadc[i]]
neg = prod[i][s_dadc[i]]
n_pos = tf.shape(pos)[0]
n_neg = tf.shape(neg)[0]
pos = tf.transpose(tf.broadcast_to(pos,[n_neg,n_pos])) #np.transpose(np.tile(pos,[n_neg,1]))
neg = tf.broadcast_to(neg,[n_pos,n_neg])#np.tile(neg,[n_pos,1])
exp=tf.clip_by_value(tf.math.exp(neg - pos),clip_value_min=0,clip_value_max=9e6)
loss_sadc += tf.reduce_sum(tf.math.log(1 + tf.reduce_sum(exp,axis=1)))
#loss_dasc
pos = prod[i][s_dasc[i]]
neg = prod[i][s_dadc[i]]
n_pos = tf.shape(pos)[0]
n_neg = tf.shape(neg)[0]
pos = tf.transpose(tf.broadcast_to(pos,[n_neg,n_pos])) #np.transpose(np.tile(pos,[n_neg,1]))
neg = tf.broadcast_to(neg,[n_pos,n_neg])#np.tile(neg,[n_pos,1])
exp=tf.clip_by_value(tf.math.exp(neg - pos),clip_value_min=0,clip_value_max=9e6)
loss_dasc += tf.reduce_sum(tf.math.log(1 + tf.reduce_sum(exp,axis=1)))
return (loss_sasc + loss_sadc + loss_dasc) / n
then, for training :
#tf.function
def train_step(x,y):
with tf.GradientTape() as tape:
fc1,fc2,y_pred=model(x,training=True)
stacked=tf.stack([fc1,fc2],axis=1)
layerLoss=npair(stacked,y)
loss=cce(y, y_pred) +0.001*layerLoss
grads=tape.gradient(loss,model.trainable_variables)
opt.apply_gradients(zip(grads,model.trainable_variables))
return loss
model=OSME_network(weight="imagenet",nbrclass=10,input_tensor=(32, 32, 3))
model.compile(optimizer=opt, loss=categorical_crossentropy, metrics=["acc"])
model.build(input_shape=(None,32,32,3))
cce = tf.keras.losses.CategoricalCrossentropy(from_logits=True,name='categorical_crossentropy')
npair=NPairLoss()
for each batch :
x=tf.Variable(x_train[start:end])
y=tf.Variable(y_train[start:end])
train_loss=train_step(x,y)
Thanks for any help :)

You can use tensorflow's add_loss.
model.compile() loss functions in Tensorflow always take two parameters y_true and y_pred. Using model.add_loss() has no such restriction and allows you to write much more complex losses that depend on many other tensors, but it has the inconvenience of being more dependent on the model, whereas the standard loss functions work with just any model.
You can find the official documentation of add_loss here. Add loss tensor(s), potentially dependent on layer inputs. This method can be used inside a subclassed layer or model's call function, in which case losses should be a Tensor or list of Tensors. There are few example in the documentation to explain the add_loss.
This method can also be called directly on a Functional Model during construction. In this case, any loss Tensors passed to this Model must be symbolic and be able to be traced back to the model's Inputs. These losses become part of the model's topology and are tracked in get_config.
Example :
inputs = tf.keras.Input(shape=(10,))
x = tf.keras.layers.Dense(10)(inputs)
outputs = tf.keras.layers.Dense(1)(x)
model = tf.keras.Model(inputs, outputs)
# Activity regularization.
model.add_loss(tf.abs(tf.reduce_mean(x)))
You can call self.add_loss(loss_value) from inside the call method of a custom layer. Here's a simple example that adds activity regularization.
Example:
class ActivityRegularizationLayer(layers.Layer):
def call(self, inputs):
self.add_loss(tf.reduce_sum(inputs) * 0.1)
return inputs # Pass-through layer.
inputs = keras.Input(shape=(784,), name='digits')
x = layers.Dense(64, activation='relu', name='dense_1')(inputs)
# Insert activity regularization as a layer
x = ActivityRegularizationLayer()(x)
x = layers.Dense(64, activation='relu', name='dense_2')(x)
outputs = layers.Dense(10, name='predictions')(x)
model = keras.Model(inputs=inputs, outputs=outputs)
model.compile(optimizer=keras.optimizers.RMSprop(learning_rate=1e-3),
loss=keras.losses.SparseCategoricalCrossentropy(from_logits=True))
# The displayed loss will be much higher than before
# due to the regularization component.
model.fit(x_train, y_train,
batch_size=64,
epochs=1)
You can find good example using add_loss here and here with explanations.
Hope this answers your question. Happy Learning.

Batch training DAE with LSTM Encoder, Decoder properly in PyTorch

def denoise_train(x: DataLoader):
loss = 0
x_padded = list(map(lambda s: pad_string(s), x))
x_idx_tensor = strings_to_index_tensor(x_padded)
noisy_x = list(map(lambda s: noise_name(s), x))
noisy_x_padded = list(map(lambda s: pad_string(s), noisy_x))
noisy_x_idx_tensor = strings_to_index_tensor(noisy_x_padded)
noisy_x_rnn_tensor = to_rnn_tensor(noisy_x_idx_tensor)
batch_sz = len(x)
encoder_hidden = encoder.init_hidden(batch_size=batch_sz)
for i in range(noisy_x_rnn_tensor.shape[0]):
_, encoder_hidden = encoder(noisy_x_rnn_tensor[i].unsqueeze(0), encoder_hidden)
decoder_input = strings_to_tensor([SOS] * batch_sz)
decoder_hidden = encoder_hidden
names = [''] * batch_sz
for i in range(x_idx_tensor.shape[0]):
decoder_probs, decoder_hidden = decoder(decoder_input, decoder_hidden)
nonzero_indexes = x_idx_tensor[i]
best_indexes = torch.squeeze(torch.argmax(decoder_probs, dim=2), dim=0)
decoder_probs = torch.squeeze(decoder_probs, dim=0)
best_chars = list(map(lambda idx: index_to_char(int(idx)), best_indexes))
loss += criterion(decoder_probs, nonzero_indexes.type(torch.LongTensor))
for i, char in enumerate(best_chars):
names[i] += char
decoder_input = strings_to_tensor(best_chars)
loss.backward()
return names, noisy_x, loss.item()
I have this code for a denoising autoencoder on first names. It takes in a first name then pre-pads it with PAD char which I denote. I set it up like this so I could do batch training, but after training it for a couple days, I ran a name through it and it just printed out pads, which makes sense because it backprops on every iteration of a char so it’s being rewarded for just generating pads. How do I set it up not to learn this? I need the padding for batch training, because the names aren’t all consistent length so the padding allows them to be a consistent length. Currently right now after training if I put a name through it just generates pads.

Using multi-output labels in keras ImageDataGenerator.flow() and using model.fit_generator()

I have a single-input,multi-output Neural Network model whose last layers are
out1 = Dense(168, activation = 'softmax')(dense)
out2 = Dense(11, activation = 'softmax')(dense)
out3 = Dense(7, activation = 'softmax')(dense)
model = Model(inputs=inputs, outputs=[out1,out2,out3])
the Y-labels for each image are as follows
train
>>
image_id class_1 class_2 class_3
0 Train_0 15 9 5
1 Train_1 159 0 0
...
...
...
453651 Train_453651 0 15 34
453652 Train_453652 18 0 7
EDIT:-
train.iloc[:,1:4].nunique()
>>
class_1 168
class_2 11
class_3 7
dtype: int64
So looking at these different range of classes, should I use categorical_crossentropy or sparse_categorical_crossentropy? and how should I use the Y_labels in flow for the code given below?
imgs_arr = df.iloc[:,1:].values.reshape(df.shape[0],137,236,1)
# 32332 columns representing pixels of 137*236 and single channel images.
# converting it to (samples,w,h,c) format
Y = train.iloc[:,1:].values #need help from here
image_data_gen = ImageDataGenerator(validation_split=0.25)
train_gen = image_data_gen.flow(x=imgs_arr, y=Y, batch_size=32,subset='training')
valid_gen = image_data_gen.flow(x=imgs_arr,y=Y,subset='validation')
is this this the right way to pass Yor use Y=[y1,y2,y3] where
y1=train.iloc[:,1].values
y2=train.iloc[:,2].values
y3=train.iloc[:,3].values

Ouch....
By the message given in your flow, you will need a single output. So you need to make the separation inside your model. (Keras failed to follow its own standards there)
This means something like:
Y = train.iloc[:,1:].values #shape = (50210, 3)
With a single output like:
out = Dense(168+11+7, activation='linear')(dense)
And a loss function that handles the separation:
def custom_loss(y_true, y_pred):
true1 = y_true[:,0:1]
true2 = y_true[:,1:2]
true3 = y_true[:,2:3]
out1 = y_pred[:,0:168]
out2 = y_pred[:,168:168+11]
out3 = y_pred[:,168+11:]
out1 = K.softmax(out1, axis=-1)
out2 = K.softmax(out2, axis=-1)
out3 = K.softmax(out3, axis=-1)
loss1 = K.sparse_categorical_crossentropy(true1, out1, from_logits=False, axis=-1)
loss2 = K.sparse_categorical_crossentropy(true2, out2, from_logits=False, axis=-1)
loss3 = K.sparse_categorical_crossentropy(true3, out3, from_logits=False, axis=-1)
return loss1+loss2+loss3
Compile the model with loss=custom_loss.
Then the flow should stop complaining when you do flow.
Just make sure X and Y are exactly in the same order: imgs_arr[i] corresponds to Y[i] correctly.

Another workaround is:
Make an array of tuple, then pass it to the ImageDataGenerator flow method.
Make an iterator method that accepts the iterator made by the previous step. This iterator converts back the array of tuple to list of arrays.
Here is the methods to implement the steps above:
def make_array_of_tuple(tuple_of_arrays):
array_0 = tuple_of_arrays[0]
array_of_tuple = np.empty(array_0.shape[0], dtype=np.object)
for i, tuple_of_array_elements in enumerate(zip(*tuple_of_arrays)):
array_of_tuple[i] = tuple_of_array_elements
return array_of_tuple
def convert_to_list_of_arrays(array_of_tuple):
array_length = array_of_tuple.shape[0]
tuple_length = len(array_of_tuple[0])
array_list = [
np.empty(array_length, dtype=np.uint8) for i in range(tuple_length) ]
for i, array_element_tuple in enumerate(array_of_tuple):
for array, tuple_element in zip(array_list, array_element_tuple):
array[i] = tuple_element
return array_list
def tuple_of_arrays_flow(original_flow):
while True:
(X, array_of_tuple) = next(original_flow)
list_of_arrays = convert_to_list_of_arrays(array_of_tuple)
yield X, list_of_arrays
To call the ImageDataGenerator flow() method and get the flow used for the model:
y_train = make_array_of_tuple((y_train_1, y_train_2, y_train_3))
orig_image_flow = train_image_generator.flow(X_train, y=y_train)
train_image_flow = tuple_of_arrays_flow(orig_image_flow)
The size of y_train is the same as X_train, so it should be accepted.
'train_image_flow' returns list of arrays
that should be accepted by the Keras multi-output model.
ADDED (2019/01/26)
One another idea, simpler than the above one:
Pass array of indices, which contains 0, 1, 2, ..., to ImageDataGenerator.flow().
In the iterator, select the elements in the arrays for the multiple output by using the returned indices array from the original flow.
Here is the implementation:
def make_multi_output_flow(image_gen, X, y_list, batch_size):
y_item_0 = y_list[0]
y_indices = np.arange(y_item_0.shape[0])
orig_flow = image_gen.flow(X, y=y_indices, batch_size=batch_size)
while True:
(X, y_next_i) = next(orig_flow)
y_next = [ y_item[y_next_i] for y_item in y_list ]
yield X, y_next
This is an example to call the method above.
y_train = [y_train_1, y_train_2, y_train_3]
multi_output_flow = make_multi_output_flow(
image_data_generator, X_train, y_train, batch_size)