I am trying to transform a TensorFlow script into Keras. I have a classification model which predicts the presence of digits and 4 digits. I have two different losses and two outputs: one for the presence and one for the digits.
This is my model :
def get_training_model2():
inp, x = convolutional_layers2()
x = Flatten()(x)
x = Dense(2048)(x)
x = Activation('relu')(x)
y = Dense(1+5*len(DIGITS), activation='softmax')(x)
out1 = Lambda(lambda a: a[:,:1])(y)
out2 = Lambda(lambda a: a[:,1:])(y)
model = Model(inp, [out1, out2])
return model
This is my digit loss :
def digits_loss(y_true, y_pred):
digits_loss = losses.sparse_categorical_crossentropy(K.reshape(y_true, [-1, len(DIGITS)]),
K.reshape(y_pred, [-1, len(DIGITS)]))
digits_loss = np.sum(digits_loss)
return digits_loss
# Target
target = [y_train[:,:1], y_train[:,1:]]
When I launch :
model = model_keras.get_training_model2()
opt = Adam(lr=learn_rate)
model.compile(loss=[presence_loss, digits_loss],
optimizer=opt,
metrics=['accuracy'])
model.fit(X_train, target, batch_size=50,
epochs=50 ...)
I have this error :
File "train_keras.py", line 96, in digits_loss
K.reshape(y_pred, [-1, len(DIGITS)]))
logits and labels must have the same first dimension, got logits shape [250,11] and labels shape [2750]
I don't understand why I have this error because I am reshaping both labels and logits in the same way. Moreover, I don't understand why the shape is [250,11] although it's should be [50, 11] (batch_size).
Thank you for any help.
Related
Say I have a classification problem that has 30 potential binary labels. These labels are not mutually exclusive. The labels tend to be sparse--there is, on average, 1 positive label per all 30 labels but sometimes more than only 1. In the following code, how can I penalize the model from predicting all zeros? The accuracy will be high, but recall will be awful!
import numpy as np
from tensorflow.keras.layers import Input, Dense
from tensorflow.keras.models import Model
OUTPUT_NODES = 30
np.random.seed(0)
def get_dataset():
"""
Get a dataset of X and y. This is a learnable problem as there is some signal in the features. 10% of the time, a
positive-output's index will also have a positive feature for that index
:return: X and y data for training
"""
n_observations = 30000
y = np.random.rand(n_observations, OUTPUT_NODES)
y = (y <= (1 / OUTPUT_NODES)).astype(int) # Makes a sparse output where there is roughly 1 positive label: ((1 / OUTPUT_NODES) * OUTPUT_NODES ≈ 1)
X = np.zeros((n_observations, OUTPUT_NODES))
for i in range(len(y)):
for j, feature in enumerate(y[i]):
if feature == 1:
X[i][j] = 1 if np.random.rand(1) > 0.9 else 0 # Makes the input features more noisy
# X[i][j] = 1 # Using this instead will make the model perform very well
return X, y
def create_model():
input_layer = Input(shape=(OUTPUT_NODES, ))
dense1 = Dense(100, activation='relu')(input_layer)
dense2 = Dense(100, activation='relu')(dense1)
output_layer = Dense(30, activation='sigmoid')(dense2)
model = Model(inputs=input_layer, outputs=output_layer)
model.compile(optimizer='adam', loss='binary_crossentropy', metrics=['Recall'])
return model
def main():
X, y = get_dataset()
model = create_model()
model.fit(X, y, epochs=10, batch_size=10)
X_pred = np.random.randint(0, 2, (100, OUTPUT_NODES))
y_pred = model.predict(X_pred)
print(X_pred)
print(y_pred.round(1))
if __name__ == '__main__':
main()
I believe I read here that I could use:
weighted_cross_entropy_with_logits
to address this issue. How would that affect my final output layer's activation functions? Would I have to have an activation function? How do I specify a penalty to misclassifications of a true positive class?
Ok, it is an interesting problem
First you need to define a weighted cross entropy loss wrapper:
def wce_logits(positive_class_weight=1.):
def mylossw(y_true, logits):
cross_entropy = tf.reduce_mean(tf.nn.weighted_cross_entropy_with_logits(logits=logits, labels=tf.cast(y_true, dtype=tf.float32), pos_weight=positive_class_weight))
return cross_entropy
return mylossw
The positive_class_weight is applied to the positive class data. You need this wrapper for tf.nn.weighted_cross_entropy_with_logits to get a loss function that takes y_true and y_pred (only) as inputs.
Note that you must cast y_true to float32.
Second, you can not use the predefined Recall, because it does not work with logits. I found a workaround in this discussion
class Recall(tf.keras.metrics.Recall):
def __init__(self, from_logits=False, *args, **kwargs):
super().__init__(*args, **kwargs)
self._from_logits = from_logits
def update_state(self, y_true, y_pred, sample_weight=None):
if self._from_logits:
super(Recall, self).update_state(y_true, tf.nn.sigmoid(y_pred), sample_weight)
else:
super(Recall, self).update_state(y_true, y_pred, sample_weight)
Finally, you need to remove the sigmoid activation from the last layer as you are using logits
def create_model():
input_layer = Input(shape=(OUTPUT_NODES, ))
dense1 = Dense(100, activation='relu')(input_layer)
dense2 = Dense(100, activation='relu')(dense1)
output_layer = Dense(30)(dense2)
model = Model(inputs=input_layer, outputs=output_layer)
model.compile(optimizer='adam', loss=wce_logits(positive_class_weight=27.), metrics=[Recall(from_logits=True)])
return model
Note that the positive weight is set to 27 here. You can read a discussion on how to correctly calculate the weight
Hello I'm trying to understand the following thing:
I have created the following neural network model using PyTorch to run a regression task.
class Model(nn.Module):
def __init__(self, in_features, h1, h2, out_features=0):
super(Model, self).__init__()
self.fc1 = nn.Linear(in_features,h1) # input layer
self.fc2 = nn.Linear(h1, h2) # hidden layer
self.out = nn.Linear(h2, out_features) # output layer
def forward(self, x):
x = F.relu(self.fc1(x))
x = F.relu(self.fc2(x))
x = self.out(x)
return x
model = Model(in_features=59, h1=64, h2=32, out_features=1)
Then we get to the training where I run the following code:
epochs = 300
losses = []
for i in range(epochs):
y_pred = model(X_train)
loss = criterion(y_pred, y_train)
losses.append(loss.detach().numpy())
optimizer.zero_grad()
loss.backward()
optimizer.step()
Everything works fine but through the model's forward() method my y_pred gets the shape [1359, 1] (I guess it should be [1359] cause my y_train matches this shape and I get the following warning:
C:\Users\hp\anaconda3\lib\site-packages\torch\nn\modules\loss.py:528: UserWarning: Using a target size (torch.Size([1359])) that is different to the input size (torch.Size([1359, 1])). This will likely lead to incorrect results due to broadcasting. Please ensure they have the same size.
return F.mse_loss(input, target, reduction=self.reduction)
This also happens when I try to evaluate my model
with torch.no_grad():
y_val = model(X_test)
loss = criterion(y_val.flatten(), y_test)
print(loss)
Indeed you have a shape mismatch, your model will output a tensor of shape (batch_size, 1) while your target is shaped (batch_size,). You have to explicitly broadcast your tensor such that the inputs of your criterion have shape.
Either by reshaping the prediction y_val itself:
>>> loss = criterion(y_val[:,0], y_test)
Or the target tensor y_test:
>>> loss = criterion(y_val, y_test[:,None])
I'm trying to implement a network with keras and tensorflow back-end, I'm using transfer learning model (VGG16), my dataset is a medical images dataset so instead of having only one image, I have a series of slices, so my dataset is organized in a folder and each serie is a np.array() with size (nb_slices,512,512,3).
My dataset is composed by 1130 train samples and 120 valid samples, so I don't think that datas is the problem.
I tried to create a dataGenerator to load my image series in my model without a batch-size problem (I used this Training a Keras model from batches of .npy files using generator? to make my generator class) , (and I reshaped my volumes with size (nb_slices, 224,224,3))
then I tried to use transfer learning, and custom a VGG16 network with 1 more convolution layer, MaxPooling, Flatten, Dense, Dropout and final Dense layer.
When I start training, it seems there is no problem, but at a moment it returns IndexError: list index out of range, and I saw that DataGenerator iterates more than the size of dataset but I don't know why...
Which part could cause it ?
Here is my DataGenerator
INPUT_DIM = 224
MAX_PIXEL_VAL = 255
MEAN = 58.09
STDDEV = 49.73
class DataGenerator(keras.utils.Sequence):
def __init__(self, file_list, labels, data_loc):
self.listIDs = file_list
self.labels = labels
self.data_loc = data_loc
self.on_epoch_end()
def __len__(self):
return int(len(self.listIDs))
def __getitem__(self, index):
indexes = self.indexes[index:(index + 1)]
list_IDS_temp = [self.listIDs[k] for k in indexes]
X, y = self.__data_generation(list_IDS_temp)
return X, y
def on_epoch_end(self):
self.indexes = np.arange(len(self.listIDs))
def __data_generation(self, list_IDS_temp):
for ID in list_IDS_temp:
vol = np.load(self.data_loc + ID + '.npy')
nb_slices = vol.shape[0]
pad = int((vol.shape[2] - INPUT_DIM) / 2)
vol = vol[:, pad:-pad, pad:-pad]
# standardize
vol = (vol - np.min(vol)) / (np.max(vol) - np.min(vol)) * MAX_PIXEL_VAL
# normalize
vol = (vol - MEAN) / STDDEV
# convert to RGB
vol = np.stack((vol,) * 3, axis=3)
y = np.empty(nb_slices, dtype=int)
# y = self.labels[ID]
for i in range(nb_slices):
y[i] = self.labels[int(ID)]
return vol, keras.utils.to_categorical(y, num_classes=2)
and my model:
train_set = DataGenerator(df_train['exams'].tolist(), df_train['labels'].tolist(), all_file_loc_train)
valid_set = DataGenerator(df_val['exams'].tolist(), df_val['labels'].tolist(), all_file_loc_val)
model = VGG16(weights='imagenet', include_top=False, input_shape=(224, 224, 3)) # , input_shape=(224, 224, 3)
layer_dict = dict([(layer.name, layer) for layer in model.layers])
x = layer_dict['block2_pool'].output
x = Conv2D(filters=64, kernel_size=(3, 3), activation='relu')(x)
x = MaxPooling2D(pool_size=(2, 2))(x)
x = Flatten()(x)
x = Dense(256, activation='relu')(x)
x = Dropout(0.4)(x)
x = Dense(2, activation='softmax')(x)
custom_model = Model(inputs=model.input, outputs=x)
for layer in custom_model.layers[:7]:
layer.trainable = False
custom_model.compile(loss='categorical_crossentropy', optimizer=optimizers.SGD(lr=0.0001, momentum=0.9),
metrics=["accuracy"])
results = custom_model.fit_generator(generator=train_set, validation_data=valid_set, epochs=50, verbose=2)
I expected accuracy around 80-90% of accuracy but it seems that something goes wrong in my DataGenerator and I don't know what. Please I need help...
I want to split my training samples of size [batchSize, 2, 16] into 16 tensors of size [batchSize, 2] and input them to the same model. How can I accomplish them in keras?
I have first implemented this in following way,
def functionA(x_Config):
y = layers.Input(shape=(2,16,))
hidden1 = 5
hidden2 = 10
x_input = layers.Input(shape=(2,))
hidden_layer1 = Dense(hidden1, activation='relu')(x_input)
hidden_layer2 = Dense(hidden2, activation='relu')(hidden_layer1)
x_output = Dense(x_Config.m, activation='linear')(hidden_layer2)
model_x= Model(inputs=x_input, outputs=x_output)
for i in range(16):
x_input = Lambda(lambda x: x[:, :, i])(y)
if i == 0:
x_output = model_x(x_input)
else:
x_output = layers.concatenate([x_output,
model_x(x_input)])
x_output = Lambda(lambda x: x[:, :tf.cast(N, tf.int32)])(x_output)
final_model = Model(y, x_output)
return final_model
PS: I have a trained model for the same NN architecture for model_x for [batchSize, 2] inputs (without the need for splitting). This model performs very well. When I tried to load the weights of that model to above mentioned code, model_x, it does not give good performance at all and does not train well.
So I believe my problems lies inside the loop in the Lamda layer. How can I use tf.split or tf.slice for this?
you can use tf.unstack function:
tensors = tf.unstack(data, axis=2)
i have an error:
ValueError: Error when checking model target: expected dense_4 to have shape (None, 2) but got array with shape (12956, 1)
When i run this script.
def image_text_model(image_features, text_features, n_classes):
# fine-tune the last layer
image_features = Input(shape=image_features.shape[1:], dtype='float32')
n_text_features = text_features.shape[1]
text_features = Input(shape=text_features.shape[1:], dtype='float32')
# text model
x_text = Dense(256, activation='elu', kernel_regularizer=l2(1e-5))(text_features)
x_text = Dropout(0.5)(x_text)
# image model
x_img = Dense(256, activation='elu')(image_features)
x_img = Dropout(0.5)(x_img)
x_img = Dense(256, activation='elu')(x_img)
x_img = Dropout(0.5)(x_img)
merged = concatenate([x_img, x_text])
predictions = Dense(n_classes, activation='softmax')(merged)
model = Model(inputs=[image_features, text_features], outputs=[predictions])
model.compile(optimizer='adam',
loss='categorical_crossentropy',
metrics=['accuracy'])
return model
# dev
df = pd.read_csv(os.path.join(data_dir, 'amazon_products_dev.csv'))
dev_image_list = df['image_file'].values
dev_text = df['title'].values.tolist()
dev_categories = df['product_category'].values
# encode labels (binary labels)
encoder = LabelBinarizer()
train_labels = encoder.fit_transform(train_categories)
dev_labels = encoder.transform(dev_categories)
# get features from a pre-trained resnet model
vec = ResNetVectorizer(batch_size=500,
image_dir=image_dir,
use_cache=True,
cache_dir=cache_dir)
train_image_features = vec.transform(train_image_list)
dev_image_features = vec.transform(dev_image_list)
# get text features
tfidf = TfidfVectorizer(ngram_range=(1,1), stop_words='english', max_features=5000)
train_text_features = tfidf.fit_transform(train_text)
dev_text_features = tfidf.transform(dev_text).toarray()
# fine-tune the last layer
n_classes = encoder.classes_.shape[0]
model = image_text_model(train_image_features, train_text_features, n_classes)
data_gen = sparse_batch_generator(train_image_features, train_text_features, train_labels, shuffle=True)
steps_per_epoch = int(np.ceil(train_image_features.shape[0]/32.))
model.fit_generator(data_gen,
steps_per_epoch=steps_per_epoch,
epochs=50,
validation_data=[[dev_image_features, dev_text_features], dev_labels])
I See this topic : ValueError: Error when checking model target: expected dense_4 to have shape (None, 4) but got array with shape (13252, 1)
But i don't know how can use it into my script.
Thank you in advance for your answer.
Currently you must only have two classes as you output is expecting (None, 2). However, when working with two classes, you matrix structure can either be
[[0,1],
[1,0],
[1,0]]
or
[[0],
[1],
[1]]
Sklearns LabelBinarizer converts a matrix with two classes into a column of zeros and one's. 0 for class one and 1 for class two. So your output layer should just be
predictions = Dense(1, activation='sigmoid')(merged)