Related
Images that I would like to use to train the network are about the size of 4000px*3000px and about 40k of them, sorted in 250 classes.
I have made a CNN shown below:
model = keras.Sequential([
layers.Input((imgHeight, imgWidth, 1)),
layers.Conv2D(16, 3, padding = 'same'), # filters, kernel_size
layers.Conv2D(32, 3, padding = 'same'),
layers.MaxPooling2D(),
layers.Flatten(),
layers.Dense(250),
])
How do I figure out what layers.Conv2D(*16*, ...), a value I need?
How do I figure out what layers.Dense(*250*), a value I need?
Because I can't start the training process, I'm running out of memory.
The output shape of the Flatten() layer is 96 Million, and so the final dense layer of your model has 24 Billion parameters, this is why you are running out of memory. There are some steps you can take to fix this issue:
Try resizing your images to a smaller shape, if 4000x3000x1 isn't necessary, 160x160x1 would be a good choice.
Try using more Conv2D layers followed by a MaxPool2D layer to decrease the size of the input, and then finally at the end, use a Flatten layer followed by a Dense layer.
For example:
model = keras.Sequential([
layers.Input((160, 160, 1)),
layers.Conv2D(32, 3, padding = 'same'),
layers.Conv2D(32, 3, padding = 'same'),
layers.MaxPooling2D((2,2)),
layers.Conv2D(64, 3, padding = 'same'),
layers.Conv2D(64, 3, padding = 'same'),
layers.MaxPooling2D((2,2)),
layers.Conv2D(128, 3, padding = 'same'),
layers.Conv2D(128, 3, padding = 'same'),
layers.MaxPooling2D((2,2)),
layers.Conv2D(256, 3, padding = 'same'),
layers.Conv2D(256, 3, padding = 'same'),
layers.MaxPooling2D((2,2)),
layers.Flatten(),
layers.Dense(512),
layers.Dense(250),
])
This type of architecture will work well if you are doing a classification task, and will not run out of memory.
I am trying to online-train a neural network. I want to use the Tensorflow Keras train_on_batch function on a convolutional neural network. Here it is:
look_back=1600
inputTensor = keras.layers.Input([look_back+3,2])
inputTensorReshaped = tf.reshape(inputTensor, [1, look_back + 3, 2, 1])
#split into 2 groups
inputgroup1 = keras.layers.Lambda(lambda x: x[:, :3], output_shape=((1, 3, 2, 1)))(inputTensorReshaped)
inputgroup2 = keras.layers.Lambda(lambda x: x[:, 3:look_back + 3], output_shape=((1, look_back,2, 1)))(inputTensorReshaped)
conv1 = keras.layers.Conv2D(filters=1024, kernel_size=(10, 2), activation='relu')(inputgroup2)#10
pool1 = keras.layers.MaxPooling2D(pool_size=(2, 1))(conv1)
dropout1 = keras.layers.Dropout(rate=0.1)(pool1)
norm1 = keras.layers.LayerNormalization()(dropout1)
conv2 = keras.layers.Conv2D(filters=512, kernel_size=(8, 1), activation='relu')(norm1)
pool2 = keras.layers.MaxPooling2D(pool_size=(2, 1))(conv2)
dropout2 = keras.layers.Dropout(rate=0.1)(pool2)
norm2 = keras.layers.LayerNormalization()(dropout2)
conv3 = keras.layers.Conv2D(filters=256, kernel_size=(6, 1), activation='relu')(norm2)
pool3 = keras.layers.MaxPooling2D(pool_size=(2, 1))(conv3)
dropout3 = keras.layers.Dropout(rate=0.1)(pool3)
norm3 = keras.layers.LayerNormalization()(dropout3)
conv4 = keras.layers.Conv2D(filters=128, kernel_size=(4, 1), activation='relu')(norm3)
pool4 = keras.layers.MaxPooling2D(pool_size=(2, 1))(conv4)
dropout4 = keras.layers.Dropout(rate=0.1)(pool4)
norm4 = keras.layers.LayerNormalization()(dropout4)
conv5 = keras.layers.Conv2D(filters=64, kernel_size=(2, 1), activation='relu')(norm4)
pool5 = keras.layers.MaxPooling2D(pool_size=(2, 1))(conv5)
dropout5 = keras.layers.Dropout(rate=0.1)(pool5)
norm5 = keras.layers.LayerNormalization()(dropout5)
flatten1 = keras.layers.Flatten()(norm5)
dense1 = keras.layers.Dense(32, activation='relu')(flatten1)
misclayer1 = keras.layers.Dense(32, activation='relu')(inputgroup1)
miscdropout1 = keras.layers.Dropout(rate=0.1)(misclayer1)
miscnorm1 = keras.layers.LayerNormalization()(miscdropout1)
misclayer2 = keras.layers.Dense(128, activation='relu')(miscnorm1)
miscdropout2 = keras.layers.Dropout(rate=0.1)(misclayer2)
miscnorm2 = keras.layers.LayerNormalization()(miscdropout2)
misclayer3 = keras.layers.Dense(32, activation='relu')(miscnorm2)
miscdropout3 = keras.layers.Dropout(rate=0.1)(misclayer3)
miscnorm3 = keras.layers.LayerNormalization()(miscdropout3)
miscflatten1 = keras.layers.Flatten()(miscnorm3)
misclayer4 = keras.layers.Dense(32, activation='relu')(miscflatten1)
rejoinlayer = keras.layers.Concatenate()([dense1, misclayer4])
processing1 = keras.layers.Dense(64, activation='relu')(rejoinlayer)
totalnorm1 = keras.layers.LayerNormalization()(processing1)
processing2 = keras.layers.Dense(32, activation='relu')(totalnorm1)
totaldropout1 = keras.layers.Dropout(rate=0.2)(processing2)
processing3 = keras.layers.Dense(16, activation='relu')(totaldropout1)
totalnorm2 = keras.layers.LayerNormalization()(processing3)
processing4 = keras.layers.Dense(8, activation='relu')(totalnorm2)
totaldropout2 = keras.layers.Dropout(rate=0.2)(processing4)
processing5 = keras.layers.Dense(4, activation='relu')(totaldropout2)
output = keras.layers.Dense(1, activation='linear')(processing5)
model = keras.Model(inputTensor,output)
model.compile(optimizer=keras.optimizers.SGD(learning_rate=0.00005, momentum=0.1, nesterov=True), loss="mean_squared_error")
#trains the model with the 1st state, action, and value
def train():
global qtable
x = []
y = []
for i in range(0, 8):
state = qtable.loc[qtable.index[i], "state"]
action = [qtable.loc[qtable.index[i], "action"], qtable.loc[qtable.index[0], "action"]]
x.append([action])
x[i].extend(state)
y.append([qtable.loc[qtable.index[i], "value"]])
print("training...loss:")
with tf.device('/gpu:0'):
print(model.train_on_batch(np.nan_to_num(np.array(x)), np.nan_to_num(np.array(y))))
In this case the variable "state" would be a 1202-by-2 list [[a,b],[c,d],[e,f],...] and the variable "action" would be a 1-by-2 list [a,b] before being appended/extended to x. In theory, the training I want is a batch size of 8 with a 1203-by-2 input shape. However, I get this error:
ValueError: Cannot reshape a tensor with 19248 elements to shape [1,1203,2,1] (2406 elements) for '{{node model/tf.reshape/Reshape}} = Reshape[T=DT_FLOAT, Tshape=DT_INT32](IteratorGetNext, model/tf.reshape/Reshape/shape)' with input shapes: [8,1203,2], [4] and with input tensors computed as partial shapes: input[1] = [1,1203,2,1].
This shows that all the inputs and outputs are being put into the CNN at once which is not what I want. Instead, I want the data to be in a batch of 8. How can I do this??? Am I even using "train_on_batch" correctly
batch_size:: Integer or None. Number of samples per batch of
computation. If unspecified, batch_size will default to 32. Do not
specify the batch_size if your data is in the form of a dataset,
generators, or keras.utils.Sequence instances (since they generate
batches).
Find the below example with batch_size
num_classes = 5
model = Sequential([
layers.experimental.preprocessing.Rescaling(1./255, input_shape=(img_height, img_width, 3)),
layers.Conv2D(16, 3, padding='same', activation='relu'),
layers.MaxPooling2D(),
layers.Conv2D(32, 3, padding='same', activation='relu'),
layers.MaxPooling2D(),
layers.Conv2D(64, 3, padding='same', activation='relu'),
layers.MaxPooling2D(),
layers.Flatten(),
layers.Dense(128, activation='relu'),
layers.Dense(num_classes)
])
model.compile(optimizer='adam',
loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),
metrics=['accuracy'])
history = model.fit(
train_ds,
validation_data=val_ds,
batch_size = 32
epochs=10
)
I'm trying to separate close objects as was shown in the U-Net paper (here). For this, one generates weight maps which can be used for pixel-wise losses. The following code describes the network I use from this blog post.
x_train_val = # list of images (imgs, 256, 256, 3)
y_train_val = # list of masks (imgs, 256, 256, 1)
y_weights = # list of weight maps (imgs, 256, 256, 1) according to the blog post
# visual inspection confirms the correct calculation of these maps
# Blog posts' loss function
def my_loss(target, output):
return - tf.reduce_sum(target * output,
len(output.get_shape()) - 1)
# Standard Unet model from blog post
_epsilon = tf.convert_to_tensor(K.epsilon(), np.float32)
def make_weighted_loss_unet(input_shape, n_classes):
ip = L.Input(shape=input_shape)
weight_ip = L.Input(shape=input_shape[:2] + (n_classes,))
conv1 = L.Conv2D(64, 3, activation='relu', padding='same', kernel_initializer='he_normal')(ip)
conv1 = L.Conv2D(64, 3, activation='relu', padding='same', kernel_initializer='he_normal')(conv1)
conv1 = L.Dropout(0.1)(conv1)
mpool1 = L.MaxPool2D()(conv1)
conv2 = L.Conv2D(128, 3, activation='relu', padding='same', kernel_initializer='he_normal')(mpool1)
conv2 = L.Conv2D(128, 3, activation='relu', padding='same', kernel_initializer='he_normal')(conv2)
conv2 = L.Dropout(0.2)(conv2)
mpool2 = L.MaxPool2D()(conv2)
conv3 = L.Conv2D(256, 3, activation='relu', padding='same', kernel_initializer='he_normal')(mpool2)
conv3 = L.Conv2D(256, 3, activation='relu', padding='same', kernel_initializer='he_normal')(conv3)
conv3 = L.Dropout(0.3)(conv3)
mpool3 = L.MaxPool2D()(conv3)
conv4 = L.Conv2D(512, 3, activation='relu', padding='same', kernel_initializer='he_normal')(mpool3)
conv4 = L.Conv2D(512, 3, activation='relu', padding='same', kernel_initializer='he_normal')(conv4)
conv4 = L.Dropout(0.4)(conv4)
mpool4 = L.MaxPool2D()(conv4)
conv5 = L.Conv2D(1024, 3, activation='relu', padding='same', kernel_initializer='he_normal')(mpool4)
conv5 = L.Conv2D(1024, 3, activation='relu', padding='same', kernel_initializer='he_normal')(conv5)
conv5 = L.Dropout(0.5)(conv5)
up6 = L.Conv2DTranspose(512, 2, strides=2, kernel_initializer='he_normal', padding='same')(conv5)
conv6 = L.Concatenate()([up6, conv4])
conv6 = L.Conv2D(512, 3, activation='relu', padding='same', kernel_initializer='he_normal')(conv6)
conv6 = L.Conv2D(512, 3, activation='relu', padding='same', kernel_initializer='he_normal')(conv6)
conv6 = L.Dropout(0.4)(conv6)
up7 = L.Conv2DTranspose(256, 2, strides=2, kernel_initializer='he_normal', padding='same')(conv6)
conv7 = L.Concatenate()([up7, conv3])
conv7 = L.Conv2D(256, 3, activation='relu', padding='same', kernel_initializer='he_normal')(conv7)
conv7 = L.Conv2D(256, 3, activation='relu', padding='same', kernel_initializer='he_normal')(conv7)
conv7 = L.Dropout(0.3)(conv7)
up8 = L.Conv2DTranspose(128, 2, strides=2, kernel_initializer='he_normal', padding='same')(conv7)
conv8 = L.Concatenate()([up8, conv2])
conv8 = L.Conv2D(128, 3, activation='relu', padding='same', kernel_initializer='he_normal')(conv8)
conv8 = L.Conv2D(128, 3, activation='relu', padding='same', kernel_initializer='he_normal')(conv8)
conv8 = L.Dropout(0.2)(conv8)
up9 = L.Conv2DTranspose(64, 2, strides=2, kernel_initializer='he_normal', padding='same')(conv8)
conv9 = L.Concatenate()([up9, conv1])
conv9 = L.Conv2D(64, 3, activation='relu', padding='same', kernel_initializer='he_normal')(conv9)
conv9 = L.Conv2D(64, 3, activation='relu', padding='same', kernel_initializer='he_normal')(conv9)
conv9 = L.Dropout(0.1)(conv9)
c10 = L.Conv2D(n_classes, 1, activation='softmax', kernel_initializer='he_normal')(conv9)
# Mimic crossentropy loss
c11 = L.Lambda(lambda x: x / tf.reduce_sum(x, len(x.get_shape()) - 1, True))(c10)
c11 = L.Lambda(lambda x: tf.clip_by_value(x, _epsilon, 1. - _epsilon))(c11)
c11 = L.Lambda(lambda x: K.log(x))(c11)
weighted_sm = L.multiply([c11, weight_ip])
model = Model(inputs=[ip, weight_ip], outputs=[weighted_sm])
return model
I then compile and fit the model as is shown below:
model = make_weighted_loss_unet((256, 256, 3), 1) # shape of input, number of classes
model.compile(optimizer='adam', loss=my_loss, metrics=['acc'])
model.fit([x_train_val, y_weights], y_train_val, validation_split=0.1, epochs=1)
The model can then train as usual. However, the loss doesn't seem to improve much. Furthermore, when I try to predict on new images, I obviously don't have the weight maps (because they are calculated on the labeled masks). I tried to use empty / zero arrays shaped like the weight map but that only yields in blank / zero predictions. I also tried different metrics and more standards losses without any success.
Did anyone face the same issue or have an alternative in implementing this weighted loss? Thanks in advance. BBQuercus
A simpler way to write custom loss with pixel weights
In your code, the loss is scattered around, between my_loss and make_weighted_loss_unet functions. You can add targets as an input and use model.add_loss to structure the code better :
def make_weighted_loss_unet(input_shape, n_classes):
ip = L.Input(shape=input_shape)
weight_ip = L.Input(shape=input_shape[:2] + (n_classes,))
targets = L.input(shape=input_shape[:2] + (n_classes,))
# .... rest of your model definition code ...
c10 = L.Conv2D(n_classes, 1, activation='softmax', kernel_initializer='he_normal')(conv9)
model.add_loss(pixel_weighted_cross_entropy(weights_ip, targets, c10))
# .... return Model .... NO NEED to specify loss in model.compile
def pixel_weighted_cross_entropy(weights, targets, predictions)
loss_val = keras.losses.categorical_crossentropy(targets, predictions)
weighted_loss_val = weights * loss_val
return K.mean(weighted_loss_val)
If you don't refactor your code to the above approach, next section shows how to still run inference without issues
How to run your model in inference
Option 1 : Use another Model object for inference
You can create a Model used for training and another used for inference. Both are largely the same except that the inference Model does not take weights_ip, and gives an early output c10.
Here's an example code that adds an argument is_training=True to decide which Model to return :
def make_weighted_loss_unet(input_shape, n_classes, is_training=True):
ip = L.Input(shape=input_shape)
conv1 = L.Conv2D(64, 3, activation='relu', padding='same', kernel_initializer='he_normal')(ip)
# .... rest of your model definition code ...
c10 = L.Conv2D(n_classes, 1, activation='softmax', kernel_initializer='he_normal')(conv9)
if is_training:
# Mimic crossentropy loss
c11 = L.Lambda(lambda x: x / tf.reduce_sum(x, len(x.get_shape()) - 1, True))(c10)
c11 = L.Lambda(lambda x: tf.clip_by_value(x, _epsilon, 1. - _epsilon))(c11)
c11 = L.Lambda(lambda x: K.log(x))(c11)
weight_ip = L.Input(shape=input_shape[:2] + (n_classes,))
weighted_sm = L.multiply([c11, weight_ip])
return Model(inputs=[ip, weight_ip], outputs=[weighted_sm])
else:
return Model(inputs=[ip], outputs=[c10])
return model
Option 2 : Use K.function
If you don't want to mess with your Model definition method (make_weighted_loss_unet) and want to achieve the same result outside, you can use a function that extracts the subgraph relevant for inference.
In your inference function:
from keras import backend as K
model = make_weighted_loss_unet(input_shape, n_classes)
inference_function = K.function([model.get_layer("input_layer").input],
[model.get_layer("output_softmax_layer").output])
predicted_heatmap = inference_function(new_image)
Note that you'll have to give name= to your ip layer and c10 layer to be able to retrieve them via model.get_layer(name) :
ip = L.Input(shape=input_shape, name="input_layer")
and
c10 = L.Conv2D(n_classes, 1, activation='softmax', kernel_initializer='he_normal', name="output_softmax_layer")(conv9)
My input is 299,299,3
My graphics card is 1070 (8 gigs of ram)
Other Specs: Python 3.6, Keras 2.xx, Tensorflow-backend(1.4), Windows 7
Even batch size of 1 isn't working.
I feel like my card should handle a batch of size one --
Here is my code:
def full_model():
#model layers
input_img = Input(shape=(299, 299, 3))
tower_1 = Conv2D(64, (1, 1,), padding='same', activation='relu')(input_img)
tower_1 = Conv2D(64, (3, 3), padding='same', activation='relu')(tower_1)
tower_2 = Conv2D(64, (1, 1), padding='same', activation='relu')(input_img)
tower_2 = Conv2D(64, (5, 5), padding='same', activation='relu')(tower_2)
concatenated_layer = keras.layers.concatenate([tower_1, tower_2], axis=3)
bottleneck = MaxPooling2D((2, 2), strides=(2, 2), padding='same')(concatenated_layer)
flatten = Flatten()(bottleneck)
dense_1 = Dense(500, activation = 'relu')(flatten)
predictions = Dense(12, activation = 'softmax')(dense_1)
model = Model(inputs= input_img, output = predictions)
SGD =keras.optimizers.SGD(lr=0.1, momentum=0.0, decay=0.0, nesterov=False)
model.compile(optimizer=SGD,
loss='categorical_crossentropy',
metrics=['accuracy'])
return model
hdf5_path =r'C:\Users\Moondra\Desktop\Keras Applications\training.hdf5'
model = full_model()
def run_model( hdf5_path,
epochs = 10,
steps_per_epoch =8,
classes =12,
batch_size =1, model= model ):
for i in range(epochs):
batches = loading_hdf5_files.load_batches(batch_size =1,
hdf5_path=hdf5_path ,
classes = classes)
for i in range(steps_per_epoch):
x,y = next(batches)
#plt.imshow(x[0])
#plt.show()
x = (x/255).astype('float32')
print(x.shape)
data =model.train_on_batch(x,y)
print('loss : {:.5}, accuracy : {:.2%}'.format(*data))
return model
I can't seem to handle even a batch of size one.
Here is the last part of the error:
ResourceExhaustedError (see above for traceback): OOM when allocating tensor of shape [] and type float
[[Node: conv2d_4/random_uniform/sub = Const[dtype=DT_FLOAT, value=Tensor<type: float shape: [] values: 0.0866025388>, _device="/job:localhost/replica:0/task:0/device:GPU:0"]()]]
It turns out I have way too many parameters.
After running print(model.summary()), I had a billion plus parameters.
I increased size of MaxPooling and no more problems.
I have Keras model that accepts inputs which have 4D shapes as (n, height, width, channel).
However, my data generator is producing 2D arrays as(n, width*height). So, the predict function of Keras is expecting inputs as 4D. I have no chance to change the data generator because the model will be tested by someone else. So, is there a way to override the predict function of Keras.
My model structure
a = Input(shape=(width*height,))
d1 = 16 # depth of filter kernel each layer
d2 = 16
d3 = 64
d4 = 128
d5 = 256
drop_out = 0.25
patch_size = (3, 3)
k_size = (2, 2)
reshape = Reshape((height, width, 1))(a)
conv1 = Conv2D(filters=d1, kernel_size=patch_size, padding='same', activation='relu')(reshape)
conv1 = MaxPooling2D(pool_size=k_size, padding='same')(conv1)
conv2 = Convolution2D(filters=d2, kernel_size=patch_size, padding='same', activation='relu')(conv1)
conv2 = MaxPooling2D(pool_size=k_size, padding='same')(conv2)
conv3 = Convolution2D(filters=d3, kernel_size=patch_size, padding='same', activation='relu')(conv2)
conv3 = MaxPooling2D(pool_size=k_size, padding='same')(conv3)
conv4 = Convolution2D(filters=d4, kernel_size=patch_size, padding='same', activation='relu')(conv3)
conv4 = MaxPooling2D(pool_size=k_size, padding='same')(conv4)
conv5 = Convolution2D(filters=d5, kernel_size=patch_size, padding='same', activation='relu')(conv4)
conv5 = MaxPooling2D(pool_size=k_size, padding='same')(conv5)
x = Flatten()(conv5)
x = Dropout(drop_out)(x)
node = 32
x_1 = Dense(node, activation='relu')(x) # connect the flatten layer to five classifier,each one comes to a digit.
x_2 = Dense(node, activation='relu')(x)
x_3 = Dense(node, activation='relu')(x)
x_4 = Dense(node, activation='relu')(x)
x_5 = Dense(node, activation='relu')(x)
d1 = Dense(n_class, activation='softmax')(x_1)
d2 = Dense(n_class, activation='softmax')(x_2)
d3 = Dense(n_class, activation='softmax')(x_3)
d4 = Dense(n_class, activation='softmax')(x_4)
d5 = Dense(n_class, activation='softmax')(x_5)
outputs = [d1, d2, d3, d4, d5]
model = Model(a, outputs)
model.compile(loss='categorical_crossentropy', optimizer='adadelta', metrics=['accuracy'])
model.fit(raw_train_data, raw_train_target, batch_size=200, epochs=5, validation_split=0.2)
You don't override the predict, you simply add a Reshape layer at the beginning of your model.
With the functional API:
from keras.layers import *
inp = Input((width*heigth,))
first = Reshape((width,height,1))(inp)
..... other layers.....
model = Model(inp, outputFromTheLastLayer)
With a sequential model:
model = Sequential()
model.add(Reshape((width,height,1), input_shape = (width*height,)))
model.add(otherlayers)
About the output shape.
Since you have 5 outputs, you need your target array to be a list of five arrays:
raw_train_target = [target1,target2,target3,target4,target5]
If you cannot do that, and raw_train_target is one single arary with the targets all following a sequence, you can try to use a concatenate layer at the end:
output = Concatenate()(outputs)