I am using Python 3.X and TensorFlow 2.0 along with "tensorflow_model_optimization" package for neural network pruning. The code I have is as follows-
from tensorflow_model_optimization.sparsity import keras as sparsity
l = tf.keras.layers
# Original model without pruning-
model = Sequential()
model.add(l.InputLayer(input_shape = (784, )))
model.add(Flatten())
model.add(Dense(units = 300, activation='relu', kernel_initializer = tf.initializers.GlorotUniform()))
model.add(l.Dropout(0.2))
model.add(Dense(units = 100, activation='relu', kernel_initializer = tf.initializers.GlorotUniform()))
model.add(l.Dropout(0.1))
model.add(Dense(units = num_classes, activation='softmax'))
# Define callbacks-
callbacks = [
# tf.keras.callbacks.TensorBoard(log_dir=logdir, profile_batch = 0),
tf.keras.callbacks.EarlyStopping(monitor='val_loss', patience = 3)
]
# Compile designed Neural Network-
model.compile(
loss = tf.keras.losses.categorical_crossentropy,
optimizer = 'adam',
metrics = ['accuracy'])
# Save untrained and initial weights to disk-
model.save_weights("Initial_non_trained_weights.h5")
epochs = 12
num_train_samples = X_train.shape[0]
end_step = np.ceil(1.0 * num_train_samples / batch_size).astype(np.int32) * epochs
print("end_step parameter for this dataset = {0}".format(end_step))
# end_step = 5628
# Specify the parameters to be used for layer-wise pruning:
pruning_params = {
'pruning_schedule': sparsity.PolynomialDecay(
initial_sparsity=0.50, final_sparsity=0.90,
begin_step=2000, end_step=end_step, frequency=100)
}
# Neural network which is to be pruned-
pruned_model = Sequential()
pruned_model.add(l.InputLayer(input_shape=(784, )))
pruned_model.add(Flatten())
pruned_model.add(sparsity.prune_low_magnitude(Dense(units = 300, activation='relu', kernel_initializer=tf.initializers.GlorotUniform()),
**pruning_params))
pruned_model.add(l.Dropout(0.2))
pruned_model.add(sparsity.prune_low_magnitude(Dense(units = 100, activation='relu', kernel_initializer=tf.initializers.GlorotUniform()),
**pruning_params))
pruned_model.add(l.Dropout(0.1))
pruned_model.add(sparsity.prune_low_magnitude(Dense(units = num_classes, activation='softmax'), **pruning_params))
# Compile pruned CNN-
pruned_model.compile(
loss=tf.keras.losses.categorical_crossentropy,
optimizer='adam',
metrics=['accuracy'])
# Load weights from before-
pruned_model.load_weights("Initial_non_trained_weights.h5")
This last line of loading initial weights into the pruned model gives me error:
ValueError: Layer #0 (named "prune_low_magnitude_dense_9" in the current model) was found to correspond to layer dense in the save file.
However the new layer prune_low_magnitude_dense_9 expects 5 weights, but the saved weights have 2 elements.
What's going wrong?
Thanks!
Related
I have a large csv file that I want to train a tensorflow model on, so I am creating a PrefetchDataset object to read from the csv file in epochs.
dataset = tf.data.experimental.make_csv_dataset("test.csv", batch_size = 2, label_name="next_movement", shuffle = False, num_epochs=1)
But when I train to train the model
X_dim = pd.read_csv("test.csv", nrows = 1).shape[1]
optimizer = Adam(learning_rate = 1e-4)
model = Sequential()
model.add(Dense(128, activation="relu", kernel_regularizer=regularizers.l2(.0001),
input_shape=(X_dim,)))
#model.add(layers.Dropout(.5))
model.add(Dense(64, activation="relu", kernel_regularizer=regularizers.l2(.0001)))
#model.add(layers.Dropout(.5))
model.add(Dense(32, activation="relu",kernel_regularizer=regularizers.l2(.0001)))
#model.add(layers.Dropout(.5))
model.add(Dense(8, activation = "relu", kernel_regularizer=regularizers.l2(.0001)))
#model.add(layers.Dropout(.5))
model.add(Dense(1, activation="sigmoid", kernel_regularizer=regularizers.l2(.0001)))
#model = keras.Model(inputs, model(x))
model.compile(optimizer=optimizer,
loss="binary_crossentropy",
metrics = ['accuracy'])
model.fit(dataset)
I get this error
ValueError: Missing data for input "dense_7_input". You passed a data dictionary with keys ['', 'bid_prc1', 'bid_vol1', 'ask_prc1', 'ask_vol1', 'bid_prc2'... ] Expected the following keys: ['dense_7_input']
I am new to machine learning and I am trying to make a semantic segmentation model for detecting couches in images. However, my validation loss and training loss are stuck at around 0.25, and my validation binary IoU and training binary IoU are stuck at around 0.7. Here is the code for my model and my training parameters.
def upsample(filters, size, norm_type='batchnorm', apply_dropout=False, kernel_regularizer=None):
"""Upsamples an input.
Conv2DTranspose => Batchnorm => Dropout => Relu
Args:
filters: number of filters
size: filter size
norm_type: Normalization type; either 'batchnorm' or 'instancenorm'.
apply_dropout: If True, adds the dropout layer
Returns:
Upsample Sequential Model
"""
initializer = tf.random_normal_initializer(0., 0.02)
result = tf.keras.Sequential()
result.add(
tf.keras.layers.Conv2DTranspose(filters, size, strides=2,
padding='same',
kernel_initializer=initializer,
kernel_regularizer=kernel_regularizer,
use_bias=False))
if norm_type.lower() == 'batchnorm':
result.add(tf.keras.layers.BatchNormalization())
elif norm_type.lower() == 'instancenorm':
result.add(InstanceNormalization())
if apply_dropout:
result.add(tf.keras.layers.Dropout(0.3))
result.add(tf.keras.layers.ReLU())
return result
class Unet:
output_channels = 1
def __init__(self):
base_model = tf.keras.applications.MobileNetV2(input_shape=[224, 224, 3], include_top=False)
# Use the activations of these layers
layer_names = [
'block_1_expand_relu',
'block_3_expand_relu',
'block_6_expand_relu',
'block_13_expand_relu',
'block_16_project',
]
base_model_outputs = [base_model.get_layer(name).output for name in layer_names]
# Create the feature extraction model
down_stack = tf.keras.Model(inputs=base_model.input, outputs=base_model_outputs)
down_stack.trainable = False
up_stack = [
upsample(1024, 3, norm_type='batchnorm', apply_dropout=True, kernel_regularizer="l2"),
upsample(512, 3, norm_type='batchnorm', apply_dropout=True, kernel_regularizer="l2"),
upsample(256, 3, norm_type='batchnorm', apply_dropout=True, kernel_regularizer="l2"),
upsample(128, 3, norm_type='batchnorm', apply_dropout=False, kernel_regularizer="l2"),
upsample(64, 3, norm_type='batchnorm', apply_dropout=True, kernel_regularizer="l2"),
]
inputs = tf.keras.layers.Input(shape=[224, 224, 3])
# Downsampling through the model
skips = down_stack(inputs)
x = skips[-1]
skips = reversed(skips[:-1])
# Upsampling and establishing the skip connections
for up, skip in zip(up_stack, skips):
x = up(x)
concat = tf.keras.layers.Concatenate()
x = concat([x, skip])
last = tf.keras.layers.Conv2DTranspose(
self.output_channels,
kernel_size=3, strides=2,
padding='same', activation='sigmoid')
x = last(x)
optimizer = tf.keras.optimizers.Nadam(1e-4)
metrics = [tf.keras.metrics.BinaryIoU(target_class_ids=[0, 1], threshold=0.5)]
self.model = tf.keras.Model(inputs=inputs, outputs=x)
self.model.compile(
optimizer=optimizer,
loss=tf.keras.losses.BinaryCrossentropy(),
metrics=metrics)
def train():
batch_size = 4
EPOCHS = 250
VAL_SUBSPLITS = 2
VALIDATION_STEPS = dataset_size // batch_size // VAL_SUBSPLITS
STEPS_PER_EPOCH = train_dataset_size // batch_size
unet.load_weights('./checkpoints/my_checkpoint')
history = unet.fit(train_gen, epochs=EPOCHS,
steps_per_epoch=STEPS_PER_EPOCH,
validation_steps=VALIDATION_STEPS,
validation_data=val_gen,
callbacks=[
ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=4),
EarlyStopping(monitor='val_loss', patience=10, restore_best_weights=False)
],
)
Are there any tips to improve my model?
Current output: https://imgur.com/a/5Jqn1oX
What I've tried:
Augmented my data generator which is based from the COCO dataset
Used an Adam optimizer with an exponential decaying learning rate schedule
Increased batch size from 4 to 32
Used learning rates from 0.01 to 0.0000001
I am trying to add autoencoder layer to LSTM neural network. The input data is the pandas DataFrame with numerical features.
To do this task, I am using Keras and Python My current code in Python is given below.
I cannot compile the model because I seem to mix Keras and Tensorflow:
TypeError: The added layer must be an instance of class Layer. Found: Tensor("input_2:0", shape=(?, 22), dtype=float32)
I am quite new to both packages, and I'd appreciate if somebody could tell me how to fix this error.
nb_features = X_train.shape[2]
hidden_neurons = nb_classes*3
timestamps = X_train.shape[1]
NUM_CLASSES = 3
BATCH_SIZE = 32
input_size = len(col_names)
hidden_size = int(input_size/2)
code_size = int(input_size/4)
model = Sequential()
model.add(LSTM(
units=hidden_neurons,
return_sequences=True,
input_shape=(timestamps, nb_features),
dropout=0.15,
recurrent_dropout=0.20
)
)
input_vec = Input(shape=(input_size,))
# Encoder
hidden_1 = Dense(hidden_size, activation='relu')(input_vec)
code = Dense(code_size, activation='relu')(hidden_1)
# Decoder
hidden_2 = Dense(hidden_size, activation='relu')(code)
output_vec = Dense(input_size, activation='relu')(hidden_2)
model.add(input_vec)
model.add(hidden_1)
model.add(code)
model.add(hidden_2)
model.add(output_vec)
model.add(Dense(units=100,
kernel_initializer='normal'))
model.add(LeakyReLU(alpha=0.5))
model.add(Dropout(0.20))
model.add(Dense(units=200,
kernel_initializer='normal',
activation='relu'))
model.add(Flatten())
model.add(Dense(units=200,
kernel_initializer='uniform',
activation='relu'))
model.add(Dropout(0.10))
model.add(Dense(units=NUM_CLASSES,
activation='softmax'))
model.compile(loss="categorical_crossentropy",
metrics = ["accuracy"],
optimizer='adam')
The issue is that you are mixing Keras' sequential API with its functional API. To fix your issue, you must replace:
input_vec = Input(shape=(input_size,))
# Encoder
hidden_1 = Dense(hidden_size, activation='relu')(input_vec)
code = Dense(code_size, activation='relu')(hidden_1)
# Decoder
hidden_2 = Dense(hidden_size, activation='relu')(code)
output_vec = Dense(input_size, activation='relu')(hidden_2)
With:
# Encoder
model.add(Dense(hidden_size, activation='relu'))
model.add(Dense(code_size, activation='relu'))
# Decoder
model.add(Dense(hidden_size, activation='relu'))
model.add(Dense(input_size, activation='relu'))
Or convert everything to the functional API
I am trying to learn how vgg16 works. Below is my code, using vgg16 for another classification.
# Generate a model with all layers (with top)
model_vgg16_conv = VGG16(weights='imagenet', include_top=False)
model_vgg16_conv.summary()
# create your own input format
input = Input(shape=(128,128,3),name = 'image_input')
# Use the generated model
output_vgg16_conv = model_vgg16_conv(input)
# Add the fully-connected layers
x = Flatten(name='flatten')(output_vgg16_conv)
x = Dense(4096, activation='relu', name='fc1')(x)
x = Dense(4096, activation='relu', name='fc2')(x)
x = Dense(5, activation='softmax', name='predictions')(x)
#Create your own model
model = Model(input=input, output=x)
#In the summary, weights and layers from VGG part will be hidden, but they will be fit during the training
model.summary()
# Specify an optimizer to use
adam = Adam(lr=0.001, beta_1=0.9, beta_2=0.999, epsilon=1e-08, decay=0.0)
# Choose loss function, optimization method, and metrics (which results to display)
model.compile(
optimizer = adam,
loss='categorical_crossentropy',
metrics=['accuracy']
)
model.fit(X_train,y_train,epochs=10,batch_size=10,verbose=2)
# model.fit(X_train,y_train,epochs=30,batch_size=100,verbose=2)
result = model.predict(y_test) # same result
For some reason, using different epoch size and batch size generate exactly the same result. Am I doing something wrong?
Suppose I have dataset 100k x 400. I created this model:
model = Sequential()
model.add(Dense(200, input_dim = 400, init = init_weights))
model.add(BatchNormalization())
model.add(SReLU())
model.add(Dropout(0.5))
model.add(Dense(200, init = init_weights))
model.add(BatchNormalization())
model.add(SReLU())
model.add(Dropout(0.5))
model.add(Dense(1, activation = 'linear', init = init_weights))
Than I call
model.compile(loss = ..
And
model.fit(input_matrix,..
After training I can call model.predict(.. for predictions.
What I would like to get is prediction matrix from model without last linear layer..
So something like:
model.remove_last_layer
pred_matrix = model.predict(input_matrix)
where output is 100k x 200 array, how can I do this with keras? thx a lot
thx to the link to docs I found this
layer_name = 'dropout_2'
intermediate_layer_model = Model(input = model.input, output = model.get_layer(layer_name).output)
intermediate_output = intermediate_layer_model.predict(matrix_test)