I use the following code to tune the hyperparameters (hidden layers, hidden neurons, batch size, optimizer) of an ANN.
## Part 2 - Tuning the ANN
from keras.wrappers.scikit_learn import KerasRegressor
from sklearn.model_selection import GridSearchCV
from keras.models import Sequential
from keras.layers import Dense
def build_regressor(hidden_nodes, hidden_layers, optimizer):
regressor = Sequential()
regressor.add(Dense(units = hidden_nodes, kernel_initializer = 'uniform', activation = 'relu', input_dim = 7))
for layer_size in range(hidden_layers):
regressor.add(Dense(hidden_nodes, kernel_initializer = 'uniform', activation = 'relu'))
regressor.add(Dense(units = 1, kernel_initializer = 'uniform', activation = 'linear'))
regressor.compile(optimizer = optimizer, loss = 'mse', metrics = ['mse'])
return regressor
regressor = KerasRegressor(build_fn = build_regressor, epochs = 100)
# Create a dictionary of tuning parameters
parameters = {'hidden_nodes': list(range(2,101)), 'hidden_layers': [4,5,6,7], 'batch_size': [25,32], 'optimizer' : ['adam', 'nadam','RMSprop', 'adamax']}
grid_search = GridSearchCV(estimator = regressor, param_grid = parameters, scoring = 'neg_mean_squared_error', cv = 10, n_jobs = 4)
start = time.time()
grid_search = grid_search.fit(X_train, y_train)
end = time.time()
elapsed = (end - start)/3600
Now I want to add a droppout layer after each of the hidden layer like this:
regressor1 = Sequential()
regressor1.add(Dense(units = 41, kernel_initializer = 'uniform', activation = 'relu', input_dim = 7))
regressor1.add(Dropout(0.1))
regressor1.add(Dense(units = 41, kernel_initializer = 'uniform', activation = 'relu'))
regressor1.add(Dropout(0.1))
regressor1.add(Dense(units = 41, kernel_initializer = 'uniform', activation = 'relu'))
regressor1.add(Dropout(0.1))
regressor1.add(Dense(units = 41, kernel_initializer = 'uniform', activation = 'relu'))
regressor1.add(Dropout(0.1))
regressor1.add(Dense(units = 41, kernel_initializer = 'uniform', activation = 'relu'))
regressor1.add(Dropout(0.1))
regressor1.add(Dense(units = 1, kernel_initializer = 'uniform', activation = 'linear'))
regressor1.compile(optimizer = 'nadam', loss = 'mse', metrics = ['mse'])
history = regressor1.fit(X_train, y_train, batch_size = 25, epochs = 500, validation_data = (X_test, y_test), callbacks = [EarlyStopping(patience = 10)])
Is there a way to tune the number of dropout layers (the same number of hidden layers) and the dropout rates together with my current code?
Thank you so much in advance,
Here's your solution
https://machinelearningmastery.com/grid-search-hyperparameters-deep-learning-models-python-keras/
Tip: I would suggest you you to look for similar models created by other people and play with the dropout values they've used this will save you a lot of time
Related
I got the following error: RuntimeError: You must compile your model before training/testing. Use model.compile(optimizer, loss). However, i have already compiled my model so i don't understand what the problem is.
`vgg = VGG16(weights='imagenet',include_top=False,input_shape=(224,224,3))
for layer in vgg.layers:
layer.trainable = False #making all the layers non-trainable
x = Flatten()(vgg.output) #flattening out the last layer
predictions = Dense(2,activation='sigmoid')(x) #Dense layer to predict wether there is pneumonia or not
model = Model(inputs=vgg.input, outputs=predictions)
early_stopping_callbacks = tensorflow.keras.callbacks.EarlyStopping(patience = 15,
restore_best_weights = True,
verbose = 1)
base_model1 = VGG16(include_top = False, weights = "imagenet", input_shape = (224, 224, 3), pooling = "max",
classes = 2)
#base_model1.load_weights("../input/vgg16/vgg16_weights_tf_dim_ordering_tf_kernels_notop.h5")
base_model1.summary()
model2 = Sequential()
model2.add(base_model1)
model2.add(Flatten())
model2.add(Dense(128, activation = "relu"))
model2.add(Dense(64, activation = "relu"))
model2.add(Dense(32, activation = "relu"))
model2.add(Dense(1, activation = "sigmoid"))
# freeze the layers
for layer in base_model1.layers:
layer.trainable = False
model2.compile(optimizer = "adam", loss = "binary_crossentropy", metrics = ["accuracy"])
history = model2.fit_generator(train_generator, epochs = EPOCH, validation_data = val_generator, steps_per_epoch = 10,
callbacks = [early_stopping_callbacks])
test_loss, test_accuracy = base_model1.evaluate(test_generator, steps = 50)
print("The testing accuracy is: ", test_accuracy * 100, "%")
print("The testing loss is: ", test_loss * 100, "%")`
base_model1 is my vgg16 model and test_generator is my test set
I'm trying to predict joint torque from 8 input features with an LSTM model.
I've tried using TimeseriesGenerator but kept getting an error
Key: 10 when I tried running the .fit_generator function in the model.
I also sliced my dataset into sub time frames, but am pretty confused on this whole concept.
The dataset is at this link.
Here's my code:
file = r'/content/drive/MyDrive/only_force.csv'
df = pd.read_csv(file)
X = df.iloc[:, :9]
y = df.iloc[:,9]
first_slice = X[:1081]
second_slice = X[1081:2076]
third_slice = X[2076:3122]
fourth_slice = X[3122:4038]
fifth_slice = X[4038:5186]
sixth_slice = X[5186:6270]
seventh_slice = X[6270:7464]
eighth_slice = X[7464:]
from keras.preprocessing.sequence import TimeseriesGenerator
look_back = 10
train_generator = TimeseriesGenerator(X_train, X_train, length = look_back, batch_size = 32)
test_generatory = TimeseriesGenerator(X_test, X_test, length = look_back, batch_size = 32)
[verbose, epochs, batch_size] = [1, 500, 32]
input_shape = (X_train.shape[1],1)
model = Sequential()
# LSTM
model.add(LSTM(64, input_shape=input_shape, return_sequences = False))
model.add(Dropout(0.2))
model.add(Dense(64, activation='relu', kernel_regularizer=keras.regularizers.l2(0.001)))
#model.add(Dropout(0.2))
model.add(Dense(32, activation='relu', kernel_regularizer=keras.regularizers.l2(0.001)))
model.add(Dense(1,activation='relu'))
earlystopper = EarlyStopping(monitor='val_loss', min_delta=0, patience = 30, verbose =1, mode = 'auto')
model.summary()
model.compile(loss = 'mse', optimizer = Adam(learning_rate = 0.0005), metrics=[tf.keras.metrics.RootMeanSquaredError()])
history = model.fit(X_train, y_train, batch_size = batch_size, epochs = epochs, verbose = verbose, validation_data=(X_test,y_test), callbacks = [earlys
#model.fit_generator(train_generator, epochs=epochs, verbose=1)
Who can suggest how to improve the model?
The regular model in sklearn LinearRegression() predicts temperature with an error of 1 and the error of the model built manually on tensorflow won't drop below 5.5, no matter the activation function, the number of layers, or epochs.
The data was both standardized and derived into positive values
def createModelG(inputShape, dropout, initW):
model = Sequential()
model.add(Dense(4096,
kernel_regularizer=keras.regularizers.l2(0.001),
activation = 'elu',
kernel_initializer = initW,
input_dim = inputShape
))
model.add(Dropout(dropout))
#for i in range(3):
# model.add(Dense(512, activation = 'relu'))
# model.add(Dropout(dropout))
model.add(Dense(1024,
kernel_regularizer=keras.regularizers.l2(0.001),
activation = 'elu'
))
model.add(Dropout(dropout))
model.add(Dense(1))
model.compile(
loss = 'mae',
optimizer = tf.keras.optimizers.Adam(learning_rate = 0.0000005),
metrics = ['mse', 'mae']
)
return model
startModelTest = crossValdation(createModelG, trainDataXS, 0.01, 'truncated_normal', 'VancouverT', PrintDot())
modelTest = startModelTest[1]
hist = startModelTest[2]
startModelTest[0]
loss mse mae val_loss val_mse val_mae
0 22.6255 737.889 21.3214 7.32549 55.3201 6.02149
1 21.6446 677.313 20.3387 7.83092 64.0345 6.5251
2 21.1013 646.857 19.7952 7.00224 49.6842 5.69622
3 22.3446 712.008 21.0386 8.07596 68.7968 6.77008
4 24.2565 874.824 22.9531 7.71605 65.3973 6.41274
0 --- --- --- --- --- ---
0 22.3945
link to all code and result of my keras model and ready sklearn models:
https://www.kaggle.com/alihanurumov/weather-prediction-network
def createModelG(inputShape):
model = Sequential()
model.add(Dense(4096, input_dim = inputShape,
kernel_initializer = initializers.glorot_uniform(seed = 1),
kernel_regularizer = keras.regularizers.l2(0.01), activation = "relu"))
model.add(Dense(2048,
kernel_initializer = initializers.glorot_uniform(seed = 1), activation = "relu"))
model.add(Dense(2048,
kernel_initializer = initializers.glorot_uniform(seed = 1), activation = "relu"))
model.add(Dense(1024,
kernel_initializer = initializers.glorot_uniform(seed = 1), activation = "relu"))
model.add(Dense(1024,
kernel_initializer = initializers.glorot_uniform(seed = 1), activation = "relu"))
model.add(layers.Dropout(0.05))
model.add(Dense(1))
optimizer = tf.keras.optimizers.Adam(learning_rate = 0.000001)
model.compile(loss = 'mse', optimizer = optimizer, metrics = ["mse", "mae"])
return model
I want to conduct a hyperparameter tuning for the learning rate. However, I got the error that I do not know how to solve.
I used the Tensorflow.Keras package.
import tensorflow as tf
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.datasets.mnist import load_data
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import (Flatten, BatchNormalization, Dropout, Dense)
from keras.wrappers.scikit_learn import KerasClassifier
(x_train_all, y_train_all), (x_test, y_test) = load_data()
x_train, x_valid, x_test = x_train_all[5000:]/255.0, x_train_all[:5000]/255.0, x_test/255.0
y_train, y_valid = y_train_all[5000:], y_train_all[:5000]
tf.cast(x_train, tf.float32)
tf.cast(x_valid, tf.float32)
tf.cast(x_test, tf.float32)
def my_model(learning_rate = 5e-3):
model = Sequential([
Flatten(input_shape = (28, 28)),
BatchNormalization(),
Dropout(rate = 0.2),
Dense(300, activation = "elu", kernel_initializer = "he_normal"),
Dropout(rate = 0.2),
BatchNormalization(),
Dense(300, activation = "elu", kernel_initializer = "he_normal"),
Dropout(rate = 0.2),
BatchNormalization(),
Dense(10, activation = "softmax",kernel_initializer = "he_normal")])
opt = Adam(lr = learning_rate)
model.summary()
model.compile(loss = "sparse_categorical_crossentropy", optimizer = opt, learning_rate = learning_rate, metrics = ["accuracy"])
return model
from sklearn.model_selection import RandomizedSearchCV
keras_classifier = KerasClassifier(my_model)
param_distribs = {"learning_rate": [1e-5, 5e-5, 1e-4, 5e-4, 1e-3, 5e-3]}
rnd_search_cv = RandomizedSearchCV(keras_classifier, param_distribs, n_iter = 10, cv = 3)
rnd_search_cv.fit(x_train, y_train, epochs = 10, validation_data = (x_valid, y_valid))
I got the value error as the following:
ValueError: Session keyword arguments are not support during eager
execution. You passed: {'learning_rate': 1e-05}
Mentioning the Solution in this Section (even though it is present in Comment's section), for the benefit of the community.
The issue is resolved by removing learning_rate = learning_rate in model.compile.
Correct Code is mentioned below:
def my_model(learning_rate = 5e-3):
model = Sequential([
Flatten(input_shape = (28, 28)),
BatchNormalization(),
Dropout(rate = 0.2),
Dense(300, activation = "elu", kernel_initializer = "he_normal"),
Dropout(rate = 0.2),
BatchNormalization(),
Dense(300, activation = "elu", kernel_initializer = "he_normal"),
Dropout(rate = 0.2),
BatchNormalization(),
Dense(10, activation = "softmax",kernel_initializer = "he_normal")])
opt = Adam(lr = learning_rate)
model.summary()
model.compile(loss = "sparse_categorical_crossentropy", optimizer = opt, metrics = ["accuracy"])
return model
I have one question.
I want to print a confusion matrix.
my model is functional api of keras.
and
model = Model(inputs=[data_input], outputs=[output_1, output_2])
output_1 = 9 classes
output_2 = 5 classes
My multi-classification model
data_input = Input(shape=(trainX.shape[1], trainX.shape[2]))
Conv1 = Conv1D(filters=50, kernel_size=4, padding='valid', activation='relu', strides=1)(data_input)
Conv1 = MaxPooling1D(pool_size=2)(Conv1)
Conv2 = Conv1D(filters=50, kernel_size=4, padding='valid', activation='relu', strides=1)(Conv1)
Conv2 = MaxPooling1D(pool_size=2)(Conv2)
Conv3 = Conv1D(filters=50, kernel_size=4, padding='valid', activation='relu', strides=1)(Conv2)
Conv3 = MaxPooling1D(pool_size=2)(Conv3)
Classification1 = LSTM(128, input_shape=(47, 50), return_sequences=False)(Conv3)
Classification2 = GRU(128, input_shape=(47, 50), return_sequences=False)(Conv3)
activity = Dense(9)(Classification1)
activity = Activation('softmax')(activity)
speed = Dense(5)(Classification2)
speed = Activation('softmax')(speed)
model = Model(inputs=[data_input], outputs=[activity, speed])
model.compile(loss= 'categorical_crossentropy' , optimizer='adam', metrics=[ 'accuracy' ])
print(model.summary())
history = model.fit(trainX, {'activation_1': trainY_Activity, 'activation_2': trainY_Speed},
validation_data=(testX, {'activation_1': testY_Activity, 'activation_2': testY_Speed}),
epochs=epochs, batch_size=batch_size, verbose=1, shuffle=False)