Why is the testing accuracy higher than my training accuracy? This is not the case for the validation accuracy. Is it because of the way I am splitting my dataset?
Modifying the network did not work so I am guessing I am doing something wrong in the dataset preparation part.
The dataset is composed of packet captures of malware and normal activities.. dataset.txt file contains total of 777 rows and 28 columns.
#converting dataset and labels to numpy arrays
x = np.genfromtxt("dataset.txt", delimiter=",")
y = np.genfromtxt("label.txt", delimiter=",")
#handling missing values
x[np.isnan(x)] = 0
#shuffling the data
indices = np.arange(x.shape[0])
np.random.shuffle(indices)
x = x[indices]
y = y[indices]
#dividing the dataset into train and test
x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.2, random_state=0)
#building the model
def build_model():
model = models.Sequential()
model.add(layers.Dense(32, activation='relu', input_shape=(28,)))
model.add(layers.Dense(32, activation='relu'))
model.add(layers.Dense(32, activation='relu'))
model.add(Dropout(0.2))
model.add(layers.Dense(1, activation='sigmoid'))
model.compile(optimizer='rmsprop', loss='binary_crossentropy',
metrics=['accuracy'])
return model
'''cross validation
k = 5
num_val_samples = len(x_train) // k
all_scores = []
for i in range(k):
print('fold #', i)
x_val = x_train[i * num_val_samples: (i + 1) * num_val_samples]
y_val = y_train[i * num_val_samples: (i + 1) * num_val_samples]
partial_x_train = np.concatenate([x_train[:i * num_val_samples],
x_train[(i + 1) * num_val_samples:]], axis=0)
partial_y_train = np.concatenate([y_train[:i * num_val_samples],
y_train[(i + 1) * num_val_samples:]], axis=0)
model = build_model()
model.fit(partial_x_train, partial_y_train,epochs=20, batch_size=16,
verbose=0)
val_loss, val_acc = model.evaluate(x_val, y_val, verbose=0)
all_scores.append(val_acc)
print(all_scores)
val_acc = np.mean(all_scores)
print(val_loss , val_acc)
'''
#training the model with the entire training dataset
model = build_model()
model.fit(x_train, y_train, epochs=20, batch_size=16)
#confusion matrix
y_pred = model.predict(x_test)
y_pred = (y_pred > 0.5)
result = confusion_matrix(y_test, y_pred)
print ('Confusion Matrix:')
print(result)
#calculating the test accuracy
model_acc = accuracy_score(y_test, y_pred)
print('Test Accuracy:')
print(model_acc)
This is because keras reports running average accuracy for each epoch. For small number of epochs this means that by the end of an epoch your model is better than it was on average during this epoch.
This could also be due to randomly having 'easier' samples in the test set, but this would not happen each run if you split it randomly in the same portion of the code.
Related
I try to predict LTC coin price on 100 steps in future. All code works good, I create model architecture, save model, load model, load 1000 records for training.
Everising work normal, but after several cycles model show just simple line.
Here is code:
df = df[['y', 'h', 'o', 'l']] # ,'t'
df2 = df.values
print(len(df))
training = int(np.ceil(len(df) * .95))
print(training)
# quit()
# prepere data for tensorflow
# MinMaxScaler expecting like 1 feature
scaler = MinMaxScaler(feature_range=(0, 1))
scaled_data = scaler.fit_transform(df)
print(f"scaled_data {len(scaled_data)}")
# How many past days of data we want to use to predict the next day price
prediction_days = 500
train_data = scaled_data[0:int(training), :]
print(f"train_data {len(train_data)}")
# Preparing the Training data
X_train = []
y_train = []
X_test = []
y_test = []
for x in range(prediction_days, len(train_data)):
X_train.append(scaled_data[x - prediction_days:x, 0])
y_train.append(scaled_data[x, 0])
X_test.append(scaled_data[x - prediction_days:x, 0])
y_test.append(scaled_data[x, 0])
X_train, y_train = np.array(X_train), np.array(y_train)
X_test, y_test = np.array(X_test), np.array(y_test)
# Reshaping so that it will work in Neural net
X_train = np.reshape(X_train, (X_train.shape[0], X_train.shape[1], 1))
X_test = np.reshape(X_test, (X_test.shape[0], X_test.shape[1], 1))
print(X_train.shape)
print(y_train.shape)
print("Files")
print(os.path.isfile('model.h5'))
if os.path.isfile('model.h5') is False:
model = Sequential()
model.add(LSTM(units=50, return_sequences=True, input_shape=(X_train.shape[1], 1)))
model.add(Dropout(0.2))
model.add(LSTM(units=50, return_sequences=True))
model.add(Dropout(0.2))
model.add(LSTM(units=50))
model.add(Dropout(0.2))
model.add(Dense(units=100))
# define the optimization algorithm
# best learning rate for
opt = Adam(lr=0.01, beta_1=0.9, beta_2=0.999, epsilon=1e-08, decay=0.0)
model.compile(optimizer=opt, loss='mean_squared_error')
model.fit(X_train, y_train, epochs=5, validation_data=(X_test, y_test)) #, callbacks=[keras.callbacks.LearningRateScheduler(lambda epoch: 1e-8 * 10 ** (epoch / 30))]
# evaluate the model
model.save('model.h5')
#del model
model = load_model('model.h5')
# loss, accuracy = my_model.evaluate(X_test, y_test)
# print(f"accuracy: {accuracy * 100:.2f}%")
model_json = model.to_json()
with open("model.json", "w") as json_file:
json_file.write(model_json)
model.save_weights('model_weight.h5')
model.load_weights('model_weight.h5')
else:
# load json and create model
json_file = open('model.json', 'r')
loaded_model_json = json_file.read()
json_file.close()
model = model_from_json(loaded_model_json)
# load weights into new model
model.load_weights("model.h5")
print("Loaded model from disk")
opt = Adam(lr=0.01, beta_1=0.9, beta_2=0.999, epsilon=1e-08, decay=0.0)
model.compile(optimizer=opt, loss='mean_squared_error')
# train the model, iterating on the data in batches
model.fit(X_train, y_train, epochs=5, validation_data=(X_test, y_test)) # callbacks=[keras.callbacks.LearningRateScheduler(lambda epoch: 1e-8 * 10 ** (epoch / 30))]
# evaluate the model
model_json = model.to_json()
with open("model.json", "w") as json_file:
json_file.write(model_json)
model.save_weights('model_weight.h5')
model.load_weights('model_weight.h5')
test_data = 60
# actual_prices = df.values
total_dataset = df.values
print(f"Counter")
model_inputs = total_dataset[len(total_dataset) - test_data - prediction_days:]
model_inputs = model_inputs.reshape(-1, 1)
model_inputs = scaler.fit_transform( model_inputs) # ValueError: X has 1 features, but MinMaxScaler is expecting 4 features as input.
real_data = [model_inputs[len(model_inputs) - prediction_days: len(model_inputs) + 1, 0]]
# len of real_data = 101
real_data = np.array(real_data)
real_data = np.reshape(real_data, (real_data.shape[0], real_data.shape[1], 1))
# reshape real_data to (100, 100, 1)
prediction = model.predict(real_data)
# evaluation metrics
prediction = scaler.inverse_transform(prediction)
prediction = prediction.reshape(-1, 1)
# plot
# plt.plot(df, color='green')
plt.plot(prediction, color='green')
plt.legend()
plt.show()
Here is first cycle of model:
Here is 7-10 cycle of model:
Maybe some problem with learning process?
Need to use not linear optimizer
from keras.optimizers import SGD
....
sgd = SGD(learning_rate=0.01)
model.compile(loss='mse', optimizer=sgd)
I want to binary classify breast cancer histopathological images from the BreakHis dataset (https://www.kaggle.com/ambarish/breakhis) using transfer learning and the Inception Resnet v2. The goal is to freeze all layers and train the fully connected layer by adding two neurons to the model. In particular, initially I want to consider the images related to the magnificant factor 40X (Benign: 625, Malignant: 1370). Here is a summary of what I do:
I read the images and resize them to 150x150
I partition the dataset into training, validation and test set
I load the pre-trained network Inception Resnet v2
I freeze all the layers I add the two neurons for binary
classification (1 = "benign", 0 = "malignant")
I compile the model using as activation function the Adam method
I carry out the training
I make the prediction
I calculate the accuracy
This is the code:
data = dataset[dataset["Magnificant"]=="40X"]
def preprocessing(dataset, img_size):
# images
X = []
# labels
y = []
i = 0
for image in list(dataset["Path"]):
# Ridimensiono e leggo le immagini
X.append(cv2.resize(cv2.imread(image, cv2.IMREAD_COLOR),
(img_size, img_size), interpolation=cv2.INTER_CUBIC))
basename = os.path.basename(image)
# Get labels
if dataset.loc[i][2] == "benign":
y.append(1)
else:
y.append(0)
i = i+1
return X, y
X, y = preprocessing(data, 150)
X = np.array(X)
y = np.array(y)
# Splitting
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, stratify = y_40, shuffle=True, random_state=1)
X_train, X_val, y_train, y_val = train_test_split(X_train, y_train, test_size=0.25, stratify = y_train, shuffle=True, random_state=1)
conv_base = InceptionResNetV2(weights='imagenet', include_top=False, input_shape=[150, 150, 3])
# Freezing
for layer in conv_base.layers:
layer.trainable = False
model = models.Sequential()
model.add(conv_base)
model.add(layers.Flatten())
model.add(layers.Dropout(0.5))
model.add(layers.Dense(1, activation='sigmoid'))
opt = tf.keras.optimizers.Adam(learning_rate=0.0002)
loss = tf.keras.losses.BinaryCrossentropy(from_logits=False)
model.compile(loss=loss, optimizer=opt, metrics = ["accuracy", tf.metrics.AUC()])
batch_size = 32
train_datagen = ImageDataGenerator(rescale=1./255)
val_datagen = ImageDataGenerator(rescale=1./255)
train_generator = train_datagen.flow(X_train, y_train, batch_size=batch_size)
val_generator = val_datagen.flow(X_val, y_val, batch_size=batch_size)
callback = tf.keras.callbacks.EarlyStopping(monitor='loss', patience=3)
ntrain =len(X_train)
nval = len(X_val)
len(y_train)
epochs = 70
history = model.fit_generator(train_generator,
steps_per_epoch=ntrain // batch_size,
epochs=epochs,
validation_data=val_generator,
validation_steps=nval // batch_size, callbacks=[callback])
This is the output of the training at the last epoch:
Epoch 70/70
32/32 [==============================] - 3s 84ms/step - loss: 0.0499 - accuracy: 0.9903 - auc_5: 0.9996 - val_loss: 0.5661 - val_accuracy: 0.8250 - val_auc_5: 0.8521
I make the prediction:
test_datagen = ImageDataGenerator(rescale=1./255)
x = X_test
y_pred = model.predict(test_datagen.flow(x))
y_p = []
for i in range(len(y_pred)):
if y_pred[i] > 0.5:
y_p.append(1)
else:
y_p.append(0)
I calculate the accuracy:
from sklearn.metrics import accuracy_score
accuracy = accuracy_score(y_test, y_p)
print(accuracy)
This is the accuracy value I get: 0.5459098497495827
Why do I get such low accuracy, I have done several tests but I always get similar results? (HELP ME)
When doing transfer learning, especially with frozen weights, it is extremely important to do the same pre-processing as was used when the network was originally trained.
For the InceptionResNetV2 network the pre-processing type is "tf" in the tensorflow / keras libraries, which corresponds to dividing by 127 then subtracting 1 for the imagenet weights. You are instead dividing by 255.
Fortunately you do not have to wade through the code to find out what function was used, as they are exposed in the API. Simply do
train_datagen = ImageDataGenerator(preprocessing_function=tf.keras.applications.inception_resnet_v2.preprocess_input)
and so on for validation and test
This question already has answers here:
Keras, how do I predict after I trained a model?
(6 answers)
Closed 2 years ago.
I built the DNN model for predicting the survivability of breast cancer patients, I Want to predict class label (Died or Survive)for new cases, how to implement it? In summary: What I would like to apply is to predict a new case, without conducting the training process directly, just I want to show the result of the class label either died from breast cancer (1) or survived(0).
dataset=pd.read_csv("C:/Users/User/Desktop/mixed 7 12-2-2020.csv",encoding='cp1252')
array = dataset.values
X = array[:, 0:33]
y = array[:, 33]
n_features, n_outputs = X.shape[1], 1
def create_model():
model = Sequential()
model.add(Input(n_features))
model.add(BatchNormalization())
model.add(Dense(51, activation='relu'))
model.add(Dropout(0.5))
model.add(Dense(68, activation='relu'))
model.add(Dropout(0.5))
model.add(Dense(85, activation='relu'))
model.add(Dropout(0.5))
model.add(Dense(85, activation='relu'))
model.add(Dropout(0.5))
model.add(Dense(68, activation='relu'))
model.add(Dropout(0.5))
model.add(Dense(51, activation='relu'))
model.add(Dropout(0.5))
model.add(Dense(n_outputs, activation='sigmoid'))
model.compile(loss='binary_crossentropy',
optimizer='Adam',
metrics=['accuracy'])
#model.summary()
return model
checkpoint = tf.keras.callbacks.ModelCheckpoint(
"model.{epoch:02d}-{val_loss:.2f}.hdf5",
monitor='val_accuracy',
verbose=1,
save_best_only=True)
EPOCHS = 50
BATCH_SIZE = 128
N_FOLDS = 10
ACC_array = np.array([])
sensitivity_array = np.array([])
specificity_array = np.array([])
kf = KFold(n_splits=N_FOLDS, shuffle=True)
kf = kf.split(X, y)
for train_index, valid_index in kf:
X_train, X_valid = X[train_index], X[valid_index]
y_train, y_valid = y[train_index], y[valid_index]
model = create_model()
# fit network
model.fit(X_train, y_train, validation_data=(X_valid, y_valid),
epochs=EPOCHS, batch_size=BATCH_SIZE,
# callbacks=[checkpoint]
)
#model.load_weights("model.best.hdf5")
X_valid = X_valid.astype(np.float32)
predictions = model.predict(X_valid, batch_size=BATCH_SIZE)
y_pred = np.round(predictions[:, 0])
confusion = skl.metrics.confusion_matrix(y_valid, y_pred)
TP = confusion[1, 1]
TN = confusion[0, 0]
FP = confusion[0, 1]
FN = confusion[1, 0]
# Accuracy
accuracy = (TP + TN) / float(TP + TN + FP + FN)
# Sensitivity
sensitivity = TP / float(FN + TP)
# Specificity
specificity = TN / float(TN + FP)
ACC_array = np.append(ACC_array, accuracy)
sensitivity_array = np.append(sensitivity_array, sensitivity)
specificity_array = np.append(specificity_array, specificity)
ACC_mean = np.mean(ACC_array, axis=0)
print('mean Accuracy', ACC_mean * 100)
sensitivity_mean = np.mean(sensitivity_array, axis=0)
print('mean sensitivity', sensitivity_mean * 100)
specificity_mean = np.mean(specificity_array, axis=0)
print('mean specificity', specificity_mean * 100)
This answer: Get class labels from Keras functional model may be what you are looking for.
I'm assuming you want to convert the raw probabilities output by your model into a discrete class (1 or 0).
Hello I'm getting this error for my prediction model. I'm getting datas from an excel file with 4 input and 4 output. I'm new in deep learning. I can see it's about y_test but I dont know what I should write exactly. Here's my code. Thanks for any help in advance
df = pd.read_excel ("C:/Users/hayri/Desktop/aa.xlsx")
xdata = df
print(xdata)
min_max_scaler = preprocessing.MinMaxScaler()
x_scaled = min_max_scaler.fit_transform(xdata)
df = pd.read_excel ("C:/Users/hayri/Desktop/bb.xlsx")
ydata = df
min_max_scaler = preprocessing.MinMaxScaler()
y_scaled = min_max_scaler.fit_transform(ydata)
(x_train, x_test, y_train, y_test)= train_test_split(x_scaled,y_scaled,test_size = 0.2,random_state=0)
conv = Sequential()
conv.add(Conv1D(filters=32, kernel_size=4, activation='relu', input_shape=(4, 4)))
conv.add(Dropout(0.5))
conv.add(MaxPooling1D(3))
conv.add(Flatten())
conv.add(Dense(4, activation = 'sigmoid'))
sgd = optimizers.SGD(lr=0.3, momentum = 0.6, decay = 0, nesterov = False)
conv.compile(loss = 'binary_crossentropy', optimizer ='sgd', metrics = ['accuracy'])
history=conv.fit(x_train, y_train, batch_size =10, epochs =200,validation_data=(x_test,y_test), verbose = 1)
score = conv.evaluate(x_test, y_test, batch_size=10)
y_test=np.argmax(y_test, axis=1)
y_pred = conv.predict(x_test,batch_size=64)
y_pred = np.argmax(y_pred, axis=1)
results = confusion_matrix(y_test, y_pred)
sns.heatmap(results,cmap="Blues")
accuracy = conv.evaluate(x_test, y_test)
print('Accuracy: %.2f' % (accuracy*100))
model.compile(loss = 'binary_crossentropy', optimizer ='adam', metrics = ['accuracy'])
history=model.fit(x_train, y_train, batch_size =10, epochs =200,validation_data=(x_test,y_test), verbose = 1)
score = model.evaluate(x_test, y_test, batch_size=10)
y_pred = model.predict(x_test,batch_size=64)
y_pred = np.argmax(y_pred, axis=1)
I am having some trouble with my ANN. It is only predicting '0.' The dataset is imbalanced (10:1), ALTHOUGH, I undersampled the training dataset, so I am unsure of what is going on. I am getting 92-93% accuracy on the balanced training set, although on testing (on an unbalanced test set) it just predicts zeroes. Unsure of where to go from here. Anything helps. The data has been one hot encoded and scaled.
#create 80/20 train-test split
train, test = train_test_split(selection, test_size=0.2)
# Class count
count_class_0, count_class_1 = train.AUDITED_FLAG.value_counts()
# Divide by class
df_class_0 = train[train['AUDITED_FLAG'] == 0]
df_class_1 = train[train['AUDITED_FLAG'] == 1]
df_class_0_under = df_class_0.sample(count_class_1)
train_under = pd.concat([df_class_0_under, df_class_1], axis=0)
print('Random under-sampling:')
print(train_under.AUDITED_FLAG.value_counts())
train_under.AUDITED_FLAG.value_counts().plot(kind='bar', title='Count (target)');
Random under-sampling:
1.0 112384
0.0 112384
#split features and labels
y_train = np.array(train_under['AUDITED_FLAG'])
X_train = train_under.drop('AUDITED_FLAG', axis=1)
y_test = np.array(test['AUDITED_FLAG'])
X_test = test.drop('AUDITED_FLAG', axis=1)
y_train = y_train.astype(int)
y_test = y_test.astype(int)
# define model
model = Sequential()
model.add(Dense(6, input_dim=179, activation='relu'))
model.add(Dense(30, activation='relu'))
model.add(Dense(1, activation='sigmoid'))
model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
# fit model
history = model.fit(X_train, y_train, epochs=5, batch_size=16, verbose=1)
#validate
test_loss, test_acc = model.evaluate(X_test, y_test)
# evaluate the model
_, train_acc = model.evaluate(X_train, y_train, verbose=0)
_, test_acc = model.evaluate(X_test, y_test, verbose=0)
print('Train: %.3f, Test: %.3f' % (train_acc, test_acc))
print('test_acc:', test_acc)
# plot history
pyplot.plot(history.history['acc'], label='train')
#pyplot.plot(history.history['val_acc'], label='test')
Train: 0.931, Test: 0.921
#preds
y_pred = model.predict(X_test)
y_pred_bool = np.argmax(y_pred, axis=1)
# #plot confusion matrix
y_actu = pd.Series(y_test, name='Actual')
y_pred_bool = pd.Series(y_pred_bool, name='Predicted')
print(pd.crosstab(y_actu, y_pred_bool))
'''
Predicted 0
Actual
0 300011
1 28030
This is not right:
y_pred_bool = np.argmax(y_pred, axis=1)
Argmax is only used with categorical cross-entropy loss and softmax outputs. For binary cross-entropy and sigmoid outputs, you should round the outputs, which is equivalent to thresholding predictions > 0.5:
y_pred_bool = np.round(y_pred)
This is what Keras does to compute binary accuracy.