We Keep Coding

High accuracy in mode.fit but low precision and recall

I've been training a CNN with keras. A binary classificator where it says if a depth image has a manhole or not. I've manually labeled the datasets with 0 (no manhole) and 1 (it has a manhole). I have 2 datasets 1 with 45k images to train the CNN and one with 26k images to test the CNN.
Both datasets are unbalanced double of negatives images than positives.
This is the code:
# dimensions of our images.
img_width, img_height = 80, 60
n_positives_img, n_negatives_img = 17874, 26308
n_total_img = 44182
#Labeled arrays for datasets
arrayceros = np.zeros(n_negatives_img)
arrayunos = np.ones(n_positives_img)
#Reshaping of datasets to convert separate them
arraynegativos= ds_negatives.reshape(( n_negatives_img, img_height, img_width,1))
arraypositivos= ds_positives.reshape((n_positives_img, img_height, img_width,1))
#Labeling datasets with the arrays
ds_negatives_target = tf.data.Dataset.from_tensor_slices((arraynegativos, arrayceros))
ds_positives_target = tf.data.Dataset.from_tensor_slices((arraypositivos, arrayunos))
#Concatenate 2 datasets and shuffle them
ds_concatenate = ds_negatives_target.concatenate(ds_positives_target)
datasetfinal = ds_concatenate.shuffle(n_total_img)
Then I have the same for the second dataset for testing.
#Adding batch dimension to datasets 4dim
valid_ds = datasetfinal2.batch(12)
train_ds = datasetfinal.batch(12)
#Defining model
model = Sequential()
model.add(Conv2D(5, kernel_size=(5, 5),activation='relu',input_shape=(60,80,1),padding='same'))
model.add(BatchNormalization())
model.add(MaxPooling2D((5, 5),padding='same'))
model.add(Dropout(0.3))
model.add(Conv2D(5, (5, 5), activation='relu',padding='same'))
model.add(BatchNormalization())
model.add(MaxPooling2D(pool_size=(2, 2),padding='same'))
model.add(Dropout(0.3))
model.add(Conv2D(5, (5, 5), activation='relu',padding='same'))
model.add(BatchNormalization())
model.add(MaxPooling2D(pool_size=(2, 2),padding='same'))
model.add(Dropout(0.3))
model.add(Conv2D(5, (5, 5), activation='relu',padding='same'))
model.add(BatchNormalization())
model.add(Flatten())
model.add(Dropout(0.5))
model.add(Dense(100, activation='relu'))
model.add(Dropout(0.2))
model.add(Dense(50, activation='relu'))
model.add(Dropout(0.2))
model.add(Dense(1, activation='sigmoid'))
#Compiling model
model.summary()
initial_learning_rate = 0.001
lr_schedule = keras.optimizers.schedules.ExponentialDecay(
initial_learning_rate, decay_steps=100000, decay_rate=0.96, staircase=True
)
model.compile(
loss="binary_crossentropy",
optimizer=keras.optimizers.Adam(learning_rate=lr_schedule),
metrics=["acc"],
)
# Define callbacks.
checkpoint_cb = keras.callbacks.ModelCheckpoint(
"2d_image_classification.h5", save_best_only=True
)
early_stopping_cb = keras.callbacks.EarlyStopping(monitor="val_acc", patience=15)
#Fitting the model
history= model.fit(train_ds, validation_data=valid_ds, batch_size=100, epochs=5,callbacks=[checkpoint_cb, early_stopping_cb])
This gives me 99% of acc in train dataset and 95% in test dataset.
But when i do this it gives me 60% precision for negatives images and 45% for positives:
#Get the real labels of valid dataset
valid_labels = list(valid_ds.flat_map(lambda x, y: tf.data.Dataset.from_tensor_slices((x, y))).as_numpy_iterator())
valid_labels = [y for x, y in valid_labels]
y_pred = model.predict(valid_ds)
y_pred = (y_pred > 0.5).astype(float)
from sklearn.metrics import classification_report
print(classification_report(valid_labels, y_pred))
Why this? I have printed both predicted labels and true labels and it look likes its random. It has no sense.
https://colab.research.google.com/drive/1bhrntDItqoeT0KLb-aKp0W8cV6LOQOtP?usp=sharing
If u need more information, just ask me.
Thanks!!!!

Tensorflow CNN only 1 output predict value

I've already looked into similar topics but none of the tips helped me. My model predicts and outputs only 1 class, even in the console I see only 1 array value. I have to check if the font in the account number is fake or real. It prints an accuracy of 0.99 even 1.00 but after manually checking it with model.predicts it only outputs 0's. I train it on 1000 pictures of each class. Any solutions? My code:
train = tf.keras.preprocessing.image.ImageDataGenerator(rescale=1/255)
validation = train_set = tf.keras.preprocessing.image.ImageDataGenerator(rescale=1/255)
validation_set = validation.flow_from_directory('Samples', class_mode='binary', batch_size=30, target_size=(500, 50), shuffle=True, seed=42, color_mode='rgb')
train_set = train.flow_from_directory(directory='Train', class_mode='binary', batch_size=30, target_size=(500, 50), shuffle=True, seed=42, color_mode='rgb')
print(train_set.classes)
model = tf.keras.models.Sequential([
tf.keras.layers.Conv2D(16, kernel_size = (3, 3), activation='relu', input_shape=(500, 50, 3)),
tf.keras.layers.MaxPool2D(pool_size = (2, 2)),
tf.keras.layers.Conv2D(32, kernel_size = (3, 3), activation='relu'),
tf.keras.layers.MaxPool2D(pool_size = (2, 2)),
tf.keras.layers.Conv2D(64, kernel_size = (3, 3), activation='relu'),
tf.keras.layers.MaxPool2D(pool_size = (2, 2)),
tf.keras.layers.Flatten(),
tf.keras.layers.Dense(512, activation='relu'),
tf.keras.layers.Dense(1, activation='sigmoid')
])
print(train_set.class_indices)
model.compile(loss='binary_crossentropy', optimizer='adam', metrics= ['accuracy'])
model.fit(train_set, epochs=2, validation_data=validation_set)
path = 'Samples/'
DIRECTORIES = ['Fake', 'Real']
for dir in DIRECTORIES:
for file in os.listdir(path+dir):
img = tf.keras.preprocessing.image.load_img(path+dir+'/'+file)
img = tf.keras.preprocessing.image.img_to_array(img)
img = np.expand_dims(img, axis=0)
images = np.vstack([img])
val = np.argmax(model.predict(images))
print(val)
Photos to compare:
Real
Fake
Output

You're using your model as if you had two output neurons in your output layer. np.argmax(model.predict(images)) would return the index of the neuron with the maximum value, but since you only have 1, it will always return 0. Just check if the value returned by predict exceeds the threshhold or alternativly use two neurons.

Validation accurary doesn't get better

I'm doing a project on Neural network and was trying a python code using keras and tensorflow package. Currently, I'm experiencing a problem of not getting the validation accurary to going up at all. I have a training set of 9815 images and 200 test set images. I'm really stuck here please help.
Right now, the validation result is at exactly 0.5000 for almost all 100 epoch and not going up at all.
#Image Processing Stage
train_data = ImageDataGenerator(rescale = 1./255, shear_range = 0.2, zoom_range = 0.2, horizontal_flip = True)
test_data = ImageDataGenerator(rescale = 1./255)
training_set = train_data.flow_from_directory('dataset/train_data', target_size = (128, 128), batch_size = 42, class_mode = 'binary')
test_set = test_data.flow_from_directory('dataset/test_data', target_size = (128, 128), batch_size = 42, class_mode = 'binary')
# Starting Convolutional Neural Network
start_cnn = load_model('CNN.h5')
start_cnn.get_weights()
start_cnn = Sequential()
start_cnn.add(Conv2D(32, (3, 3), input_shape = (128, 128, 3), activation = 'relu', padding='same')) #3*3*3*32+32
start_cnn.add(Conv2D(32, (3, 3), activation = 'relu'))
start_cnn.add(MaxPooling2D(pool_size = (2, 2)))
for i in range(0,2):
start_cnn.add(Conv2D(128, (3, 3), activation = 'relu', padding='same'))
start_cnn.add(MaxPooling2D(pool_size = (2, 2)))
for i in range(0,2):
start_cnn.add(Conv2D(128, (3, 3), activation = 'relu', padding='same'))
start_cnn.add(MaxPooling2D(pool_size = (2, 2)))
# Flattening
start_cnn.add(Flatten())
# Step 4 - Full connection
start_cnn.add(Dense(activation="relu", units=128))
start_cnn.add(Dense(activation="relu", units=64))
start_cnn.add(Dense(activation="relu", units=32))
start_cnn.add(Dense(activation="softmax", units=1))
start_cnn.summary()
# Compiling the CNN
start_cnn.compile(Adam(learning_rate=0.001), loss = 'binary_crossentropy', metrics = ['accuracy'])
start_cnn.fit(training_set, steps_per_epoch=234, epochs = 100, validation_data = test_set)
start_cnn.save('CNN.h5')

You cannot use the softmax activation with one neuron, like you are doing here:
start_cnn.add(Dense(activation="softmax", units=1))
To do binary classification with one neuron, you have to use the sigmoid activation:
start_cnn.add(Dense(activation="sigmoid", units=1))

Predictive model giving only one class on holdout set keras

Currently, using a CNN to predict a model for some black and white images. The model is working well for what I want it to do. This is the model:
lr = 0.2
Adam(lr=lr, beta_1=0.9, beta_2=0.999, epsilon=1e-08, decay=0.001)
classifier = Sequential()
# Step 1 - Convolution
classifier.add(Conv2D(32, (3, 3), input_shape = (64, 64, 3), activation = 'relu'))
dropout = 0.3
# Step 2 - Pooling
classifier.add(MaxPooling2D(pool_size = (2, 2)))
classifier.add(Dropout(rate=dropout))
# Adding a second convolutional layer
classifier.add(Conv2D(32, (3, 3), activation = 'sigmoid'))
classifier.add(MaxPooling2D(pool_size = (2, 2)))
classifier.add(Dropout(rate=dropout))
classifier.add(Conv2D(32, (3, 3), activation = 'sigmoid'))
classifier.add(MaxPooling2D(pool_size = (2, 2)))
# Step 3 - Flattening
classifier.add(Flatten())
# Step 4 - Full connection
classifier.add(Dense(units = 128, activation = 'relu'))
classifier.add(Dense(units = 1, activation = 'sigmoid'))
# Compiling the CNN
classifier.compile(optimizer = 'adam', loss = 'binary_crossentropy', metrics = ['accuracy'])
epoch_number = 40
epochs = 25
steps_validation = 100
history = classifier.fit_generator(training_set,
steps_per_epoch = epoch_number,
epochs = epochs,
validation_data = test_set,
validation_steps = steps_validation)
classifier.save('epoch40epochs25softmax.h5')
Now, after I train this model I want to use it in the 'real world' so I have a holdout set that I use it on. The problem is it only predicts class 1 when I do a classifier.predict_classes(test_image) I figured I was doing something wrong so I tried doing a classifier.predict_classes(test_image) for all the elements in the testing set since I knew I was getting 75% accuracy on that set and still upon doing a:
for test_image in test_set_dir:
classifier.predict_classes(test_image)
I am still only getting numbers very close to [[1]]. My question is am I using keras correctly, or if not, what is going on? I should get some [[0]] classifications.
I am absolutely sure that the model is not predicting just class 1 since I have a 50% - 50% split on the two classes and I am getting 75% accuracy.
Edit:
I am normalizing the images thusly
train_datagen = ImageDataGenerator(rescale = 1./255,
shear_range = 0,
zoom_range = 0,
horizontal_flip = True)
test_datagen = ImageDataGenerator(rescale = 1./255)
training_set = train_datagen.flow_from_directory('datasetI/training_set',
target_size = (64, 64),
batch_size = 32,
class_mode = 'binary')
test_set = test_datagen.flow_from_directory('datasetI/test_set',
target_size = (64, 64),
batch_size = 32,
class_mode = 'binary')
If I want to get the class prediction on the test_set what should I do to get it?

Low accuracy on captcha recognition

I am generating at most 4 digits captchas with the following method:
def genData(n=30000, max_digs=4, width=150):
capgen = ImageCaptcha()
data = []
target = []
for i in range(n):
x = np.random.randint(0, 10 ** max_digs)
img = misc.imread(capgen.generate(str(x)))
img = np.mean(img, axis=2)[:, :width]
data.append(img.flatten())
target.append(x)
return np.array(data), np.array(target)
Then I am processing data like following
train_data, train_target = genData()
test_data, test_target = genData(1000)
train_data = train_data.reshape(train_data.shape[0], 1, 150, 60)
test_data = test_data.reshape(test_data.shape[0], 1, 150, 60)
train_data = train_data.astype('float32')
test_data = test_data.astype('float32')
train_data /= 255
test_data /= 255
My model structure is as follows:
def get_model():
# create model
model = Sequential()
model.add(Conv2D(30, (5, 5), input_shape=(1, 150, 60), activation='relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Conv2D(15, (3, 3), activation='relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Dropout(0.2))
model.add(Flatten())
model.add(Dense(128, activation='relu'))
model.add(Dense(50, activation='relu'))
model.add(Dense(10 ** 4, activation='softmax'))
# Compile model
model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
return model
Then I am training the model
model = get_model()
# Fit the model
model.fit(train_data, train_target, validation_data=(test_data, test_target), epochs=10, batch_size=200)
# Final evaluation of the model
scores = model.evaluate(test_data, test_target, verbose=0)
print("Large CNN Error: %.2f%%" % (100 - scores[1] * 100))
I don't know which part that I am doing wrong but my accuracy cannot reach even %1.

You have 10000(!) classes. How long do you train? How much training data do you have per class?
Your approach is almost certainly the problem. While you can solve problems "brute force" like this, it is a very bad way to do so. You should first try to detect single digits and then classify each digit with a 10-class classifier.