We Keep Coding

How to input numerical data into Tensorflow ResNet50 model for regression?

I currently have a TensorFlow model (ResNet50), which takes a single image input and outputs a continuous value via regression (ranges from 0.8 - 2.0). The dataset has 3000 different patients, each of which has an image and several numerical data points (specifically age, gender, weight). I'm able to get decent accuracy by training on the images, but I was wondering how I could also add the numerical data points as separate inputs. The numerical data is in a csv file where each row is a seperate patients and different columns contain different values. Here is what I have for 1 image:
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
import tensorflow as tf
from tensorflow.keras.preprocessing.image import ImageDataGenerator
from tensorflow.keras.applications.resnet import ResNet50
from tensorflow.keras.applications.resnet_rs import ResNetRS50
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import GlobalAveragePooling2D, Dense, Dropout, Flatten, GlobalMaxPooling2D
from tensorflow.keras.optimizers import Adam
import warnings
labels = pd.read_csv('[path goes here]')
labels.head()
def load_train(path):
"""
It loads the train part of dataset from path
"""
#1 / 255
labels = pd.read_csv('[Target Dataframe path]')
train_datagen = ImageDataGenerator(validation_split=0.2, rescale=None)
train_gen_flow = train_datagen.flow_from_dataframe(
dataframe=labels,
directory='[Images folder path]',
x_col='ID',
y_col='Value',
target_size=(224, 224),
batch_size=32,
class_mode='raw',
subset = 'training',
seed=1234)
return train_gen_flow
def load_test(path):
"""
It loads the validation/test part of dataset from path
"""
labels = pd.read_csv('[Target Dataframe path]')
validation_datagen = ImageDataGenerator(validation_split=0.2, rescale=None)
test_gen_flow = validation_datagen.flow_from_dataframe(
dataframe = labels,
directory='[Images folder path]',
x_col="ID",
y_col="Value",
class_mode="raw",
target_size=(224,224),
batch_size=32,
subset = "validation",
seed=1234,
)
return test_gen_flow
def create_model(input_shape):
"""
It defines the model
"""
backbone = ResNetRS50(input_shape=input_shape, weights='imagenet', include_top=False)
model = Sequential()
model.add(backbone)
model.add(Dropout(0.3))
model.add(GlobalMaxPooling2D())
model.add(Dense(1, activation='linear'))
optimizer = Adam(learning_rate=0.0003)
model.compile(optimizer=optimizer, loss='mae', metrics=['mae'])
print(model.summary())
return model
def train_model(model, train_data, test_data, batch_size=32, epochs=100,
steps_per_epoch=None, validation_steps=None):
"""
Trains the model given the parameters
"""
history = model.fit(train_data, validation_data=test_data, batch_size=batch_size,
epochs=epochs, steps_per_epoch=steps_per_epoch,
validation_steps=validation_steps, verbose=2)
# Get training and test loss histories
training_loss = history.history['loss']
test_loss = history.history['val_loss']
# Create count of the number of epochs
epoch_count = range(1, len(training_loss) + 1)
# Visualize loss history
plt.plot(epoch_count, training_loss, 'r--')
plt.plot(epoch_count, test_loss, 'b-')
plt.legend(['Training Loss', 'Test Loss'])
plt.xlabel('Epoch')
plt.ylabel('Loss')
plt.show();
return model
path = '[Full set path (contains image folder, target csv, and numerical feature csv]'
train_data = load_train(path)
test_data = load_test(path)
#build a model
model = create_model(input_shape = (224, 224, 3))
model = train_model(model, train_data, test_data)

What does keras default to when the batch size is larger than the testing set size?

I am running my model and my testing set size is 77. When I set the batch size to a value larger than the testing set size, my model still runs and there are much less ups and downs in the mean squared error over each epoch. As I increase the batch size, the model works much more efficiently which is exactly what I want,but I am curious what keras defaults to doing in this scenario.
import numpy as np
import pandas
import keras
from keras.models import Sequential
from keras.layers import Dense
import matplotlib.pyplot as plt
from pandas import read_csv
# load dataset
dataframe = read_csv("new_1501_na_replaced.csv", header=None)
dataset = dataframe.values
# split into input (X) and output (Y) variables
X = dataset[:,0:1500]
Y = dataset[:,1500]
model = Sequential()
model.add(Dense(150, input_dim=1500, kernel_initializer='normal', activation='relu'))
model.add(Dense(150,kernel_initializer = 'normal', activation = 'relu'))
model.add(Dense(1, kernel_initializer='normal'))
# Compile model asking for accuracy, too:
model.compile(loss='mean_squared_error', optimizer='sgd', metrics=['mean_squared_error'])
model.fit(X, Y, validation_split=0.33,
batch_size=100,
epochs=100,
verbose=1)
score = model.evaluate(X, Y, verbose=0)
history = model.fit(X, Y, validation_split=0.33,
batch_size=500,
epochs=300,
verbose=1)
print(score)
# Plot
plt.plot(history.history['mean_squared_error'])
plt.title('mean_squared_error')
plt.show()
plt.plot(history.history['loss'])
plt.plot(history.history['val_loss'])
plt.legend(['train', 'test'], loc='upper left')
plt.title('model loss')
plt.show()

Tensorflow always gives me the same result while the outputs are normal

I hope you are having a great day!
I recently tried to train a regression model by using TensorFlow and I completed my code by following the instruction in here.
data = pd.read_csv('regret.csv')
max_regret = data['regret'].max()
data['regret'] = data['regret'] / max_regret # Normalize Regrets
regret_labels = data.pop('regret')
def build_model():
model = keras.Sequential([
layers.Dense(64, activation='relu', input_shape=[len(data.keys())]),
layers.Dense(64, activation='relu'),
layers.Dense(1)
])
optimizer = tf.keras.optimizers.RMSprop(0.001)
model.compile(loss='mse',
optimizer=optimizer,
metrics=['mae', 'mse'])
return model
model = build_model()
class PrintDot(keras.callbacks.Callback):
def on_epoch_end(self, epoch, logs):
if epoch % 100 == 0: print('')
print('.', end='')
EPOCHS = 1000
# early_stop = keras.callbacks.EarlyStopping(monitor='val_loss', patience=100)
history = model.fit(
data, regret_labels,
epochs=EPOCHS, validation_split=0.2, verbose=0,
callbacks=[PrintDot()])
test = model.predict(data)
loss, mae, mse = model.evaluate(data, regret_labels, verbose=2)
However, I encountered a problem that all the predictions were the same, even though the model.evaluate() gave me different statistics by trials.
I also attached the file via this link.
Would you take a look at it and give me some ideas to solve it? Thanks in advance!

You can try this approach as below which split your data set into training and test set before fitting into the model.
You can try this approach as below which split your data set into training and test set before fitting into the model.
import pandas as pd
import keras
from keras.models import Sequential
from keras.layers.core import Dense
from keras.optimizers import RMSprop
data = pd.read_csv('regret.csv')
max_regret = data['regret'].max()
data['regret'] = data['regret'] / max_regret
len(data.keys())
data
train_dataset = data.sample(frac=0.8,random_state=0)
test_dataset = data.drop(train_dataset.index)
train_labels = train_dataset.pop('regret')
test_labels = test_dataset.pop('regret')
def build_model():
model = Sequential()
model.add(Dense(64, activation='relu', input_shape=(27,)))
model.add(Dense(64, activation='relu'))
model.add(Dense(1))
optimizer = RMSprop(0.001)
model.compile(loss='mse',
optimizer=optimizer,
metrics=['mae', 'mse'])
return model
model = build_model()
class PrintDot(keras.callbacks.Callback):
def on_epoch_end(self, epoch, logs):
if epoch % 100 == 0: print('')
print('.', end='')
EPOCHS = 1000
early_stop = keras.callbacks.EarlyStopping(monitor='val_loss', patience=100)
history = model.fit(
train_dataset, train_labels,
epochs=EPOCHS, validation_split=0.2, verbose=0,
callbacks=[PrintDot()])
test = model.predict(test_dataset)
The result have slightly changes as below:
enter image description here
What you could do better is min max scaling all the attributes.
Hope it can help you.

Tensor math with tensorflow backend

I was trying add custom metrics while training my LSTM using keras. See code below:
from keras.models import Sequential
from keras.layers import Dense, LSTM, Masking, Dropout
from keras.optimizers import SGD, Adam, RMSprop
import keras.backend as K
import numpy as np
_Xtrain = np.random.rand(1000,21,47)
_ytrain = np.random.randint(2, size=1000)
_Xtest = np.random.rand(200,21,47)
_ytest = np.random.randint(1, size=200)
def t1(y_pred, y_true):
return K.tf.count_nonzero((1 - y_true))
def t2(y_pred, y_true):
return K.tf.count_nonzero(y_true)
def build_model():
model = Sequential()
model.add(Masking(mask_value=0, input_shape=(21, _Xtrain[0].shape[1])))
model.add(LSTM(32, return_sequences=True))
model.add(LSTM(64, return_sequences=False))
model.add(Dense(1, activation='sigmoid'))
rms = RMSprop(lr=.001, decay=.001)
model.compile(loss='binary_crossentropy', optimizer=rms, metrics=[t1, t2])
return model
model = build_model()
hist = model.fit(_Xtrain, _ytrain, epochs=1, batch_size=5, validation_data=(_Xtest, _ytest), shuffle=True)
The output of the above code is as follows:
Train on 1000 samples, validate on 200 samples
Epoch 1/1
1000/1000 [==============================] - 5s - loss: 0.6958 - t1: 5.0000 - t2: 5.0000 - val_loss: 0.6975 - val_t1: 5.0000 - val_t2: 5.0000
So it appears that both methods t1 and t2 are producing the exact same output and it is baffling me. What could be going wrong and how could I get the complementary tensor to y_true?
Backstory: I was trying to write custom metrics (F1 score) in particular for my model. Keras does not seems to have those readily available. If anyone knows a better way, please help me get pointed to the right direction.

One easy way to handle this issue is to use a callback instead. Following the logic from this issue, you could specify a metrics call back that calculates any metric using sci-kit learn. For example, if you wanted to calculate f1, you could do the following:
from keras.models import Sequential
from keras.layers import Dense, LSTM, Masking, Dropout
from keras.optimizers import SGD, Adam, RMSprop
import keras.backend as K
from keras.callbacks import Callback
import numpy as np
from sklearn.metrics import f1_score
_Xtrain = np.random.rand(1000,21,47)
_ytrain = np.random.randint(2, size=1000)
_Xtest = np.random.rand(200,21,47)
_ytest = np.random.randint(2, size=200)
class MetricsCallback(Callback):
def __init__(self, train_data, validation_data):
super().__init__()
self.validation_data = validation_data
self.train_data = train_data
self.f1_scores = []
self.cutoff = .5
def on_epoch_end(self, epoch, logs={}):
X_val = self.validation_data[0]
y_val = self.validation_data[1]
preds = self.model.predict(X_val)
f1 = f1_score(y_val, (preds > self.cutoff).astype(int))
self.f1_scores.append(f1)
def build_model():
model = Sequential()
model.add(Masking(mask_value=0, input_shape=(21, _Xtrain[0].shape[1])))
model.add(LSTM(32, return_sequences=True))
model.add(LSTM(64, return_sequences=False))
model.add(Dense(1, activation='sigmoid'))
rms = RMSprop(lr=.001, decay=.001)
model.compile(loss='binary_crossentropy', optimizer=rms, metrics=['acc'])
return model
model = build_model()
hist = model.fit(_Xtrain, _ytrain, epochs=2, batch_size=5, validation_data=(_Xtest, _ytest), shuffle=True,
callbacks=[MetricsCallback((_Xtrain, _ytrain), (_Xtest, _ytest))])

How to compute Receiving Operating Characteristic (ROC) and AUC in keras?

I have a multi output(200) binary classification model which I wrote in keras.
In this model I want to add additional metrics such as ROC and AUC but to my knowledge keras dosen't have in-built ROC and AUC metric functions.
I tried to import ROC, AUC functions from scikit-learn
from sklearn.metrics import roc_curve, auc
from keras.models import Sequential
from keras.layers import Dense
.
.
.
model.add(Dense(200, activation='relu'))
model.add(Dense(300, activation='relu'))
model.add(Dense(400, activation='relu'))
model.add(Dense(300, activation='relu'))
model.add(Dense(200,init='normal', activation='softmax')) #outputlayer
model.compile(loss='categorical_crossentropy', optimizer='adam',metrics=['accuracy','roc_curve','auc'])
but it's giving this error:
Exception: Invalid metric: roc_curve
How should I add ROC, AUC to keras?

Due to that you can't calculate ROC&AUC by mini-batches, you can only calculate it on the end of one epoch. There is a solution from jamartinh, I patch the code below for convenience:
from sklearn.metrics import roc_auc_score
from keras.callbacks import Callback
class RocCallback(Callback):
def __init__(self,training_data,validation_data):
self.x = training_data[0]
self.y = training_data[1]
self.x_val = validation_data[0]
self.y_val = validation_data[1]
def on_train_begin(self, logs={}):
return
def on_train_end(self, logs={}):
return
def on_epoch_begin(self, epoch, logs={}):
return
def on_epoch_end(self, epoch, logs={}):
y_pred_train = self.model.predict_proba(self.x)
roc_train = roc_auc_score(self.y, y_pred_train)
y_pred_val = self.model.predict_proba(self.x_val)
roc_val = roc_auc_score(self.y_val, y_pred_val)
print('\rroc-auc_train: %s - roc-auc_val: %s' % (str(round(roc_train,4)),str(round(roc_val,4))),end=100*' '+'\n')
return
def on_batch_begin(self, batch, logs={}):
return
def on_batch_end(self, batch, logs={}):
return
roc = RocCallback(training_data=(X_train, y_train),
validation_data=(X_test, y_test))
model.fit(X_train, y_train,
validation_data=(X_test, y_test),
callbacks=[roc])
A more hackable way using tf.contrib.metrics.streaming_auc:
import numpy as np
import tensorflow as tf
from sklearn.metrics import roc_auc_score
from sklearn.datasets import make_classification
from keras.models import Sequential
from keras.layers import Dense
from keras.utils import np_utils
from keras.callbacks import Callback, EarlyStopping
# define roc_callback, inspired by https://github.com/keras-team/keras/issues/6050#issuecomment-329996505
def auc_roc(y_true, y_pred):
# any tensorflow metric
value, update_op = tf.contrib.metrics.streaming_auc(y_pred, y_true)
# find all variables created for this metric
metric_vars = [i for i in tf.local_variables() if 'auc_roc' in i.name.split('/')[1]]
# Add metric variables to GLOBAL_VARIABLES collection.
# They will be initialized for new session.
for v in metric_vars:
tf.add_to_collection(tf.GraphKeys.GLOBAL_VARIABLES, v)
# force to update metric values
with tf.control_dependencies([update_op]):
value = tf.identity(value)
return value
# generation a small dataset
N_all = 10000
N_tr = int(0.7 * N_all)
N_te = N_all - N_tr
X, y = make_classification(n_samples=N_all, n_features=20, n_classes=2)
y = np_utils.to_categorical(y, num_classes=2)
X_train, X_valid = X[:N_tr, :], X[N_tr:, :]
y_train, y_valid = y[:N_tr, :], y[N_tr:, :]
# model & train
model = Sequential()
model.add(Dense(2, activation="softmax", input_shape=(X.shape[1],)))
model.compile(loss='categorical_crossentropy',
optimizer='adam',
metrics=['accuracy', auc_roc])
my_callbacks = [EarlyStopping(monitor='auc_roc', patience=300, verbose=1, mode='max')]
model.fit(X, y,
validation_split=0.3,
shuffle=True,
batch_size=32, nb_epoch=5, verbose=1,
callbacks=my_callbacks)
# # or use independent valid set
# model.fit(X_train, y_train,
# validation_data=(X_valid, y_valid),
# batch_size=32, nb_epoch=5, verbose=1,
# callbacks=my_callbacks)

Like you, I prefer using scikit-learn's built in methods to evaluate AUROC. I find that the best and easiest way to do this in keras is to create a custom metric. If tensorflow is your backend, implementing this can be done in very few lines of code:
import tensorflow as tf
from sklearn.metrics import roc_auc_score
def auroc(y_true, y_pred):
return tf.py_func(roc_auc_score, (y_true, y_pred), tf.double)
# Build Model...
model.compile(loss='categorical_crossentropy', optimizer='adam',metrics=['accuracy', auroc])
Creating a custom Callback as mentioned in other answers will not work for your case since your model has multiple ouputs, but this will work. Additionally, this methods allows the metric to be evaluated on both training and validation data whereas a keras callback does not have access to the training data and can thus only be used to evaluate performance on the training data.

The following solution worked for me:
import tensorflow as tf
from keras import backend as K
def auc(y_true, y_pred):
auc = tf.metrics.auc(y_true, y_pred)[1]
K.get_session().run(tf.local_variables_initializer())
return auc
model.compile(loss="binary_crossentropy", optimizer='adam', metrics=[auc])

I solved my problem this way
consider you have testing dataset x_test for features and y_test for its corresponding targets.
first we predict targets from feature using our trained model
y_pred = model.predict_proba(x_test)
then from sklearn we import roc_auc_score function and then simple pass the original targets and predicted targets to the function.
roc_auc_score(y_test, y_pred)

You can monitor auc during training by providing metrics the following way:
METRICS = [
keras.metrics.TruePositives(name='tp'),
keras.metrics.FalsePositives(name='fp'),
keras.metrics.TrueNegatives(name='tn'),
keras.metrics.FalseNegatives(name='fn'),
keras.metrics.BinaryAccuracy(name='accuracy'),
keras.metrics.Precision(name='precision'),
keras.metrics.Recall(name='recall'),
keras.metrics.AUC(name='auc'),
]
model = keras.Sequential([
keras.layers.Dense(16, activation='relu', input_shape=(train_features.shape[-1],)),
keras.layers.Dense(1, activation='sigmoid'),
])
model.compile(
optimizer=keras.optimizers.Adam(lr=1e-3)
loss=keras.losses.BinaryCrossentropy(),
metrics=METRICS)
for a more detailed tutorial see:
https://www.tensorflow.org/tutorials/structured_data/imbalanced_data

'roc_curve','auc' are not standard metrics you can't pass them like that to metrics variable, this is not allowed.
You can pass something like 'fmeasure' which is a standard metric.
Review the available metrics here: https://keras.io/metrics/
You may also want to have a look at making your own custom metric: https://keras.io/metrics/#custom-metrics
Also have a look at generate_results method mentioned in this blog for ROC, AUC...
https://vkolachalama.blogspot.in/2016/05/keras-implementation-of-mlp-neural.html

Adding to above answers, I got the error "ValueError: bad input shape ...", so I specify the vector of probabilities as follows:
y_pred = model.predict_proba(x_test)[:,1]
auc = roc_auc_score(y_test, y_pred)
print(auc)

Set your model architecture with tf.keras.metrics.AUC():
Read the Keras documentation on Classification metrics based on True/False positives & negatives.
def model_architecture_ann(in_dim,lr=0.0001):
model = Sequential()
model.add(Dense(512, input_dim=X_train_filtered.shape[1], activation='relu'))
model.add(Dense(1, activation='sigmoid'))
opt = keras.optimizers.SGD(learning_rate=0.001)
auc=tf.keras.metrics.AUC()
model.compile(loss='binary_crossentropy', optimizer=opt, metrics=[tf.keras.metrics.AUC(name='auc')])
model.summary()
return model