We Keep Coding

Loss is always nan when training a deep learning model from tabular data

I'm trying to train a model from a dataset of about a few thousands of entries with 51 numerical features and a labeled column, Example:
when training the model to predict the 3 labels (candidate, false positive, confirmed) the loss is always nan and the accuracy stabilizes very fast on a specific value.
The code:
import tensorflow as tf
import numpy as np
import pandas as pd
import sklearn.preprocessing
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import LabelEncoder, OneHotEncoder, StandardScaler, RobustScaler
from sklearn.preprocessing import OrdinalEncoder
from tensorflow import optimizers
from tensorflow.python.keras.layers import Dense, Dropout, Normalization
from tensorflow.python.keras.models import Sequential, Model
def load_dataset(data_folder_csv):
# load the dataset as a pandas DataFrame
data = pd.read_csv(data_folder_csv, header=0)
# retrieve numpy array
dataset = data.values
# split into input (X) and output (y) variables
X = dataset[:, :-1]
y = dataset[:, -1]
print(y)
# format all fields as floats
X = X.astype(np.float)
# reshape the output variable to be one column (e.g. a 2D shape)
y = y.reshape((len(y), 1))
return X, y
# prepare input data using min/max scaler.
def prepare_inputs(X_train, X_test):
oe = RobustScaler().fit_transform(X_train)
X_train_enc = oe.transform(X_train)
X_test_enc = oe.transform(X_test)
return X_train_enc, X_test_enc
# prepare target
def prepare_targets(y_train, y_test):
le = LabelEncoder()
ohe = OneHotEncoder()
le.fit(y_train)
le.fit(y_test)
y_train_enc = ohe.fit_transform(y_train).toarray()
y_test_enc = ohe.fit_transform(y_test).toarray()
return y_train_enc, y_test_enc
X, y = load_dataset("csv_ready.csv")
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.33, random_state=1)
print('Train', X_train.shape, y_train.shape)
print('Test', X_test.shape, y_test.shape)
X_train_enc, X_test_enc = X_train, X_test
print('Finished preparing inputs.'
# prepare output data
y_train_enc, y_test_enc = prepare_targets(y_train, y_test)
norm_layer = Normalization()
norm_layer.adapt(X)
model = Sequential()
model.add(Dense(128, input_dim=X_train.shape[1], activation="tanh", kernel_initializer='he_normal'))
model.add(Dropout(0.2))
model.add(Dense(64, input_dim=X_train.shape[1], activation='relu'))
model.add(Dropout(0.2))
model.add(Dense(32, input_dim=X_train.shape[1], activation='relu'))
model.add(Dense(3, activation='sigmoid'))
opt = optimizers.Adam(lr=0.01, decay=1e-6)
model.compile(loss='categorical_crossentropy', optimizer=opt, metrics=['accuracy'])
model.summary()
model.fit(X_train, y_train_enc, epochs=20, batch_size=128, verbose=1, use_multiprocessing=True)
_, accuracy = model.evaluate(X_test, y_test_enc, verbose=0)
print('Accuracy: %.2f' % (accuracy * 100))
I tried increasing/decreasing the learning rate, changing the optimizer, lowering and increasing the number of neurons and layers, and playing with batch sizes but nothing seems to bring the model to get good results. I think I'm missing something here but can't put my finger on it.
Result example:
EDIT: More lines from the csv:
EDIT2: Tried l2 regularization also and didnt did anything.

One of the reasons:
Check whether your dataset have NaN values or not. NaN values can cause problem to the model while learning.
Some of the major bugs in your code:
You are using sigmoid activation function instead of softmax for output layer having 3 neurons
You are fitting both train and test set while using encoders which is wrong. You should fit_transform for your train data and only use transform for test sets
Also you are using input for all layers which is wrong, Only the first layer should accept the input tensor.
You forgot to use prepare_inputs function for X_train and X_test
Your model should be fit with X_train_enc not X_train
Use this instead
import tensorflow as tf
import numpy as np
import pandas as pd
import sklearn.preprocessing
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import LabelEncoder, OneHotEncoder, StandardScaler, MinMaxScaler
from sklearn.preprocessing import OrdinalEncoder
from tensorflow import optimizers
from tensorflow.python.keras.layers import Dense, Dropout, Normalization
from tensorflow.python.keras.models import Sequential, Model
def load_dataset(data_folder_csv):
# load the dataset as a pandas DataFrame
data = pd.read_csv(data_folder_csv, header=0)
# retrieve numpy array
dataset = data.values
# split into input (X) and output (y) variables
X = dataset[:, :-1]
y = dataset[:, -1]
print(y)
# format all fields as floats
X = X.astype(np.float)
# reshape the output variable to be one column (e.g. a 2D shape)
y = y.reshape((len(y), 1))
return X, y
# prepare input data using min/max scaler.
def prepare_inputs(X_train, X_test):
oe = MinMaxScaler()
X_train_enc = oe.fit_transform(X_train)
X_test_enc = oe.transform(X_test)
return X_train_enc, X_test_enc
# prepare target
def prepare_targets(y_train, y_test):
le = LabelEncoder()
ohe = OneHotEncoder()
y_train = le.fit_transform(y_train)
y_test = le.transform(y_test)
y_train_enc = ohe.fit_transform(y_train).toarray()
y_test_enc = ohe.transform(y_test).toarray()
return y_train_enc, y_test_enc
X, y = load_dataset("csv_ready.csv")
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.33, random_state=1)
print('Train', X_train.shape, y_train.shape)
print('Test', X_test.shape, y_test.shape)
#prepare_input function missing here
X_train_enc, X_test_enc = prepare_inputs(X_train, X_test)
print('Finished preparing inputs.')
# prepare output data
y_train_enc, y_test_enc = prepare_targets(y_train, y_test)
model = Sequential()
model.add(Dense(128, input_dim=X_train.shape[1], activation="relu"))
model.add(Dropout(0.2))
model.add(Dense(128, activation='relu'))
model.add(Dropout(0.2))
model.add(Dense(128, activation='relu'))
model.add(Dense(3, activation='softmax'))
#opt = optimizers.Adam(lr=0.01, decay=1e-6)
model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
model.summary()
model.fit(X_train_enc, y_train_enc, epochs=20, batch_size=32, verbose=1, use_multiprocessing=True)
_, accuracy = model.evaluate(X_test_enc, y_test_enc, verbose=0)
print('Accuracy: %.2f' % (accuracy * 100))

You want to change your model definition to this:
model = Sequential()
model.add(Dense(128, input_shape=X_train.shape[1:], activation="tanh", kernel_initializer='he_normal'))
model.add(Dropout(0.2))
model.add(Dense(64, activation='relu'))
model.add(Dropout(0.2))
model.add(Dense(32, activation='relu'))
model.add(Dense(3, activation='softmax'))
You only need to define the input shape for the first layer, Keras will automatically determine the proper shape for the subsequent layers. You leave out the batch size when defining the input_shape, which is the first dimension, hence input_shape=X_train.shape[1:].
A sigmoid activation will actually work (because the output will vary between 0 and 1), but what you really want is a softmax activation (which makes sure all the outputs sum to 1, which is what probability dictates -- the probability that something happened is 100%, not the 120% that sigmoid could end up giving you).
Also, you're not using your LabelEncoder anywhere. I think what you mean to do is this:
def prepare_targets(y_train, y_test):
le = LabelEncoder()
ohe = OneHotEncoder()
# teach the label encoder our labels
le.fit(y_train)
# turn our strings into integers
y_train_transformed = le.transform(y_train)
y_test_transformed = le.transform(y_test)
# turn our integers into one-hot-encoded arrays
y_train_enc = ohe.fit_transform(y_train_transformed).toarray()
y_test_enc = ohe.transform(y_test_transformed).toarray()
return y_train_enc, y_test_enc

ValueError: Input 0 of layer sequential_2 is incompatible with the layer

I have the following code:
import tensorflow as tf
import keras
from keras.datasets import cifar10
(x_train, y_train), (x_test, y_test) = cifar10.load_data()
import numpy as np
x_train = np.reshape(x_train, (x_train.shape[0], x_train.shape[1], x_train.shape[2], 3))
print(x_train.shape)
x_test = np.reshape(x_test, (x_test.shape[0], x_test.shape[1], x_test.shape[2], 3))
print(x_test.shape)
x_train = x_train.astype('float32')/255.0
x_test = x_test.astype('float32')/255.0
from keras.utils import to_categorical
y_train = to_categorical(y_train, num_classes = 10)
y_test = to_categorical(y_test, num_classes = 10)
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, Flatten
model = Sequential()
#Defining layers of the model
model.add(Dense(2056, activation='relu', input_shape = (3072,)))
model.add(Dense(10, activation='softmax'))
model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'])
model.summary()
history = model.fit(x_train, y_train, batch_size=1000, epochs=50)
And I am facing the following error:
ValueError: Input 0 of layer sequential_2 is incompatible with the layer: expected axis -1 of input shape to have value 3072 but received input with shape (1000, 32, 32, 3)
I want to keep the input_shape as 3072 only. How can I reshape my y_test to solve this?

You should Flatten your input data before passing them to Dense layer.
model = Sequential()
#Defining layers of the model
model.add(Flatten(input_shape=(32,32,3)) # 32*32*3 = 3072
model.add(Dense(2056, activation='relu'))
model.add(Dense(10, activation='softmax'))
This should fix the problem.

Incompatible shapes: [64,4,4] vs. [64,4] - Time Series with 4 variables as input

I am trying to train a LSTM Autoencoder with multivariate time series data. The shape of data is:
print(X_train.shape)
print(y_train.shape)
print(X_test.shape)
print(y_test.shape)
(160573, 4, 4)
(160573, 4)
(17838, 4, 4)
(17838, 4)
I wanted to transfer my model for univariate time series into a model for multivariate time series but somehow I do not know what to change:
model = keras.Sequential()
model.add(keras.layers.LSTM(
units=64,
input_shape=(X_train.shape[1], X_train.shape[2])
))
model.add(keras.layers.Dropout(rate=0.2))
model.add(keras.layers.RepeatVector(n=X_train.shape[1]))
model.add(keras.layers.LSTM(units=64, return_sequences=True))
model.add(keras.layers.Dropout(rate=0.2))
model.add(
keras.layers.TimeDistributed(
keras.layers.Dense(units=X_train.shape[2])
)
)
model.compile(loss='mse', optimizer='adam')
training = model.fit(
X_train, y_train,
epochs=10,
batch_size=64,
validation_split=0.1,
shuffle=False
)
The error is:
InvalidArgumentError: Incompatible shapes: [64,4,4] vs. [64,4]
[[node gradient_tape/mean_squared_error/BroadcastGradientArgs (defined at <ipython-input-83-6205cceab3d0>:1) ]] [Op:__inference_train_function_96522]
Function call stack:
train_function
Is there any solution for multivariate time series?
Thank you and best,
Can

Assuming you're not trying to predict a sequence, you need to remove return_sequences=True to remove the time step. Same goes for the TimeDistributed layer, remove it.
Corrected, minimal example:
import os
os.environ['CUDA_VISIBLE_DEVICES'] = '-1'
from tensorflow import keras
import numpy as np
X_train = np.random.rand(160, 4, 4)
y_train = np.random.rand(160, 4)
X_test = np.random.rand(17, 4, 4)
y_test = np.random.rand(17, 4)
model = keras.Sequential()
model.add(keras.layers.LSTM(
units=4,
input_shape=(X_train.shape[1], X_train.shape[2])
))
model.add(keras.layers.Dropout(rate=0.2))
model.add(keras.layers.RepeatVector(n=X_train.shape[1]))
model.add(keras.layers.LSTM(units=4))
model.add(keras.layers.Dropout(rate=0.2))
model.add(keras.layers.Dense(units=X_train.shape[2]))
model.compile(loss='mse', optimizer='adam')
training = model.fit(
X_train, y_train,
epochs=1,
batch_size=8,
validation_split=0.1,
shuffle=False)

Using conv1D “Error when checking input: expected conv1d_input to have 3 dimensions, but got array with shape (213412, 36)”

My input is simply a csv file with 237124 rows and 37 columns :
The first 36 columns as features
The last column is a Binary class label
I am trying to train my data on the conv1D model.
I have tried to build a CNN with one layer, but I have some problems with it.
The compiler outputs:
ValueError:Error when checking input: expected conv1d_9_input to have shape
(213412, 36) but got array with shape (36, 1)
Code:
import pandas as pd
import numpy as np
import sklearn
from sklearn import metrics
from sklearn.model_selection import KFold
from sklearn.metrics import confusion_matrix
from sklearn.preprocessing import StandardScaler
import keras
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, Flatten
from tensorflow.keras.layers import Conv2D,Conv1D, MaxPooling2D,MaxPooling1D
from tensorflow.keras.layers import Activation
from tensorflow.keras.layers import Dropout,BatchNormalization
dataset=pd.read_csv("C:/Users/User/Desktop/data.csv",encoding='cp1252')
dataset.shape
#output: (237124, 37)
array = dataset.values
X = array[:,0:36]
Y = array[:,36]
kf = KFold(n_splits=10)
kf.get_n_splits(X)
for trainindex, testindex in kf.split(X):
Xtrain, Xtest = X[trainindex], X[testindex]
Ytrain, Ytest = Y[trainindex], Y[testindex]
Xtrain.shape[0]
#output: 213412
Xtrain.shape[1]
#output: 36
Ytrain.shape[0]
#output: 213412
n_timesteps, n_features, n_outputs =Xtrain.shape[0], Xtrain.shape[1],
Ytrain.shape[0]
model = Sequential()
model.add(Conv1D(filters=64, kernel_size=1,
activation='relu',input_shape=(n_timesteps,n_features)))
model.add(Conv1D(filters=64, kernel_size=1, activation='relu'))
model.add(Dropout(0.5))
model.add(MaxPooling1D(pool_size=2))
model.add(Flatten())
model.add(Dense(100, activation='relu'))
model.add(Dense(n_outputs, activation='softmax'))
model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=
['accuracy'])
# fit network
model.fit(Xtrain, Ytrain, epochs=10, batch_size=32, verbose=0)
# Testing CNN model BY X test
Predictions = model.predict(Xtest,batch_size =100)
rounded = [round(x[0]) for x in Predictions]
Y_predection = pd.DataFrame(rounded)
Y_predection = Y_predection.iloc[:, 0]
.
.
.
I tried to modify the code this way:
Xtrain = np.expand_dims(Xtrain, axis=2)
But the error remains the same.

There's a couple of problems I notice with your code.
Xtrain - Needs to be a 3D tensor. Because anything else, Conv1D cannot process. So if you have 2D data you need to add a new dimension to make it 3D.
Your input_shape needs to be changed to reflect that. For example, if you added only a single channel, it should be [n_features, 1].
# Here I'm assuming some dummy data
# Xtrain => [213412, 36, 1] (Note that you need Xtrain to be 3D not 2D - So we're adding a channel dimension of 1)
Xtrain = np.expand_dims(np.random.normal(size=(213412, 36)),axis=-1)
# Ytrain => [213412, 10]
Ytrain = np.random.choice([0,1], size=(213412,10))
n_timesteps, n_features, n_outputs =Xtrain.shape[0], Xtrain.shape[1], Ytrain.shape[1]
model = Sequential()
model.add(Conv1D(filters=64, kernel_size=1,
activation='relu',input_shape=(n_features,1)))
model.add(Conv1D(filters=64, kernel_size=1, activation='relu'))
model.add(Dropout(0.5))
model.add(MaxPooling1D(pool_size=2))
model.add(Flatten())
model.add(Dense(100, activation='relu'))
model.add(Dense(n_outputs, activation='softmax'))
model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
# fit network
model.fit(Xtrain, Ytrain, epochs=10, batch_size=32, verbose=0)

You need to specifi only how many dimension X has, not how many samples you will pass for the input layer.
model.add(Conv1D(filters=64, kernel_size=3, activation='relu', input_shape=(n_features,)))
This means that the input will be N samples of shape n_features
For the last layer you should change the number of units to how many classes you have instead of how many rows your data has.

I get an error in CNN optimization (learning) implemented by keras, but I do not know the solution

An error occurred while creating the convolutional layer of the CNN model.
I do not know the cause of the error and how to handle it. I'm in the process of implementing CNN using keras and cifar10 for machine learning with Python 3.5.
I get the following error in the learning phase:
Error when checking target: expected dense_46 to have shape (10,) but got array with shape (1,)
It does not work well with reshaping the array of labels (y_train).
#import
import tensorflow as tf
import keras
import numpy as np
import matplotlib.pyplot as plt
#get data
from keras.datasets import cifar10
(x_train, y_train), (x_test, y_test) = cifar10.load_data()
#Image normalization
x_train = x_train / 255.0
x_test = x_test / 255.0
#Build a CNN mode
def CNN_model():
model = keras.models.Sequential()
#1st convolutional layer
model.add(keras.layers.Conv2D(96, 11, strides=(4, 4), padding='valid', data_format='channels_last', activation='relu', bias_initializer='ones', input_shape=(32,32,3)))
model.add(keras.layers.normalization.BatchNormalization(axis=1))
model.add(keras.layers.MaxPooling2D(pool_size=(3, 3), strides=(2, 2), padding='valid', data_format='channels_last'))
#2nd convolutional layer
model.add(keras.layers.Conv2D(256, 2, strides=(4, 4), padding='valid', data_format='channels_last', activation='relu', bias_initializer='zeros'))
model.add(keras.layers.normalization.BatchNormalization(axis=1))
model.add(keras.layers.MaxPooling2D(pool_size=(1, 1), strides=(2, 2), padding='valid', data_format='channels_last'))
#3rd layer
model.add(keras.layers.Flatten())
model.add(keras.layers.Dense(4096))
model.add(keras.layers.Dropout(0.5))
model.add(keras.layers.Dense(4096))
model.add(keras.layers.Dropout(0.5))
#4th Softmax layer
model.add(keras.layers.Dense(10, activation='softmax'))
#compile
model.compile(optimizer=keras.optimizers.SGD(lr=0.01), loss='categorical_crossentropy', metrics=['accuracy'])
return model
#Summary
model = CNN_model()
model.summary()
#Fitting
early_stopping = keras.callbacks.EarlyStopping(monitor='val_loss', patience=3, verbose=1, mode='auto')
history = model.fit(x_train, y_train, epochs=15, batch_size=128, shuffle=True, validation_split=0.25, callbacks=[early_stopping])

The problem here lies in your target aka y_train. Yours is probably a 1 x n_samples long vector with 10 different lables in. The way Keras needs it is in a One-Hot-Encoded way. You can do this by:
y_train = to_categorical(y_train)
Hope it helps!