Cant understand why keras linear regression model is not working. Using Boston Housing data.Get Loss as nan
path='/Users/admin/Desktop/airfoil_self_noise.csv'
df=pd.read_csv(path,sep='\t',header=None)
y=df[5] #TARGET
df2=df.iloc[:,:-1]
X_train, X_test, y_train, y_test = train_test_split(df2, y, test_size=0.2)
p = Sequential()
p.add(Dense(units=20, activation='relu', input_dim=5))
p.add(Dense(units=20, activation='relu'))
p.add(Dense(units=1))
p.compile(loss='mean_squared_error',
optimizer='sgd')
p.fit(X_train, y_train, epochs=10, batch_size=32)
this yeilds:
Epoch 1/10
1202/1202 [==============================] - 0s 172us/step - loss: nan
Epoch 2/10
1202/1202 [==============================] - 0s 37us/step - loss: nan
Epoch 3/10
1202/1202 [==============================] - 0s 38us/step - loss: nan
Epoch 4/10
1202/1202 [==============================] - 0s 36us/step - loss: nan
Epoch 5/10
1202/1202 [==============================] - 0s 36us/step - loss: nan
Epoch 6/10
1202/1202 [==============================] - 0s 40us/step - loss: nan
Just to get you started, building on the top of NaN loss when training regression network
import pandas as pd
import keras
from keras.layers import Dense, Input
from keras import Sequential
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler
sc = StandardScaler()
#Grabbing these 2 lines from your example
path='/Users/admin/Desktop/airfoil_self_noise.csv'
df = pd.read_csv("airfoil_self_noise.csv", sep = '\t', header = None)
y = df[5]
df2 = df.iloc[:, :-1]
#preprocessing. Vectorization and Scaling
X_train, X_test, y_train, y_test = train_test_split(df2.values, y.values, test_size = 0.2)
X_train = sc.fit_transform(X_train)
X_test = sc.fit_transform(X_test)
p = Sequential()
p.add(Dense(units = 20, activation ='relu', input_dim = 5))
p.add(Dense(units = 20, activation ='relu'))
p.add(Dense(units = 1))
p.compile(loss = 'mean_squared_error', optimizer = 'adam')
print(p.fit(X_train, y_train, epochs = 100, batch_size = 64))
Related
After training the below model and plotting the train and validation accuracy I'm getting two straight horizontal lines (picture attached).
These are the parameters
Params:
mid_units: 256.0
activation: relu
dropout: 0.34943936277356535
optimizer: adam
batch_size: 64.0
for cls in os.listdir(path):
for sound in tqdm(os.listdir(os.path.join(path, cls))):
wav = librosa.load(os.path.join(os.path.join(path, cls, sound)), sr=16000)[0].astype(np.float32)
tmp_samples.append(wav)
tmp_labels.append(cls)
X_train, X_test, y_train , y_test = train_test_split( tmp_samples, tmp_labels , test_size=0.60,shuffle=True)
X_test,X_valid, y_test , y_valid = train_test_split( X_test, y_test , test_size=0.50,shuffle=True)
for x,y in zip(X_train,y_train):
extract_features_with_aug(x, y, model, samples , labels )
for x,y in zip(X_test,y_test):
extract_features(x, y, model, plain_samples , plain_labels )
for x,y in zip(X_valid,y_valid):
extract_features(x, y, model, valid_sample,valid_label)
X_train = np.asarray(samples)
y_train = np.asarray(labels)
X_test = np.asarray(plain_samples)
y_test=np.asarray(plain_labels)
X_valid = np.asarray(valid_sample)
y_valid=np.asarray(valid_label)
X_train = shuffle(samples)
y_train = shuffle(labels)
X_test = shuffle(plain_samples)
y_test=shuffle(plain_labels)
X_valid = shuffle(valid_sample)
y_valid=shuffle(valid_label)
return X_train, y_train , X_test , y_test ,X_valid,y_valid
Model:
input = layers.Input( batch_shape=(None,1024,1),dtype=tf.float32,name='audio')
drop=layers.Dropout( dropout_rate ) (input)
fl= layers.Flatten() (drop)
l= layers.Dense( mid_units , activation= activation )(fl)
ba=layers.BatchNormalization() (l)
drop2=layers.Dropout( dropout_rate ) (ba)
net=layers.Dense( 5, activation= activation )(drop2)
model = Model(inputs=input, outputs=net)
model.summary()
return model
def train_model(
X_train, y_train , X_test , y_test , X_valid,y_valid,
fname, # Path where to save the model
mid_units,
activation ,
dropout ,
batch_size ,
optimizer
):
# Generate the model
general_model = create_model( mid_units, activation , dropout )
general_model.compile(optimizer= optimizer , loss='categorical_crossentropy',
metrics=['accuracy'])
# Create some callbacks
callbacks = [tf.keras.callbacks.ModelCheckpoint(filepath=fname, monitor='val_loss', save_best_only=True),
tf.keras.callbacks.ReduceLROnPlateau(monitor='val_loss', factor=0.95, patience=5, verbose=1,
min_lr=0.000001)]
################
history = general_model.fit(X_train, y_train, epochs=EPOCHS, validation_data = ( X_valid,y_valid ), batch_size= batch_size ,
callbacks=callbacks, verbose=1)
For the training history I'm getting fixed values
3027/3027 [==============================] - 29s 9ms/step - loss: nan - accuracy: 0.2150 - val_loss: nan - val_accuracy: 0.2266
Epoch 97/100
3027/3027 [==============================] - 31s 10ms/step - loss: nan - accuracy: 0.2150 - val_loss: nan - val_accuracy: 0.2266
Epoch 98/100
3027/3027 [==============================] - 41s 14ms/step - loss: nan - accuracy: 0.2150 - val_loss: nan - val_accuracy: 0.2266
Epoch 99/100
3027/3027 [==============================] - 32s 11ms/step - loss: nan - accuracy: 0.2150 - val_loss: nan - val_accuracy: 0.2266
Epoch 100/100
Code:
import tensorflow.keras as tfk
import pandas as pd
import numpy as np
dataset = pd.read_csv("translator.csv")
x_train, x_test = dataset[["Afrikaans Woorde", "English Words"]]
y_train, y_test = dataset[["Total Letter Amount", "Incommon Letters"]]
x_train = np.array(x_train)
y_train = np.array(y_train)
x_test = np.array(x_test)
y_test = np.array(y_test)
model = tfk.models.Sequential()
input_layer = model.add(tfk.layers.Flatten())
hidden_layer1 = model.add(tfk.layers.Dense(128, activation="relu"))
hidden_layer2 = model.add(tfk.layers.Dense(128, activation="relu"))
output_layer = model.add(tfk.layers.Dense(1))
compiler = model.compile(optimizer="adam", loss="spare_categorical_crossentropy", metrics=["accuracy"])
fitter = model.fit(x_train, y_train, epochs=10)
val_loss, val_acc = model.evaluate(x_test, y_test)
print(f"Percentage loss {val_loss * 100}%", f"Percentage accuracy {val_acc * 100}%")
Error:
IndexError: list index out of range
---------------------------------------------------------------------------
IndexError Traceback (most recent call last)
~\AppData\Local\Temp/ipykernel_5408/3773670894.py in <module>
22 compiler = model.compile(optimizer="adam", loss="spare_categorical_crossentropy", metrics=["accuracy"])
23
---> 24 fitter = model.fit(x_train, y_train, epochs=10)
25
26 val_loss, val_acc = model.evaluate(x_test, y_test)
Question:
I have tried everything, I am not sure what to do? I have, even converted the dataset to an numpy array, yet it still gives me the error.
This specific model is to see if I can build a Translator just from a couple of words.
I tried with random input, your model architecture outputs 1, which means binary classification.
Working sample code
import tensorflow.keras as tfk
import numpy as np
import tensorflow as tf
X_train = np.random.random((1512,18))
y_train = np.random.random((1512,1))
dataset = tf.data.Dataset.from_tensor_slices((X_train, y_train))
train_data = dataset.shuffle(len(X_train)).batch(32)
train_data = train_data.prefetch(
buffer_size=tf.data.experimental.AUTOTUNE)
model = tfk.models.Sequential()
input = model.add(tfk.layers.Dense(15, activation=tf.nn.relu, input_shape=(18,)))
input_layer = model.add(tfk.layers.Flatten())
hidden_layer1 = model.add(tfk.layers.Dense(128, activation="relu"))
hidden_layer2 = model.add(tfk.layers.Dense(128, activation="relu"))
output_layer = model.add(tfk.layers.Dense(1))
model.compile(optimizer='adam',
loss=tf.keras.losses.CategoricalCrossentropy(),
metrics=['accuracy'])
fitter = model.fit(train_data, epochs=5, batch_size=5, verbose=1)
Output
Epoch 1/5
48/48 [==============================] - 4s 5ms/step - loss: 5.9153e-08 - accuracy: 0.0000e+00
Epoch 2/5
48/48 [==============================] - 0s 4ms/step - loss: 5.9153e-08 - accuracy: 0.0000e+00
Epoch 3/5
48/48 [==============================] - 0s 5ms/step - loss: 5.9153e-08 - accuracy: 0.0000e+00
Epoch 4/5
48/48 [==============================] - 0s 6ms/step - loss: 5.9153e-08 - accuracy: 0.0000e+00
Epoch 5/5
48/48 [==============================] - 0s 5ms/step - loss: 5.9153e-08 - accuracy: 0.0000e+00
I'm making a model (from this dataset: https://www.kaggle.com/karangadiya/fifa19) which guesses the "overall" of a player given some of his statistics. I've done the required data cleaning but the loss and accuracy is coming as "loss: nan - accuracy: 0.0000e+00."
Below is my code for data cleaning:-
%matplotlib notebook
import matplotlib.pyplot as plt
import pandas as pd
import numpy as np
import tensorflow as tf
from tensorflow import keras
from sklearn.model_selection import train_test_split
data = pd.read_csv("./data.csv")
data.dropna().replace(np.nan, 0)
data.columns = [x.lower() for x in data.columns]
data = data.select_dtypes(include="number").drop(columns = ["id", "unnamed: 0"], axis=1)
y = data["overall"]
X = data[["crossing", "headingaccuracy", "standingtackle", "gkreflexes"]]
X_train, X_test, y_train, y_test = train_test_split(X,y, random_state=45)
X_train = (X_train - np.max(X_train))/(np.max(X_train) - np.min(X_train))
y_train = np.array(y_train, dtype = 'float32')
And here's my model's code:-
model = keras.models.Sequential([
keras.layers.Dense(4, input_shape=(4,)),
keras.layers.Dense(128, activation=tf.nn.relu),
keras.layers.Dense(1, activation=tf.nn.relu)
])
model.compile(optimizer="sgd", loss="mean_squared_error", metrics=["accuracy"])
model.fit(X_train, y_train, epochs=5)
This was the output while model training:-
Epoch 1/5
427/427 [==============================] - 1s 2ms/step - loss: nan - accuracy: 0.0000e+00
Epoch 2/5
427/427 [==============================] - 1s 1ms/step - loss: nan - accuracy: 0.0000e+00
Epoch 3/5
427/427 [==============================] - 1s 1ms/step - loss: nan - accuracy: 0.0000e+00
Epoch 4/5
427/427 [==============================] - 1s 2ms/step - loss: nan - accuracy: 0.0000e+00
Epoch 5/5
427/427 [==============================] - 1s 2ms/step - loss: nan - accuracy: 0.0000e+00
<tensorflow.python.keras.callbacks.History at 0x13b16c6e280>
Thank you for your help!
i am pretty new to ML and trying to do an typical fashion_mnist Classification. The Problem is that the accuracy Score after I run the code is only 0.1 and the loss is below 0. So i guess the ML is not learning but I dont know what the Problem is?
Thx
from tensorflow.keras.datasets import fashion_mnist
(x_train, y_train), (x_test, y_test) = fashion_mnist.load_data()
x_train = x_train.astype('float32')
print(type(x_train))
x_train =x_train.reshape(60000,784)
x_train = x_train / 255.0
x_test =x_test.reshape(60000,784)
x_test= x_test/ 255.0
from tensorflow.keras import Sequential
from tensorflow.keras.layers import Dense
model= Sequential()
model.add(Dense(100, activation="sigmoid", input_shape=(784,)))
model.add(Dense(1, activation="sigmoid"))
model.compile(optimizer='sgd', loss="binary_crossentropy", metrics=["accuracy"])
model.fit(
x_train,
y_train,
epochs=10,
batch_size=1000)
Output:
Multiple issues with your code -
You have some error in the reshape x_test = x_test.reshape(10000,784) as it has 10000 images only.
You are using a sigmoid activation in the first dense layer, which is not a good practice. Instead, use relu.
Your output Dense has only 1 node. You are working with a dataset that has 10 unique classes. The output has to be Dense(10). Please understand that even though the y_train has classes 0-10, a neural network can't predict integer values with a softmax or sigmoid activation. Instead what you are trying to do is predict the probability values for EACH of the 10 classes.
You are using the incorrect activation in the final layer for multi-class classification. Use softmax.
You are using the incorrect loss function. For multi-class classification use categorical_crossentropy. Since your output is a 10-dimensional probability distribution, but your y_train is a single value for each class label, you can use sparse_categorical_crossentropy instead which is the same thing but handles label encoded y.
Try using a better optimizer to avoid getting stuck in local minima, such as adam.
It's preferred to use CNNs for image data since a simple Dense layer will not be able to capture the spatial features that make up the image. Since the images are small (28,28) and this is a toy example, it's ok the way it is.
Please refer to this table for checking out what to use. You have to ensure you know what problem you are solving in the first place though.
In your case, you want to do a multi-class single label classification but you are instead doing a multi-class multi-label classification by using the incorrect loss and output layer activation.
from tensorflow.keras.datasets import fashion_mnist
from tensorflow.keras import Sequential
from tensorflow.keras.layers import Dense
#Load data
(x_train, y_train), (x_test, y_test) = fashion_mnist.load_data()
#Normalize
x_train = x_train.astype('float32')
x_test = x_test.astype('float32')
#Reshape
x_train = x_train.reshape(60000,784)
x_train = x_train / 255.0
x_test = x_test.reshape(10000,784)
x_test = x_test / 255.0
print('Data shapes->',[i.shape for i in [x_train, y_train, x_test, y_test]])
#Contruct computation graph
model = Sequential()
model.add(Dense(100, activation="relu", input_shape=(784,)))
model.add(Dense(10, activation="softmax"))
#Compile with loss as cross_entropy and optimizer as adam
model.compile(optimizer='adam', loss="sparse_categorical_crossentropy", metrics=["accuracy"])
#Fit model
model.fit(x_train, y_train, epochs=10, batch_size=1000)
Data shapes-> [(60000, 784), (60000,), (10000, 784), (10000,)]
Epoch 1/10
60/60 [==============================] - 0s 5ms/step - loss: 0.8832 - accuracy: 0.7118
Epoch 2/10
60/60 [==============================] - 0s 6ms/step - loss: 0.5125 - accuracy: 0.8281
Epoch 3/10
60/60 [==============================] - 0s 6ms/step - loss: 0.4585 - accuracy: 0.8425
Epoch 4/10
60/60 [==============================] - 0s 6ms/step - loss: 0.4238 - accuracy: 0.8547
Epoch 5/10
60/60 [==============================] - 0s 7ms/step - loss: 0.4038 - accuracy: 0.8608
Epoch 6/10
60/60 [==============================] - 0s 6ms/step - loss: 0.3886 - accuracy: 0.8656
Epoch 7/10
60/60 [==============================] - 0s 6ms/step - loss: 0.3788 - accuracy: 0.8689
Epoch 8/10
60/60 [==============================] - 0s 6ms/step - loss: 0.3669 - accuracy: 0.8725
Epoch 9/10
60/60 [==============================] - 0s 6ms/step - loss: 0.3560 - accuracy: 0.8753
Epoch 10/10
60/60 [==============================] - 0s 6ms/step - loss: 0.3451 - accuracy: 0.8794
I am also adding a code for your reference with Convolutional layers, using categorical_crossentropy and functional API instead of Sequential. Please read the comments inline the code for more clarity. This should help you get an idea of some good practices when working with Keras.
from tensorflow.keras.datasets import fashion_mnist
from tensorflow.keras import layers, Model, utils
#Load data
(x_train, y_train), (x_test, y_test) = fashion_mnist.load_data()
#Normalize
x_train = x_train.astype('float32')
x_test = x_test.astype('float32')
#Reshape
x_train = x_train.reshape(60000,28,28,1)
x_train = x_train / 255.0
x_test = x_test.reshape(10000,28,28,1)
x_test = x_test / 255.0
#Set y to onehot instead of label encoded
y_train = utils.to_categorical(y_train)
y_test = utils.to_categorical(y_test)
#print([i.shape for i in [x_train, y_train, x_test, y_test]])
#Contruct computation graph
inp = layers.Input((28,28,1))
x = layers.Conv2D(32, (3,3), activation='relu', padding='same')(inp)
x = layers.MaxPooling2D((2,2))(x)
x = layers.Conv2D(32, (3,3), activation='relu', padding='same')(x)
x = layers.MaxPooling2D((2,2))(x)
x = layers.Flatten()(x)
out = Dense(10, activation='softmax')(x)
#Define model
model = Model(inp, out)
#Compile with loss as cross_entropy and optimizer as adam
model.compile(optimizer='adam', loss="categorical_crossentropy", metrics=["accuracy"])
#Fit model
model.fit(x_train, y_train, epochs=10, batch_size=1000)
utils.plot_model(model, show_layer_names=False, show_shapes=True)
I trained LSTM classification model, but got weird results (0 accuracy). Here is my dataset with preprocessing steps:
import pandas as pd
from sklearn.model_selection import train_test_split
import tensorflow as tf
from tensorflow import keras
import numpy as np
url = 'https://raw.githubusercontent.com/MislavSag/trademl/master/trademl/modeling/random_forest/X_TEST.csv'
X_TEST = pd.read_csv(url, sep=',')
url = 'https://raw.githubusercontent.com/MislavSag/trademl/master/trademl/modeling/random_forest/labeling_info_TEST.csv'
labeling_info_TEST = pd.read_csv(url, sep=',')
# TRAIN TEST SPLIT
X_train, X_test, y_train, y_test = train_test_split(
X_TEST.drop(columns=['close_orig']), labeling_info_TEST['bin'],
test_size=0.10, shuffle=False, stratify=None)
### PREPARE LSTM
x = X_train['close'].values.reshape(-1, 1)
y = y_train.values.reshape(-1, 1)
x_test = X_test['close'].values.reshape(-1, 1)
y_test = y_test.values.reshape(-1, 1)
train_val_index_split = 0.75
train_generator = keras.preprocessing.sequence.TimeseriesGenerator(
data=x,
targets=y,
length=30,
sampling_rate=1,
stride=1,
start_index=0,
end_index=int(train_val_index_split*X_TEST.shape[0]),
shuffle=False,
reverse=False,
batch_size=128
)
validation_generator = keras.preprocessing.sequence.TimeseriesGenerator(
data=x,
targets=y,
length=30,
sampling_rate=1,
stride=1,
start_index=int((train_val_index_split*X_TEST.shape[0] + 1)),
end_index=None, #int(train_test_index_split*X.shape[0])
shuffle=False,
reverse=False,
batch_size=128
)
test_generator = keras.preprocessing.sequence.TimeseriesGenerator(
data=x_test,
targets=y_test,
length=30,
sampling_rate=1,
stride=1,
start_index=0,
end_index=None,
shuffle=False,
reverse=False,
batch_size=128
)
# convert generator to inmemory 3D series (if enough RAM)
def generator_to_obj(generator):
xlist = []
ylist = []
for i in range(len(generator)):
x, y = train_generator[i]
xlist.append(x)
ylist.append(y)
X_train = np.concatenate(xlist, axis=0)
y_train = np.concatenate(ylist, axis=0)
return X_train, y_train
X_train_lstm, y_train_lstm = generator_to_obj(train_generator)
X_val_lstm, y_val_lstm = generator_to_obj(validation_generator)
X_test_lstm, y_test_lstm = generator_to_obj(test_generator)
# test for shapes
print('X and y shape train: ', X_train_lstm.shape, y_train_lstm.shape)
print('X and y shape validate: ', X_val_lstm.shape, y_val_lstm.shape)
print('X and y shape test: ', X_test_lstm.shape, y_test_lstm.shape)
and here is my model with resuslts:
### MODEL
model = keras.models.Sequential([
keras.layers.LSTM(124, return_sequences=True, input_shape=[None, 1]),
keras.layers.LSTM(258),
keras.layers.Dense(1, activation='sigmoid')
])
model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
history = model.fit(X_train_lstm, y_train_lstm, epochs=10, batch_size=128,
validation_data=[X_val_lstm, y_val_lstm])
# history = model.fit_generator(train_generator, epochs=40, validation_data=validation_generator, verbose=1)
score, acc = model.evaluate(X_val_lstm, y_val_lstm,
batch_size=128)
historydf = pd.DataFrame(history.history)
historydf.head(10)
Why do I get 0 accuracy?
You're using sigmoid activation, which means your labels must be in range 0 and 1. But in your case, the labels are 1. and -1.
Just replace -1 with 0.
for i, y in enumerate(y_train_lstm):
if y == -1.:
y_train_lstm[i,:] = 0.
for i, y in enumerate(y_val_lstm):
if y == -1.:
y_val_lstm[i,:] = 0.
for i, y in enumerate(y_test_lstm):
if y == -1.:
y_test_lstm[i,:] = 0.
Sidenote:
The signals are very close, it would be hard to distinguish them. So, probably accuracy won't be high with simple models.
After training with 0. and 1. labels,
model = keras.models.Sequential([
keras.layers.LSTM(124, return_sequences=True, input_shape=(30, 1)),
keras.layers.LSTM(258),
keras.layers.Dense(1, activation='sigmoid')
])
model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
history = model.fit(X_train_lstm, y_train_lstm, epochs=5, batch_size=128,
validation_data=(X_val_lstm, y_val_lstm))
# history = model.fit_generator(train_generator, epochs=40, validation_data=validation_generator, verbose=1)
score, acc = model.evaluate(X_val_lstm, y_val_lstm,
batch_size=128)
historydf = pd.DataFrame(history.history)
historydf.head(10)
Epoch 1/5
12/12 [==============================] - 5s 378ms/step - loss: 0.7386 - accuracy: 0.4990 - val_loss: 0.6959 - val_accuracy: 0.4896
Epoch 2/5
12/12 [==============================] - 4s 318ms/step - loss: 0.6947 - accuracy: 0.5133 - val_loss: 0.6959 - val_accuracy: 0.5104
Epoch 3/5
12/12 [==============================] - 4s 318ms/step - loss: 0.6941 - accuracy: 0.4895 - val_loss: 0.6930 - val_accuracy: 0.5104
Epoch 4/5
12/12 [==============================] - 4s 332ms/step - loss: 0.6946 - accuracy: 0.5269 - val_loss: 0.6946 - val_accuracy: 0.5104
Epoch 5/5
12/12 [==============================] - 4s 334ms/step - loss: 0.6931 - accuracy: 0.4901 - val_loss: 0.6929 - val_accuracy: 0.5104
3/3 [==============================] - 0s 73ms/step - loss: 0.6929 - accuracy: 0.5104
loss accuracy val_loss val_accuracy
0 0.738649 0.498980 0.695888 0.489583
1 0.694708 0.513256 0.695942 0.510417
2 0.694117 0.489463 0.692987 0.510417
3 0.694554 0.526852 0.694613 0.510417
4 0.693118 0.490143 0.692936 0.510417
Source code in colab: https://colab.research.google.com/drive/10yRf4TfGDnp_4F2HYoxPyTlF18no-8Dr?usp=sharing