I have a problem using Tensorboard especially with histogram_freq not zero in a Hyperas model.
I only added a Hyperas example with the tensorboard-callback. If histogram_freq=0 everything works fine. But if it is different I got the error:
InvalidArgumentError (see above for traceback): Shape [-1,784] has negative dimensions
[[Node: dense_1_input = Placeholderdtype=DT_FLOAT, shape=[?,784], _device="/job:localhost/replica:0/task:0/cpu:0"]]
I'm using:
Windows 7
tensorflow 1.3.0
tensorflow-tensorboard 0.1.6
hyperas 0.4
hyperopt 0.1
python 3.5.3
I tried on different machines (Windows 10) and also with tensorflow-version 1.2.1. Has anybody an idea how to fix it?
Example.py:
from __future__ import print_function
from hyperopt import Trials, STATUS_OK, tpe
from keras.datasets import mnist
from keras.layers.core import Dense, Dropout, Activation
from keras.models import Sequential
from keras.utils import np_utils
import keras.callbacks as callbacks
from hyperas import optim
from hyperas.distributions import choice, uniform, conditional
def data():
"""`
Data providing function:
This function is separated from model() so that hyperopt
won't reload data for each evaluation run.
"""
(x_train, y_train), (x_test, y_test) = mnist.load_data()
x_train = x_train.reshape(60000, 784)
x_test = x_test.reshape(10000, 784)
x_train = x_train.astype('float32')
x_test = x_test.astype('float32')
x_train /= 255
x_test /= 255
nb_classes = 10
y_train = np_utils.to_categorical(y_train, nb_classes)
y_test = np_utils.to_categorical(y_test, nb_classes)
return x_train, y_train, x_test, y_test
def model(x_train, y_train, x_test, y_test):
"""
Model providing function:
Create Keras model with double curly brackets dropped-in as needed.
Return value has to be a valid python dictionary with two customary keys:
- loss: Specify a numeric evaluation metric to be minimized
- status: Just use STATUS_OK and see hyperopt documentation if not feasible
The last one is optional, though recommended, namely:
- model: specify the model just created so that we can later use it again.
"""
model = Sequential()
model.add(Dense(512, input_shape=(784,)))
model.add(Activation('relu'))
model.add(Dropout({{uniform(0, 1)}}))
model.add(Dense({{choice([256, 512, 1024])}}))
model.add(Activation({{choice(['relu', 'sigmoid'])}}))
model.add(Dropout({{uniform(0, 1)}}))
# If we choose 'four', add an additional fourth layer
if conditional({{choice(['three', 'four'])}}) == 'four':
model.add(Dense(100))
# We can also choose between complete sets of layers
model.add({{choice([Dropout(0.5), Activation('linear')])}})
model.add(Activation('relu'))
model.add(Dense(10))
model.add(Activation('softmax'))
model.compile(loss='categorical_crossentropy', metrics=['accuracy'],
optimizer={{choice(['rmsprop', 'adam', 'sgd'])}})
tbCallBack = callbacks.TensorBoard(log_dir='./Graph', histogram_freq=1,
write_graph=True, write_images=True)
model.fit(x_train, y_train,
batch_size={{choice([64, 128])}},
epochs=3,
verbose=2,
validation_data=(x_test, y_test),
callbacks=[tbCallBack])
score, acc = model.evaluate(x_test, y_test, verbose=0)
print('Test accuracy:', acc)
return {'loss': -acc, 'status': STATUS_OK, 'model': model}
if __name__ == '__main__':
best_run, best_model = optim.minimize(model=model,
data=data,
algo=tpe.suggest,
max_evals=2,
trials=Trials())
X_train, Y_train, X_test, Y_test = data()
print("Evalutation of best performing model:")
print(best_model.evaluate(X_test, Y_test))
print("Best performing model chosen hyper-parameters:")
print(best_run)
Related
I'm trying to calculate cross-entropy losses using the Iris dataset, but when I ran my model and fired up my plots, both my losses and validation losses remained a straight line at zero. I don't know what I'm doing wrong. This is my code:
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler
from tensorflow import keras
from keras import Sequential
from keras.layers import BatchNormalization, Dense, Dropout
from keras.callbacks import EarlyStopping
iris = sns.load_dataset('iris')
X = iris.iloc[:,:4]
y = iris.species.replace({'setosa': 0, 'versicolor': 1, 'virginica': 2})
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=.3, random_state=69)
sc = StandardScaler()
sc.fit_transform(X_train)
sc.fit_transform(X_test)
nn_model = Sequential([Dense(4, activation='relu', input_shape=[X.shape[1]]),
BatchNormalization(),
Dropout(.3),
Dense(4, activation='relu'),
BatchNormalization(),
Dropout(.3),
Dense(1, activation='sigmoid')])
nn_model.compile(optimizer='sgd', loss='categorical_crossentropy', metrics=['categorical_accuracy'])
early_stopping = EarlyStopping(min_delta=1e-3, patience=10, restore_best_weights=True)
fit = nn_model.fit(X_train, y_train, validation_data=(X_test,y_test),
batch_size=16, epochs=200, callbacks=[early_stopping], verbose=1)
losses = pd.DataFrame(fit.history)
And this is what the plots look like:
Any reason why it's doing this?
The fit transformation of StandardScaler() isn't in-place operation. You have to do as follows
sc = StandardScaler()
X_train = sc.fit_transform(X_train)
X_test = sc.transform(X_test)
Also, you have 3 outputs (check: y_train.value_counts()), so the output layer should be:
nn_model = Sequential([ ...,
Dropout(.3),
Dense(3, activation='softmax')])
Lastly, the loss function should be sparse_categorical_crossentropy for your integer target.
nn_model.compile(optimizer='sgd',
loss='sparse_categorical_crossentropy',
metrics=['accuracy'])
enter image description hereenter image description here
There is 1000 data, and in this 1000, 670 is train and 330 is test.
model.fit(x_train, y_train, epochs = 250)
when I write this it should be 670 train data but in my case it’s 21.
CODE
import pandas as pd
dataFrame = pd.read_excel("bisiklet_fiyatlari.xlsx")
from sklearn.model_selection import train_test_split
y=dataFrame["Fiyat"].values
x=dataFrame[["BisikletOzellik1","BisikletOzellik2"]].values
x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.33, random_state=15)
from sklearn.preprocessing import MinMaxScaler
scaler = MinMaxScaler()
scaler.fit(x_train)
x_train=scaler.transform(x_train)
x_test=scaler.transform(x_test)
import tensorflow as tf
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense
model = Sequential()
model.add(Dense(4, activation = "relu"))
model.add(Dense(4, activation = "relu"))
model.add(Dense(4, activation = "relu"))
model.add(Dense(1))
model.compile(optimizer="rmsprop",loss="mse")
model.fit(x_train, y_train, epochs = 250)
The reason you are getting 21/21 in the step count of the fit method is because, as you didn't specified a batch_size argument to the method, it defaults to 32 (see the documentation of the fit method), and 670/32 = 20,94 that approximates to 21.
Regarding why your loss is so high, I cannot tell with the data provided, you will need to show which is the data, the code used to preprocess it, etc.
I'm trying to train my Deep Neural Network to recognize handwritten
numbers but I keep getting the error stated previously in the title It
gives me an error saying: ValueError: Input arrays should have the
same number of samples as target arrays. Found 60000 input samples and
10000 target samples. How can i fix this? (i already tried
train_test_split and transport but nothing worked)
# Imports
import keras
from keras.datasets import mnist
from keras.models import Sequential
from keras.layers import Dense
from keras.utils import to_categorical
# Configuration options
feature_vector_length = 784
num_classes = 60000
# Load the data
(X_train, Y_train), (X_test, Y_test) = mnist.load_data()
# Reshape the data - MLPs do not understand such things as '2D'.
# Reshape to 28 x 28 pixels = 784 features
X_train = X_train.reshape(X_train.shape[0], feature_vector_length)
X_test = X_test.reshape(X_test.shape[0], feature_vector_length)
# Convert into greyscale
X_train = X_train.astype('float32')
X_test = X_test.astype('float32')
X_train /= 255
X_test /= 255
# Convert target classes to categorical ones
Y_train = to_categorical(Y_train, num_classes)
Y_test = to_categorical(Y_test, num_classes)
# Load the data
(X_train, Y_train), (X_test, Y_test) = mnist.load_data()
# Visualize one sample
import matplotlib.pyplot as plt
plt.imshow(X_train[0], cmap='Greys')
plt.show()
# Set the input shape
input_shape = (feature_vector_length,)
print(f'Feature shape: {input_shape}')
# Create the model
# Using sigmoid instead of relu function
model = Sequential()
model.add(Flatten())
model.add(Dense(350, input_shape=input_shape, activation="sigmoid",
kernel_initializer=init))
model.add(Dense(50, activation="sigmoid", kernel_initializer=init))
model.add(Dense(num_classes, activation="sigmoid",
kernel_initializer=init))
# Configure the model and start training
model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=
['accuracy'])
model.fit(X_train, Y_train, epochs=10, batch_size=250, verbose=1,
validation_split=0.2)
# Test the model after training
test_results = model.evaluate(X_test, Y_test, verbose=1)
print(f'Test results - Loss: {test_results[0]} - Accuracy:
{test_results[1]}%')
If you want a solution to your problem, here it is:
# Imports
import keras
from keras.datasets import mnist
from keras.models import Sequential
from keras.layers import Dense, Flatten
from keras.utils import to_categorical
# Configuration options
feature_vector_length = 784
num_classes = 10
# Load the data
(X_train, Y_train), (X_test, Y_test) = mnist.load_data()
# Reshape the data - MLPs do not understand such things as '2D'.
# Reshape to 28 x 28 pixels = 784 features
# X_train = X_train.reshape(X_train.shape[0], feature_vector_length)
# X_test = X_test.reshape(X_test.shape[0], feature_vector_length)
# Convert into greyscale
X_train = X_train.astype('float32')
X_test = X_test.astype('float32')
X_train /= 255
X_test /= 255
# Convert target classes to categorical ones
Y_train = to_categorical(Y_train, num_classes)
Y_test = to_categorical(Y_test, num_classes)
# Create the model
# Using sigmoid instead of relu function
model = Sequential()
model.add(Flatten())
model.add(Dense(350, input_shape=input_shape, activation="relu"))
model.add(Dense(50, activation="relu"))
model.add(Dense(num_classes, activation="softmax"))
# Configure the model and start training
model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=
['accuracy'])
model.fit(X_train, Y_train, epochs=10, batch_size=250, verbose=1,
validation_split=0.2)
# Test the model after training
test_results = model.evaluate(X_test, Y_test, verbose=1)
But you should really do some research and understand what every line in your code is supposed to do and what every parameter means. For example, the choice of sigmoid activation function being wrong, especially in the final layer is the very first thing to understand. That's one of the many things that you should do research about. Then there is:
understanding why and when to reshape your data,
what is the purpose of flatten layer
and most importantly, understand what is num_classes and why it is 10 and not 1000 or 60000
So, I understand that normalization is important to train a neural network.
I also understand that I have to normalize validation- and test-set with the parameters from the training set (see e.g. this discussion: https://stats.stackexchange.com/questions/77350/perform-feature-normalization-before-or-within-model-validation)
My question is: How do I do this in Keras?
What I'm currently doing is:
import numpy as np
from keras.models import Sequential
from keras.layers import Dense
from keras.callbacks import EarlyStopping
def Normalize(data):
mean_data = np.mean(data)
std_data = np.std(data)
norm_data = (data-mean_data)/std_data
return norm_data
input_data, targets = np.loadtxt(fname='data', delimiter=';')
norm_input = Normalize(input_data)
model = Sequential()
model.add(Dense(25, input_dim=20, activation='relu'))
model.add(Dense(1, activation='sigmoid'))
model.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy'])
early_stopping = EarlyStopping(monitor='val_acc', patience=50)
model.fit(norm_input, targets, validation_split=0.2, batch_size=15, callbacks=[early_stopping], verbose=1)
But here, I first normalize the data w.r.t. the whole data set and then split up the validation set, which is wrong according to the above mentioned discussion.
It wouldn't be a big deal to save the mean and standard deviation from the training set(training_mean and training_std), but how can I apply the normalization with the training_mean and training_std on the validation set separately?
Following code makes exactly what you want:
import numpy as np
def normalize(x_train, x_test):
mu = np.mean(x_train, axis=0)
std = np.std(x_train, axis=0)
x_train_normalized = (x_train - mu) / std
x_test_normalized = (x_test - mu) / std
return x_train_normalized, x_test_normalized
Then you can use it with keras like this:
from keras.datasets import boston_housing
(x_train, y_train), (x_test, y_test) = boston_housing.load_data()
x_train, x_test = normalize(x_train, x_test)
The Wilmar's answer is not correct.
You can split your data into a training and testing dataset manually before fitting the model with sklearn.model_selection.train_test_split. Afterwards, normalize the training and testing data based on the mean and standard deviation of the training data. Finally, call model.fit with the validation_data argument.
Code example
import numpy as np
from sklearn.model_selection import train_test_split
data = np.random.randint(0,100,200).reshape(20,10)
target = np.random.randint(0,1,20)
X_train, X_test, y_train, y_test = train_test_split(data, target, test_size=0.2)
def Normalize(data, mean_data =None, std_data =None):
if not mean_data:
mean_data = np.mean(data)
if not std_data:
std_data = np.std(data)
norm_data = (data-mean_data)/std_data
return norm_data, mean_data, std_data
X_train, mean_data, std_data = Normalize(X_train)
X_test, _, _ = Normalize(X_test, mean_data, std_data)
model.fit(X_train, y_train, validation_data=(X_test,y_test), batch_size=15, callbacks=[early_stopping], verbose=1)
I am trying to run the complete example from https://github.com/maxpumperla/hyperas (copied below).
Code from the example:
from __future__ import print_function
from hyperopt import Trials, STATUS_OK, tpe
from keras.datasets import mnist
from keras.layers.core import Dense, Dropout, Activation
from keras.models import Sequential
from keras.utils import np_utils
from hyperas import optim
from hyperas.distributions import choice, uniform, conditional
def data():
"""
Data providing function:
This function is separated from model() so that hyperopt
won't reload data for each evaluation run.
"""
(x_train, y_train), (x_test, y_test) = mnist.load_data()
x_train = x_train.reshape(60000, 784)
x_test = x_test.reshape(10000, 784)
x_train = x_train.astype('float32')
x_test = x_test.astype('float32')
x_train /= 255
x_test /= 255
nb_classes = 10
y_train = np_utils.to_categorical(y_train, nb_classes)
y_test = np_utils.to_categorical(y_test, nb_classes)
return x_train, y_train, x_test, y_test
def model(x_train, y_train, x_test, y_test):
"""
Model providing function:
Create Keras model with double curly brackets dropped-in as needed.
Return value has to be a valid python dictionary with two customary keys:
- loss: Specify a numeric evaluation metric to be minimized
- status: Just use STATUS_OK and see hyperopt documentation if not feasible
The last one is optional, though recommended, namely:
- model: specify the model just created so that we can later use it again.
"""
model = Sequential()
model.add(Dense(512, input_shape=(784,)))
model.add(Activation('relu'))
model.add(Dropout({{uniform(0, 1)}}))
model.add(Dense({{choice([256, 512, 1024])}}))
model.add(Activation({{choice(['relu', 'sigmoid'])}}))
model.add(Dropout({{uniform(0, 1)}}))
# If we choose 'four', add an additional fourth layer
if conditional({{choice(['three', 'four'])}}) == 'four':
model.add(Dense(100))
# We can also choose between complete sets of layers
model.add({{choice([Dropout(0.5), Activation('linear')])}})
model.add(Activation('relu'))
model.add(Dense(10))
model.add(Activation('softmax'))
model.compile(loss='categorical_crossentropy', metrics=['accuracy'],
optimizer={{choice(['rmsprop', 'adam', 'sgd'])}})
model.fit(x_train, y_train,
batch_size={{choice([64, 128])}},
epochs=1,
verbose=2,
validation_data=(x_test, y_test))
score, acc = model.evaluate(x_test, y_test, verbose=0)
print('Test accuracy:', acc)
return {'loss': -acc, 'status': STATUS_OK, 'model': model}
if __name__ == '__main__':
best_run, best_model = optim.minimize(model=model,
data=data,
algo=tpe.suggest,
max_evals=5,
trials=Trials())
X_train, Y_train, X_test, Y_test = data()
print("Evalutation of best performing model:")
print(best_model.evaluate(X_test, Y_test))
print("Best performing model chosen hyper-parameters:")
print(best_run)
I'm working with
hyperopt: 0.0.4
keras: 2.0.3
hyperas: 0.2 (not the dev version, which is 0.3)
Python 2.7.12, Anaconda
The error I get is:
Traceback (most recent call last):
File "hyperas_test.py", line 82, in <module>
trials=Trials())
File "/user/pkgs/anaconda2/lib/python2.7/site-packages/hyperas/optim.py", line 32, in minimize
best_run = base_minimizer(model, data, algo, max_evals, trials, rseed)
File "/user/pkgs/anaconda2/lib/python2.7/site-packages/hyperas/optim.py", line 120, in base_minimizer
rstate=np.random.RandomState(rseed))
File "/user/pkgs/anaconda2/lib/python2.7/site-packages/hyperopt-0.0.4-py2.7.egg/hyperopt/fmin.py", line 306, in fmin
File "/user/pkgs/anaconda2/lib/python2.7/site-packages/hyperopt-0.0.4-py2.7.egg/hyperopt/base.py", line 633, in fmin
TypeError: 'module' object is not callable
This error occurs in the __main__ section.
The part trials=Trials() looks like a deliberate calling of the module Trials, imported via from hyperopt import Trials.
Any idea what I'm doing wrong?
EDIT: Possibly interestingly, I got this simpler example running no problem, as long as I used Canopy instead of Anaconda.