The rmse computed by tensorflow does not match with the rmse computed manually by me. The relevant code has been pasted below :
# Train a linear regression model.
tf.logging.set_verbosity(tf.logging.INFO)
OUTDIR = 'sample_model_metadata'
import shutil
shutil.rmtree(OUTDIR, ignore_errors=True)
model = tf.estimator.LinearRegressor(feature_columns=make_feature_cols(), model_dir=OUTDIR)
model.train(make_train_input_fn(train_data, num_epochs=1))
#Make predictions on the validation data set.
predictions_vals = np.zeros(len(validation_data))
predictions = model.predict(input_fn = make_train_input_fn(validation_data, 1))
i =0
for items in predictions:
predictions_vals[i] = items['predictions'][0]
i += 1
evaluated_rmse = np.sqrt(mean_squared_error(predictions_vals, validation_data['Y']))
print(evaluated_rmse)
def print_rmse(model, df):
metrics = model.evaluate(input_fn = make_train_input_fn(df, 1))
print('RMSE on dataset = {}'.format(np.sqrt(metrics['average_loss'])))
print_rmse(model, validation_data)
i = 0
for items in predictions:
predictions_vals[i] = items['predictions'][0]
You are saving all the predictions into the same location of np array predictions_vals i.e. at i = 0. You missing i += 1 here ! unless you copied the code wrongly here.
The problem was with my input function, which was having shuffling turned on by default, causing the validation data to get jumbled and hence resulting in an erroneous validation score.
def make_train_input_fn(df, num_epochs=1, shuffle=True):
return tf.estimator.inputs.pandas_input_fn(
x=df,
y=df['Y'],
batch_size=128,
num_epochs=num_epochs,
shuffle=shuffle,
queue_capacity=2000)
I have made sure that shuffling is turned off while doing validation and this has fixed the issue.
predictions = model.predict(input_fn = make_train_input_fn(validation_data, 1, False))
Thanks
Related
I created the following function to use as an evaluation metric to tune hyper parameter
# function to calculate the RMSLE
def get_msle(true, predicted) :
return np.sqrt(msle(true, predicted))
# custom evaluation metric function for the LightGBM
def custom_eval(preds, dtrain):
labels = dtrain.get_label().astype(np.int)
preds = preds.clip(min=0)
return [('rmsle', get_msle(labels, preds))]
I created the following function to train and validate the hyper parameter
def get_n_estimators(evaluation_set, min_r, max_r):
results = []
for n_est in (range(min_r, max_r, 20)):
x = {}
SCORE_TRAIN = []
SCORE_VALID = []
for train, valid in (evaluation_set):
# seperate the independent and target variable from the train and validation set
train_data_x = train.drop(columns= ['WEEK_END_DATE', 'UNITS'])
train_data_y = train['UNITS']
valid_data_x = valid.drop(columns= ['WEEK_END_DATE', 'UNITS'])
valid_data_y = valid['UNITS']
# evaluation sets
# we will evaluate our model on both train and validation data
e_set = [(train_data_x, train_data_y), (valid_data_x, valid_data_y)]
# define the lgbmRegressor Model
model = lgb.LGBMRegressor(n_estimators = n_est,
learning_rate = 0.01,
n_jobs=4,
random_state=0,
objective='regression')
# fit the model
model.fit(train_data_x, train_data_y, eval_metric= custom_eval ,eval_set= e_set, verbose=False)
# store the RMSLE on train and validation sets in different lists
# so that at the end we can calculate the mean of results at the end
SCORE_TRAIN.append(model.evals_result_['validation_0']['rmsle'][-1])
SCORE_VALID.append(model.evals_result_['validation_1']['rmsle'][-1])
# calculate the mean rmsle on train and valid
mean_score_train = np.mean(SCORE_TRAIN)
mean_score_valid = np.mean(SCORE_VALID)
print('With N_ESTIMATORS:\t'+ str(n_est) + '\tMEAN RMSLE TRAIN:\t' + str(mean_score_train) + "\tMEAN RMSLE VALID: "+str(mean_score_valid))
x['n_estimators'] = n_est
x['mean_rmsle_train'] = mean_score_train
x['mean_rmsle_valid'] = mean_score_valid
results.append(x)
return pd.DataFrame.from_dict(results)
However, when I am trying to implement the function I am getting an error, this is the implementation code.
n_estimators_result = get_n_estimators(evaluation_set,min_r = 20, max_r = 901)
This is the error I am getting
'numpy.ndarray' object has no attribute 'get_label'
can you please help me in resolving this error ? I am struck with this for two days now
I am very new to neural networks. And I'm learning some basics and I'm stuck at this point.
I have played with many parameters included the inital values of the dynamic system. I dropped the learning rate and changed the number of epoches and the batch size. Also I changed the number of samples I give to the fit function. I added a hidden layer and removed one again. After all nothing really helped. Sometimes the predicted value is pretty close and sometimes its really far away from the tested value. Do I miss something? Or how can I improve neural networks in general to get what I want to predict? Do I have to find a "sweet spot"?
Below you can find my code. I would be glad for new ideas to improve the network. I am a beginner and this is my first StackOverflow post.
######################### import stuff ##########################
import numpy as np
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras import layers
import time
import matplotlib.pyplot as p
from keras import backend as K
###Model
k_dim =1000 # Number of timestamps
seq_dim = 1000 # Number of samples
def g_func(x):
return np.power(x,2)
input1 = keras.Input(shape=(k_dim,))
input2 = keras.Input(shape=(k_dim,))
merged = layers.concatenate([input1,input2])
alpha_pred = layers.Dense(32,input_dim=2, activation='relu')(merged)
x = layers.Dense(16,activation='sigmoid')(alpha_pred)
output_a = layers.Dense(1)(x)
model = keras.Model(inputs=[input1,input2], outputs=output_a)
model.compile(loss='mean_absolute_error', optimizer='sgd', metrics=['mean_squared_error'])
model.summary()
######################## Simulating Input and Output data ########################
n = 2
alpha = np.random.rand(1,seq_dim)
m = 1
X_train = np.random.rand(seq_dim,k_dim)
X_train[:,0] = 0
u = [float] * seq_dim
y_train = np.zeros((seq_dim,k_dim))
for j in range (seq_dim):
u = X_train[j,:]
for k in range(k_dim-1):
for i in range(n-1):
y_train[j,k+1] = alpha[0,i] * y_train[j, k-i] + g_func(u[k])
alpha = np.transpose(alpha)
print('Learning rate before first fit:', model.optimizer.learning_rate.numpy())
history = model.fit([X_train,y_train], alpha, batch_size=64, epochs=3000)
print("Learning rate before second fit:", model.optimizer.learning_rate.numpy())
K.set_value(model.optimizer.learning_rate, 0.001)
history = model.fit([X_train,y_train], alpha, batch_size=64, epochs=1000)
# Plot the lossfunction and mean squared error
p.plot(history.history['loss'], 'b-', label='LOSS')
p.plot(history.history['mean_squared_error'], 'r-', label='Mean squared error')
p.legend(loc='best')
p.draw()
print('Model trained...')
time.sleep(2)
alpha = None
X_train = None
y_train = None
u = None
seq_dim = 1
#####
###Model has been trained. Lets test with new x and y to get one alpha###
####
X_train = np.random.rand(seq_dim,k_dim)
u = [float] * seq_dim
y_train = np.zeros((seq_dim,k_dim))
alpha = np.array([0.9])
for j in range (seq_dim):
u = X_train[j, :]
for k in range(k_dim-1):
for i in range(n-1):
y_train[j,k+1] = alpha[i] * y_train[j,k-i] + g_func(u[k])
z = model.predict([X_train, y_train])
#Compare the real value with the predicted value
print('Comparing real value with predicted value')
for i,j in zip(alpha,z):
print('{} => {}'.format(i,j))
p.show()
The problem might be using sigmoid. This causes vanishing gradients too often and explodes gradients sometimes while backpropogating because its derivates are in range 0-0.25.
You have to know how your data is spread using plot. Also remove outliers in regression problems.
If the input data is too spread in your plot then the model will not predict close to the correct value all the time.
At the moment I'm trying to follow a example of Temperature Forecasting in Keras (as given in chapter 6.3 of F. Chollet's "Deep Learning with Python" book). I'm having some issues with prediction using the generator that is specified. My understanding is that I should be using model.predict_generator for prediction, but I'm unsure how to use the steps parameter for this method and how to get back predictions that are the correct "shape" for my original data.
Ideally, I would like to be able to plot the test set (indices 300001 until the end) and also plot my predictions for this test set (i.e. an array of the same length with predicted values).
An example (Dataset available here: https://s3.amazonaws.com/keras-datasets/jena_climate_2009_2016.csv.zip) is as follows:
import numpy as np
# Read in data
fname = ('jena_climate_2009_2016.csv')
f = open(fname)
data = f.read()
f.close()
lines = data.split('\n')
col_names = lines[0].split(',')
col_names = [i.replace('"', "") for i in col_names]
# Normalize the data
float_data = np.array(df.iloc[:, 1:])
temp = float_data[:, 1]
mean = float_data[:200000].mean(axis=0)
float_data -= mean
std = float_data[:200000].std(axis=0)
float_data /= std
def generator(data, lookback, delay, min_index, max_index, shuffle=False, batch_size=128, step=6):
if max_index is None:
max_index = len(data) - delay - 1
i = min_index + lookback
while 1:
if shuffle:
rows = np.random.randint(
min_index + lookback, max_index, size=batch_size)
else:
if i + batch_size >= max_index:
i = min_index + lookback
rows = np.arange(i, min(i + batch_size, max_index))
i += len(rows)
samples = np.zeros((len(rows),
lookback // step,
data.shape[-1]))
targets = np.zeros((len(rows),))
for j, row in enumerate(rows):
indices = range(rows[j] - lookback, rows[j], step)
samples[j] = data[indices]
targets[j] = data[rows[j] + delay][1]
yield(samples, targets)
lookback = 720
step = 6
delay = 144
train_gen = generator(float_data, lookback=lookback, delay=delay,
min_index=0, max_index=200000, shuffle=True,
step=step, batch_size=batch_size)
val_gen = generator(float_data, lookback=lookback, delay=delay,
min_index=200001, max_index=300000, step=step,
batch_size=batch_size)
test_gen = generator(float_data, lookback=lookback, delay=delay,
min_index=300001, max_index=None, step=step,
batch_size=batch_size)
val_steps = (300000 - 200001 - lookback)
test_steps = (len(float_data) - 300001 - lookback)
from keras.models import Sequential
from keras import layers
from keras.optimizers import RMSprop
model = Sequential()
model.add(layers.Flatten(input_shape=(lookback // step, float_data.shape[-1])))
model.add(layers.Dense(32, activation='relu'))
model.add(layers.Dense(1))
model.compile(optimizer=RMSprop(), loss='mae')
model.fit_generator(train_gen, steps_per_epoch=500,
epochs=20, validation_data=val_gen,
validation_steps=val_steps)
After some searching around online, I tried some techniques similar to the following:
pred = model.predict_generator(test_gen, steps=test_steps // batch_size)
However the prediction array that I got back was far too long and didn't match up to my original data at all. Has anyone got any suggestions?
For anyone looking at the question now, we are not required to specify the steps parameter when using predict_generator for the newer versions of keras. Ref: https://github.com/keras-team/keras/issues/11902
If a value is provided, predictions for step*batch_size examples will be generated. This may result in exclusion of len(test)%batch_size rows, as mentioned by OP.
Also, it seems to me that setting batch_size=1 defeats the purpose of using the generator, as it is equivalent to iterating over the test data one by one.
Similarly setting steps=1 (when batch_size is not set in test_generator) will read the entire test data at once, which is not ideal for large test data.
In predict_generator for steps divide number of images you have in test path with whatever batchsize you have provided in test_gen
EX: i have 50 images and i provided batch size of 10 than steps would be 5
#first seperate the `test images` and `test labels`
test_images,test_labels = next(test_gen)
#get the class indices
test_labels = test_labels[:,0] #this should give you array of labels
predictions = model.predict_generator(test_gen,steps = number of images/batchsize,verbose=0)
predictions[:,0] #this is your actual predictions
Your original code looks correct:
pred = model.predict_generator(test_gen, steps=test_steps // batch_size)
I tried and did not see any problem generating a pred of length around 120k. What size did you get?
Actually both of the steps in the code are incorrect. They should be:
val_steps = (300000 - 200001 - lookback) // batch_size
test_steps = (len(float_data) - 300001 - lookback) // batch_size
(Didn't it take forever for your validation to run for each epoch?)
Of course with this correction you can simply use
pred = model.predict_generator(test_gen, steps=test_steps)
As I arrived at a semi-acceptable version of an answer to my own question, I decided to post it for posterity:
test_gen = generator(float_data, lookback=lookback, delay=delay,
min_index=300001, max_index=None, step=step,
batch_size=1) # "reset" the generator
pred = model.predict_generator(test_gen, steps=test_steps)
This now has the shape I want to plot it against my original test set. I could also use a more manual approach inspired somewhat by this answer:
test_gen = generator(float_data, lookback=lookback, delay=delay,
min_index=300001, max_index=None, step=step,
batch_size=1) # "reset" the generator
truth = []
pred = []
for i in range(test_steps):
x, y = next(test_gen)
pred.append(model.pred(x))
truth.append(y)
pred = np.concatenate(pred)
truth = np.concatenate(truth)
I've stored the coefficients of intercept, AR, MA off ARIMA model of statsmodel package
x = df_sku
x_train = x['Weekly_Volume_Sales']
x_train_log = np.log(x_train)
x_train_log[x_train_log == -np.inf] = 0
x_train_mat = x_train_log.as_matrix()
model = ARIMA(x_train_mat, order=(1,1,1))
model_fit = model.fit(disp=0)
res = model_fit.predict(start=1, end=137, exog=None, dynamic=False)
print(res)
params = model_fit.params
But I'm unable to find any documentation on statsmodel that lets me refit the model parameters onto a set of new data and predict N steps.
Has anyone been able to accomplishing refitting the model and predicting out of time samples ?
I'm trying to accomplish something similar to R:
# Refit the old model with testData
new_model <- Arima(as.ts(testData.zoo), model = old_model)
Here is a code you can use:
def ARIMAForecasting(data, best_pdq, start_params, step):
model = ARIMA(data, order=best_pdq)
model_fit = model.fit(start_params = start_params)
prediction = model_fit.forecast(steps=step)[0]
#This returns only last step
return prediction[-1], model_fit.params
#Get the starting parameters on train data
best_pdq = (3,1,3) #It is fixed, but you can search for the best parameters
model = ARIMA(train_data, best_pdq)
model_fit = model.fit()
start_params = model_fit.params
data = train_data
predictions = list()
for t in range(len(test_data)):
real_value = data[t]
prediction = ARIMAForecasting(data, best_pdq, start_params, step)
predictions.append(prediction)
data.append(real_value)
#After you can compare test_data with predictions
Details you can check here:
https://www.statsmodels.org/dev/generated/statsmodels.tsa.arima_model.ARIMA.fit.html#statsmodels.tsa.arima_model.ARIMA.fit
Great question. I have found such example: https://alkaline-ml.com/pmdarima/develop/auto_examples/arima/example_add_new_samples.html
briefly:
import pmdarima as pmd
...
### split data as train/test:
train, test = ...
### fit initial model on `train` data:
arima = pmd.auto_arima(train)
...
### update initial fit with `test` data:
arima.update(test)
...
### create forecast using updated fit for N steps:
new_preds = arima.predict(n_periods=10)
I want to train, evaluate the accuracy and eventually predict with my model. This is my first time using high level APIs such as tf.estimator.
I'm getting a value error from estimator.train(train_input_fn):
'ValueError: features should be a dictionary of `Tensor's. Given type: '
I'm not sure what is going on here. My model is taking 3 inputs and producing a binary output from one neuron.
Before this error I was getting an error about the requested shape not equal to the actual shape, or something along those lines. I fixed it by reducing the batchSize down to 1, instead of 100. I'm sure this isn't going to do so well when it comes to training though.
Any ideas? Heres my code:
import tensorflow as tf
import numpy as np
import sys
sys.path.insert(0, '/Users/blairburns/Documents/DeepLearning/BackgroundColourPredictor/Dataset/Testing/')
sys.path.insert(0, '/Users/blairburns/Documents/DeepLearning/BackgroundColourPredictor/Dataset/Training/')
#other files
from TestDataNormaliser import *
from TrainDataNormaliser import *
learning_rate = 0.01
trainingIteration = 15
batchSize = 1
displayStep = 2
#Layers using tf.layers
def get_logits(features):
l1 = tf.layers.dense(features, 3, activation=tf.nn.relu)
l2 = tf.layers.dense(l1, 4, activation=tf.nn.relu)
l3 = tf.layers.dense(l2, 1, activation=None)
a = l3
return a
#cost function
def get_loss(a, labels):
#cross_entropy = tf.reduce_mean(-tf.reduce_sum(y * tf.log(a)))
return tf.nn.sigmoid_cross_entropy_with_logits(logits=a, labels=labels)
#cross_entropy = tf.reduce_mean((l3 - y)**2)
#cross_entropy = -tf.reduce_sum(y*tf.log(a))-tf.reduce_sum((1-y)*tf.log(1-a))
#optimizer
def get_train_op(loss):
learning_rate = 1e-3
optimizer = tf.train.RMSPropOptimizer(learning_rate)
return optimizer.minimize(loss, global_step=tf.train.get_global_step())
#training
####
def get_inputs(feature_data, label_data, batch_size, n_epochs=None, shuffle=True):
dataset = tf.data.Dataset.from_tensor_slices(
(feature_data, label_data))
dataset = dataset.repeat(n_epochs)
if shuffle:
dataset = dataset.shuffle(len(feature_data))
dataset = dataset.batch(batch_size)
features, labels = dataset.make_one_shot_iterator().get_next()
return features, labels
def model_fn(features, labels, mode):
a = get_logits(features)
loss = get_loss(a, labels)
train_op = get_train_op(loss)
predictions = tf.greater(a, 0)
accuracy = tf.metrics.accuracy(labels, predictions)
return tf.estimator.EstimatorSpec(
mode=mode,
loss=loss,
train_op=train_op,
eval_metric_ops={'Accuracy': accuracy},
predictions=predictions
)
def train_input_fn():
return get_inputs(
trainArrayValues,
trainArrayLabels,
batchSize
)
def eval_input_fn():
return get_inputs(
testArrayValues,
testArrayLabels,
batchSize,
n_epochs=1,
shuffle=False
)
model_dir = './savedModel'
estimator = tf.estimator.LinearRegressor(feature_columns=[model_fn, model_dir])
#estimator.train(train_input_fn, max_steps=1)
estimator.train(train_input_fn)
estimator.evaluate(eval_input_fn)
Your problem is this line:
estimator = tf.estimator.LinearRegressor(feature_columns=[model_fn, model_dir])
You need to set the feature_columns argument to an array of feature columns. A feature column tells the estimator about the data you're feeding it.
It looks like all your input data is numeric, so I'd call tf.feature_column.numeric_column to create your feature column(s). The documentation is here. For example, the following code creates a numeric feature column containing x-coordinates:
xcol = tf.feature_column.numeric_column('x')
If all your estimator needs are x-coordinates, then you could create the estimator with the following code:
estimator = tf.estimator.LinearRegressor(feature_columns=[xcol])