I am trying to make a deep neural network with the low level API of tensorflow although when I train the model and test it, the loss and mae of the testing set and the training set it very similar and very high compared to other models I have tried(e.g. Random Forest, AdaBoost Decision Tree). I even made the same dnn using keras and it gave me much better results I don't understand the problem, I do not have to much experience in machine learning
Tensorflow code
# load libraries
import tensorflow as tf
# reset graph
tf.reset_default_graph()
# define variables
n_hidden1 = 200
n_outputs = 1
with tf.device("/gpu:0"):
X = tf.placeholder(tf.float32, shape=(None, n_features), name="X")
y = tf.placeholder(tf.float32, shape=(None), name="y")
with tf.name_scope("dnn"):
hidden1 = tf.layers.dense(X, n_hidden1, name="hidden1",
activation=tf.nn.leaky_relu)
logits = tf.layers.dense(hidden1, n_outputs, name="logits")
with tf.device("/cpu:0"):
with tf.name_scope("loss"):
loss = tf.reduce_mean(tf.abs(logits - y), name="loss")
with tf.device("cpu:0"):
with tf.name_scope("learning_rate"):
learning_rate = 0.001
with tf.device("/gpu:0"):
with tf.name_scope("train"):
optimizer = tf.train.AdamOptimizer(learning_rate)
training_op = optimizer.minimize(loss)
with tf.device("/gpu:0"):
with tf.name_scope("eval"):
mae = tf.reduce_mean(tf.abs(logits - y), name="mae")
def shuffle_batch(X, y, batch_size):
rnd_idx = np.random.permutation(len(X))
n_batches = len(X) // batch_size
for batch_idx in np.array_split(rnd_idx, n_batches):
X_batch, y_batch = X[batch_idx], y[batch_idx]
yield X_batch, y_batch
n_epochs = 200
batch_size = 1000
n_batches = int(np.ceil(X_train.shape[0] / batch_size))
# create graph variables initializer
init = tf.global_variables_initializer()
# create model saver
saver = tf.train.Saver()
# set device to gpu
with tf.device("/gpu:0"):
with tf.Session() as sess:
sess.run(init)
for epoch in range(n_epochs):
print("Epoch:", str(epoch) + "/" + str(n_epochs))
batch_index = 0
for X_batch, y_batch in shuffle_batch(X_train, np.array(y_train).reshape(-1), batch_size):
sess.run(training_op, feed_dict={X: X_batch, y: y_batch})
acc_batch = mae.eval(feed_dict={X: X_batch, y: y_batch})
acc_val = mae.eval(feed_dict={X: X_test, y: np.array(y_test).reshape(-1)})
loss_batch = loss.eval(feed_dict={X: X_batch, y: y_batch})
loss_val = loss.eval(feed_dict={X: X_test, y: np.array(y_test).reshape(-1)})
print("Batch mae:", acc_batch, "Val mae:", acc_val)
print("Batch loss:", loss_batch, "Val loss:", loss_val)
Keras Code
import tensorflow as tf
from keras import layers
from keras import models
from keras import optimizers
from keras import initializers
from keras import regularizers
network = models.Sequential()
network.add(layers.Dense(units=200,
activation=tf.nn.leaky_relu,
input_shape=(X_train.shape[1], )))
# output layer
network.add(layers.Dense(units=1))
network.compile(loss="mae",
optimizer=optimizers.Adam(lr=0.001),
metrics=["mae"])
history = network.fit(X_train,
np.array(y_train).reshape(-1),
epochs=200,
verbose=1,
batch_size=1000,
validation_data=(X_test, np.array(y_test).reshape(-1)),
shuffle=True)
Related
So I have a CNN in Tensorflow 1.9 and want to make a prediction to a picture. I have this code:
# ...
x = tf.placeholder(tf.float32, shape=[None, 32, 32, 3], name='images_in')
y = tf.placeholder(tf.float32, [None, 10], name='labels_in')
def cnn(x):
# define my cnn
# build the network, input comes from the 'x' placeholder
logits = cnn(x)
prediction = tf.nn.softmax(logits, name='prediction')
# softmax cross entropy loss function
loss = tf.reduce_mean(tf.losses.softmax_cross_entropy(logits=logits, onehot_labels=y))
# Adaptive Momentum optimizer - minimize the loss
optimizer = tf.train.AdamOptimizer(learning_rate=LEARN_RATE, name='Adam').minimize(loss)
# Check to see if the prediction matches the label
correct_prediction = tf.equal(tf.argmax(prediction, 1), tf.argmax(y, 1))
# Calculate accuracy as mean of the correct predictions
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
# TensorBoard data collection
tf.summary.scalar('cross_entropy_loss', loss)
tf.summary.scalar('accuracy', accuracy)
tf.summary.image('input_images', x)
# set up saver object
saver = tf.train.Saver()
with tf.Session() as sess:
sess.run(tf.initializers.global_variables())
# ... batch up, etc.
_, s = sess.run([optimizer, tb_summary], feed_dict={x: batch_x, y: batch_y})
# ...
# Save chkpts, graph, etc.
and then, in the with tf.Session() as sess: environment, I would like to predict on my x_train or x_test, or better, an image I've loaded on my own. Is this the right code?
predictions = sess.run(correct_prediction, feed_dict={x: x_valid, y: y_valid})
I am getting a answer that 'tuple' object has no attribute 'train'. I can not understand this error(I am using google colab). Please, help me, and explain more detail (training part) as you can. my code is in the below. Thanks a lot in advance
%tensorflow_version 1.x
## loading nessecery functions and CIFAR10 dataset
from __future__ import print_function
import tensorflow as tf
from tensorflow.keras.datasets import cifar10
tf.__version__
((train_X, train_y), (test_X, test_y)) = cifar10.load_data()
print(f"train_X: {train_X.shape}, test_X = {test_X.shape}")
cifar10 = cifar10.load_data()
# define placeholder for inputs to network
X = tf.placeholder(tf.float32, [None, 3072])/255.0 # 32x32x3
Y = tf.placeholder(tf.float32, [None, 10])
keep_prob = tf.placeholder(tf.float32)
learning_rate = 0.001
training_epochs =10
batch_size = 30
# weights & bias for nn layers
W = tf.Variable(tf.random_normal([3072, 10]))
b = tf.Variable(tf.random_normal([10]))
hypothesis = tf.matmul(X, W) + b
# define cost/loss & optimizer
cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=hypothesis, labels=Y))
optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(cost)
# initialize
sess = tf.Session()
sess.run(tf.global_variables_initializer())
# train my model
for epoch in range(training_epochs):
avg_cost = 0
num_examples = 50000
total_batch = int(num_examples / batch_size)
My question is here
for i in range(total_batch):
batch_xs, batch_ys = cifar10.train.next_batch(batch_size)
feed_dict = {X: batch_xs, Y: batch_ys}
c, _ = sess.run([cost, optimizer], feed_dict=feed_dict)
avg_cost += c / total_batch
print('Epoch:', '%04d' % (epoch + 1), 'cost =', '{:.9f}'.format(avg_cost))
print('Learning Finished!')
# Test model and check accuracy
correct_prediction = tf.equal(tf.argmax(hypothesis, 1), tf.argmax(Y, 1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
print('Accuracy:', sess.run(accuracy, feed_dict={X: cifar10.test.images, Y: cifar10.test.labels}))
You're trying to access a train attribute from a tuple resulting from cifar10.load_data(). You've already loaded the data correctly on a previous step:
(x_train, y_train), (x_test, y_test) = cifar10.load_data()
cifar10.load_data() is a data loader, which returns the train and test sets of the dataset.
If you want to implement a next_batch method to do as above, you'll need to define a custom helper class, which is something quite commonly used. Here's an example case.
I am starting to work again on tensorflow. I was relaunching some codes I did a few years ago, it's not working though.
Old version
from tensorflow.examples.tutorials.mnist import input_data
mnist = input_data.read_data_sets("MNIST_data", one_hot=True)
import tensorflow as tf
# Parameters
learning_rate = 0.001
training_epochs = 20
batch_size = 128 # Decrease batch size if you don't have enough memory
display_step = 1
n_input = 784 # MNIST data input (img shape: 28*28)
n_classes = 10 # MNIST total classes (0-9 digits)
n_hidden_layer = 256 # layer number of features
# Store layers weight & bias
weights = {
'hidden_layer': tf.Variable(tf.random_normal([n_input, n_hidden_layer])),
'out': tf.Variable(tf.random_normal([n_hidden_layer, n_classes]))
}
biases = {
'hidden_layer': tf.Variable(tf.random_normal([n_hidden_layer])),
'out': tf.Variable(tf.random_normal([n_classes]))
}
# tf Graph input
x = tf.placeholder("float", [None, 28, 28, 1])
y = tf.placeholder("float", [None, n_classes])
x_flat = tf.reshape(x, [-1, n_input])
# Hidden layer with RELU activation
layer_1 = tf.add(tf.matmul(x_flat, weights['hidden_layer']),\
biases['hidden_layer'])
layer_1 = tf.nn.relu(layer_1)
# Output layer with linear activation
logits = tf.add(tf.matmul(layer_1, weights['out']), biases['out'])
# Define loss and optimizer
cost = tf.reduce_mean(\
tf.nn.softmax_cross_entropy_with_logits_v2(logits=logits, labels=y))
optimizer = tf.train.GradientDescentOptimizer(learning_rate=learning_rate)\
.minimize(cost)
# Initializing the variables
init = tf.global_variables_initializer()
# Launch the graph
with tf.Session() as sess:
sess.run(init)
# Training cycle
for epoch in range(training_epochs):
total_batch = int(mnist.train.num_examples/batch_size)
# Loop over all batches
for i in range(total_batch):
batch_x, batch_y = mnist.train.next_batch(batch_size)
# Run optimization op (backprop) and cost op (to get loss value)
sess.run(optimizer, feed_dict={x: batch_x, y: batch_y})
From what I have understood the cause comes from the "read_data_sets" and I should use "tf.data". The problem with "tf.data" is that I cannot use that anymore:
mnist.train.num_examples
mnist.train.next_batch
And the data is not one encoded.
I have tried something like that:
import tensorflow_datasets as tfds
import tensorflow as tf
import matplotlib.pyplot as plt
import numpy as np
# Mandatory: to launch
#tf.enable_eager_execution()
mnist_data, info = tfds.load("mnist", with_info=True, as_supervised=True)
mnist_train, mnist_test = mnist_data["train"], mnist_data["test"]
And, mnist_train.batch instead of mnist.train.next_batch
# Launch the graph
with tf.Session() as sess:
sess.run(init)
# Training cycle
for epoch in range(training_epochs):
total_batch = int(info.splits["train"].num_examples/batch_size)
print(total_batch)
# Loop over all batches
for i in range(total_batch):
batch_x, batch_y = mnist_train.batch(batch_size)
# Run optimization op (backprop) and cost op (to get loss value)
sess.run(optimizer, feed_dict={x: batch_x, y: batch_y})
With the error:
RuntimeError: dataset.__iter__() is only supported when eager execution is enabled.
And if I do :
tf.enable_eager_execution()
I cannot use
tf.placeholder()
New version
import tensorflow_datasets as tfds
import tensorflow as tf
import matplotlib.pyplot as plt
import numpy as np
# Mandatory: to launch
#tf.enable_eager_execution()
mnist_data, info = tfds.load("mnist", with_info=True, as_supervised=True)
mnist_train, mnist_test = mnist_data["train"], mnist_data["test"]
import tensorflow as tf
# Parameters
learning_rate = 0.001
training_epochs = 20
batch_size = 128 # Decrease batch size if you don't have enough memory
display_step = 1
n_input = 784 # MNIST data input (img shape: 28*28)
n_classes = 10 # MNIST total classes (0-9 digits)
n_hidden_layer = 256 # layer number of features
# Store layers weight & bias
weights = {
'hidden_layer': tf.Variable(tf.random_normal([n_input, n_hidden_layer])),
'out': tf.Variable(tf.random_normal([n_hidden_layer, n_classes]))
}
biases = {
'hidden_layer': tf.Variable(tf.random_normal([n_hidden_layer])),
'out': tf.Variable(tf.random_normal([n_classes]))
}
# tf Graph input
x = tf.placeholder("float", [None, 28, 28, 1])
y = tf.placeholder("float", [None, n_classes])
x_flat = tf.reshape(x, [-1, n_input])
# Hidden layer with RELU activation
layer_1 = tf.add(tf.matmul(x_flat, weights['hidden_layer']),\
biases['hidden_layer'])
layer_1 = tf.nn.relu(layer_1)
# Output layer with linear activation
logits = tf.add(tf.matmul(layer_1, weights['out']), biases['out'])
# Define loss and optimizer
cost = tf.reduce_mean(\
tf.nn.softmax_cross_entropy_with_logits_v2(logits=logits, labels=y))
optimizer = tf.train.GradientDescentOptimizer(learning_rate=learning_rate)\
.minimize(cost)
# Initializing the variables
init = tf.global_variables_initializer()
# Launch the graph
with tf.Session() as sess:
sess.run(init)
# Training cycle
for epoch in range(training_epochs):
total_batch = int(info.splits["train"].num_examples/batch_size)
print(total_batch)
# Loop over all batches
for i in range(total_batch):
batch_x, batch_y = mnist_train.batch(batch_size)
# Run optimization op (backprop) and cost op (to get loss value)
sess.run(optimizer, feed_dict={x: batch_x, y: batch_y})
When you load data using tfds.load you get an instance of tf.data.Dataset. You cannot feed this directly to a feed_dict but rather have to make an iterator and feed the values you obtain from the iterator at each step into the feedable input.You can do something roughly like this:
# one hot encode for 10 MNIST classes
def my_one_hot(feature, label):
return feature, tf.one_hot(label, depth=10)
# load your data from tfds
mnist_train, train_info = tfds.load(name="mnist", with_info=True, as_supervised=True, split=tfds.Split.TRAIN)
# convert your labels in one-hot
mnist_train = mnist_train.map(my_one_hot)
# you can batch your data here
mnist_train = mnist_train.batch(8)
Then you can launch your graph (and initialize your iterator)
# Launch the graph
with tf.Session() as sess:
sess.run(init)
# make an iterator
train_iterator = mnist_train.make_initializable_iterator()
next_element = train_iterator.get_next()
# Training cycle
for epoch in range(training_epochs):
sess.run(train_iterator.initializer)
batch_train_x, batch_train_y = sess.run(next_element)
total_batch = int(train_info.splits["train"].num_examples/batch_size)
print(total_batch)
# Loop over all batches
for i in range(total_batch):
# Run optimization op (backprop) and cost op (to get loss value)
sess.run(optimizer, feed_dict={x: batch_train_x, y: batch_train_y})
For more infor on tf.data.Dataset look at the guide. Hope it helps!
I am novice in Machine Learning (ML) and I'm trying to implement algorithm to understand basic syntax of ML frameworks etc. Now I am working on MNIST database of handwritten digits dataset.
I implemented just one layer (I mean: Input layer has 784 inputs, Hidden layer has 512 nodes, Output layer has 10 outputs) Neural Network using TensorFlow framework, no data preprocessing, 128 batch size, 10 epochs, ADAM optimizer. And the algorithm achieved about 0.95 accuracy on train set.
After that I tried to implement exactly the same architecture in Keras. However, the accuracy (train set) is about 0.3. I tried to find many different implementations founded on the internet but I still cannot find where is the issue. I believe that it is something stupid (as always is) :-/
I presume that the same architecture in Keras should give the same results as the implementation in TensorFlow, am I correct?
My Keras implementation is:
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import tensorflow as tf
from keras.layers import Input, Dense
from keras.models import Model
from keras.utils.np_utils import to_categorical
df_train = pd.read_csv('datasets/MNIST_train.csv', delimiter=',', header=0)
Y_train, X_train = np.split(df_train.values, [1], axis=1)
m, n_x = X_train.shape
n_y = len(np.unique(Y_train))
n_layer1 = 512
batch_size = 128
num_epochs = 10
Y_train = to_categorical(Y_train)
X_input = Input(shape=(n_x,), name='input')
X = Dense(n_layer1, activation='relu', name='hidden')(X_input)
X = Dense(n_y, activation='softmax', name='output')(X)
model = Model(inputs=X_input, outputs=X, name='Neural Network')
model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'])
model.fit(X_train, Y_train, epochs=num_epochs, batch_size=batch_size)
My TensorFlow implementation is:
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import tensorflow as tf
def one_hot(a, num_classes):
return np.eye(num_classes)[a.reshape(-1)]
def get_minibatches(batch_size, m, X, Y):
output_batches = []
for index in range(0, m, batch_size):
index_end = index + batch_size
batch = [X[index:index_end], Y[index:index_end]]
output_batches.append(batch)
return output_batches
def dense_layer(input, channels_in, channels_out, activation=None):
initializer = tf.contrib.layers.xavier_initializer()
w = tf.Variable(initializer([channels_in, channels_out]), name="w")
b = tf.Variable(tf.zeros([1, channels_out]), name="b")
if (activation == 'relu'):
a = tf.nn.relu(tf.matmul(input, w) + b)
return a
else:
z = tf.matmul(input, w) + b
return z
df_train = pd.read_csv('datasets/MNIST_train.csv', delimiter=',', header=0)
Y_train, X_train = np.split(df_train.values, [1], axis=1)
m, n_x = X_train.shape
n_y = len(np.unique(Y_train))
n_layer1 = 512
batch_size = 128
num_epochs = 10
Y_train = one_hot(Y_train, n_y)
X = tf.placeholder(tf.float32, [None, n_x], name="X")
Y = tf.placeholder(tf.float32, [None, n_y], name="Y")
hidden = dense_layer(X, n_x, n_layer1, 'relu')
output = dense_layer(hidden, n_layer1, n_y)
loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits_v2(logits=output, labels=Y))
optimizer = tf.train.AdamOptimizer().minimize(loss)
predict = tf.argmax(output, 1)
correct_prediction = tf.equal(predict, tf.argmax(Y, 1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
minibatches = get_minibatches(batch_size, m, X_train, Y_train)
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
current_cost = sess.run(loss, feed_dict={X: X_train, Y: Y_train})
train_accuracy = sess.run(accuracy, feed_dict={X: X_train, Y: Y_train})
print('Epoch: {:<4} - Loss: {:<8.3} Train Accuracy: {:<5.3} '.format(0, current_cost, train_accuracy))
for epoch in range(num_epochs):
for minibatch in minibatches:
minibatch_X, minibatch_Y = minibatch
sess.run(optimizer, feed_dict={ X: minibatch_X, Y: minibatch_Y })
current_cost = sess.run(loss, feed_dict={X: X_train, Y: Y_train})
train_accuracy = sess.run(accuracy, feed_dict={X: X_train, Y: Y_train})
print('Epoch: {:<4} - Loss: {:<8.3} Train Accuracy: {:<5.3} '.format(epoch + 1, current_cost, train_accuracy))
Could you help me and advice what I am doing wrong?
Thank you
Petr
I figured it out. At least partially. I standardized the input ((x - xmean) / xstd) and the Keras implementation has been started to return similar results as TensorFlow implementation…
I just checked that my computer is using GPU to run it.
But the running time is roughly the same with my CPU.
I'm using Windows10, i7-7700, NV GTX1050, Python 3.6, cuda9.0.
Is there any code that doesn't support GPU?
Or how should I fix it? Thanks!
X = tf.placeholder(tf.float32,[None, n_steps, n_inputs])
y = tf.placeholder(tf.int32, [None])
lstm_cells = [tf.contrib.rnn.LSTMCell(num_units = n_neurons, use_peepholes=True) for layer in range(n_layers)]
multi_cell = tf.contrib.rnn.MultiRNNCell(lstm_cells)
outputs, states = tf.nn.dynamic_rnn(multi_cell,X, dtype= tf.float32)
top_layer_h_state = states[-1][1]
logits = tf.layers.dense(top_layer_h_state, n_outputs, name="softmax")
xentropy = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=y, logits=logits)
loss = tf.reduce_mean(xentropy, name="loss")
optimizer = tf.train.AdamOptimizer(learning_rate = learning_rate)
training_op = optimizer.minimize(loss)
correct = tf.nn.in_top_k(logits, y, 1)
accuracy = tf.reduce_mean(tf.cast(correct, tf.float32))
init = tf.global_variables_initializer()
saver = tf.train.Saver()
n_epochs = 2000
with tf.Session() as sess:
init.run()
for epoch in range(n_epochs):
for iteration in range(n_examples):
X_batch, y_batch = next_batch(iteration)
sess.run(training_op, feed_dict={X: X_batch, y: y_batch})
if epoch % 100 == 0 :
X_test, y_test, batch_num = test_batch(n_examples)
acc_test = accuracy.eval(feed_dict={X: X_test, y: y_test})
print(epoch, "Test accuracy:", acc_test)