I'm trying to do a prediction on new text examples, where I want the prediction to return a probability output for each example.
This is my learning model:
with tf.name_scope('Placeholders'):
input_x = tf.placeholder(tf.int32, [None, sequence_length], name='input_x')
input_y = tf.placeholder(tf.float32, [None, n_classes], name='input_y')
drops = tf.placeholder(tf.float32, name='dropout_keep_prob')
with tf.name_scope('Embedding_layer'):
embeddings_v = tf.Variable(tf.random_uniform([vocabulary_size, embedding_size], -1.0, 1.0))
embeddings = tf.nn.embedding_lookup(params=embeddings_v, ids=input_x)
# Bi_directional LSTM
with tf.name_scope('Bi_directional_LSTM'):
rnn_outputs, _ = bi_rnn(LSTMCell(hidden_unit), LSTMCell(hidden_unit),inputs=embeddings, dtype=tf.float32)
tf.summary.histogram('Bi_directional_LSTM', rnn_outputs)
# Attention layer
with tf.name_scope('Attention_layer'):
attention_output, alphas = attention(rnn_outputs, attention_size, return_alphas=True)
tf.summary.histogram('alphas', alphas)
with tf.name_scope('Dropout'):
drop = tf.nn.dropout(attention_output, drops)
with tf.name_scope('Fully_connected_layer'):
W = tf.Variable(tf.truncated_normal([hidden_unit * 2, n_classes],stddev=0.1))
b = tf.Variable(tf.constant(0., shape=[n_classes]))
y_hat = tf.nn.xw_plus_b(drop, W,b)
predictions = tf.argmax(input=y_hat, axis=1, name='predictions')
with tf.name_scope('Loss'):
loss = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(logits=y_hat, labels=input_y))
global_step = tf.Variable(0.1, name="global_step", trainable=False)
optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(loss, global_step=global_step)
probs = tf.nn.sigmoid(y_hat)
tf.summary.scalar('loss', loss)
with tf.name_scope('Accuracy'):
correct_pred = tf.equal(tf.argmax(y_hat,1), tf.argmax(input_y,1))
accuracy = tf.reduce_mean(input_tensor=tf.cast(correct_pred, tf.float32), name='accuracy')
num_correct = tf.reduce_sum(input_tensor=tf.cast(correct_pred, 'float'), name='correct_predictions')
tf.summary.scalar('accuracy', accuracy)
merged = tf.summary.merge_all()
I then run the following piece of code to train the model:
def batch_generator(X, y, batch_size):
"""batch generator"""
size = X.shape[0]
X_copy = X.copy()
y_copy = y.copy()
indices = np.arange(size)
np.random.shuffle(indices)
X_copy = X_copy[indices]
y_copy = y_copy[indices]
i = 0
while True:
if i + batch_size <= size:
yield X_copy[i:i + batch_size], y_copy[i:i + batch_size]
i += batch_size
else:
i = 0
indices = np.arange(size)
np.random.shuffle(indices)
X_copy = X_copy[indices]
y_copy = y_copy[indices]
continue
train_batch_generator = batch_generator(x_train, y_train, batch_size)
test_batch_generator = batch_generator(x_dev, y_dev, batch_size)
predict_generator = batch_generator(x_test, y_test, batch_size)
saver = tf.train.Saver()
with tf.Session(config=tf.ConfigProto(gpu_options=tf.GPUOptions(allow_growth=True))) as sess:
sess.run(tf.global_variables_initializer())
print("Start learning...")
for epoch in range(epochs):
loss_train = 0
loss_val = 0
loss_test = 0
accuracy_train = 0
accuracy_val = 0
accuracy_test = 0
train_loss_l = []
val_loss_l = []
print("epoch: {}\t".format(epoch), end="")
# Training
num_batches = x_train.shape[0] // batch_size
for b in tqdm(range(num_batches)):
x_batch, y_batch = next(train_batch_generator)
loss_tr, acc, _, summary = sess.run([loss, accuracy, optimizer,merged],
feed_dict={input_x: x_batch,
input_y: y_batch,
drops: 0.5})
train_loss_l.append(loss_tr)
accuracy_train += acc
loss_train = loss_tr * DELTA + loss_train * (1 - DELTA)
accuracy_train /= num_batches
# Validation
num_batches = x_dev.shape[0] // batch_size
for b in tqdm(range(num_batches)):
x_batch, y_batch = next(test_batch_generator)
val_loss, val_acc, summary = sess.run([loss, accuracy,merged],
feed_dict={input_x: x_batch,
input_y: y_batch,
drops: 0.5})
val_loss_l.append(val_loss)
accuracy_val += val_acc
loss_val += val_loss
accuracy_val /= num_batches
loss_val /= num_batches
print("loss: {:.3f}, val_loss: {:.3f}, acc: {:.3f}, val_acc: {:.3f}".format(loss_train, loss_val, accuracy_train, accuracy_val))
# predict x_test
num_batches = x_test.shape[0] // batch_size
print("n batches",num_batches)
predict_correct = 0
for batch in tqdm(range(num_batches)):
x_batch, yx_batch = next(predict_generator)
y_true = np.argmax(yx_batch,1)
loss_pred, acc_pred, n_correct, y_pred = sess.run([loss,accuracy,num_correct,predictions], feed_dict={input_x: x_batch,input_y: y_batch,drops : 0.5 })
print("Precision", sk.metrics.precision_score(y_true, y_pred,average='weighted'))
print("Recall", sk.metrics.recall_score(y_true, y_pred,average='weighted'))
print("f1_score", sk.metrics.f1_score(y_true, y_pred,average='weighted'))
print("confusion_matrix")
print(sk.metrics.confusion_matrix(y_true, y_pred))
saver.save(sess, MODEL_PATH)
sess.run(predictions, feed_dict={x: x_test})
The code runs fine until it hits the prediction part where I get the following error:
TypeError: unhashable type: 'numpy.ndarray'
Any chance someone could explain this to me please?
Just answered my own question...
This is just in case someone else is attempting to do this. If you feed tensors to feed_dict, just use the actual placeholder name you used before.
In my case this would be:
print(sess.run(predictions, feed_dict={input_x: x_test,drops:0.5}))
Related
In a weight tensor, which is a matrix for suppose, how do I select some of the elements from the weight matrix and add it to a list of variables to be froze and rest of elements of matrix to be trained in tensorflow?
example: created a variable W: of size 20*20
how can I pick out few elements like W[0][1],W[13][15] and freeze them in optimizer
........
def rnn_cell(rnn_input, state, weight):
with tf.variable_scope('rnn_cell', reuse=True):
W = tf.get_variable('W', [n_inputs + n_neurons, n_neurons])
b = tf.get_variable('b', [1, n_neurons],
initializer=tf.constant_initializer(0.0))
return (tf.tanh(tf.matmul(tf.concat([rnn_input, state], 1), weight) + b))
part_W = tf.scatter_nd([[0,0]], [W[0][0]], [178,150])
W_2 = part_W + tf.stop_gradient(-part_W + W)
state = init_state
rnn_outputs = []
for rnn_input in rnn_inputs:
state = rnn_cell(rnn_input, state, W_2)
rnn_outputs.append(state)
final_state = rnn_outputs[-1]
logits = fully_connected(final_state, n_outputs, activation_fn=None)
xentropy = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=y,
logits=logits)
loss = tf.reduce_mean(xentropy)
optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate)
training_op = optimizer.minimize(loss)
correct = tf.nn.in_top_k(logits, y, 1)
accuracy = tf.reduce_mean(tf.cast(correct, tf.float32))
init = tf.global_variables_initializer()
with tf.Session() as sess:
init.run()
for epoch in range(n_epochs):
for iteration in range(mnist.train.num_examples //
batch_size):
X_batch, y_batch =
mnist.train.next_batch(batch_size)
X_batch = X_batch.reshape((-1, n_steps,
n_inputs))
h=np.zeros([batch_size,n_neurons])
sess.run(training_op, feed_dict={X: X_batch, y:
y_batch, p:h})
acc_train = accuracy.eval(feed_dict={X: X_batch, y: y_batch,
p:h})
q=np.zeros([10000,n_neurons])
acc_test = accuracy.eval(feed_dict={X: X_test, y: y_test,p:q})
print(epoch, "Train accuracy:", acc_train, "Test
accuracy:",acc_test)
I would recombine the weights with a copy of it-self going through tf.stop_gradient. For example,
import tensorflow as tf
w = tf.Variable(tf.zeros((10, 10)))
mask = tf.cast(tf.random_uniform((10, 10), 0, 2, dtype=tf.int32), tf.bool)
w = tf.where(mask, w, tf.stop_gradient(w))
I just checked that my computer is using GPU to run it.
But the running time is roughly the same with my CPU.
I'm using Windows10, i7-7700, NV GTX1050, Python 3.6, cuda9.0.
Is there any code that doesn't support GPU?
Or how should I fix it? Thanks!
X = tf.placeholder(tf.float32,[None, n_steps, n_inputs])
y = tf.placeholder(tf.int32, [None])
lstm_cells = [tf.contrib.rnn.LSTMCell(num_units = n_neurons, use_peepholes=True) for layer in range(n_layers)]
multi_cell = tf.contrib.rnn.MultiRNNCell(lstm_cells)
outputs, states = tf.nn.dynamic_rnn(multi_cell,X, dtype= tf.float32)
top_layer_h_state = states[-1][1]
logits = tf.layers.dense(top_layer_h_state, n_outputs, name="softmax")
xentropy = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=y, logits=logits)
loss = tf.reduce_mean(xentropy, name="loss")
optimizer = tf.train.AdamOptimizer(learning_rate = learning_rate)
training_op = optimizer.minimize(loss)
correct = tf.nn.in_top_k(logits, y, 1)
accuracy = tf.reduce_mean(tf.cast(correct, tf.float32))
init = tf.global_variables_initializer()
saver = tf.train.Saver()
n_epochs = 2000
with tf.Session() as sess:
init.run()
for epoch in range(n_epochs):
for iteration in range(n_examples):
X_batch, y_batch = next_batch(iteration)
sess.run(training_op, feed_dict={X: X_batch, y: y_batch})
if epoch % 100 == 0 :
X_test, y_test, batch_num = test_batch(n_examples)
acc_test = accuracy.eval(feed_dict={X: X_test, y: y_test})
print(epoch, "Test accuracy:", acc_test)
the input does not have any NaN value, but the accuracy is always 0.
n_nodes_hl1 = 3000
n_nodes_hl2 = 1500
n_nodes_hl3 = 1000
n_classes = 3
batch_size = 5
hm_epochs = 5
def train_neural_network(x):
prediction=neural_network_model(x)
cost=tf.nn.softmax_cross_entropy_with_logits_v2(logits = prediction, labels = y)
optimizer=tf.train.AdamOptimizer(learning_rate=0.001).minimize(cost)
saver = tf.train.Saver()
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
for epoch in range(hm_epochs):
epoch_loss = 0
i = 0
#while i < len(train_x):
t = len(train_x)
f = t%batch_size
while i < (t-f):
start = i
end = i+batch_size
batch_x = np.array(train_x[start:end])
batch_y = np.array(train_y[start:end])
_, c = sess.run([optimizer, cost], feed_dict={x: batch_x, y: batch_y})
epoch_loss += c
#epoch_loss = epoch_loss + c
i+=batch_size
#i = i + batch_size
print('Epoch =', epoch+1, '/',hm_epochs,'loss:',epoch_loss)
save_path = saver.save(sess, "sesionestensorflow/model1802.ckpt")
print("Model saved in path: %s" % save_path)
correct = tf.equal(tf.argmax(prediction, 1), tf.argmax(y, 1))
accuracy = tf.reduce_mean(tf.cast(correct, 'float'))
print('Accuracy:',accuracy.eval({x:test_x, y:test_y}))
i understand that it could be the loss function and if it is the case, the neural network need a clip by value procedure.
Background: We try to produce a model that will predict whether a case is happening or not, based on data numbers in rnn model
The problem: We reached a situation in which the LOSS decreases but the accuracy remains at 0.5
tf.initialize_all_variables()
ops.reset_default_graph()
sess = tf.InteractiveSession()
np_labels = np.array(labels)
np_labels = np_labels.reshape([np_labels.shape[0], 1])
print([np_labels.shape[0], 1])
############################ modle ############
cellsize = 100
possible_chars = 1
x = tf.placeholder(tf.float32, [None, 91, 10])
y = tf.placeholder(tf.float32, [None, 1])
lstm_cell = tf.nn.rnn_cell.BasicLSTMCell(cellsize, forget_bias=0.0)
output, _ = tf.nn.dynamic_rnn(lstm_cell, x, dtype=tf.float32)
output = tf.transpose(output, [1, 0, 2])
last = output[-1]
W = tf.Variable(tf.truncated_normal([cellsize, possible_chars], stddev=0.1))
b = tf.Variable(tf.constant(0.1))
z = tf.matmul(last, W) + b
res = tf.nn.sigmoid(z)
cross_entropy = tf.reduce_mean(-tf.reduce_sum(y * tf.log(res), reduction_indices=[1]))
train_step = tf.train.GradientDescentOptimizer(0.1).minimize(cross_entropy)
sess = tf.InteractiveSession()
sess.run(tf.global_variables_initializer())
correct_prediction = tf.greater(res, 0.1)
correct = tf.equal(correct_prediction, tf.equal(y, 1))
accuracy = tf.reduce_mean(tf.cast(correct, tf.float32))
############################ startTheTrain ################
num_of_epochs = 10
for ephoch in range(num_of_epochs):
_, loss = sess.run([train_step, cross_entropy], feed_dict={x: fvecs, y:np_labels})
a = accuracy.eval(feed_dict={x: fvecs, y: np_labels}) #
print("step %d, loss: %g, training accuracy %g" % (ephoch, loss, a))
After I run the code I get this output:
step 0, loss: 0.338716, training accuracy 0.5
step 1, loss: 0.130198, training accuracy 0.5
step 8, loss: 0.0502876, training accuracy 0.5
step 9, loss: 0.0468232, training accuracy 0.5
I am trying to fix the code so that accuracy it will work
I'm implementing a neural network to map one vector of 194 values to another vector of 5 values. Values are float numbers from an interval of -10, 10 on.
The problem with my code below is that the avg_cost printed in the end is always Nan. I've read somewhere that it happened because the guy didn't set the activation function of the output layer correctly, but I've set mine as it is (linear activation which is the default one). I'm a newbie in this area and to Tensorflow and so I'm not sure what I'm doing wrong here. Is the linear activation function a correct decision? Or is something else wrong in my code?
def neural_net(x):
layer_1 = tf.layers.dense(inputs=x, units=97, activation=tf.nn.relu)
out_layer = tf.layers.dense(inputs=layer_1, units=5)
return out_layer
train_x = pd.read_csv("x_data.csv", sep=" ")
train_y = pd.read_csv("y_data.csv", sep=" ")
train_size = 0.9
train_cnt = int(floor(train_x.shape[0] * train_size))
x_train = train_x.iloc[0:train_cnt].values
y_train = train_y.iloc[0:train_cnt].values
x_test = train_x.iloc[train_cnt:].values
y_test = train_y.iloc[train_cnt:].values
x = tf.placeholder("float", [None, 194])
y = tf.placeholder("float", [None, 5])
nn_output = neural_net(x)
cost = tf.reduce_mean(tf.losses.mean_squared_error(labels=y, predictions=nn_output))
optimizer = tf.train.GradientDescentOptimizer(learning_rate=0.0001).minimize(cost)
training_epochs = 5000
display_step = 1000
batch_size = 30
keep_prob = tf.placeholder("float")
saver = tf.train.Saver()
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
for epoch in range(training_epochs):
avg_cost = 0.0
total_batch = int(len(x_train) / batch_size)
x_batches = np.array_split(x_train, total_batch)
y_batches = np.array_split(y_train, total_batch)
for i in range(total_batch):
batch_x, batch_y = x_batches[i], y_batches[i]
_, c = sess.run([optimizer, cost],
feed_dict={
x: batch_x,
y: batch_y,
keep_prob: 0.8
})
avg_cost += c / total_batch
if epoch % display_step == 0:
print("Epoch:", '%04d' % (epoch+1), "cost=", \
"{:.9f}".format(avg_cost))