Related
I am trying to build this LSTM network and I ran into this error constantly. I looked it up on Google and am still not sure what is going on. I have tried adding this:with tf.variable_scope("cell1"). Still not working. Any help is greatly appreciated.
Here is the code of the main LSTM structure
def lstm_structure(self):
train_dataset, train_labels, valid_dataset, valid_labels, test_dataset, test_labels = self.data_preprocessing(
'SSE_Composite_Index.csv')
graph=tf.Graph()
with graph.as_default():
'''Placeholders'''
X=tf.placeholder(tf.float32,shape=[None,self.num_steps,self.input_size])
y=tf.placeholder(tf.float32,shape=[None,self.num_classes])
valid=tf.constant(valid_dataset)
test=tf.constant(test_dataset)
'''Weights'''
weights={'in':tf.Variable(tf.random_normal([self.input_size,self.num_neurons])),'out':tf.Variable(tf.random_normal([self.num_neurons,self.num_classes]))}
'''Biases'''
biases={'in':tf.Variable(tf.zeros(shape=[self.num_neurons,])),'out':tf.Variable(tf.zeros(shape=[self.num_classes,]))}
def lstm(X, weights, biases, reuse=True
):
'''from input to cell'''
with tf.variable_scope("foo") as f:
if reuse:
f.reuse_variables()
X = tf.reshape(X, shape=[-1, self.input_size])
X_in = tf.matmul(X, weights['in']) + biases['in']
X_in = tf.reshape(X_in, shape=[-1, self.num_steps, self.num_neurons])
'''cell'''
cell_ = tf.contrib.rnn.BasicLSTMCell(self.num_neurons, forget_bias=1, state_is_tuple=True)
_init_state = cell_.zero_state(self.batch_size, dtype=tf.float32)
outputs, states = tf.nn.dynamic_rnn(cell_, X_in, initial_state=_init_state, time_major=False, dtype=tf.float32)
'''from cell to output'''
results = tf.matmul(states[1], weights['out']) + biases['out']
return results
logits=lstm(X,weights,biases,False)
loss=tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=logits,labels=y))
optimizer = tf.train.AdamOptimizer(learning_rate=self.learning_rate).minimize(loss)
train_prediction=tf.nn.softmax(logits=logits)
valid_prediction=tf.nn.softmax(lstm(valid,weights,biases,True))
test_prediction=tf.nn.softmax(lstm(test,weights,biases,True))
with tf.Session(graph=graph) as session:
tf.initialize_all_variables().run()
print('Initialised')
loss_list = []
for step in range(self.num_epochs):
if step==0:
i = 0
while i < len(train_dataset):
start = i
end = i + self.batch_size
batch_data = train_dataset[start:end, :]
batch_label = train_labels[start:end, :]
i += self.batch_size
a, b, prediction = session.run([optimizer, loss, train_prediction],
feed_dict={X: batch_data, y: batch_label})
loss_list.append(b)
else:
i = 0
while i < len(train_dataset):
start = i
end = i + self.batch_size
batch_data = train_dataset[start:end, :]
batch_label = train_labels[start:end, :]
i += self.batch_size
a, b, prediction = session.run([optimizer, loss, train_prediction],
feed_dict={X: batch_data, y: batch_label})
loss_list.append(b)
if step % 10 == 0:
print('Step', step, 'Loss', b)
print('Training Accuracy', self.accuracy(prediction, batch_label), '%')
print('Validation Accuracy',
self.accuracy(valid_prediction.eval(), valid_labels), '%')
print('test Accuracy', self.accuracy(test_prediction.eval(), test_labels), '%')
print('Finished')
The error message that I am getting is:
Traceback (most recent call last):
File "C:/Users/LiXin/PycharmProjects/PythonProjects/LSTM_CLASSIFIER.py", line 114, in <module>
lstm.LSTM_structure()
File "C:/Users/LiXin/PycharmProjects/PythonProjects/LSTM_CLASSIFIER.py", line 87, in LSTM_structure
valid_prediction=tf.nn.softmax(LSTM(valid,weights,biases))
File "C:/Users/LiXin/PycharmProjects/PythonProjects/LSTM_CLASSIFIER.py", line 75, in LSTM
outputs,states=tf.nn.dynamic_rnn(cell_,X_in,initial_state=_init_state,time_major=False,dtype=tf.float32)
File "C:\Users\LiXin\PycharmProjects\PythonProjects\venv\lib\site-packages\tensorflow\python\ops\rnn.py", line 614, in dynamic_rnn
dtype=dtype)
File "C:\Users\LiXin\PycharmProjects\PythonProjects\venv\lib\site-packages\tensorflow\python\ops\rnn.py", line 777, in _dynamic_rnn_loop
swap_memory=swap_memory)
File "C:\Users\LiXin\PycharmProjects\PythonProjects\venv\lib\site-packages\tensorflow\python\ops\control_flow_ops.py", line 2816, in while_loop
result = loop_context.BuildLoop(cond, body, loop_vars, shape_invariants)
File "C:\Users\LiXin\PycharmProjects\PythonProjects\venv\lib\site-packages\tensorflow\python\ops\control_flow_ops.py", line 2640, in BuildLoop
pred, body, original_loop_vars, loop_vars, shape_invariants)
File "C:\Users\LiXin\PycharmProjects\PythonProjects\venv\lib\site-packages\tensorflow\python\ops\control_flow_ops.py", line 2590, in _BuildLoop
body_result = body(*packed_vars_for_body)
File "C:\Users\LiXin\PycharmProjects\PythonProjects\venv\lib\site-packages\tensorflow\python\ops\rnn.py", line 762, in _time_step
(output, new_state) = call_cell()
File "C:\Users\LiXin\PycharmProjects\PythonProjects\venv\lib\site-packages\tensorflow\python\ops\rnn.py", line 748, in <lambda>
call_cell = lambda: cell(input_t, state)
File "C:\Users\LiXin\PycharmProjects\PythonProjects\venv\lib\site-packages\tensorflow\python\ops\rnn_cell_impl.py", line 183, in __call__
return super(RNNCell, self).__call__(inputs, state)
File "C:\Users\LiXin\PycharmProjects\PythonProjects\venv\lib\site-packages\tensorflow\python\layers\base.py", line 575, in __call__
outputs = self.call(inputs, *args, **kwargs)
File "C:\Users\LiXin\PycharmProjects\PythonProjects\venv\lib\site-packages\tensorflow\python\ops\rnn_cell_impl.py", line 438, in call
self._linear = _Linear([inputs, h], 4 * self._num_units, True)
File "C:\Users\LiXin\PycharmProjects\PythonProjects\venv\lib\site-packages\tensorflow\python\ops\rnn_cell_impl.py", line 1171, in __init__
initializer=kernel_initializer)
File "C:\Users\LiXin\PycharmProjects\PythonProjects\venv\lib\site-packages\tensorflow\python\ops\variable_scope.py", line 1203, in get_variable
constraint=constraint)
File "C:\Users\LiXin\PycharmProjects\PythonProjects\venv\lib\site-packages\tensorflow\python\ops\variable_scope.py", line 1092, in get_variable
constraint=constraint)
File "C:\Users\LiXin\PycharmProjects\PythonProjects\venv\lib\site-packages\tensorflow\python\ops\variable_scope.py", line 417, in get_variable
return custom_getter(**custom_getter_kwargs)
File "C:\Users\LiXin\PycharmProjects\PythonProjects\venv\lib\site-packages\tensorflow\python\ops\rnn_cell_impl.py", line 186, in _rnn_get_variable
variable = getter(*args, **kwargs)
File "C:\Users\LiXin\PycharmProjects\PythonProjects\venv\lib\site-packages\tensorflow\python\ops\variable_scope.py", line 394, in _true_getter
use_resource=use_resource, constraint=constraint)
File "C:\Users\LiXin\PycharmProjects\PythonProjects\venv\lib\site-packages\tensorflow\python\ops\variable_scope.py", line 742, in _get_single_variable
name, "".join(traceback.format_list(tb))))
ValueError: Variable rnn/basic_lstm_cell/kernel already exists, disallowed. Did you mean to set reuse=True or reuse=tf.AUTO_REUSE in VarScope? Originally defined at:
File "C:\Users\LiXin\PycharmProjects\PythonProjects\venv\lib\site-packages\tensorflow\python\framework\ops.py", line 1470, in __init__
self._traceback = self._graph._extract_stack() # pylint: disable=protected-access
File "C:\Users\LiXin\PycharmProjects\PythonProjects\venv\lib\site-packages\tensorflow\python\framework\ops.py", line 2956, in create_op
op_def=op_def)
File "C:\Users\LiXin\PycharmProjects\PythonProjects\venv\lib\site-packages\tensorflow\python\framework\op_def_library.py", line 787, in _apply_op_helper
op_def=op_def)
The following code works, however.
def RNN_neural_network_model(x):
layer={'weights':tf.Variable(tf.random_normal([rnn_size,n_classes])),
'biases':tf.Variable(tf.random_normal([n_classes]))}
x=tf.transpose(x,[1,0,2])
x=tf.reshape(x,[-1,chunk_size])
x=tf.split(x,n_chunks,0)
lstm_cell=tf.contrib.rnn.BasicLSTMCell(rnn_size,state_is_tuple=True)
# stacked_lstm = tf.contrib.rnn.MultiRNNCell(
# [lstmcell(rnn_size) for _ in range(num_layers)])
outputs, states=rnn.static_rnn(lstm_cell,x,dtype=tf.float32)
output = tf.matmul(outputs[-1], layer['weights'])+ layer['biases']
return output
You should wrap your LSTM definition inside a variable scope and then reuse it for validation and testing. Try the following
def LSTM(X,weights,biases, reuse=reuse):
'''from input to cell'''
with tf.variable_scope("foo") as f:
if reuse:
f.reuse_variables()
X=tf.reshape(X,shape=[-1,self.input_size])
X_in=tf.matmul(X,weights['in'])+biases['in']
X_in=tf.reshape(X_in,shape=[-1,self.num_steps,self.num_neurons])
'''cell'''
cell_=tf.contrib.rnn.BasicLSTMCell(self.num_neurons,forget_bias=1,state_is_tuple=True)
_init_state=cell_.zero_state(self.batch_size,dtype=tf.float32)
outputs,states=tf.nn.dynamic_rnn(cell_,X_in,initial_state=_init_state,time_major=False,dtype=tf.float32)
'''from cell to output'''
results=tf.matmul(states[1],weights['out'])+biases['out']
return results
Change your training, testing code as below
logits=LSTM(X,weights,biases,False)
loss=tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=logits,labels=y))
optimizer = tf.train.AdamOptimizer(learning_rate=self.learning_rate).minimize(loss)
train_prediction=tf.nn.softmax(logits=logits)
valid_prediction=tf.nn.softmax(LSTM(valid,weights,biases, True))
test_prediction=tf.nn.softmax(LSTM(test,weights,biases, True))
I just run a simple code and want to get accuracy after training. I load the model that I saved, but when I want to get accuracy, I get something wrong. Why?
# coding=utf-8
from color_1 import read_and_decode, get_batch, get_test_batch
import AlexNet
import cv2
import os
import time
import numpy as np
import tensorflow as tf
import AlexNet_train
import math
batch_size=128
num_examples = 1000
crop_size=56
def evaluate(test_x, test_y):
image_holder = tf.placeholder(tf.float32, [batch_size, 56, 56, 3], name='x-input')
label_holder = tf.placeholder(tf.int32, [batch_size], name='y-input')
y = AlexNet.inference(image_holder,evaluate,None)
correct_prediction=tf.equal(tf.argmax(y,1),tf.argmax(label_holder,1))
accuracy=tf.reduce_mean(tf.cast(correct_prediction,tf.float32))
saver = tf.train.Saver()
with tf.Session() as sess:
init_op = tf.group(tf.global_variables_initializer(), tf.local_variables_initializer())
coord = tf.train.Coordinator()
sess.run(init_op)
threads = tf.train.start_queue_runners(sess=sess, coord=coord)
ckpt=tf.train.get_checkpoint_state(AlexNet_train.MODEL_SAVE_PATH)
if ckpt and ckpt.model_checkpoint_path:
ckpt_name = os.path.basename(ckpt.model_checkpoint_path)
global_step = ckpt.model_checkpoint_path.split('/')[-1].split('-')[-1]
saver.restore(sess, os.path.join(AlexNet_train.MODEL_SAVE_PATH, ckpt_name))
print('Loading success, global_step is %s' % global_step)
step=0
image_batch, label_batch = sess.run([test_x, test_y])
accuracy_score=sess.run(accuracy,feed_dict={image_holder: image_batch,
label_holder: label_batch})
print("After %s training step(s),validation "
"precision=%g" % (global_step, accuracy_score))
coord.request_stop()
coord.join(threads)
def main(argv=None):
test_image, test_label = read_and_decode('val.tfrecords')
test_images, test_labels = get_test_batch(test_image, test_label, batch_size, crop_size)
evaluate(test_images, test_labels)
if __name__=='__main__':
tf.app.run()
And here is error,it said that this line in my code is wrong:" correct_prediction=tf.equal(tf.argmax(y,1),tf.argmax(label_holder,1))"
Traceback (most recent call last):
File "/home/vrview/tensorflow/example/char/tfrecords/AlexNet/Alex_save/AlexNet_test.py", line 80, in <module>
tf.app.run()
File "/home/vrview/tensorflow/local/lib/python2.7/site-packages/tensorflow/python/platform/app.py", line 44, in run
_sys.exit(main(_sys.argv[:1] + flags_passthrough))
File "/home/vrview/tensorflow/example/char/tfrecords/AlexNet/Alex_save/AlexNet_test.py", line 76, in main
evaluate(test_images, test_labels)
File "/home/vrview/tensorflow/example/char/tfrecords/AlexNet/Alex_save/AlexNet_test.py", line 45, in evaluate
label_holder: label_batch})
File "/home/vrview/tensorflow/local/lib/python2.7/site-packages/tensorflow/python/client/session.py", line 767, in run
run_metadata_ptr)
File "/home/vrview/tensorflow/local/lib/python2.7/site-packages/tensorflow/python/client/session.py", line 965, in _run
feed_dict_string, options, run_metadata)
File "/home/vrview/tensorflow/local/lib/python2.7/site-packages/tensorflow/python/client/session.py", line 1015, in _do_run
target_list, options, run_metadata)
File "/home/vrview/tensorflow/local/lib/python2.7/site-packages/tensorflow/python/client/session.py", line 1035, in _do_call
raise type(e)(node_def, op, message)
tensorflow.python.framework.errors_impl.InvalidArgumentError: Expected dimension in the range [-1, 1), but got 1
[[Node: ArgMax_1 = ArgMax[T=DT_INT32, Tidx=DT_INT32, _device="/job:localhost/replica:0/task:0/cpu:0"](_recv_y-input_0, ArgMax_1/dimension)]]
Caused by op u'ArgMax_1', defined at:
File "/home/vrview/tensorflow/example/char/tfrecords/AlexNet/Alex_save/AlexNet_test.py", line 80, in <module>
tf.app.run()
File "/home/vrview/tensorflow/local/lib/python2.7/site-packages/tensorflow/python/platform/app.py", line 44, in run
_sys.exit(main(_sys.argv[:1] + flags_passthrough))
File "/home/vrview/tensorflow/example/char/tfrecords/AlexNet/Alex_save/AlexNet_test.py", line 76, in main
evaluate(test_images, test_labels)
File "/home/vrview/tensorflow/example/char/tfrecords/AlexNet/Alex_save/AlexNet_test.py", line 22, in evaluate
correct_prediction=tf.equal(tf.argmax(y,1),tf.argmax(label_holder,1))
File "/home/vrview/tensorflow/local/lib/python2.7/site-packages/tensorflow/python/ops/math_ops.py", line 263, in argmax
return gen_math_ops.arg_max(input, axis, name)
File "/home/vrview/tensorflow/local/lib/python2.7/site-packages/tensorflow/python/ops/gen_math_ops.py", line 168, in arg_max
name=name)
File "/home/vrview/tensorflow/local/lib/python2.7/site-packages/tensorflow/python/framework/op_def_library.py", line 763, in apply_op
op_def=op_def)
File "/home/vrview/tensorflow/local/lib/python2.7/site-packages/tensorflow/python/framework/ops.py", line 2395, in create_op
original_op=self._default_original_op, op_def=op_def)
File "/home/vrview/tensorflow/local/lib/python2.7/site-packages/tensorflow/python/framework/ops.py", line 1264, in __init__
self._traceback = _extract_stack()
InvalidArgumentError (see above for traceback): Expected dimension in the range [-1, 1), but got 1
[[Node: ArgMax_1 = ArgMax[T=DT_INT32, Tidx=DT_INT32, _device="/job:localhost/replica:0/task:0/cpu:0"](_recv_y-input_0, ArgMax_1/dimension)]]
How to solve it?
Taking the part of this answer related to the problem here:
tf.argmax's definition states:
axis: A Tensor. Must be one of the following types: int32, int64.
int32, 0 <= axis < rank(input). Describes which axis of the input
Tensor to reduce across.
It seems, then, that the only way to run argmax on the last axis of the tensor is by giving it axis=-1, because of the "strictly less than" sign in the definition of the function.
I have issues using an exported tensorflow model. It doesn't allow me to evaluate the dataset I provided it with. If I run the evaluation in the same session as the training, there are no issues, however, that defeats the purpose of saving the model, if I have to retrain my model just to test with another dataset. The python file for generating the model is as such:
x = tf.placeholder(tf.float32, shape=[None, 1024], name = "x")
y_ = tf.placeholder(tf.float32, shape=[None, 10], name = "y_")
#===Model===
#Train
cross_entropy = tf.reduce_mean(
tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y_conv))
train_step = tf.train.AdamOptimizer(1e-4).minimize(cross_entropy)
correct_prediction = tf.equal(tf.argmax(y_conv,1), tf.argmax(y_,1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32), name= "accuracy")
#Create Saver
saver = tf.train.Saver()
sess.run(tf.global_variables_initializer())
for i in range(40000):
batch = shvn_data.nextbatch(100)
if i%100 == 0:
train_accuracy = accuracy.eval(feed_dict={x:batch[0], y_: batch[1], keep_prob: 1.0})
print("step %d, training accuracy %f"%(i, train_accuracy))
train_step.run(feed_dict={x: batch[0], y_: batch[1], keep_prob: 0.5})
#Save
saver.save(sess,'svhn_model1')
I saved input variables x and y_, to be fed through function 'accuracy', so that I can run accuracy.eval() to obtain the accuracy of prediction. I evaluated the dataset in batches of 100 images, then summed the final prediction. The python file to evaluate the model in another session is as such:
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
sess = tf.Session(config = config)
shvn_data = DataLoader()
saver = tf.train.import_meta_graph('svhn_model1.meta')
saver.restore(sess,tf.train.latest_checkpoint('./'))
#sess.run(tf.global_variables_initializer())
#Variables to use with model
graph = tf.get_default_graph()
x = graph.get_tensor_by_name("x:0")
y_ = graph.get_tensor_by_name("y_:0")
accuracy = graph.get_tensor_by_name("accuracy:0")
keep_prob = tf.placeholder(tf.float32)
img_whole = np.reshape(shvn_data.test_images,(-1,1024))
batch_whole = np.asarray(shvn_data.test_label.eval(), dtype = np.float32)
total_accuracy = 0
test_count = shvn_data.TEST_COUNT
batch_size = 100
steps = int(math.ceil(test_count/float(batch_size)))
for j in range(steps):
start = j*batch_size
if (j+1)*batch_size > shvn_data.TEST_COUNT:
end = test_count
else:
end = (j+1)*batch_size
img_batch = img_whole[start:end]
label_batch = batch_whole[start:end]
batch_accuracy = accuracy.eval(session = sess, feed_dict={ x: img_batch, y_: label_batch, keep_prob: 1.0}) #ISSUE LIES HERE
print("Test batch %d:%d accuracy %g"%(start,end,batch_accuracy))
total_accuracy += batch_accuracy
print ("Total Accuracy: %f" %(total_accuracy/steps))
The error is as follows.
File "/home/lwenyao/anaconda2/lib/python2.7/site-packages/tensorflow/python/client/session.py", line 1052, in _do_call
raise type(e)(node_def, op, message)
InvalidArgumentError: You must feed a value for placeholder tensor 'Placeholder' with dtype float
[[Node: Placeholder = Placeholder[dtype=DT_FLOAT, shape=[], _device="/job:localhost/replica:0/task:0/gpu:0"]()]]
Caused by op u'Placeholder', defined at:
File "/home/lwenyao/anaconda2/lib/python2.7/site-packages/spyder/utils/ipython/start_kernel.py", line 227, in <module>
main()
File "/home/lwenyao/anaconda2/lib/python2.7/site-packages/spyder/utils/ipython/start_kernel.py", line 223, in main
kernel.start()
File "/home/lwenyao/anaconda2/lib/python2.7/site-packages/ipykernel/kernelapp.py", line 474, in start
ioloop.IOLoop.instance().start()
File "/home/lwenyao/anaconda2/lib/python2.7/site-packages/zmq/eventloop/ioloop.py", line 177, in start
super(ZMQIOLoop, self).start()
File "/home/lwenyao/anaconda2/lib/python2.7/site-packages/tornado/ioloop.py", line 887, in start
handler_func(fd_obj, events)
File "/home/lwenyao/anaconda2/lib/python2.7/site-packages/tornado/stack_context.py", line 275, in null_wrapper
return fn(*args, **kwargs)
File "/home/lwenyao/anaconda2/lib/python2.7/site-packages/zmq/eventloop/zmqstream.py", line 440, in _handle_events
self._handle_recv()
File "/home/lwenyao/anaconda2/lib/python2.7/site-packages/zmq/eventloop/zmqstream.py", line 472, in _handle_recv
self._run_callback(callback, msg)
File "/home/lwenyao/anaconda2/lib/python2.7/site-packages/zmq/eventloop/zmqstream.py", line 414, in _run_callback
callback(*args, **kwargs)
File "/home/lwenyao/anaconda2/lib/python2.7/site-packages/tornado/stack_context.py", line 275, in null_wrapper
return fn(*args, **kwargs)
File "/home/lwenyao/anaconda2/lib/python2.7/site-packages/ipykernel/kernelbase.py", line 276, in dispatcher
return self.dispatch_shell(stream, msg)
File "/home/lwenyao/anaconda2/lib/python2.7/site-packages/ipykernel/kernelbase.py", line 228, in dispatch_shell
handler(stream, idents, msg)
File "/home/lwenyao/anaconda2/lib/python2.7/site-packages/ipykernel/kernelbase.py", line 390, in execute_request
user_expressions, allow_stdin)
File "/home/lwenyao/anaconda2/lib/python2.7/site-packages/ipykernel/ipkernel.py", line 196, in do_execute
res = shell.run_cell(code, store_history=store_history, silent=silent)
File "/home/lwenyao/anaconda2/lib/python2.7/site-packages/ipykernel/zmqshell.py", line 501, in run_cell
return super(ZMQInteractiveShell, self).run_cell(*args, **kwargs)
File "/home/lwenyao/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2717, in run_cell
interactivity=interactivity, compiler=compiler, result=result)
File "/home/lwenyao/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2827, in run_ast_nodes
if self.run_code(code, result):
File "/home/lwenyao/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2881, in run_code
exec(code_obj, self.user_global_ns, self.user_ns)
File "<ipython-input-1-33e4fce19d34>", line 1, in <module>
runfile('/home/lwenyao/Desktop/Python/Import_Model.py', wdir='/home/lwenyao/Desktop/Python')
File "/home/lwenyao/anaconda2/lib/python2.7/site-packages/spyder/utils/site/sitecustomize.py", line 866, in runfile
execfile(filename, namespace)
File "/home/lwenyao/anaconda2/lib/python2.7/site-packages/spyder/utils/site/sitecustomize.py", line 94, in execfile
builtins.execfile(filename, *where)
File "/home/lwenyao/Desktop/Python/Import_Model.py", line 63, in <module>
saver = tf.train.import_meta_graph('svhn_model1.meta')
File "/home/lwenyao/anaconda2/lib/python2.7/site-packages/tensorflow/python/training/saver.py", line 1595, in import_meta_graph
**kwargs)
File "/home/lwenyao/anaconda2/lib/python2.7/site-packages/tensorflow/python/framework/meta_graph.py", line 499, in import_scoped_meta_graph
producer_op_list=producer_op_list)
File "/home/lwenyao/anaconda2/lib/python2.7/site-packages/tensorflow/python/framework/importer.py", line 308, in import_graph_def
op_def=op_def)
File "/home/lwenyao/anaconda2/lib/python2.7/site-packages/tensorflow/python/framework/ops.py", line 2336, in create_op
original_op=self._default_original_op, op_def=op_def)
File "/home/lwenyao/anaconda2/lib/python2.7/site-packages/tensorflow/python/framework/ops.py", line 1228, in __init__
self._traceback = _extract_stack()
InvalidArgumentError (see above for traceback): You must feed a value for placeholder tensor 'Placeholder' with dtype float
[[Node: Placeholder = Placeholder[dtype=DT_FLOAT, shape=[], _device="/job:localhost/replica:0/task:0/gpu:0"]()]]
As previously mentioned, evaluation has no issues if I run it in the same session when training the model. The only changes that I made were i added the argument session = sess each time I called .eval() when using the imported model. Sorry for the long post!
Alright, it appears that the error was caused from attempting to create and use another keep_prob variable in the test script, after importing the model. I.e. I created keep_prob = tf.placeholder(tf.float32,) in the training file. However,accuracy.eval() in the testing file was trying to look for keep_prob specifically from the model. I created another keep_prob = tf.placeholder(tf.float32,) in testing file, thinking it would be the same, but it was not.
I modified my code in the training file by adding the label:
keep_prob = tf.placeholder(tf.float32, name="keep_prob")
and in my testing file, called for the model's variable:
#Variables to use with model
graph = tf.get_default_graph()
x = graph.get_tensor_by_name("x:0")
y_ = graph.get_tensor_by_name("y_:0")
keep_prob = graph.get_tensor_by_name("keep_prob:0")#Changed this
accuracy = graph.get_tensor_by_name("accuracy:0")
And now it works fine. My code is modified from Deep MNIST for Experts from tensorflow.
tf.contrib.crf doesn't seem to support sequences of length 1.
For example, if I run the example on https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/crf (mirror) and replace num_words = 20 by num_words = 1:
import numpy as np
import tensorflow as tf
# Data settings.
num_examples = 10
num_words = 1
num_features = 100
num_tags = 5
# Random features.
x = np.random.rand(num_examples, num_words, num_features).astype(np.float32)
# Random tag indices representing the gold sequence.
y = np.random.randint(num_tags, size=[num_examples, num_words]).astype(np.int32)
# All sequences in this example have the same length, but they can be variable in a real model.
sequence_lengths = np.full(num_examples, num_words - 1, dtype=np.int32)
# Train and evaluate the model.
with tf.Graph().as_default():
with tf.Session() as session:
# Add the data to the TensorFlow graph.
x_t = tf.constant(x)
y_t = tf.constant(y)
sequence_lengths_t = tf.constant(sequence_lengths)
# Compute unary scores from a linear layer.
weights = tf.get_variable("weights", [num_features, num_tags])
matricized_x_t = tf.reshape(x_t, [-1, num_features])
matricized_unary_scores = tf.matmul(matricized_x_t, weights)
unary_scores = tf.reshape(matricized_unary_scores,
[num_examples, num_words, num_tags])
# Compute the log-likelihood of the gold sequences and keep the transition
# params for inference at test time.
log_likelihood, transition_params = tf.contrib.crf.crf_log_likelihood(
unary_scores, y_t, sequence_lengths_t)
# Add a training op to tune the parameters.
loss = tf.reduce_mean(-log_likelihood)
train_op = tf.train.GradientDescentOptimizer(0.01).minimize(loss)
# Train for a fixed number of iterations.
session.run(tf.global_variables_initializer())
for i in range(1000):
tf_unary_scores, tf_transition_params, _ = session.run(
[unary_scores, transition_params, train_op])
if i % 100 == 0:
correct_labels = 0
total_labels = 0
for tf_unary_scores_, y_, sequence_length_ in zip(tf_unary_scores, y,
sequence_lengths):
# Remove padding from the scores and tag sequence.
tf_unary_scores_ = tf_unary_scores_[:sequence_length_]
y_ = y_[:sequence_length_]
# Compute the highest scoring sequence.
viterbi_sequence, _ = tf.contrib.crf.viterbi_decode(
tf_unary_scores_, tf_transition_params)
# Evaluate word-level accuracy.
correct_labels += np.sum(np.equal(viterbi_sequence, y_))
total_labels += sequence_length_
accuracy = 100.0 * correct_labels / float(total_labels)
print("Accuracy: %.2f%%" % accuracy)
I get the error message:
Traceback (most recent call last):
File "C:\Anaconda\envs\py35\lib\site-packages\tensorflow\python\client\session.py", line 1022, in _do_call
return fn(*args)
File "C:\Anaconda\envs\py35\lib\site-packages\tensorflow\python\client\session.py", line 1004, in _run_fn
status, run_metadata)
File "C:\Anaconda\envs\py35\lib\contextlib.py", line 66, in __exit__
next(self.gen)
File "C:\Anaconda\envs\py35\lib\site-packages\tensorflow\python\framework\errors_impl.py", line 469, in raise_exception_on_not_ok_status
pywrap_tensorflow.TF_GetCode(status))
tensorflow.python.framework.errors_impl.UnimplementedError: TensorArray has size zero, but element shape <unknown> is not fully defined. Currently only static shapes are supported when packing zero-size TensorArrays.
[[Node: gradients/rnn/TensorArrayUnstack/TensorArrayScatter/TensorArrayScatterV3_grad/TensorArrayGatherV3 = TensorArrayGatherV3[_class=["loc:#rnn/TensorArray_1"], dtype=DT_FLOAT, element_shape=<unknown>, _device="/job:localhost/replica:0/task:0/cpu:0"](gradients/rnn/TensorArrayUnstack/TensorArrayScatter/TensorArrayScatterV3_grad/TensorArrayGrad/TensorArrayGradV3, rnn/TensorArrayUnstack/range, gradients/rnn/TensorArrayUnstack/TensorArrayScatter/TensorArrayScatterV3_grad/TensorArrayGrad/gradient_flow)]]
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "C:\Users\Francky\Documents\GitHub\nlp\neurodeid\test\CRF_v2.py", line 47, in <module>
[unary_scores, transition_params, train_op])
File "C:\Anaconda\envs\py35\lib\site-packages\tensorflow\python\client\session.py", line 767, in run
run_metadata_ptr)
File "C:\Anaconda\envs\py35\lib\site-packages\tensorflow\python\client\session.py", line 965, in _run
feed_dict_string, options, run_metadata)
File "C:\Anaconda\envs\py35\lib\site-packages\tensorflow\python\client\session.py", line 1015, in _do_run
target_list, options, run_metadata)
File "C:\Anaconda\envs\py35\lib\site-packages\tensorflow\python\client\session.py", line 1035, in _do_call
raise type(e)(node_def, op, message)
tensorflow.python.framework.errors_impl.UnimplementedError: TensorArray has size zero, but element shape <unknown> is not fully defined. Currently only static shapes are supported when packing zero-size TensorArrays.
[[Node: gradients/rnn/TensorArrayUnstack/TensorArrayScatter/TensorArrayScatterV3_grad/TensorArrayGatherV3 = TensorArrayGatherV3[_class=["loc:#rnn/TensorArray_1"], dtype=DT_FLOAT, element_shape=<unknown>, _device="/job:localhost/replica:0/task:0/cpu:0"](gradients/rnn/TensorArrayUnstack/TensorArrayScatter/TensorArrayScatterV3_grad/TensorArrayGrad/TensorArrayGradV3, rnn/TensorArrayUnstack/range, gradients/rnn/TensorArrayUnstack/TensorArrayScatter/TensorArrayScatterV3_grad/TensorArrayGrad/gradient_flow)]]
Caused by op 'gradients/rnn/TensorArrayUnstack/TensorArrayScatter/TensorArrayScatterV3_grad/TensorArrayGatherV3', defined at:
File "C:\Users\Francky\Documents\GitHub\nlp\neurodeid\test\CRF_v2.py", line 41, in <module>
train_op = tf.train.GradientDescentOptimizer(0.01).minimize(loss)
File "C:\Anaconda\envs\py35\lib\site-packages\tensorflow\python\training\optimizer.py", line 288, in minimize
grad_loss=grad_loss)
File "C:\Anaconda\envs\py35\lib\site-packages\tensorflow\python\training\optimizer.py", line 354, in compute_gradients
colocate_gradients_with_ops=colocate_gradients_with_ops)
File "C:\Anaconda\envs\py35\lib\site-packages\tensorflow\python\ops\gradients_impl.py", line 482, in gradients
in_grads = grad_fn(op, *out_grads)
File "C:\Anaconda\envs\py35\lib\site-packages\tensorflow\python\ops\tensor_array_grad.py", line 186, in _TensorArrayScatterGrad
grad = g.gather(indices)
File "C:\Anaconda\envs\py35\lib\site-packages\tensorflow\python\ops\tensor_array_ops.py", line 348, in gather
element_shape=element_shape)
File "C:\Anaconda\envs\py35\lib\site-packages\tensorflow\python\ops\gen_data_flow_ops.py", line 2226, in _tensor_array_gather_v3
element_shape=element_shape, name=name)
File "C:\Anaconda\envs\py35\lib\site-packages\tensorflow\python\framework\op_def_library.py", line 763, in apply_op
op_def=op_def)
File "C:\Anaconda\envs\py35\lib\site-packages\tensorflow\python\framework\ops.py", line 2395, in create_op
original_op=self._default_original_op, op_def=op_def)
File "C:\Anaconda\envs\py35\lib\site-packages\tensorflow\python\framework\ops.py", line 1264, in __init__
self._traceback = _extract_stack()
...which was originally created as op 'rnn/TensorArrayUnstack/TensorArrayScatter/TensorArrayScatterV3', defined at:
File "C:\Users\Francky\Documents\GitHub\nlp\neurodeid\test\CRF_v2.py", line 37, in <module>
unary_scores, y_t, sequence_lengths_t)
File "C:\Anaconda\envs\py35\lib\site-packages\tensorflow\contrib\crf\python\ops\crf.py", line 156, in crf_log_likelihood
log_norm = crf_log_norm(inputs, sequence_lengths, transition_params)
File "C:\Anaconda\envs\py35\lib\site-packages\tensorflow\contrib\crf\python\ops\crf.py", line 123, in crf_log_norm
dtype=dtypes.float32)
File "C:\Anaconda\envs\py35\lib\site-packages\tensorflow\python\ops\rnn.py", line 545, in dynamic_rnn
dtype=dtype)
File "C:\Anaconda\envs\py35\lib\site-packages\tensorflow\python\ops\rnn.py", line 663, in _dynamic_rnn_loop
for ta, input_ in zip(input_ta, flat_input))
File "C:\Anaconda\envs\py35\lib\site-packages\tensorflow\python\ops\rnn.py", line 663, in <genexpr>
for ta, input_ in zip(input_ta, flat_input))
File "C:\Anaconda\envs\py35\lib\site-packages\tensorflow\python\ops\tensor_array_ops.py", line 400, in unstack
indices=math_ops.range(0, num_elements), value=value, name=name)
File "C:\Anaconda\envs\py35\lib\site-packages\tensorflow\python\ops\tensor_array_ops.py", line 428, in scatter
name=name)
File "C:\Anaconda\envs\py35\lib\site-packages\tensorflow\python\ops\gen_data_flow_ops.py", line 2492, in _tensor_array_scatter_v3
name=name)
File "C:\Anaconda\envs\py35\lib\site-packages\tensorflow\python\framework\op_def_library.py", line 763, in apply_op
op_def=op_def)
UnimplementedError (see above for traceback): TensorArray has size zero, but element shape <unknown> is not fully defined. Currently only static shapes are supported when packing zero-size TensorArrays.
[[Node: gradients/rnn/TensorArrayUnstack/TensorArrayScatter/TensorArrayScatterV3_grad/TensorArrayGatherV3 = TensorArrayGatherV3[_class=["loc:#rnn/TensorArray_1"], dtype=DT_FLOAT, element_shape=<unknown>, _device="/job:localhost/replica:0/task:0/cpu:0"](gradients/rnn/TensorArrayUnstack/TensorArrayScatter/TensorArrayScatterV3_grad/TensorArrayGrad/TensorArrayGradV3, rnn/TensorArrayUnstack/range, gradients/rnn/TensorArrayUnstack/TensorArrayScatter/TensorArrayScatterV3_grad/TensorArrayGrad/gradient_flow)]]
num_words = 2 or num_words = 5 work. I see on
https://github.com/tensorflow/tensorflow/blob/e121667dc609de978a223c56ee906368d2c4ceef/tensorflow/contrib/crf/python/ops/crf.py#L121 (mirror) is already decrementing the sequence_length by 1:
# Compute the alpha values in the forward algorithm in order to get the
# partition function.
forward_cell = CrfForwardRnnCell(transition_params)
_, alphas = rnn.dynamic_rnn(
cell=forward_cell,
inputs=rest_of_input,
sequence_length=sequence_lengths - 1,
initial_state=first_input,
dtype=dtypes.float32)
log_norm = math_ops.reduce_logsumexp(alphas, [1])
return log_norm
However, changing sequence_lengths = np.full(num_examples, num_words - 1, dtype=np.int32) to sequence_lengths = np.full(num_examples, num_words, dtype=np.int32) does not solve the issue when num_words = 1 .
How to fix this issue so that the CRF layer supports sequences of length 1?
Tested with TensorFlow 1.0.0 on Windows 7 SP1 x64 Ultimate and TensorFlow-GPU 1.0.0 on Ubuntu 14.04.4 LTS x64. I created an issue in the TensorFlow repository but it got closed due to inactivity: https://github.com/tensorflow/tensorflow/issues/7751 (mirror)
I came across this so posting showing us how to setup the code to read in csv files using a queue. However, each time I run it, I run into an error. I've tried debugging it, but can't figure out what the error means. Can anyone help me out?
The code I'm using is almost verbatim what was posted in the above post:
import tensorflow as tf
dataset = '/Users/hdadmin/Data/actions/testing.csv'
def file_len(fname):
with open(fname) as f:
for i, l in enumerate(f):
pass
return i + 1
def read_from_csv(filename_queue):
reader = tf.TextLineReader(skip_header_lines=1)
_, csv_row = reader.read(filename_queue)
record_defaults = [[0],[0],[0],[0],[0]]
colHour,colQuarter,colAction,colUser,colLabel = tf.decode_csv(csv_row, record_defaults=record_defaults)
features = tf.pack([colHour,colQuarter,colAction,colUser])
label = tf.pack([colLabel])
return features, label
def input_pipeline(batch_size, num_epochs=None):
filename_queue = tf.train.string_input_producer([dataset], num_epochs=num_epochs, shuffle=True)
example, label = read_from_csv(filename_queue)
min_after_dequeue = 1000
capacity = min_after_dequeue + 3 * batch_size
example_batch, label_batch = tf.train.shuffle_batch(
[example, label], batch_size=batch_size, capacity=capacity,
min_after_dequeue=min_after_dequeue)
return example_batch, label_batch
file_length = file_len(dataset) - 1
examples, labels = input_pipeline(file_length, 1)
with tf.Session() as sess:
tf.initialize_all_variables().run()
# start populating filename queue
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(coord=coord)
try:
while not coord.should_stop():
example_batch, label_batch = sess.run([examples, labels])
print(example_batch)
except tf.errors.OutOfRangeError:
print('Done training, epoch reached')
finally:
coord.request_stop()
coord.join(threads)
The error I'm getting is:
E tensorflow/core/client/tensor_c_api.cc:485] Attempting to use uninitialized value input_producer/limit_epochs/epochs
[[Node: input_producer/limit_epochs/CountUpTo = CountUpTo[T=DT_INT64, _class=["loc:#input_producer/limit_epochs/epochs"], limit=1, _device="/job:localhost/replica:0/task:0/cpu:0"](input_producer/limit_epochs/epochs)]]
E tensorflow/core/client/tensor_c_api.cc:485] RandomShuffleQueue '_2_shuffle_batch/random_shuffle_queue' is closed and has insufficient elements (requested 10000, current size 0)
[[Node: shuffle_batch = QueueDequeueMany[_class=["loc:#shuffle_batch/random_shuffle_queue"], component_types=[DT_INT32, DT_INT32], timeout_ms=-1, _device="/job:localhost/replica:0/task:0/cpu:0"](shuffle_batch/random_shuffle_queue, shuffle_batch/n)]]
Done training, epoch reached
E tensorflow/core/client/tensor_c_api.cc:485] FIFOQueue '_0_input_producer' is closed and has insufficient elements (requested 1, current size 0)
[[Node: ReaderRead = ReaderRead[_class=["loc:#TextLineReader", "loc:#input_producer"], _device="/job:localhost/replica:0/task:0/cpu:0"](TextLineReader, input_producer)]]
E tensorflow/core/client/tensor_c_api.cc:485] Queue '_2_shuffle_batch/random_shuffle_queue' is already closed.
[[Node: shuffle_batch/random_shuffle_queue_Close = QueueClose[_class=["loc:#shuffle_batch/random_shuffle_queue"], cancel_pending_enqueues=false, _device="/job:localhost/replica:0/task:0/cpu:0"](shuffle_batch/random_shuffle_queue)]]
Traceback (most recent call last):
File "csv_test.py", line 49, in <module>
coord.join(threads)
File "/usr/local/anaconda2/lib/python2.7/site-packages/tensorflow/python/training/coordinator.py", line 357, in join
six.reraise(*self._exc_info_to_raise)
File "/usr/local/anaconda2/lib/python2.7/site-packages/tensorflow/python/training/queue_runner.py", line 185, in _run
sess.run(enqueue_op)
File "/usr/local/anaconda2/lib/python2.7/site-packages/tensorflow/python/client/session.py", line 382, in run
run_metadata_ptr)
File "/usr/local/anaconda2/lib/python2.7/site-packages/tensorflow/python/client/session.py", line 655, in _run
feed_dict_string, options, run_metadata)
File "/usr/local/anaconda2/lib/python2.7/site-packages/tensorflow/python/client/session.py", line 723, in _do_run
target_list, options, run_metadata)
File "/usr/local/anaconda2/lib/python2.7/site-packages/tensorflow/python/client/session.py", line 743, in _do_call
raise type(e)(node_def, op, message)
tensorflow.python.framework.errors.FailedPreconditionError: Attempting to use uninitialized value input_producer/limit_epochs/epochs
[[Node: input_producer/limit_epochs/CountUpTo = CountUpTo[T=DT_INT64, _class=["loc:#input_producer/limit_epochs/epochs"], limit=1, _device="/job:localhost/replica:0/task:0/cpu:0"](input_producer/limit_epochs/epochs)]]
Caused by op u'input_producer/limit_epochs/CountUpTo', defined at:
File "csv_test.py", line 31, in <module>
examples, labels = input_pipeline(file_length, 1)
File "csv_test.py", line 21, in input_pipeline
filename_queue = tf.train.string_input_producer([dataset], num_epochs=num_epochs, shuffle=True)
File "/usr/local/anaconda2/lib/python2.7/site-packages/tensorflow/python/training/input.py", line 194, in string_input_producer
summary_name="fraction_of_%d_full" % capacity)
File "/usr/local/anaconda2/lib/python2.7/site-packages/tensorflow/python/training/input.py", line 133, in input_producer
input_tensor = limit_epochs(input_tensor, num_epochs)
File "/usr/local/anaconda2/lib/python2.7/site-packages/tensorflow/python/training/input.py", line 84, in limit_epochs
counter = epochs.count_up_to(num_epochs)
File "/usr/local/anaconda2/lib/python2.7/site-packages/tensorflow/python/ops/variables.py", line 577, in count_up_to
return state_ops.count_up_to(self._variable, limit=limit)
File "/usr/local/anaconda2/lib/python2.7/site-packages/tensorflow/python/ops/gen_state_ops.py", line 127, in count_up_to
result = _op_def_lib.apply_op("CountUpTo", ref=ref, limit=limit, name=name)
File "/usr/local/anaconda2/lib/python2.7/site-packages/tensorflow/python/framework/op_def_library.py", line 703, in apply_op
op_def=op_def)
File "/usr/local/anaconda2/lib/python2.7/site-packages/tensorflow/python/framework/ops.py", line 2310, in create_op
original_op=self._default_original_op, op_def=op_def)
File "/usr/local/anaconda2/lib/python2.7/site-packages/tensorflow/python/framework/ops.py", line 1232, in __init__
self._traceback = _extract_stack()
I made up data comprised of five columns to match with the example. It's something along the lines of:
"v1","v2","v3","v4","v5"
1,1,1,3,10
4,2,1,10,8
1,4,1,9,3
3,3,1,1,5
3,4,1,4,3
3,2,1,5,8
1,1,1,9,7
4,1,1,4,9
2,3,1,8,4
Thanks ahead of time.
I think what you are missing is initialization of local variables (e.g. input_producer/limit_epochs/epochs). Local variables are not initialized during initialization of all variables, which btw is quite confusing.
You can add the following initialization operation that will initialize everything at once:
init_op = tf.group(tf.initialize_all_variables(),
tf.initialize_local_variables())
and then:
sess.run(init_op)