I already have some code which trains a classifier from numpy arrays. However, my training data set is very large. It seems the recommended solution is to use TFRecords. My attempts to use TFRecords with my own data set have failed, so I have gradually reduced my code to a minimal toy.
Example:
import tensorflow as tf
def readsingleexample(serialized):
print("readsingleexample", serialized)
feature = dict()
feature['x'] = tf.FixedLenFeature([], tf.int64)
feature['label'] = tf.FixedLenFeature([], tf.int64)
parsed_example = tf.parse_single_example(serialized, features=feature)
print(parsed_example)
return parsed_example['x'], parsed_example['label']
def TestParse(filename):
record_iterator=tf.python_io.tf_record_iterator(path=filename)
for string_record in record_iterator:
example=tf.train.Example()
example.ParseFromString(string_record)
print(example.features)
def TestRead(filename):
record_iterator=tf.python_io.tf_record_iterator(path=filename)
for string_record in record_iterator:
feats, label = readsingleexample(string_record)
print(feats, label)
def _int64_feature(value):
return tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))
def TFRecordsTest(filename):
example=tf.train.Example(features=tf.train.Features(feature={
'x': _int64_feature(7),
'label': _int64_feature(4)
}))
writer = tf.python_io.TFRecordWriter(filename)
writer.write(example.SerializeToString())
record_iterator=tf.python_io.tf_record_iterator(path=filename)
for string_record in record_iterator:
example=tf.train.Example()
example.ParseFromString(string_record)
print(example.features)
dataset=tf.data.TFRecordDataset(filenames=[filename])
dataset=dataset.map(readsingleexample)
dataset=dataset.repeat()
def train_input_fn():
iterator=dataset.make_one_shot_iterator()
feats_tensor, labels_tensor = iterator.get_next()
return {"x":feats_tensor}, labels_tensor
feature_columns = []
feature_columns.append(tf.feature_column.numeric_column(key='x'))
classifier = tf.estimator.DNNClassifier(feature_columns=feature_columns,
hidden_units=[10, 10, 10],
n_classes=2)
classifier.train(input_fn=train_input_fn, steps=1000)
return
This results in the following output:
feature {
key: "label"
value {
int64_list {
value: 4
}
}
}
feature {
key: "x"
value {
int64_list {
value: 7
}
}
}
readsingleexample Tensor("arg0:0", shape=(), dtype=string)
{'x': <tf.Tensor 'ParseSingleExample/ParseSingleExample:1' shape=() dtype=int64>, 'label': <tf.Tensor 'ParseSingleExample/ParseSingleExample:0' shape=() dtype=int64>}
WARNING:tensorflow:Using temporary folder as model directory: C:\Users\eeark\AppData\Local\Temp\tmpcl47b2ut
Traceback (most recent call last):
File "<pyshell#2>", line 1, in <module>
tfrecords_test.TFRecordsTest(fn)
File "C:\_P4\user_feindselig\_python\tfrecords_test.py", line 60, in TFRecordsTest
classifier.train(input_fn=train_input_fn, steps=1000)
File "C:\Program Files\Python352\lib\site-packages\tensorflow\python\estimator\estimator.py", line 352, in train
loss = self._train_model(input_fn, hooks, saving_listeners)
File "C:\Program Files\Python352\lib\site-packages\tensorflow\python\estimator\estimator.py", line 812, in _train_model
features, labels, model_fn_lib.ModeKeys.TRAIN, self.config)
File "C:\Program Files\Python352\lib\site-packages\tensorflow\python\estimator\estimator.py", line 793, in _call_model_fn
model_fn_results = self._model_fn(features=features, **kwargs)
File "C:\Program Files\Python352\lib\site-packages\tensorflow\python\estimator\canned\dnn.py", line 354, in _model_fn
config=config)
File "C:\Program Files\Python352\lib\site-packages\tensorflow\python\estimator\canned\dnn.py", line 185, in _dnn_model_fn
logits = logit_fn(features=features, mode=mode)
File "C:\Program Files\Python352\lib\site-packages\tensorflow\python\estimator\canned\dnn.py", line 91, in dnn_logit_fn
features=features, feature_columns=feature_columns)
File "C:\Program Files\Python352\lib\site-packages\tensorflow\python\feature_column\feature_column.py", line 273, in input_layer
trainable, cols_to_vars)
File "C:\Program Files\Python352\lib\site-packages\tensorflow\python\feature_column\feature_column.py", line 198, in _internal_input_layer
trainable=trainable)
File "C:\Program Files\Python352\lib\site-packages\tensorflow\python\feature_column\feature_column.py", line 2080, in _get_dense_tensor
return inputs.get(self)
File "C:\Program Files\Python352\lib\site-packages\tensorflow\python\feature_column\feature_column.py", line 1883, in get
transformed = column._transform_feature(self) # pylint: disable=protected-access
File "C:\Program Files\Python352\lib\site-packages\tensorflow\python\feature_column\feature_column.py", line 2048, in _transform_feature
input_tensor = inputs.get(self.key)
File "C:\Program Files\Python352\lib\site-packages\tensorflow\python\feature_column\feature_column.py", line 1870, in get
feature_tensor = self._get_raw_feature_as_tensor(key)
File "C:\Program Files\Python352\lib\site-packages\tensorflow\python\feature_column\feature_column.py", line 1924, in _get_raw_feature_as_tensor
key, feature_tensor))
ValueError: Feature (key: x) cannot have rank 0. Give: Tensor("IteratorGetNext:0", shape=(), dtype=int64, device=/device:CPU:0)
What does the error mean? What could be going wrong?
The following appears to work: no errors are raised, at least. tf.parse_example([serialized], ...) is used instead of tf.parse_single_example(serialized, ...). (Also, the label in the synthetic data was altered to be less than the number of classes.)
import tensorflow as tf
def readsingleexample(serialized):
print("readsingleexample", serialized)
feature = dict()
feature['x'] = tf.FixedLenFeature([], tf.int64)
feature['label'] = tf.FixedLenFeature([], tf.int64)
parsed_example = tf.parse_example([serialized], features=feature)
print(parsed_example)
return parsed_example['x'], parsed_example['label']
def TestParse(filename):
record_iterator=tf.python_io.tf_record_iterator(path=filename)
for string_record in record_iterator:
example=tf.train.Example()
example.ParseFromString(string_record)
print(example.features)
def TestRead(filename):
record_iterator=tf.python_io.tf_record_iterator(path=filename)
for string_record in record_iterator:
feats, label = readsingleexample(string_record)
print(feats, label)
def _int64_feature(value):
return tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))
def TFRecordsTest(filename):
example=tf.train.Example(features=tf.train.Features(feature={
'x': _int64_feature(7),
'label': _int64_feature(0)
}))
writer = tf.python_io.TFRecordWriter(filename)
writer.write(example.SerializeToString())
record_iterator=tf.python_io.tf_record_iterator(path=filename)
for string_record in record_iterator:
example=tf.train.Example()
example.ParseFromString(string_record)
print(example.features)
dataset=tf.data.TFRecordDataset(filenames=[filename])
dataset=dataset.map(readsingleexample)
dataset=dataset.repeat()
def train_input_fn():
iterator=dataset.make_one_shot_iterator()
feats_tensor, labels_tensor = iterator.get_next()
return {'x':feats_tensor}, labels_tensor
feature_columns = []
feature_columns.append(tf.feature_column.numeric_column(key='x'))
classifier = tf.estimator.DNNClassifier(feature_columns=feature_columns,
hidden_units=[10, 10, 10],
n_classes=2)
classifier.train(input_fn=train_input_fn, steps=1000)
return
rank 0 means its a scalar
so
example=tf.train.Example(features=tf.train.Features(feature={
'x': [_int64_feature(7)],
'label': _int64_feature(4)
}))
would make it rank 1 or a vector i.e. add []
Related
I wanted to test my network on a toy data set - a few examples with two imbalanced classes (0 and 1). Unfortunately, there are problems when using the class_weight parameter to improve the balance. It looks like I forget something.
import tensorflow as tf
from tensorflow.python.keras.layers import Dense, Dropout
from tensorflow.python.keras.applications.xception import Xception, preprocess_input
from tensorflow.python.keras.models import Model
from tensorflow.python.keras.optimizers import Adam
# parsing images from TFRecords
def parse_function(proto):
example = {'image_raw': tf.FixedLenFeature([], tf.string), 'label': tf.FixedLenFeature([], tf.int64)}
parsed_example = tf.parse_single_example(proto, example)
image = tf.decode_raw(parsed_example['image_raw'], tf.uint8)
image = tf.reshape(image, [HEIGHT, WIDTH, DEPTH])
image = preprocess_input(tf.cast(image, tf.float32))
return image, parsed_example['label']
def get_data(filepath, schuffle_size=32, batch_size=8, prefetch=1, repeat=None, num_parallel_calls=1):
dataset = tf.data.TFRecordDataset(filepath)
if schuffle_size != 0:
dataset = dataset.shuffle(schuffle_size)
dataset = dataset.repeat(repeat)
dataset = dataset.map(parse_function, num_parallel_calls=num_parallel_calls)
dataset = dataset.batch(batch_size)
dataset = dataset.prefetch(prefetch)
iterator = dataset.make_one_shot_iterator()
return iterator
def build_model(number_of_neurons_in_dense_layer, dropout, learning_rate):
base_model = Xception(weights='imagenet', include_top=False, pooling='avg', input_shape=(HEIGHT, WIDTH, 3))
for layer in base_model.layers:
layer.trainable = True
x = base_model.output
x = Dropout(dropout)(x)
x = Dense(number_of_neurons_in_dense_layer, activation='relu')(x)
x = Dropout(dropout)(x)
logits = Dense(NUMBER_OF_CLASSES, activation='softmax')(x)
model = Model(inputs=base_model.input, outputs=logits)
model.compile(optimizer=Adam(lr=learning_rate), loss='sparse_categorical_crossentropy', metrics=['categorical_accuracy'])
return model
global NUMBER_OF_CLASSES, HEIGHT, WIDTH, DEPTH
NUMBER_OF_CLASSES = 2
...
CLASS_WEIGHTS = {
0: 1,
1: 7
}
model = build_model(64, 0.4, 0.001)
train = get_data(..., 8, 2, num_parallel_calls=8)
val = get_data(...., 0, 4, num_parallel_calls=8)
model.fit(train, validation_data=val, epochs=3,steps_per_epoch=8//2,
validation_steps=8//4, shuffle=False,
class_weight=CLASS_WEIGHTS)
I am getting the following errors
Original exception was:
Traceback (most recent call last):
File "/usr/local/lib/python3.6/dist-packages/numpy/core/fromnumeric.py", line 51, in _wrapfunc
return getattr(obj, method)(*args, **kwds)
AttributeError: 'Tensor' object has no attribute 'reshape'
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/usr/model.py", line 147, in main
class_weight=CLASS_WEIGHTS)
File "/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/training.py", line 776, in fit
shuffle=shuffle)
File "/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/training.py", line 2432, in _standardize_user_data
feed_sample_weight_modes)
File "/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/training.py", line 2431, in <listcomp>
for (ref, sw, cw, mode) in zip(y, sample_weights, class_weights,
File "/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/training_utils.py", line 758, in standardize_weights
y_classes = np.reshape(y, y.shape[0])
File "/usr/local/lib/python3.6/dist-packages/numpy/core/fromnumeric.py", line 279, in reshape
return _wrapfunc(a, 'reshape', newshape, order=order)
File "/usr/local/lib/python3.6/dist-packages/numpy/core/fromnumeric.py", line 61, in _wrapfunc
return _wrapit(obj, method, *args, **kwds)
File "/usr/local/lib/python3.6/dist-packages/numpy/core/fromnumeric.py", line 41, in _wrapit
result = getattr(asarray(obj), method)(*args, **kwds)
TypeError: __index__ returned non-int (type NoneType)
Without the class_weight parameter, the fit function works correctly.
Just for a future reference:
I ran into this error to and was able to resolve it by passing an array instead of a dictionary.
e.g.
CLASS_WEIGHTS = np.array([1,7])
instead of:
CLASS_WEIGHTS = {
0: 1,
1: 7
}
I don't know why I am getting this error.
I saw a some posts to change state_is_tuple=False but it was giving me some other error. I think the error is in the way I defined lstm cell but not sure what should I change? I followed this link which has similar code structure.
Here is my code:
Required placeholders
n_hidden = args.rnn_size
n_layers = args.num_layers
max_sequence_length = args.max_sequence_length
encoderEmbeddingsize = args.encoderEmbeddingsize
decoderEmbeddingsize = args.decoderEmbeddingsize
queVocabsize = len(question_vocab_to_int)
ansVocabsize = len(answer_vocab_to_int)
batch_size = args.batch_size
# Input Embedding for Encoder ## CHECK THE VOCAB SIZE!!!
encoder_input = tf.contrib.layers.embed_sequence(input_data, queVocabsize, encoderEmbeddingsize,
initializer=tf.random_uniform_initializer(0, 1))
print('encoder_input', encoder_input)
# Layers for the model
lstm_cell = rnn.BasicLSTMCell(n_hidden) # lstm layer
dropout = rnn.DropoutWrapper(lstm_cell, input_keep_prob=keep_prob) # dropout layer
# Encoder Model
# Make two layer encoder
encoder_multirnn_cell = rnn.MultiRNNCell([dropout]*n_layers)
# Make it bidirectional
print(sequence_length)
encoder_output, encoder_state = tf.nn.dynamic_rnn(encoder_multirnn_cell,
inputs=encoder_input, dtype=tf.float32) # sequence_length=sequence_length,
print('encoder_output', encoder_output)
print('encoder_state', encoder_state)
# preprocessing encoder input
initial_tensor = tf.strided_slice(target, [0, 0], [batch_size, -1], [1, 1])
decoder_input = tf.concat([tf.fill([batch_size, 1], question_vocab_to_int['<GO>']), initial_tensor], 1)
print('decoder_input', decoder_input)
## Input Embedding for the Decoder
decoder_embedding = tf.Variable(tf.random_uniform([queVocabsize+1, decoderEmbeddingsize], 0, 1))
decoder_embedded_input = tf.nn.embedding_lookup(decoder_embedding, decoder_input)
print('check')
print(decoder_embedded_input)
print(decoder_embedding)
## Decoder Model
#with tf.variable_scope("decoding") as decoding_scope:
lstm_decoder_cell = rnn.BasicLSTMCell(n_hidden) # lstm layer
dropout_decoder = rnn.DropoutWrapper(lstm_decoder_cell, input_keep_prob=keep_prob) # droput layer
# decoder
# Make two layer encoder
decoder_multirnn_cell = rnn.MultiRNNCell([dropout_decoder] * n_layers)
# weights = tf.truncated_normal_initializer(stddev=0.1)
# biases = tf.zeros_initializer()
output_layer_function = layers_core.Dense(
ansVocabsize, use_bias=False) #lambda x: tf.contrib.layers.fully_connected(x, queVocabsize, scope=decoding_scope,
# weights_initializer=weights,
# biases_initializer=biases)
#print(decoder_multirnn_cell.output_size)
#decoding_scope.reuse_variables()
print('output_kayer_function', output_layer_function)
# training vs inference!
encoder_output = tf.transpose(encoder_output, [1, 0, 2])
attention_state = tf.zeros([batch_size, 1, decoder_multirnn_cell.output_size * 2])
attention_mechanism = tf.contrib.seq2seq.BahdanauAttention(
num_units=decoder_multirnn_cell.output_size, memory=encoder_output)
lstm_decoder_cell = tf.contrib.seq2seq.AttentionWrapper(lstm_decoder_cell,
attention_mechanism=attention_mechanism)
attn_zero = lstm_decoder_cell.zero_state(batch_size=batch_size, dtype=tf.float32)
init_state = attn_zero.clone(cell_state=encoder_state)
print(('sequence!!!!!!!!1', sequence_length))
helper = tf.contrib.seq2seq.TrainingHelper(decoder_embedded_input, sequence_length)
# decoder
decoder = tf.contrib.seq2seq.BasicDecoder(lstm_decoder_cell, helper, initial_state=init_state,
output_layer= output_layer_function)
print(decoder)
final_outputs, _final_state, _final_sequence_lengths = tf.contrib.seq2seq.dynamic_decode(decoder)
train_pred_drop = tf.nn.dropout(final_outputs, keep_prob)
logits = train_pred_drop.rnn_output
Now, I am getting the error in tf.contrib.seq2seq.dynamic_decode(decoder), as shown below:
Traceback (most recent call last):
File "test_model.py", line 272, in <module>
train_logits, infer_logits = load_model(args, tf.reverse(input_data, [-1]), target, learning_rate, sequence_length, question_vocab_to_int, answer_vocab_to_int, keep_prob ) ## load model here!
File "test_model.py", line 165, in load_model
final_outputs, _final_state, _final_sequence_lengths = tf.contrib.seq2seq.dynamic_decode(decoder)
File "/home/saurabh/tfnightly/lib/python3.5/site-packages/tensorflow/contrib/seq2seq/python/ops/decoder.py", line 286, in dynamic_decode
swap_memory=swap_memory)
File "/home/saurabh/tfnightly/lib/python3.5/site-packages/tensorflow/python/ops/control_flow_ops.py", line 2816, in while_loop
result = loop_context.BuildLoop(cond, body, loop_vars, shape_invariants)
File "/home/saurabh/tfnightly/lib/python3.5/site-packages/tensorflow/python/ops/control_flow_ops.py", line 2640, in BuildLoop
pred, body, original_loop_vars, loop_vars, shape_invariants)
File "/home/saurabh/tfnightly/lib/python3.5/site-packages/tensorflow/python/ops/control_flow_ops.py", line 2590, in _BuildLoop
body_result = body(*packed_vars_for_body)
File "/home/saurabh/tfnightly/lib/python3.5/site-packages/tensorflow/contrib/seq2seq/python/ops/decoder.py", line 234, in body
decoder_finished) = decoder.step(time, inputs, state)
File "/home/saurabh/tfnightly/lib/python3.5/site-packages/tensorflow/contrib/seq2seq/python/ops/basic_decoder.py", line 138, in step
cell_outputs, cell_state = self._cell(inputs, state)
File "/home/saurabh/tfnightly/lib/python3.5/site-packages/tensorflow/python/ops/rnn_cell_impl.py", line 183, in __call__
return super(RNNCell, self).__call__(inputs, state)
File "/home/saurabh/tfnightly/lib/python3.5/site-packages/tensorflow/python/layers/base.py", line 575, in __call__
outputs = self.call(inputs, *args, **kwargs)
File "/home/saurabh/tfnightly/lib/python3.5/site-packages/tensorflow/contrib/seq2seq/python/ops/attention_wrapper.py", line 1295, in call
cell_output, next_cell_state = self._cell(cell_inputs, cell_state)
File "/home/saurabh/tfnightly/lib/python3.5/site-packages/tensorflow/python/ops/rnn_cell_impl.py", line 183, in __call__
return super(RNNCell, self).__call__(inputs, state)
File "/home/saurabh/tfnightly/lib/python3.5/site-packages/tensorflow/python/layers/base.py", line 575, in __call__
outputs = self.call(inputs, *args, **kwargs)
File "/home/saurabh/tfnightly/lib/python3.5/site-packages/tensorflow/python/ops/rnn_cell_impl.py", line 438, in call
self._linear = _Linear([inputs, h], 4 * self._num_units, True)
File "/home/saurabh/tfnightly/lib/python3.5/site-packages/tensorflow/python/ops/rnn_cell_impl.py", line 1154, in __init__
shapes = [a.get_shape() for a in args]
File "/home/saurabh/tfnightly/lib/python3.5/site-packages/tensorflow/python/ops/rnn_cell_impl.py", line 1154, in <listcomp>
shapes = [a.get_shape() for a in args]
AttributeError: 'LSTMStateTuple' object has no attribute 'get_shape'
i convert my own grayscale dataset with 60*60 pixel to tfrecords with write_tfrecord() but when i want to read and decode them it causes error . what is the problem ?
train_tfrecord_addr = './data/train.tfrecords'
test_tfrecord_addr = './data/test.tfrecords'
n_train_samples = 43990
n_test_samples = 12500
batch_size = 32 # number of batches in each iteration
keep_prob = 0.5 # Dropout, probability to keep units
n_epochs = 25
tfrecords_filename = './data/test.tfrecords'
def _bytes_feature(value):
return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))
def _int64_feature(value):
return tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))
def _float32_feature(value):
return tf.train.Feature(float_list=tf.train.FloatList(value=value))
def write_tfrecord(path):
images_addrs, images_labels = get_lable_and_image(path=path)
filename_pairs = list(zip(images_addrs, images_labels))
print(filename_pairs)
# to shuffle data
shuffle(filename_pairs)
writer = tf.python_io.TFRecordWriter(tfrecords_filename)
for img_path, label in filename_pairs:
# in this case all images are png with (32, 32) shape
img = np.array(Image.open(img_path)) # (32, 32) uint8
img_raw = img.tostring()
label_raw = label.tostring()
example = tf.train.Example(features=tf.train.Features(feature={
'image_raw': _bytes_feature(img_raw),
'label_raw': _bytes_feature(label_raw),
}))
writer.write(example.SerializeToString())
writer.close()
decode method ...
def read_and_decode(filename, batch_size, num_epochs, num_samples):
filename_queue = tf.train.string_input_producer([train_tfrecord_addr],
num_epochs=num_epochs)
reader = tf.TFRecordReader()
_, serialized_example = reader.read(filename_queue)
features = tf.parse_single_example(
serialized_example,
# Defaults are not specified since both keys are required.
features={
'image_raw': tf.FixedLenFeature([], tf.string),
'label_raw': tf.FixedLenFeature([], tf.string),
})
# Convert from a scalar string tensor to a uint8 tensor
image_raw = tf.decode_raw(features['image_raw'], tf.uint8)
image_resized = tf.reshape(image_raw, [60 * 60])
# Convert from [0, 255] -> [-0.5, 0.5] floats.
image_resized = tf.cast(image_resized, tf.float32) * (1. / 255) - 0.5
# Convert from a scalar string tensor to a uint8 tensor
label_raw = tf.decode_raw(features['label_raw'], tf.uint8)
label_resized = tf.reshape(label_raw, [2])
images, labels = tf.train.batch([image_resized, label_resized],
batch_size=batch_size,
capacity=num_samples,
num_threads=2, )
return images, labels
this is the main code that feed convolutions data .
def run_training():
"""Train ShapeNet for a number of steps."""
# Tell TensorFlow that the model will be built into the default Graph.
with tf.Graph().as_default():
# Input train images and labels.
train_images, train_labels = read_and_decode(filename=train_tfrecord_addr,
batch_size=batch_size,
num_epochs=n_epochs,
num_samples=n_train_samples)
# Input test images and labels.
# define batch_size = all test samples
test_images, test_labels = read_and_decode(filename=test_tfrecord_addr,
batch_size=n_test_samples,
num_epochs=n_epochs,
num_samples=n_test_samples)
# define placeholder for input images and labels
X = tf.placeholder(tf.float32, [None, 60 * 60])
Y = tf.placeholder(tf.float32, [None, 2])
# Build a Graph that computes predictions from the inference model.
prediction = convolutional_network_model(X)
# Backpropagation
# measure of error of our model
# this needs to be minimised by adjusting W and b
cross_entropy = -tf.reduce_sum(Y * tf.log(prediction))
# define training step which minimises cross entropy
train_op =tf.train.GradientDescentOptimizer(
learning_rate=0.001)
.minimize(cross_entropy)
# argmax gives index of highest entry in vector (1st axis of 1D tensor)
correct = tf.equal(tf.argmax(prediction, 1), tf.argmax(Y, 1))
# get mean of all entries in correct prediction, the higher the better
accuracy = tf.reduce_mean(tf.cast(correct, tf.float32))
# The op for initializing the variables.
init_op = tf.group(tf.global_variables_initializer(),
tf.local_variables_initializer())
st = time.time()
# Create a session for running operations in the Graph.
with tf.Session() as sess:
# Initialize the variables (the trained variables and the
# epoch counter).
sess.run(init_op)
# Start input enqueue threads.
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess=sess, coord=coord)
for epoch in range(n_epochs):
for itr in range(n_train_samples // batch_size):
# fetch the batch train images and labels
batch_x, batch_y = sess.run([train_images, train_labels])
sess.run([train_op], feed_dict={X: batch_x, Y: batch_y})
# fetch whole test images and labels
batch_x, batch_y = sess.run([test_images, test_labels])
# feed the model with all test images and labels
acc, _ = sess.run([accuracy, train_op],
feed_dict={X: batch_x, Y: batch_y})
print('epoch %d/%d: , accuracy = %.3f'
% (epoch, n_epochs, acc))
# When done, ask the threads to stop.
coord.request_stop()
# Wait for threads to finish.
coord.join(threads)
et = time.time()
duration = et - st
print(duration)
this is the error . i also convert my dataset in binary format but again i see the same error
2018-02-13 13:53:02.401813: I C:\tf_jenkins\workspace\rel-
win\M\windows\PY\35\tensorflow\core\platform\cpu_feature_guard.cc:137] Your
CPU supports instructions that this TensorFlow binary was not compiled to
use: AVX
Traceback (most recent call last):
File "C:\Program Files\Python35\lib\site-
packages\tensorflow\python\client\session.py", line 1350, in _do_call
return fn(*args) File "C:\Program Files\Python35\lib\site-
packages\tensorflow\python\client\session.py", line 1329, in _run_fn
status, run_metadata)
File "C:\Program Files\Python35\lib\site-
packages\tensorflow\python\framework\errors_impl.py", line 473, in __exit__
c_api.TF_GetCode(self.status.status))
tensorflow.python.framework.errors_impl.OutOfRangeError: FIFOQueue
'_1_batch/fifo_queue' is closed and has insufficient elements (requested 32,
current size 0)
[[Node: batch = QueueDequeueManyV2[component_types=[DT_FLOAT, DT_UINT8],
timeout_ms=-1, _device="/job:localhost/replica:0/task:0/device:CPU:0"]
(batch/fifo_queue, batch/n)]]
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "F:/project/python/MathNet/mathnet.py", line 232, in <module>
run_training()
File "F:/project/python/MathNet/mathnet.py", line 207, in run_training
batch_x, batch_y = sess.run([train_images, train_labels])
File "C:\Program Files\Python35\lib\site-
packages\tensorflow\python\client\session.py", line 895, in run
run_metadata_ptr)
File "C:\Program Files\Python35\lib\site-
packages\tensorflow\python\client\session.py", line 1128, in _run
feed_dict_tensor, options, run_metadata)
File "C:\Program Files\Python35\lib\site-
packages\tensorflow\python\client\session.py", line 1344, in _do_run
options, run_metadata)
File "C:\Program Files\Python35\lib\site-
packages\tensorflow\python\client\session.py", line 1363, in _do_call
raise type(e)(node_def, op, message)
tensorflow.python.framework.errors_impl.OutOfRangeError: FIFOQueue
'_1_batch/fifo_queue' is closed and has insufficient elements (requested
32, current size 0)
[[Node: batch = QueueDequeueManyV2[component_types=[DT_FLOAT,
DT_UINT8], timeout_ms=-1,
_device="/job:localhost/replica:0/task:0/device:CPU:0"](batch/fifo_queue,
batch/n)]]
Caused by op 'batch', defined at:
File "F:/project/python/MathNet/mathnet.py", line 232, in <module>
run_training()
File "F:/project/python/MathNet/mathnet.py", line 159, in run_training
num_samples=n_train_samples)
File "F:/project/python/MathNet/mathnet.py", line 105, in
read_and_decode
num_threads=2, )
File "C:\Program Files\Python35\lib\site-
packages\tensorflow\python\training\input.py", line 979, in batch
name=name)
File "C:\Program Files\Python35\lib\site-
packages\tensorflow\python\training\input.py", line 754, in _batch
dequeued = queue.dequeue_many(batch_size, name=name)
File "C:\Program Files\Python35\lib\site-
packages\tensorflow\python\ops\data_flow_ops.py", line 475, in dequeue_many
self._queue_ref, n=n, component_types=self._dtypes, name=name)
File "C:\Program Files\Python35\lib\site-
packages\tensorflow\python\ops\gen_data_flow_ops.py", line 2764, in
_queue_dequeue_many_v2
component_types=component_types, timeout_ms=timeout_ms, name=name)
File "C:\Program Files\Python35\lib\site-
packages\tensorflow\python\framework\op_def_library.py", line 787, in
_apply_op_helper
op_def=op_def)
File "C:\Program Files\Python35\lib\site-
packages\tensorflow\python\framework\ops.py", line 3160, in create_op
op_def=op_def)
File "C:\Program Files\Python35\lib\site-
packages\tensorflow\python\framework\ops.py", line 1625, in __init__
self._traceback = self._graph._extract_stack() # pylint: disable=protected-
access
OutOfRangeError (see above for traceback): FIFOQueue '_1_batch/fifo_queue'
is closed and has insufficient elements (requested 32, current size 0)
[[Node: batch = QueueDequeueManyV2[component_types=[DT_FLOAT, DT_UINT8],
timeout_ms=-1, _device="/job:localhost/replica:0/task:0/device:CPU:0"]
(batch/fifo_queue, batch/n)]]
I want to train and check accuracy of validation set in this code. I use is_training and tf.QueueBase.from_list to switch between train and compute accuracy. type of datasets are tfrecord. inference give input images and a float as keep_drop_prop.
#inference, loss, training, evaluation functions ...
train_queue = tf.train.string_input_producer([train_data_path])
test_queue = tf.train.string_input_producer([validation_data_path])
# SELECT QUEUE
is_training = tf.placeholder(tf.bool, shape=None, name="is_training")
q_selector = tf.cond(is_training,
lambda: tf.constant(0),
lambda: tf.constant(1))
q = tf.QueueBase.from_list(q_selector, [train_queue, test_queue])
if is_training==True:
feature = {'train/image': tf.FixedLenFeature([], tf.string),
'train/label': tf.FixedLenFeature([], tf.int64)}
reader = tf.TFRecordReader()
_, serialized_example1 = reader.read(q)
features = tf.parse_single_example(serialized_example, features=feature)
images = tf.decode_raw(features1['train/image'], tf.float32)
labels = tf.cast(features['train/label'], tf.int32)
images = tf.reshape(images, [50, 50, 3])
batch_Xs,batch_Ys=tf.train.shuffle_batch([images,labels],batch_size=500,capacity=500,min_after_dequeue=100)
batch_Xs = tf.cast(batch_Xs,tf.float32)/255
else:
feature = {'validation/image': tf.FixedLenFeature([], tf.string),
'validation/label': tf.FixedLenFeature([], tf.int64)}
reader = tf.TFRecordReader()
_, serialized_example = reader.read(q)
features = tf.parse_single_example(serialized_example, features=feature)
images = tf.decode_raw(features['validation/image'], tf.float32)
labels = tf.cast(features['validation/label'], tf.int32)
images = tf.reshape(images, [50, 50, 3])
batch_Xs,batch_Ys=tf.train.shuffle_batch([images,labels],batch_size=500,capacity=500,min_after_dequeue=100)
batch_Xs = tf.cast(batch_Xs,tf.float32)/255
if is_training==True:
logits=inference(batch_Xs,0.7)
total_loss = loss(logits,batch_Ys)
train_op = training(total_loss,learning_rate=LEARNING_RATE)
accuracy = evaluation(logits,batch_Ys)
else:
logits=inference(batch_Xs,1)
accuracy = evaluation(logits,batch_Ys)
sess = tf.Session()
init = tf.initialize_all_variables()
sess.run(init)
tf.train.start_queue_runners(sess=sess)
for i in range(NUM_ITER):
_,loss_value,acc=sess.run([train_op,total_loss,accuracy],feed_dict={is_training:True})
val_acc=sess.run([accuracy],feed_dict={is_training:False})
The result of this code is :
UnboundLocalError: local variable 'train_op' referenced before assignment
InvalidArgumentError: You must feed a value for placeholder tensor 'is_training' with dtype bool
[[Node: is_training = Placeholder[dtype=DT_BOOL, shape=[], _device="/job:localhost/replica:0/task:0/gpu:0"]()]]
Caused by op u'is_training', defined at:
File "/home/.../anaconda2/lib/python2.7/runpy.py", line 174, in _run_module_as_main
"__main__", fname, loader, pkg_name)
File "/home/.../anaconda2/lib/python2.7/runpy.py", line 72, in _run_code
exec code in run_globals
File "/home/.../.local/lib/python2.7/site-packages/ipykernel/__main__.py", line 3, in <module>
app.launch_new_instance()
File "/home/.../.local/lib/python2.7/site-packages/traitlets/config/application.py", line 658, in launch_instance
app.start()
File "/home/.../.local/lib/python2.7/site-packages/ipykernel/kernelapp.py", line 474, in start
ioloop.IOLoop.instance().start()
File "/home/.../.local/lib/python2.7/site-packages/zmq/eventloop/ioloop.py", line 177, in start
super(ZMQIOLoop, self).start()
File "/home/.../.local/lib/python2.7/site-packages/tornado/ioloop.py", line 887, in start
handler_func(fd_obj, events)
File "/home/.../.local/lib/python2.7/site-packages/tornado/stack_context.py", line 275, in null_wrapper
return fn(*args, **kwargs)
I think queues can't access input data ... Exactly I don't know where is the problem ...Thanks so much for your help...
Classify MNIST Digits with Tensorflow by a 2-layer RNN approach. Training works fine, but when evaluating accuracy, incompatible shape of test data is reported.
import tensorflow as tf
import inspect
import numpy as np
from tensorflow.examples.tutorials.mnist import input_data
mnist = input_data.read_data_sets("/tmp/data/", one_hot = True)
hm_epochs = 1
n_classes = 10
batch_size = 128
chunk_size = 28
n_chunks = 28
rnn_size = 128
x = tf.placeholder('float', [None, n_chunks,chunk_size])
y = tf.placeholder('float')
def lstm_cell():
if 'reuse' in inspect.getargspec(
tf.contrib.rnn.BasicLSTMCell.__init__).args:
return tf.contrib.rnn.BasicLSTMCell(
rnn_size, forget_bias=0.0, state_is_tuple=True,
reuse=tf.get_variable_scope().reuse)
else:
return tf.contrib.rnn.BasicLSTMCell(
rnn_size, forget_bias=0.0, state_is_tuple=True)
def attn_cell():
return tf.contrib.rnn.DropoutWrapper(
lstm_cell())
def recurrent_neural_network(x):
layer = {'weights':tf.Variable(tf.random_normal([rnn_size,n_classes])),
'biases':tf.Variable(tf.random_normal([n_classes]))}
x = tf.transpose(x, [1,0,2])
x = tf.reshape(x, [-1, chunk_size])
x = tf.split(x, n_chunks, 0)
stacked_lstm = tf.contrib.rnn.MultiRNNCell([attn_cell(),attn_cell()], state_is_tuple=True)
initial_state = state = stacked_lstm.zero_state(batch_size, tf.float32)
outputs, states = tf.contrib.rnn.static_rnn(stacked_lstm, x,state)
output = tf.matmul(outputs[-1],layer['weights']) + layer['biases']
return output
def train_neural_network(x):
prediction = recurrent_neural_network(x)
cost = tf.reduce_mean( tf.nn.softmax_cross_entropy_with_logits(labels=y,logits=prediction) )
optimizer = tf.train.AdamOptimizer().minimize(cost)
with tf.Session() as sess:
sess.run(tf.initialize_all_variables())
for epoch in range(hm_epochs):
epoch_loss = 0
for _ in range(int(mnist.train.num_examples/batch_size)):
epoch_x, epoch_y = mnist.train.next_batch(batch_size)
epoch_x = epoch_x.reshape((batch_size,n_chunks,chunk_size))
_, c = sess.run([optimizer, cost], feed_dict={x: epoch_x, y: epoch_y})
epoch_loss += c
print('Epoch', epoch, 'completed out of',hm_epochs,'loss:',epoch_loss)
correct = tf.equal(tf.argmax(prediction, 1), tf.argmax(y, 1))
accuracy = tf.reduce_mean(tf.cast(correct, 'float'))
testdata= np.reshape( mnist.test.images, (10000, n_chunks, chunk_size))
print("Testdata ",testdata.shape)
print("x ",x)
print('Accuracy:',accuracy.eval({x:testdata, y:mnist.test.labels}))
train_neural_network(x)
However, the shapes of test data and placeholders are printed as follows. Aren't they compatible?
Epoch 0 completed out of 1 loss: 228.159379691
Testdata (10000, 28, 28)
x Tensor("Placeholder:0", shape=(?, 28, 28), dtype=float32)
Error:
Caused by op 'rnn/rnn/multi_rnn_cell/cell_0/cell_0/basic_lstm_cell/basic_lstm_ce
ll/concat', defined at:
File "main.py", line 90, in <module>
train_neural_network(x)
File "main.py", line 59, in train_neural_network
prediction = recurrent_neural_network(x)
File "main.py", line 52, in recurrent_neural_network
outputs, states = tf.contrib.rnn.static_rnn(stacked_lstm, x,state)
File "C:\Users\henry\Anaconda3\lib\site-packages\tensorflow\python\ops\rnn.py"
, line 1212, in static_rnn
(output, state) = call_cell()
File "C:\Users\henry\Anaconda3\lib\site-packages\tensorflow\python\ops\rnn.py"
, line 1199, in <lambda>
call_cell = lambda: cell(input_, state)
File "C:\Users\henry\Anaconda3\lib\site-packages\tensorflow\python\ops\rnn_cel
l_impl.py", line 180, in __call__
return super(RNNCell, self).__call__(inputs, state)
File "C:\Users\henry\Anaconda3\lib\site-packages\tensorflow\python\layers\base
.py", line 441, in __call__
outputs = self.call(inputs, *args, **kwargs)
File "C:\Users\henry\Anaconda3\lib\site-packages\tensorflow\python\ops\rnn_cel
l_impl.py", line 916, in call
cur_inp, new_state = cell(cur_inp, cur_state)
File "C:\Users\henry\Anaconda3\lib\site-packages\tensorflow\python\ops\rnn_cel
l_impl.py", line 752, in __call__
output, new_state = self._cell(inputs, state, scope)
File "C:\Users\henry\Anaconda3\lib\site-packages\tensorflow\python\ops\rnn_cel
l_impl.py", line 180, in __call__
return super(RNNCell, self).__call__(inputs, state)
File "C:\Users\henry\Anaconda3\lib\site-packages\tensorflow\python\layers\base
.py", line 441, in __call__
outputs = self.call(inputs, *args, **kwargs)
File "C:\Users\henry\Anaconda3\lib\site-packages\tensorflow\python\ops\rnn_cel
l_impl.py", line 383, in call
concat = _linear([inputs, h], 4 * self._num_units, True)
File "C:\Users\henry\Anaconda3\lib\site-packages\tensorflow\python\ops\rnn_cel
l_impl.py", line 1021, in _linear
res = math_ops.matmul(array_ops.concat(args, 1), weights)
File "C:\Users\henry\Anaconda3\lib\site-packages\tensorflow\python\ops\array_o
ps.py", line 1048, in concat
name=name)
File "C:\Users\henry\Anaconda3\lib\site-packages\tensorflow\python\ops\gen_arr
ay_ops.py", line 495, in _concat_v2
name=name)
File "C:\Users\henry\Anaconda3\lib\site-packages\tensorflow\python\framework\o
p_def_library.py", line 767, in apply_op
op_def=op_def)
File "C:\Users\henry\Anaconda3\lib\site-packages\tensorflow\python\framework\o
ps.py", line 2506, in create_op
original_op=self._default_original_op, op_def=op_def)
File "C:\Users\henry\Anaconda3\lib\site-packages\tensorflow\python\framework\o
ps.py", line 1269, in __init__
self._traceback = _extract_stack()
InvalidArgumentError (see above for traceback): ConcatOp : Dimensions of inputs
should match: shape[0] = [10000,28] vs. shape[1] = [128,128]
[[Node: rnn/rnn/multi_rnn_cell/cell_0/cell_0/basic_lstm_cell/basic_lstm
_cell/concat = ConcatV2[N=2, T=DT_FLOAT, Tidx=DT_INT32, _device="/job:localhost/
replica:0/task:0/cpu:0"](split, MultiRNNCellZeroState/DropoutWrapperZeroState/Ba
sicLSTMCellZeroState/zeros_1, rnn/rnn/multi_rnn_cell/cell_0/cell_0/basic_lstm_ce
ll/basic_lstm_cell/concat/axis)]]
When I print the shape of training data it is (128,28,28). I am confused that why the test data leads to the error because both training data and test data are formatted in the same way, that is (?,n_chunks,chunk_size). Thanks in advance.
The issue is that you always create the initial state with shape set to the training batch size instead of the eval batch size.
This is the culprit line:
initial_state = state = stacked_lstm.zero_state(batch_size, tf.float32)