We Keep Coding

Input dataset of different shapes in tf.keras.Model with custom train_step override

I defined a custom tf.keras.Model and overrode train_step for implementing custom training logic. The dataset trainDataset is a tf.data object, each element containing (image, label) with different image sizes. I would like to perform data augmentation inside the train_step as in the code below. I believe my code including the part has no logical flaws, including the part where I use model.fit to train the model.
However, an error occurs telling me that it Cannot batch tensors with different shapes. I see that something is executed before train_step and that is blocking the training process. How could I solve this?
model=GeneralCNN(cfg, network, augmentation)
model.compile(optimizer, loss, cfg['training'])
...
trainLogs=model.fit(trainDataset.batch(cfg['batch_size']), epochs=1, validation_data=valDataset)
...(subclass of tf.keras.Model)
def train_step(self, data):
tf.print('check!')
images, labels = data
images = self.augmentation(images) # <--- includes resizing
# initialize important variables.
batch_size = tf.shape(images)[0]
# Train the network
with tf.GradientTape() as tape:
predictions = self.network(images)
loss = self.loss_fn(labels, predictions)
grads = tape.gradient(loss, self.network.trainable_weights)
self.optimizer.apply_gradients(zip(grads, self.network.trainable_weights))
# Update metrics
self.lossMetric.update_state(loss)
predictionsIndicies=tf.math.argmax(predictions, axis=1)
self.accuracyMetric.update_state((predictionsIndicies, labels))
return {"loss": self.lossMetric.result(), "accuracy": self.accuracyMetric.result()}
...
Error:
raceback (most recent call last):
File "train.py", line 116, in <module>
app.run(main)
File "/usr/local/lib/python3.7/dist-packages/absl/app.py", line 303, in run
_run_main(main, args)
File "/usr/local/lib/python3.7/dist-packages/absl/app.py", line 251, in _run_main
sys.exit(main(argv))
File "train.py", line 76, in main
trainLogs=model.fit(P, epochs=1, steps_per_epoch=1000, validation_data=valDataset)
File "/usr/local/lib/python3.7/dist-packages/tensorflow/python/keras/engine/training.py", line 1183, in fit
tmp_logs = self.train_function(iterator)
File "/usr/local/lib/python3.7/dist-packages/tensorflow/python/eager/def_function.py", line 889, in __call__
result = self._call(*args, **kwds)
File "/usr/local/lib/python3.7/dist-packages/tensorflow/python/eager/def_function.py", line 950, in _call
return self._stateless_fn(*args, **kwds)
File "/usr/local/lib/python3.7/dist-packages/tensorflow/python/eager/function.py", line 3024, in __call__
filtered_flat_args, captured_inputs=graph_function.captured_inputs) # pylint: disable=protected-access
File "/usr/local/lib/python3.7/dist-packages/tensorflow/python/eager/function.py", line 1961, in _call_flat
ctx, args, cancellation_manager=cancellation_manager))
File "/usr/local/lib/python3.7/dist-packages/tensorflow/python/eager/function.py", line 596, in call
ctx=ctx)
File "/usr/local/lib/python3.7/dist-packages/tensorflow/python/eager/execute.py", line 60, in quick_execute
inputs, attrs, num_outputs)
tensorflow.python.framework.errors_impl.InvalidArgumentError: Cannot batch tensors with different shapes in component 0. First element had shape [375,500,3] and element 1 had shape [333,500,3].
[[node IteratorGetNext (defined at train.py:76) ]] [Op:__inference_train_function_1852]
Function call stack:
train_function
Edit: augmentation code just in case
the code works when I resize the dataset before model.fit
def BuildAugmentation(cfg):
augmentationType = cfg['augmentation']
if augmentationType=='none':
return SimpleResize(cfg)
elif augmentationType=='simple':
return SimpleAugmentation(cfg)
def SimpleResize(cfg):
# resizing only w/o augmentations
model=tf.keras.models.Sequential([
tfPreprocessing.Resizing(cfg['image_size'], cfg['image_size'])
])
return model
def SimpleAugmentation(cfg):
# custom simple augmenation w/ humble augmentations
model=tf.keras.models.Sequential([
tfPreprocessing.RandomRotation(factor=0.02),
tfPreprocessing.RandomZoom(height_factor=0.2, width_factor=0.2),
tfPreprocessing.Resizing(cfg['image_size'], cfg['image_size']),
tfPreprocessing.RandomFlip("horizontal")
])
return model

Tensorflow FailedPreconditionError: Attempting to use uninitialized value beta1_power

I am trying to set up a simple convolutional network in Tensorflow. I'll try to keep the amount of code at a minimum level. Here's the class:
class ConvNet(object):
def __init__(self, input, labels, dataset):
self.input = input
self.true_labels = labels
#'dataset' is an instance of a class that
#I am using to read the training images
self.data = dataset
self.build()
self._optimize = None
self.sess = tf.Session()
self.sess.run(tf.global_variables_initializer())
def build(self):
self.conv_layers = []
# create 3 conv layers using tf.nn.conv2d
# and append them to self.conv_layers
# create flattening layer using tf.nn.reshape
self.fc_layers = []
# create 2 fully connected layers using tf.matmul
# and append them to self.fc_layers
#property
def optimize(self):
"""Return the optimize operation."""
if not self._optimize:
cross_entropy = tf.nn.softmax_cross_entropy_with_logits(logits = self.fc_layers[-1],
labels = self.true_labels)
cost = tf.reduce_mean(cross_entropy)
self._optimize = tf.train.AdamOptimizer(learning_rate=1e-4).minimize(cost)
return self._optimize
def train(self, num_epochs=1, batch_size=16):
epochs_done = 0
while not epochs_done == num_epochs:
# get next batch from training data
x_batch, y_true_batch, _, _ = self.data.train.next_batch(batch_size)
feed_dict_train = {self.input: x_batch, self.true_labels: y_true_batch}
#even if I initialize variables here, same error occurs
#self.sess.run(tf.global_variables_initializer())
self.sess.run(self.optimize, feed_dict=feed_dict_train)
if self.data.train.epochs_done > epochs_done:
#print stuff...
epochs_done += 1
self.sess.close()
I was trying to run the optimize operation to train the network on a simple dataset (e.g. images of cats vs dogs). Here is the main:
if __name__ == "__main__":
classes = ['dogs', 'cats']
num_classes = len(classes)
batch_size = 16
img_size = 128
num_channels = 3
#read training data and resize images to 128 x 128 pixels
data = dataset.read_train_sets(img_size, classes)
x = tf.placeholder(tf.float32, shape = [None, img_size, img_size, num_channels], name='x')
y_true = tf.placeholder(tf.float32, shape = [None, num_classes], name = 'y_true')
net = ConvNet(x, y_true, data)
net.train()
However I keep getting the following error (it's actually much longer than this, I can post it if necessary):
Caused by op 'beta1_power/read', defined at:
File "<string>", line 1, in <module>
File "/usr/lib/python3.5/idlelib/run.py", line 124, in main
ret = method(*args, **kwargs)
File "/usr/lib/python3.5/idlelib/run.py", line 351, in runcode
exec(code, self.locals)
File "/home/gian/face_recognition/model.py", line 366, in <module>
net.train()
File "/home/gian/face_recognition/model.py", line 309, in train
self.sess.run(self.optimize, feed_dict=feed_dict_train)
File "/home/gian/face_recognition/model.py", line 269, in optimize
self._optimize = tf.train.AdamOptimizer(learning_rate=1e-4).minimize(cost)
File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/training/optimizer.py", line 409, in minimize
name=name)
File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/training/optimizer.py", line 552, in apply_gradients
self._create_slots([_get_variable_for(v) for v in var_list])
File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/training/adam.py", line 124, in _create_slots
colocate_with=first_var)
File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/training/optimizer.py", line 665, in _create_non_slot_variable
v = variable_scope.variable(initial_value, name=name, trainable=False)
File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/ops/variable_scope.py", line 2157, in variable
use_resource=use_resource)
File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/ops/variable_scope.py", line 2147, in <lambda>
previous_getter = lambda **kwargs: default_variable_creator(None, **kwargs)
File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/ops/variable_scope.py", line 2130, in default_variable_creator
constraint=constraint)
File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/ops/variables.py", line 235, in __init__
constraint=constraint)
File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/ops/variables.py", line 391, in _init_from_args
self._snapshot = array_ops.identity(self._variable, name="read")
File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/ops/array_ops.py", line 142, in identity
return gen_array_ops.identity(input, name=name)
File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/ops/gen_array_ops.py", line 3053, in identity
"Identity", input=input, name=name)
File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/framework/op_def_library.py", line 787, in _apply_op_helper
op_def=op_def)
File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/framework/ops.py", line 3290, in create_op
op_def=op_def)
File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/framework/ops.py", line 1654, in __init__
self._traceback = self._graph._extract_stack() # pylint: disable=protected-access
FailedPreconditionError (see above for traceback): Attempting to use uninitialized value beta1_power
[[Node: beta1_power/read = Identity[T=DT_FLOAT, _class=["loc:#Variable"], _device="/job:localhost/replica:0/task:0/device:GPU:0"](beta1_power)]]
Can somebody help me figure out where the problem is? Thanks a lot

This error generally occurs when you haven't initialized the optimizer.
So just add self.optimize before you initialize all the global variables.
Your code should look like this.
def __init__(self, input, labels, dataset):
self.input = input
self.true_labels = labels
#'dataset' is an instance of a class that
#I am using to read the training images
self.data = dataset
self.build()
self._optimize = None
self.sess = tf.Session()
self.optimize()
self.sess.run(tf.global_variables_initializer())

Transfer learning with tf.estimator.Estimator framework

I'm trying to do transfer learning of an Inception-resnet v2 model pretrained on imagenet, using my own dataset and classes.
My original codebase was a modification of a tf.slim sample which I can't find anymore and now I'm trying to rewrite the same code using the tf.estimator.* framework.
I am running, however, into the problem of loading only some of the weights from the pretrained checkpoint, initializing the remaining layers with their default initializers.
Researching the problem, I found this GitHub issue and this question, both mentioning the need to use tf.train.init_from_checkpoint in my model_fn. I tried, but given the lack of examples in both, I guess I got something wrong.
This is my minimal example:
import sys
import os
os.environ['CUDA_VISIBLE_DEVICES'] = '-1'
import tensorflow as tf
import numpy as np
import inception_resnet_v2
NUM_CLASSES = 900
IMAGE_SIZE = 299
def input_fn(mode, num_classes, batch_size=1):
# some code that loads images, reshapes them to 299x299x3 and batches them
return tf.constant(np.zeros([batch_size, 299, 299, 3], np.float32)), tf.one_hot(tf.constant(np.zeros([batch_size], np.int32)), NUM_CLASSES)
def model_fn(images, labels, num_classes, mode):
with tf.contrib.slim.arg_scope(inception_resnet_v2.inception_resnet_v2_arg_scope()):
logits, end_points = inception_resnet_v2.inception_resnet_v2(images,
num_classes,
is_training=(mode==tf.estimator.ModeKeys.TRAIN))
predictions = {
'classes': tf.argmax(input=logits, axis=1),
'probabilities': tf.nn.softmax(logits, name='softmax_tensor')
}
if mode == tf.estimator.ModeKeys.PREDICT:
return tf.estimator.EstimatorSpec(mode=mode, predictions=predictions)
exclude = ['InceptionResnetV2/Logits', 'InceptionResnetV2/AuxLogits']
variables_to_restore = tf.contrib.slim.get_variables_to_restore(exclude=exclude)
scopes = { os.path.dirname(v.name) for v in variables_to_restore }
tf.train.init_from_checkpoint('inception_resnet_v2_2016_08_30.ckpt',
{s+'/':s+'/' for s in scopes})
tf.losses.softmax_cross_entropy(onehot_labels=labels, logits=logits)
total_loss = tf.losses.get_total_loss() #obtain the regularization losses as well
# Configure the training op
if mode == tf.estimator.ModeKeys.TRAIN:
global_step = tf.train.get_or_create_global_step()
optimizer = tf.train.AdamOptimizer(learning_rate=0.00002)
train_op = optimizer.minimize(total_loss, global_step)
else:
train_op = None
return tf.estimator.EstimatorSpec(
mode=mode,
predictions=predictions,
loss=total_loss,
train_op=train_op)
def main(unused_argv):
# Create the Estimator
classifier = tf.estimator.Estimator(
model_fn=lambda features, labels, mode: model_fn(features, labels, NUM_CLASSES, mode),
model_dir='model/MCVE')
# Train the model
classifier.train(
input_fn=lambda: input_fn(tf.estimator.ModeKeys.TRAIN, NUM_CLASSES, batch_size=1),
steps=1000)
# Evaluate the model and print results
eval_results = classifier.evaluate(
input_fn=lambda: input_fn(tf.estimator.ModeKeys.EVAL, NUM_CLASSES, batch_size=1))
print()
print('Evaluation results:\n %s' % eval_results)
if __name__ == '__main__':
tf.app.run(main=main, argv=[sys.argv[0]])
where inception_resnet_v2 is the model implementation in Tensorflow's models repository.
If I run this script, I get a bunch of info log from init_from_checkpoint, but then, at session creation time, it seems it attempts to load the Logits weights from the checkpoint and fails because of incompatible shapes. This is the full traceback:
Traceback (most recent call last):
File "<ipython-input-6-06fadd69ae8f>", line 1, in <module>
runfile('C:/Users/1/Desktop/transfer_learning_tutorial-master/MCVE.py', wdir='C:/Users/1/Desktop/transfer_learning_tutorial-master')
File "C:\Users\1\Anaconda3\envs\tensorflow\lib\site-packages\spyder\utils\site\sitecustomize.py", line 710, in runfile
execfile(filename, namespace)
File "C:\Users\1\Anaconda3\envs\tensorflow\lib\site-packages\spyder\utils\site\sitecustomize.py", line 101, in execfile
exec(compile(f.read(), filename, 'exec'), namespace)
File "C:/Users/1/Desktop/transfer_learning_tutorial-master/MCVE.py", line 77, in <module>
tf.app.run(main=main, argv=[sys.argv[0]])
File "C:\Users\1\Anaconda3\envs\tensorflow\lib\site-packages\tensorflow\python\platform\app.py", line 48, in run
_sys.exit(main(_sys.argv[:1] + flags_passthrough))
File "C:/Users/1/Desktop/transfer_learning_tutorial-master/MCVE.py", line 68, in main
steps=1000)
File "C:\Users\1\Anaconda3\envs\tensorflow\lib\site-packages\tensorflow\python\estimator\estimator.py", line 302, in train
loss = self._train_model(input_fn, hooks, saving_listeners)
File "C:\Users\1\Anaconda3\envs\tensorflow\lib\site-packages\tensorflow\python\estimator\estimator.py", line 780, in _train_model
log_step_count_steps=self._config.log_step_count_steps) as mon_sess:
File "C:\Users\1\Anaconda3\envs\tensorflow\lib\site-packages\tensorflow\python\training\monitored_session.py", line 368, in MonitoredTrainingSession
stop_grace_period_secs=stop_grace_period_secs)
File "C:\Users\1\Anaconda3\envs\tensorflow\lib\site-packages\tensorflow\python\training\monitored_session.py", line 673, in __init__
stop_grace_period_secs=stop_grace_period_secs)
File "C:\Users\1\Anaconda3\envs\tensorflow\lib\site-packages\tensorflow\python\training\monitored_session.py", line 493, in __init__
self._sess = _RecoverableSession(self._coordinated_creator)
File "C:\Users\1\Anaconda3\envs\tensorflow\lib\site-packages\tensorflow\python\training\monitored_session.py", line 851, in __init__
_WrappedSession.__init__(self, self._create_session())
File "C:\Users\1\Anaconda3\envs\tensorflow\lib\site-packages\tensorflow\python\training\monitored_session.py", line 856, in _create_session
return self._sess_creator.create_session()
File "C:\Users\1\Anaconda3\envs\tensorflow\lib\site-packages\tensorflow\python\training\monitored_session.py", line 554, in create_session
self.tf_sess = self._session_creator.create_session()
File "C:\Users\1\Anaconda3\envs\tensorflow\lib\site-packages\tensorflow\python\training\monitored_session.py", line 428, in create_session
init_fn=self._scaffold.init_fn)
File "C:\Users\1\Anaconda3\envs\tensorflow\lib\site-packages\tensorflow\python\training\session_manager.py", line 279, in prepare_session
sess.run(init_op, feed_dict=init_feed_dict)
File "C:\Users\1\Anaconda3\envs\tensorflow\lib\site-packages\tensorflow\python\client\session.py", line 889, in run
run_metadata_ptr)
File "C:\Users\1\Anaconda3\envs\tensorflow\lib\site-packages\tensorflow\python\client\session.py", line 1120, in _run
feed_dict_tensor, options, run_metadata)
File "C:\Users\1\Anaconda3\envs\tensorflow\lib\site-packages\tensorflow\python\client\session.py", line 1317, in _do_run
options, run_metadata)
File "C:\Users\1\Anaconda3\envs\tensorflow\lib\site-packages\tensorflow\python\client\session.py", line 1336, in _do_call
raise type(e)(node_def, op, message)
InvalidArgumentError: Assign requires shapes of both tensors to match. lhs shape= [900] rhs shape= [1001] [[Node: Assign_1145 = Assign[T=DT_FLOAT,
_class=["loc:#InceptionResnetV2/Logits/Logits/biases"], use_locking=true, validate_shape=true,
_device="/job:localhost/replica:0/task:0/device:CPU:0"](InceptionResnetV2/Logits/Logits/biases, checkpoint_initializer_1145)]]
What am I doing wrong when using init_from_checkpoint? How exactly are we supposed to "use" it in our model_fn? And why is the estimator trying to load the Logits' weights from the checkpoint when I'm explicitly telling it not to?
Update:
After the suggestion in the comments, I tried alternative ways to call tf.train.init_from_checkpoint.
Using {v.name: v.name}
If, as suggested in the comment, I replace the call with {v.name:v.name for v in variables_to_restore}, I get this error:
ValueError: Assignment map with scope only name InceptionResnetV2/Conv2d_2a_3x3 should map
to scope only InceptionResnetV2/Conv2d_2a_3x3/weights:0. Should be 'scope/': 'other_scope/'.
Using {v.name: v}
If, instead, I try using the name:variable mapping, I get the following error:
ValueError: Tensor InceptionResnetV2/Conv2d_2a_3x3/weights:0 is not found in
inception_resnet_v2_2016_08_30.ckpt checkpoint
{'InceptionResnetV2/Repeat_2/block8_4/Branch_1/Conv2d_0c_3x1/BatchNorm/moving_mean': [256],
'InceptionResnetV2/Repeat/block35_9/Branch_0/Conv2d_1x1/BatchNorm/beta': [32], ...
The error continues listing what I think are all the variable names in the checkpoint (or could it be the scopes instead?).
Update (2)
After inspecting the latest error here above, I see that InceptionResnetV2/Conv2d_2a_3x3/weights is in the list of the checkpointed variables. The problem is that :0 at the end!
I'll now verify if this does indeed solve the problem and post an answer if that's the case.

Thanks to #KathyWu's comment, I got on the right track and found the problem.
Indeed, the way I was computing the scopes would include the InceptionResnetV2/ scope, that would trigger the load of all variables "under" the scope (i.e., all variables in the network). Replacing this with the correct dictionary, however, was not trivial.
Of the possible scope modes init_from_checkpoint accepts, the one I had to use was the 'scope_variable_name': variable one, but without using the actual variable.name attribute.
The variable.name looks like: 'some_scope/variable_name:0'. That :0 is not in the checkpointed variable's name and so using scopes = {v.name:v.name for v in variables_to_restore} will raise a "Variable not found" error.
The trick to make it work was stripping the tensor index from the name:
tf.train.init_from_checkpoint('inception_resnet_v2_2016_08_30.ckpt',
{v.name.split(':')[0]: v for v in variables_to_restore})

I find out {s+'/':s+'/' for s in scopes} didn't work, just because the variables_to_restore include something like "global_step", so scopes include the global scopes which could include everything. You need to print variables_to_restore, find "global_step" thing, and put it in "exclude".

How to reuse slim.arg_scope in tensorFlow?

I'm trying to load inception_resnet_v2_2016_08_30.ckpt file and do testing.
The code works well with single image (entering oneFile() function only once).
If I call oneFile() function twice, the following error occur:
ValueError: Variable InceptionResnetV2/Conv2d_1a_3x3/weights already
exists, disallowed. Did you mean to set reuse=True in VarScope?
Originally defined at:
I found related solution on Sharing Variables
If tf.variable_scope meet the same problem, could call scope.reuse_variables() to resolve this problem.
But I can't find the slim.arg_scope version to reuse the scope.
def oneFile(filepath):
imgPath = filepath
testImage_string = tf.gfile.FastGFile(imgPath, 'rb').read()
testImage = tf.image.decode_jpeg(testImage_string, channels=3)
processed_image = inception_preprocessing.preprocess_image(testImage, image_size, image_size, is_training=False)
processed_images = tf.expand_dims(processed_image, 0)
# Create the model, use the default arg scope to configure the batch norm parameters.
with slim.arg_scope(inception_resnet_v2_arg_scope()):
#logits, end_points = inception_resnet_v2(images, num_classes = dataset.num_classes, is_training = False)
logits, _ = inception_resnet_v2(processed_images, num_classes=16, is_training=False)
probabilities = tf.nn.softmax(logits)
init_fn = slim.assign_from_checkpoint_fn(
checkpoint_file,
slim.get_model_variables(model_name))
with tf.Session() as sess:
init_fn(sess)
np_image, probabilities = sess.run([processed_images, probabilities])
probabilities = probabilities[0, 0:]
sorted_inds = [i[0] for i in sorted(enumerate(-probabilities), key=lambda x: x[1])]
#print(probabilities)
print(probabilities.argmax(axis=0))
#names = imagenet.create_readable_names_for_imagenet_labels()
#for i in range(15):
# index = sorted_inds[i]
# print((probabilities[index], names[index]))
def main():
for image_file in os.listdir(dataset_dir):
try:
image_type = imghdr.what(os.path.join(dataset_dir, image_file))
if not image_type:
continue
except IsADirectoryError:
continue
#image = Image.open(os.path.join(dataset_dir, image_file))
filepath = os.path.join(dataset_dir, image_file)
oneFile(filepath)
inception_resnet_v2_arg_scope
def inception_resnet_v2_arg_scope(weight_decay=0.00004,
batch_norm_decay=0.9997,
batch_norm_epsilon=0.001):
"""Yields the scope with the default parameters for inception_resnet_v2.
Args:
weight_decay: the weight decay for weights variables.
batch_norm_decay: decay for the moving average of batch_norm momentums.
batch_norm_epsilon: small float added to variance to avoid dividing by zero.
Returns:
a arg_scope with the parameters needed for inception_resnet_v2.
"""
# Set weight_decay for weights in conv2d and fully_connected layers.
with slim.arg_scope([slim.conv2d, slim.fully_connected],
weights_regularizer=slim.l2_regularizer(weight_decay),
biases_regularizer=slim.l2_regularizer(weight_decay)):
batch_norm_params = {
'decay': batch_norm_decay,
'epsilon': batch_norm_epsilon,
}
# Set activation_fn and parameters for batch_norm.
with slim.arg_scope([slim.conv2d], activation_fn=tf.nn.relu,
normalizer_fn=slim.batch_norm,
normalizer_params=batch_norm_params) as scope:
return scope
Complete error message:
./data/test/teeth/1/7070.jpg Traceback (most recent call last): File
"testing.py", line 111, in
main() File "testing.py", line 106, in main
cal(processed_images) File "testing.py", line 67, in cal
logits, _ = inception_resnet_v2(processed_images, num_classes=16, is_training=False) File
"/notebooks/transfer_learning_tutorial/inception_resnet_v2.py", line
123, in inception_resnet_v2
scope='Conv2d_1a_3x3') File "/usr/local/lib/python3.5/dist-packages/tensorflow/contrib/framework/python/ops/arg_scope.py",
line 181, in func_with_args
return func(*args, **current_args) File "/usr/local/lib/python3.5/dist-packages/tensorflow/contrib/layers/python/layers/layers.py",
line 918, in convolution
outputs = layer.apply(inputs) File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/layers/base.py",
line 320, in apply
return self.call(inputs, **kwargs) File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/layers/base.py",
line 286, in call
self.build(input_shapes[0]) File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/layers/convolutional.py",
line 138, in build
dtype=self.dtype) File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/ops/variable_scope.py",
line 1049, in get_variable
use_resource=use_resource, custom_getter=custom_getter) File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/ops/variable_scope.py",
line 948, in get_variable
use_resource=use_resource, custom_getter=custom_getter) File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/ops/variable_scope.py",
line 349, in get_variable
validate_shape=validate_shape, use_resource=use_resource) File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/ops/variable_scope.py",
line 1389, in wrapped_custom_getter
*args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/layers/base.py",
line 275, in variable_getter
variable_getter=functools.partial(getter, **kwargs)) File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/layers/base.py",
line 228, in _add_variable
trainable=trainable and self.trainable) File "/usr/local/lib/python3.5/dist-packages/tensorflow/contrib/layers/python/layers/layers.py",
line 1334, in layer_variable_getter
return _model_variable_getter(getter, *args, **kwargs) File "/usr/local/lib/python3.5/dist-packages/tensorflow/contrib/layers/python/layers/layers.py",
line 1326, in _model_variable_getter
custom_getter=getter, use_resource=use_resource) File "/usr/local/lib/python3.5/dist-packages/tensorflow/contrib/framework/python/ops/arg_scope.py",
line 181, in func_with_args
return func(*args, **current_args) File "/usr/local/lib/python3.5/dist-packages/tensorflow/contrib/framework/python/ops/variables.py",
line 262, in model_variable
use_resource=use_resource) File "/usr/local/lib/python3.5/dist-packages/tensorflow/contrib/framework/python/ops/arg_scope.py",
line 181, in func_with_args
return func(*args, **current_args) File "/usr/local/lib/python3.5/dist-packages/tensorflow/contrib/framework/python/ops/variables.py",
line 217, in variable
use_resource=use_resource) File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/ops/variable_scope.py",
line 341, in _true_getter
use_resource=use_resource) File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/ops/variable_scope.py",
line 653, in _get_single_variable
name, "".join(traceback.format_list(tb)))) ValueError: Variable InceptionResnetV2/Conv2d_1a_3x3/weights already exists, disallowed.
Did you mean to set reuse=True in VarScope? Originally defined at:
File
"/usr/local/lib/python3.5/dist-packages/tensorflow/contrib/framework/python/ops/variables.py",
line 217, in variable
use_resource=use_resource) File "/usr/local/lib/python3.5/dist-packages/tensorflow/contrib/framework/python/ops/arg_scope.py",
line 181, in func_with_args
return func(*args, **current_args) File "/usr/local/lib/python3.5/dist-packages/tensorflow/contrib/framework/python/ops/variables.py",
line 262, in model_variable
use_resource=use_resource)

It seems like tf.reset_default_graph() before processing each image in your oneFile() function will solve this problem, as I encountered the same issue on a very similar example code. My understanding is that once you feed the image to the neural network (NN), because of the variable scope concept TensorFlow uses, it needs to be told that the variables can be reused before you can apply the NN to another image.

My guess would be that you specified the same scope for multiple variables in the graph. This error occurs when tensorflow finds multiple variables under the same scope which is irrespective of the next image or the next batch. When you create the graph, you should create it thinking about one image or batch only. If everything works well with the first batch or first image, tensorflow will take care of the next iterations including the scoping.
So check all the scopes in your model file. I am pretty sure you used the same name twice.

Tensorflow - Reusing model InvalidArgumentError

I have issues using an exported tensorflow model. It doesn't allow me to evaluate the dataset I provided it with. If I run the evaluation in the same session as the training, there are no issues, however, that defeats the purpose of saving the model, if I have to retrain my model just to test with another dataset. The python file for generating the model is as such:
x = tf.placeholder(tf.float32, shape=[None, 1024], name = "x")
y_ = tf.placeholder(tf.float32, shape=[None, 10], name = "y_")
#===Model===
#Train
cross_entropy = tf.reduce_mean(
tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y_conv))
train_step = tf.train.AdamOptimizer(1e-4).minimize(cross_entropy)
correct_prediction = tf.equal(tf.argmax(y_conv,1), tf.argmax(y_,1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32), name= "accuracy")
#Create Saver
saver = tf.train.Saver()
sess.run(tf.global_variables_initializer())
for i in range(40000):
batch = shvn_data.nextbatch(100)
if i%100 == 0:
train_accuracy = accuracy.eval(feed_dict={x:batch[0], y_: batch[1], keep_prob: 1.0})
print("step %d, training accuracy %f"%(i, train_accuracy))
train_step.run(feed_dict={x: batch[0], y_: batch[1], keep_prob: 0.5})
#Save
saver.save(sess,'svhn_model1')
I saved input variables x and y_, to be fed through function 'accuracy', so that I can run accuracy.eval() to obtain the accuracy of prediction. I evaluated the dataset in batches of 100 images, then summed the final prediction. The python file to evaluate the model in another session is as such:
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
sess = tf.Session(config = config)
shvn_data = DataLoader()
saver = tf.train.import_meta_graph('svhn_model1.meta')
saver.restore(sess,tf.train.latest_checkpoint('./'))
#sess.run(tf.global_variables_initializer())
#Variables to use with model
graph = tf.get_default_graph()
x = graph.get_tensor_by_name("x:0")
y_ = graph.get_tensor_by_name("y_:0")
accuracy = graph.get_tensor_by_name("accuracy:0")
keep_prob = tf.placeholder(tf.float32)
img_whole = np.reshape(shvn_data.test_images,(-1,1024))
batch_whole = np.asarray(shvn_data.test_label.eval(), dtype = np.float32)
total_accuracy = 0
test_count = shvn_data.TEST_COUNT
batch_size = 100
steps = int(math.ceil(test_count/float(batch_size)))
for j in range(steps):
start = j*batch_size
if (j+1)*batch_size > shvn_data.TEST_COUNT:
end = test_count
else:
end = (j+1)*batch_size
img_batch = img_whole[start:end]
label_batch = batch_whole[start:end]
batch_accuracy = accuracy.eval(session = sess, feed_dict={ x: img_batch, y_: label_batch, keep_prob: 1.0}) #ISSUE LIES HERE
print("Test batch %d:%d accuracy %g"%(start,end,batch_accuracy))
total_accuracy += batch_accuracy
print ("Total Accuracy: %f" %(total_accuracy/steps))
The error is as follows.
File "/home/lwenyao/anaconda2/lib/python2.7/site-packages/tensorflow/python/client/session.py", line 1052, in _do_call
raise type(e)(node_def, op, message)
InvalidArgumentError: You must feed a value for placeholder tensor 'Placeholder' with dtype float
[[Node: Placeholder = Placeholder[dtype=DT_FLOAT, shape=[], _device="/job:localhost/replica:0/task:0/gpu:0"]()]]
Caused by op u'Placeholder', defined at:
File "/home/lwenyao/anaconda2/lib/python2.7/site-packages/spyder/utils/ipython/start_kernel.py", line 227, in <module>
main()
File "/home/lwenyao/anaconda2/lib/python2.7/site-packages/spyder/utils/ipython/start_kernel.py", line 223, in main
kernel.start()
File "/home/lwenyao/anaconda2/lib/python2.7/site-packages/ipykernel/kernelapp.py", line 474, in start
ioloop.IOLoop.instance().start()
File "/home/lwenyao/anaconda2/lib/python2.7/site-packages/zmq/eventloop/ioloop.py", line 177, in start
super(ZMQIOLoop, self).start()
File "/home/lwenyao/anaconda2/lib/python2.7/site-packages/tornado/ioloop.py", line 887, in start
handler_func(fd_obj, events)
File "/home/lwenyao/anaconda2/lib/python2.7/site-packages/tornado/stack_context.py", line 275, in null_wrapper
return fn(*args, **kwargs)
File "/home/lwenyao/anaconda2/lib/python2.7/site-packages/zmq/eventloop/zmqstream.py", line 440, in _handle_events
self._handle_recv()
File "/home/lwenyao/anaconda2/lib/python2.7/site-packages/zmq/eventloop/zmqstream.py", line 472, in _handle_recv
self._run_callback(callback, msg)
File "/home/lwenyao/anaconda2/lib/python2.7/site-packages/zmq/eventloop/zmqstream.py", line 414, in _run_callback
callback(*args, **kwargs)
File "/home/lwenyao/anaconda2/lib/python2.7/site-packages/tornado/stack_context.py", line 275, in null_wrapper
return fn(*args, **kwargs)
File "/home/lwenyao/anaconda2/lib/python2.7/site-packages/ipykernel/kernelbase.py", line 276, in dispatcher
return self.dispatch_shell(stream, msg)
File "/home/lwenyao/anaconda2/lib/python2.7/site-packages/ipykernel/kernelbase.py", line 228, in dispatch_shell
handler(stream, idents, msg)
File "/home/lwenyao/anaconda2/lib/python2.7/site-packages/ipykernel/kernelbase.py", line 390, in execute_request
user_expressions, allow_stdin)
File "/home/lwenyao/anaconda2/lib/python2.7/site-packages/ipykernel/ipkernel.py", line 196, in do_execute
res = shell.run_cell(code, store_history=store_history, silent=silent)
File "/home/lwenyao/anaconda2/lib/python2.7/site-packages/ipykernel/zmqshell.py", line 501, in run_cell
return super(ZMQInteractiveShell, self).run_cell(*args, **kwargs)
File "/home/lwenyao/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2717, in run_cell
interactivity=interactivity, compiler=compiler, result=result)
File "/home/lwenyao/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2827, in run_ast_nodes
if self.run_code(code, result):
File "/home/lwenyao/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2881, in run_code
exec(code_obj, self.user_global_ns, self.user_ns)
File "<ipython-input-1-33e4fce19d34>", line 1, in <module>
runfile('/home/lwenyao/Desktop/Python/Import_Model.py', wdir='/home/lwenyao/Desktop/Python')
File "/home/lwenyao/anaconda2/lib/python2.7/site-packages/spyder/utils/site/sitecustomize.py", line 866, in runfile
execfile(filename, namespace)
File "/home/lwenyao/anaconda2/lib/python2.7/site-packages/spyder/utils/site/sitecustomize.py", line 94, in execfile
builtins.execfile(filename, *where)
File "/home/lwenyao/Desktop/Python/Import_Model.py", line 63, in <module>
saver = tf.train.import_meta_graph('svhn_model1.meta')
File "/home/lwenyao/anaconda2/lib/python2.7/site-packages/tensorflow/python/training/saver.py", line 1595, in import_meta_graph
**kwargs)
File "/home/lwenyao/anaconda2/lib/python2.7/site-packages/tensorflow/python/framework/meta_graph.py", line 499, in import_scoped_meta_graph
producer_op_list=producer_op_list)
File "/home/lwenyao/anaconda2/lib/python2.7/site-packages/tensorflow/python/framework/importer.py", line 308, in import_graph_def
op_def=op_def)
File "/home/lwenyao/anaconda2/lib/python2.7/site-packages/tensorflow/python/framework/ops.py", line 2336, in create_op
original_op=self._default_original_op, op_def=op_def)
File "/home/lwenyao/anaconda2/lib/python2.7/site-packages/tensorflow/python/framework/ops.py", line 1228, in __init__
self._traceback = _extract_stack()
InvalidArgumentError (see above for traceback): You must feed a value for placeholder tensor 'Placeholder' with dtype float
[[Node: Placeholder = Placeholder[dtype=DT_FLOAT, shape=[], _device="/job:localhost/replica:0/task:0/gpu:0"]()]]
As previously mentioned, evaluation has no issues if I run it in the same session when training the model. The only changes that I made were i added the argument session = sess each time I called .eval() when using the imported model. Sorry for the long post!

Alright, it appears that the error was caused from attempting to create and use another keep_prob variable in the test script, after importing the model. I.e. I created keep_prob = tf.placeholder(tf.float32,) in the training file. However,accuracy.eval() in the testing file was trying to look for keep_prob specifically from the model. I created another keep_prob = tf.placeholder(tf.float32,) in testing file, thinking it would be the same, but it was not.
I modified my code in the training file by adding the label:
keep_prob = tf.placeholder(tf.float32, name="keep_prob")
and in my testing file, called for the model's variable:
#Variables to use with model
graph = tf.get_default_graph()
x = graph.get_tensor_by_name("x:0")
y_ = graph.get_tensor_by_name("y_:0")
keep_prob = graph.get_tensor_by_name("keep_prob:0")#Changed this
accuracy = graph.get_tensor_by_name("accuracy:0")
And now it works fine. My code is modified from Deep MNIST for Experts from tensorflow.