We Keep Coding

When switching from federated to centralized dataset -Error may indicate that you're trying to pass a Tensor to a NumPy call

I'm trying to switch from a federated setting to centralized learning. I've created a federated dataset, but I want to create a dataset for centralized learning with the create_tf_dataset_from_all_clients function. When I googled the error I found out that maybe versions of NumPy and TensorFlow are not correct for this function, my current versions are :
python == 3.9
tensorflow==2.8.2
numpy==1.21.6
tensorflow-federated==0.24.0
I haven't found some recent posts about TensorFlow 2.8 and matching NumPy version
Also, the error might come from a function that I used to create the clientData object:
def parse_image(filename):
parts = tf.strings.split(filename, os.sep)
label_str = parts[-2]
label_int = tf.where(labels_tf == label_str)[0][0]
image = tf.io.read_file(filename)
image = tf.io.decode_jpeg(image, channels=3)
image = tf.image.convert_image_dtype(image, tf.float32)
image = tf.image.resize(image, [32, 32])
image = tf.keras.applications.resnet50.preprocess_input(image)
if base_model == "VGG16":
print("-------- preprocessing image for base_model VGG16 --------")
image = tf.keras.applications.vgg16.preprocess_input(image)
elif base_model == "ResNet":
print("-------- preprocessing image for base_model ResNet --------")
image = tf.keras.applications.resnet.preprocess_input(image)
return image, label_int
def create_dataset(client_id):
df = train_set
client_id = int(client_id)
file = df.loc[df["client_id"] == client_id]
# print(file)
path = file["path"]
# print(path)
list_ds = tf.data.Dataset.list_files(path)
images_ds = list_ds.map(parse_image)
return images_ds
Error:
TypeError Traceback (most recent call last)
Input In [7], in <cell line: 1>()
----> 1 train_dataset = client_data.create_tf_dataset_from_all_clients()
File ~/master_venv/lib/python3.9/site-packages/tensorflow_federated/python/simulation/datasets/client_data.py:231, in ClientData.create_tf_dataset_from_all_clients(self, seed)
227 nested_dataset = tf.data.Dataset.from_tensor_slices(client_ids)
228 # We apply serializable_dataset_fn here to avoid loading all client datasets
229 # in memory, which is slow. Note that tf.data.Dataset.map implicitly wraps
230 # the input mapping in a tf.function.
--> 231 example_dataset = nested_dataset.flat_map(self.serializable_dataset_fn)
232 return example_dataset
File ~/master_venv/lib/python3.9/site-packages/tensorflow/python/data/ops/dataset_ops.py:2092, in DatasetV2.flat_map(self, map_func, name)
2058 def flat_map(self, map_func, name=None):
2059 """Maps `map_func` across this dataset and flattens the result.
2060
2061 The type signature is:
(...)
2090 Dataset: A `Dataset`.
2091 """
-> 2092 return FlatMapDataset(self, map_func, name=name)
File ~/master_venv/lib/python3.9/site-packages/tensorflow/python/data/ops/dataset_ops.py:5327, in FlatMapDataset.__init__(self, input_dataset, map_func, name)
5325 """See `Dataset.flat_map()` for details."""
5326 self._input_dataset = input_dataset
-> 5327 self._map_func = structured_function.StructuredFunctionWrapper(
5328 map_func, self._transformation_name(), dataset=input_dataset)
5329 if not isinstance(self._map_func.output_structure, DatasetSpec):
5330 raise TypeError(
5331 "The `map_func` argument must return a `Dataset` object. Got "
5332 f"{_get_type(self._map_func.output_structure)!r}.")
File ~/master_venv/lib/python3.9/site-packages/tensorflow/python/data/ops/structured_function.py:271, in StructuredFunctionWrapper.__init__(self, func, transformation_name, dataset, input_classes, input_shapes, input_types, input_structure, add_to_graph, use_legacy_function, defun_kwargs)
264 warnings.warn(
265 "Even though the `tf.config.experimental_run_functions_eagerly` "
266 "option is set, this option does not apply to tf.data functions. "
267 "To force eager execution of tf.data functions, please use "
268 "`tf.data.experimental.enable_debug_mode()`.")
269 fn_factory = trace_tf_function(defun_kwargs)
--> 271 self._function = fn_factory()
272 # There is no graph to add in eager mode.
273 add_to_graph &= not context.executing_eagerly()
File ~/master_venv/lib/python3.9/site-packages/tensorflow/python/eager/function.py:2567, in Function.get_concrete_function(self, *args, **kwargs)
2558 def get_concrete_function(self, *args, **kwargs):
2559 """Returns a `ConcreteFunction` specialized to inputs and execution context.
2560
2561 Args:
(...)
2565 or `tf.Tensor` or `tf.TensorSpec`.
2566 """
-> 2567 graph_function = self._get_concrete_function_garbage_collected(
2568 *args, **kwargs)
2569 graph_function._garbage_collector.release() # pylint: disable=protected-access
2570 return graph_function
File ~/master_venv/lib/python3.9/site-packages/tensorflow/python/eager/function.py:2533, in Function._get_concrete_function_garbage_collected(self, *args, **kwargs)
2531 args, kwargs = None, None
2532 with self._lock:
-> 2533 graph_function, _ = self._maybe_define_function(args, kwargs)
2534 seen_names = set()
2535 captured = object_identity.ObjectIdentitySet(
2536 graph_function.graph.internal_captures)
File ~/master_venv/lib/python3.9/site-packages/tensorflow/python/eager/function.py:2711, in Function._maybe_define_function(self, args, kwargs)
2708 cache_key = self._function_cache.generalize(cache_key)
2709 (args, kwargs) = cache_key._placeholder_value() # pylint: disable=protected-access
-> 2711 graph_function = self._create_graph_function(args, kwargs)
2712 self._function_cache.add(cache_key, cache_key_deletion_observer,
2713 graph_function)
2715 return graph_function, filtered_flat_args
File ~/master_venv/lib/python3.9/site-packages/tensorflow/python/eager/function.py:2627, in Function._create_graph_function(self, args, kwargs)
2622 missing_arg_names = [
2623 "%s_%d" % (arg, i) for i, arg in enumerate(missing_arg_names)
2624 ]
2625 arg_names = base_arg_names + missing_arg_names
2626 graph_function = ConcreteFunction(
-> 2627 func_graph_module.func_graph_from_py_func(
2628 self._name,
2629 self._python_function,
2630 args,
2631 kwargs,
2632 self.input_signature,
2633 autograph=self._autograph,
2634 autograph_options=self._autograph_options,
2635 arg_names=arg_names,
2636 capture_by_value=self._capture_by_value),
2637 self._function_attributes,
2638 spec=self.function_spec,
2639 # Tell the ConcreteFunction to clean up its graph once it goes out of
2640 # scope. This is not the default behavior since it gets used in some
2641 # places (like Keras) where the FuncGraph lives longer than the
2642 # ConcreteFunction.
2643 shared_func_graph=False)
2644 return graph_function
File ~/master_venv/lib/python3.9/site-packages/tensorflow/python/framework/func_graph.py:1141, in func_graph_from_py_func(name, python_func, args, kwargs, signature, func_graph, autograph, autograph_options, add_control_dependencies, arg_names, op_return_value, collections, capture_by_value, acd_record_initial_resource_uses)
1138 else:
1139 _, original_func = tf_decorator.unwrap(python_func)
-> 1141 func_outputs = python_func(*func_args, **func_kwargs)
1143 # invariant: `func_outputs` contains only Tensors, CompositeTensors,
1144 # TensorArrays and `None`s.
1145 func_outputs = nest.map_structure(
1146 convert, func_outputs, expand_composites=True)
File ~/master_venv/lib/python3.9/site-packages/tensorflow/python/data/ops/structured_function.py:248, in StructuredFunctionWrapper.__init__.<locals>.trace_tf_function.<locals>.wrapped_fn(*args)
242 #eager_function.defun_with_attributes(
243 input_signature=structure.get_flat_tensor_specs(
244 self._input_structure),
245 autograph=False,
246 attributes=defun_kwargs)
247 def wrapped_fn(*args): # pylint: disable=missing-docstring
--> 248 ret = wrapper_helper(*args)
249 ret = structure.to_tensor_list(self._output_structure, ret)
250 return [ops.convert_to_tensor(t) for t in ret]
File ~/master_venv/lib/python3.9/site-packages/tensorflow/python/data/ops/structured_function.py:177, in StructuredFunctionWrapper.__init__.<locals>.wrapper_helper(*args)
175 if not _should_unpack(nested_args):
176 nested_args = (nested_args,)
--> 177 ret = autograph.tf_convert(self._func, ag_ctx)(*nested_args)
178 if _should_pack(ret):
179 ret = tuple(ret)
File ~/master_venv/lib/python3.9/site-packages/tensorflow/python/autograph/impl/api.py:692, in convert.<locals>.decorator.<locals>.wrapper(*args, **kwargs)
690 except Exception as e: # pylint:disable=broad-except
691 if hasattr(e, 'ag_error_metadata'):
--> 692 raise e.ag_error_metadata.to_exception(e)
693 else:
694 raise
File ~/master_venv/lib/python3.9/site-packages/tensorflow/python/autograph/impl/api.py:689, in convert.<locals>.decorator.<locals>.wrapper(*args, **kwargs)
687 try:
688 with conversion_ctx:
--> 689 return converted_call(f, args, kwargs, options=options)
690 except Exception as e: # pylint:disable=broad-except
691 if hasattr(e, 'ag_error_metadata'):
File ~/master_venv/lib/python3.9/site-packages/tensorflow/python/autograph/impl/api.py:439, in converted_call(f, args, kwargs, caller_fn_scope, options)
437 try:
438 if kwargs is not None:
--> 439 result = converted_f(*effective_args, **kwargs)
440 else:
441 result = converted_f(*effective_args)
File /var/folders/w2/fcxhc9j52tb9hymgw1b8_dmh0000gn/T/__autograph_generated_filepc7z792y.py:11, in outer_factory.<locals>.inner_factory.<locals>.tf__create_dataset(client_id)
9 retval_ = ag__.UndefinedReturnValue()
10 client_id = ag__.converted_call(ag__.ld(int), (ag__.ld(client_id),), None, fscope)
---> 11 files = ag__.ld(df).loc[ag__.ld(df)['client_id'] == ag__.ld(client_id)]
12 path = ag__.ld(files)['path']
13 list_ds = ag__.converted_call(ag__.ld(tf).data.Dataset.list_files, (ag__.ld(path),), None, fscope)
File ~/master_venv/lib/python3.9/site-packages/pandas/core/ops/common.py:70, in _unpack_zerodim_and_defer.<locals>.new_method(self, other)
66 return NotImplemented
68 other = item_from_zerodim(other)
---> 70 return method(self, other)
File ~/master_venv/lib/python3.9/site-packages/pandas/core/arraylike.py:40, in OpsMixin.__eq__(self, other)
38 #unpack_zerodim_and_defer("__eq__")
39 def __eq__(self, other):
---> 40 return self._cmp_method(other, operator.eq)
File ~/master_venv/lib/python3.9/site-packages/pandas/core/series.py:5625, in Series._cmp_method(self, other, op)
5622 with np.errstate(all="ignore"):
5623 res_values = ops.comparison_op(lvalues, rvalues, op)
-> 5625 return self._construct_result(res_values, name=res_name)
File ~/master_venv/lib/python3.9/site-packages/pandas/core/series.py:3017, in Series._construct_result(self, result, name)
3013 return (res1, res2)
3015 # We do not pass dtype to ensure that the Series constructor
3016 # does inference in the case where `result` has object-dtype.
-> 3017 out = self._constructor(result, index=self.index)
3018 out = out.__finalize__(self)
3020 # Set the result's name after __finalize__ is called because __finalize__
3021 # would set it back to self.name
File ~/master_venv/lib/python3.9/site-packages/pandas/core/series.py:442, in Series.__init__(self, data, index, dtype, name, copy, fastpath)
440 index = default_index(len(data))
441 elif is_list_like(data):
--> 442 com.require_length_match(data, index)
444 # create/copy the manager
445 if isinstance(data, (SingleBlockManager, SingleArrayManager)):
File ~/master_venv/lib/python3.9/site-packages/pandas/core/common.py:556, in require_length_match(data, index)
552 def require_length_match(data, index: Index):
553 """
554 Check the length of data matches the length of the index.
555 """
--> 556 if len(data) != len(index):
557 raise ValueError(
558 "Length of values "
559 f"({len(data)}) "
560 "does not match length of index "
561 f"({len(index)})"
562 )
File ~/master_venv/lib/python3.9/site-packages/tensorflow/python/framework/ops.py:932, in Tensor.__len__(self)
931 def __len__(self):
--> 932 raise TypeError(f"len is not well defined for a symbolic Tensor "
933 f"({self.name}). Please call `x.shape` rather than "
934 f"`len(x)` for shape information.")
TypeError: in user code:
File "/var/folders/w2/fcxhc9j52tb9hymgw1b8_dmh0000gn/T/ipykernel_2264/3413278942.py", line 7, in create_dataset *
files = df.loc[df['client_id']==client_id]
File "/Users/admin/master_venv/lib/python3.9/site-packages/pandas/core/ops/common.py", line 70, in new_method
return method(self, other)
File "/Users/admin/master_venv/lib/python3.9/site-packages/pandas/core/arraylike.py", line 40, in __eq__
return self._cmp_method(other, operator.eq)
File "/Users/admin/master_venv/lib/python3.9/site-packages/pandas/core/series.py", line 5625, in _cmp_method
return self._construct_result(res_values, name=res_name)
File "/Users/admin/master_venv/lib/python3.9/site-packages/pandas/core/series.py", line 3017, in _construct_result
out = self._constructor(result, index=self.index)
File "/Users/admin/master_venv/lib/python3.9/site-packages/pandas/core/series.py", line 442, in __init__
com.require_length_match(data, index)
File "/Users/admin/master_venv/lib/python3.9/site-packages/pandas/core/common.py", line 556, in require_length_match
if len(data) != len(index):
TypeError: len is not well defined for a symbolic Tensor (Equal:0). Please call `x.shape` rather than `len(x)` for shape information.

TFF is generally programmed assuming that 'all' local logic is expressed in pure TensorFlow (or at least, can be hoisted into a platform-independent representation, like TorchScript, GraphDef or XLA); this is crucial for TFF's "write-once, run-everywhere" philosophy, to prevent capturing of arbitrary python code.
It is this assumption that is surfacing here. TFF passes your function directly to TensorFlow libraries which implicitly create a tf.function; you can in fact see this in the stacktrace above:
228 # We apply serializable_dataset_fn here to avoid loading all client datasets
229 # in memory, which is slow. Note that tf.data.Dataset.map implicitly wraps
230 # the input mapping in a tf.function.
--> 231 example_dataset = nested_dataset.flat_map(self.serializable_dataset_fn)
While TF is trying to create this tf.function, or invoke it, it will pass a tensor through the function and attempt to trace the logic. This makes working with Python datastructures a little difficult; e.g., we cannot use this tensor to index into a list or dict. However, it looks to me like if you construct a tf.lookup.StaticHashTable with your clients-to-files mapping, and look up with your client ID in this hash table instead of the pandas dataframe, your code may 'just work'.
In general, you can test whether your code will work or not with this usage of TFF and tf.data by wrapping the function you pass to TFF as a tf.function, invoking it, and making sure it has the behavior you expect.

Error while using VGP instead of SVGP in Heteroskedastic gpflow example (multilatent GP)

I am trying to understand why I am getting a ValueError when I try to replace SVGP with VGP in the heteroscedastic regression example (https://gpflow.readthedocs.io/en/develop/notebooks/advanced/heteroskedastic.html) in GPflow.
Here are the changes I made:
model = gpf.models.VGP(...)
loss_fn = model.training_loss_closure() instead of loss_fn = model.training_loss_closure(data)
The kernel and likelihood are the same as the example.
data = (X, Y)
model = gpf.models.VGP(
data = data,
kernel=kernel,
likelihood=likelihood,
#inducing_variable=inducing_variable,
num_latent_gps=likelihood.latent_dim,
)
loss_fn = model.training_loss_closure()
gpf.utilities.set_trainable(model.q_mu, False)
gpf.utilities.set_trainable(model.q_sqrt, False)
variational_vars = [(model.q_mu, model.q_sqrt)]
natgrad_opt = gpf.optimizers.NaturalGradient(gamma=0.1)
adam_vars = model.trainable_variables
adam_opt = tf.optimizers.Adam(0.01)
#tf.function
def optimisation_step():
natgrad_opt.minimize(loss_fn, variational_vars)
adam_opt.minimize(loss_fn, adam_vars)
epochs = 100
for epoch in range(0, epochs):
optimisation_step()
The optimization step gives me this error:
---------------------------------------------------------------------------
ValueError Traceback (most recent call last)
/tmp/ipykernel_406484/3662007586.py in <module>
3
4 for epoch in range(1, epochs + 1):
----> 5 optimisation_step()
6
7
~/miniconda3/envs/tensorflow/lib/python3.9/site-packages/tensorflow/python/eager/def_function.py in __call__(self, *args, **kwds)
887
888 with OptionalXlaContext(self._jit_compile):
--> 889 result = self._call(*args, **kwds)
890
891 new_tracing_count = self.experimental_get_tracing_count()
~/miniconda3/envs/tensorflow/lib/python3.9/site-packages/tensorflow/python/eager/def_function.py in _call(self, *args, **kwds)
931 # This is the first call of __call__, so we have to initialize.
932 initializers = []
--> 933 self._initialize(args, kwds, add_initializers_to=initializers)
934 finally:
935 # At this point we know that the initialization is complete (or less
~/miniconda3/envs/tensorflow/lib/python3.9/site-packages/tensorflow/python/eager/def_function.py in _initialize(self, args, kwds, add_initializers_to)
761 self._graph_deleter = FunctionDeleter(self._lifted_initializer_graph)
762 self._concrete_stateful_fn = (
--> 763 self._stateful_fn._get_concrete_function_internal_garbage_collected( # pylint: disable=protected-access
764 *args, **kwds))
765
~/miniconda3/envs/tensorflow/lib/python3.9/site-packages/tensorflow/python/eager/function.py in _get_concrete_function_internal_garbage_collected(self, *args, **kwargs)
3048 args, kwargs = None, None
3049 with self._lock:
-> 3050 graph_function, _ = self._maybe_define_function(args, kwargs)
3051 return graph_function
3052
~/miniconda3/envs/tensorflow/lib/python3.9/site-packages/tensorflow/python/eager/function.py in _maybe_define_function(self, args, kwargs)
3442
3443 self._function_cache.missed.add(call_context_key)
-> 3444 graph_function = self._create_graph_function(args, kwargs)
3445 self._function_cache.primary[cache_key] = graph_function
3446
~/miniconda3/envs/tensorflow/lib/python3.9/site-packages/tensorflow/python/eager/function.py in _create_graph_function(self, args, kwargs, override_flat_arg_shapes)
3277 arg_names = base_arg_names + missing_arg_names
3278 graph_function = ConcreteFunction(
-> 3279 func_graph_module.func_graph_from_py_func(
3280 self._name,
3281 self._python_function,
~/miniconda3/envs/tensorflow/lib/python3.9/site-packages/tensorflow/python/framework/func_graph.py in func_graph_from_py_func(name, python_func, args, kwargs, signature, func_graph, autograph, autograph_options, add_control_dependencies, arg_names, op_return_value, collections, capture_by_value, override_flat_arg_shapes)
997 _, original_func = tf_decorator.unwrap(python_func)
998
--> 999 func_outputs = python_func(*func_args, **func_kwargs)
1000
1001 # invariant: `func_outputs` contains only Tensors, CompositeTensors,
~/miniconda3/envs/tensorflow/lib/python3.9/site-packages/tensorflow/python/eager/def_function.py in wrapped_fn(*args, **kwds)
670 # the function a weak reference to itself to avoid a reference cycle.
671 with OptionalXlaContext(compile_with_xla):
--> 672 out = weak_wrapped_fn().__wrapped__(*args, **kwds)
673 return out
674
~/miniconda3/envs/tensorflow/lib/python3.9/site-packages/tensorflow/python/framework/func_graph.py in wrapper(*args, **kwargs)
984 except Exception as e: # pylint:disable=broad-except
985 if hasattr(e, "ag_error_metadata"):
--> 986 raise e.ag_error_metadata.to_exception(e)
987 else:
988 raise
ValueError: in user code:
ValueError: Dimensions must be equal, but are 2 and 1001 for '{{node add_2}} = AddV2[T=DT_DOUBLE](diag, mul_1)' with input shapes: [1001,2,2], [1001,1001].
Is this a bug or are the likelihood and model incompatible or am I missing something?
A workaround is to make the inducing variables = the training data and use SVGP but that makes the training incredibly slow...

Tensorflow load model and load_weights give different behavior

I have one working tensorflow RNN model (called my_model) and I save it using my_model.save("<dir_to_my_model>").
However, when I reload this model with
model = tf.keras.models.load_model("<dir_to_my_model>")
and run a few incremental learning epochs with
with tf.GradientTape() as tape:
logits = model(x_test_batch, training=True)
I got an error (very long, please see at the bottom).
But, if I save the weights of this model to a checkpoint using model.save_weights(), create a new model and load the weights using model.load_weights from that checkpoint. The code above could run successfully.
My question is why these 2 methods work differently and what should I do to make method 1 work?
InvalidArgumentError Traceback (most recent call last)
~/work/tf/lib/python3.7/site-packages/tensorflow_core/python/framework/ops.py in get_attr(self, name)
2325 with c_api_util.tf_buffer() as buf:
-> 2326 c_api.TF_OperationGetAttrValueProto(self._c_op, name, buf)
2327 data = c_api.TF_GetBuffer(buf)
InvalidArgumentError: Operation 'StatefulPartitionedCall' has no attr named '_XlaCompile'.
During handling of the above exception, another exception occurred:
ValueError Traceback (most recent call last)
~/work/tf/lib/python3.7/site-packages/tensorflow_core/python/ops/gradients_util.py in _MaybeCompile(scope, op, func, grad_fn)
330 try:
--> 331 xla_compile = op.get_attr("_XlaCompile")
332 xla_separate_compiled_gradients = op.get_attr(
~/work/tf/lib/python3.7/site-packages/tensorflow_core/python/framework/ops.py in get_attr(self, name)
2329 # Convert to ValueError for backwards compatibility.
-> 2330 raise ValueError(str(e))
2331 x = attr_value_pb2.AttrValue()
ValueError: Operation 'StatefulPartitionedCall' has no attr named '_XlaCompile'.
During handling of the above exception, another exception occurred:
InvalidArgumentError Traceback (most recent call last)
~/work/tf/lib/python3.7/site-packages/tensorflow_core/python/framework/ops.py in get_attr(self, name)
2325 with c_api_util.tf_buffer() as buf:
-> 2326 c_api.TF_OperationGetAttrValueProto(self._c_op, name, buf)
2327 data = c_api.TF_GetBuffer(buf)
InvalidArgumentError: Operation 'StatefulPartitionedCall' has no attr named '_XlaCompile'.
During handling of the above exception, another exception occurred:
ValueError Traceback (most recent call last)
~/work/tf/lib/python3.7/site-packages/tensorflow_core/python/ops/gradients_util.py in _MaybeCompile(scope, op, func, grad_fn)
330 try:
--> 331 xla_compile = op.get_attr("_XlaCompile")
332 xla_separate_compiled_gradients = op.get_attr(
~/work/tf/lib/python3.7/site-packages/tensorflow_core/python/framework/ops.py in get_attr(self, name)
2329 # Convert to ValueError for backwards compatibility.
-> 2330 raise ValueError(str(e))
2331 x = attr_value_pb2.AttrValue()
ValueError: Operation 'StatefulPartitionedCall' has no attr named '_XlaCompile'.
During handling of the above exception, another exception occurred:
LookupError Traceback (most recent call last)
~/work/tf/lib/python3.7/site-packages/tensorflow_core/python/ops/gradients_util.py in _GradientsHelper(ys, xs, grad_ys, name, colocate_gradients_with_ops, gate_gradients, aggregation_method, stop_gradients, unconnected_gradients, src_graph)
606 try:
--> 607 grad_fn = ops.get_gradient_function(op)
608 except LookupError:
~/work/tf/lib/python3.7/site-packages/tensorflow_core/python/framework/ops.py in get_gradient_function(op)
2494 op_type = op.type
-> 2495 return _gradient_registry.lookup(op_type)
2496
~/work/tf/lib/python3.7/site-packages/tensorflow_core/python/framework/registry.py in lookup(self, name)
96 raise LookupError(
---> 97 "%s registry has no entry for: %s" % (self._name, name))
LookupError: gradient registry has no entry for: While
During handling of the above exception, another exception occurred:
LookupError Traceback (most recent call last)
<ipython-input-7-55225939f96a> in <module>
10 mode = 'DNN' if 'ANN' in m else 'RNN'
11 c_datasets = continual_dataset(datasets[i], mode=mode)
---> 12 res = learning_rate_test(continual_fine_tune, mname, c_datasets, lr_list=lrs)
13 dfs.append(res)
<ipython-input-5-f4d4cc64dc17> in learning_rate_test(continual_func, model_fname, c_datasets, epochs, lr_list, loss)
14 model.save_weights('Model/tmp')
15 model.load_weights('Model/tmp')
---> 16 test_predictions = continual_func(c_datasets, model, loss_fn, lr, epochs=e, save_model='')
17 test_predictions = (np.array(test_predictions) > 0.5).astype(int)
18 epoch_test_dict[e].append(np.sum(np.equal(y_test_list, test_predictions)) / len(y_test_list) * 100)
<ipython-input-4-9ae956b3e918> in continual_fine_tune(c_datasets, model, loss_fn, lr, epochs, save_model)
12 for e in range(epochs):
13 with tf.GradientTape() as tape:
---> 14 logits = model(x_test_batch, training=True)
15 loss_value = loss_fn(y_test_batch, logits)
16 grads = tape.gradient(loss_value, model.trainable_weights)
~/work/tf/lib/python3.7/site-packages/tensorflow_core/python/keras/engine/base_layer.py in __call__(self, inputs, *args, **kwargs)
820 with base_layer_utils.autocast_context_manager(
821 self._compute_dtype):
--> 822 outputs = self.call(cast_inputs, *args, **kwargs)
823 self._handle_activity_regularization(inputs, outputs)
824 self._set_mask_metadata(inputs, outputs, input_masks)
~/work/tf/lib/python3.7/site-packages/tensorflow_core/python/keras/engine/sequential.py in call(self, inputs, training, mask)
265 if not self.built:
266 self._init_graph_network(self.inputs, self.outputs, name=self.name)
--> 267 return super(Sequential, self).call(inputs, training=training, mask=mask)
268
269 outputs = inputs # handle the corner case where self.layers is empty
~/work/tf/lib/python3.7/site-packages/tensorflow_core/python/keras/engine/network.py in call(self, inputs, training, mask)
715 return self._run_internal_graph(
716 inputs, training=training, mask=mask,
--> 717 convert_kwargs_to_constants=base_layer_utils.call_context().saving)
718
719 def compute_output_shape(self, input_shape):
~/work/tf/lib/python3.7/site-packages/tensorflow_core/python/keras/engine/network.py in _run_internal_graph(self, inputs, training, mask, convert_kwargs_to_constants)
889
890 # Compute outputs.
--> 891 output_tensors = layer(computed_tensors, **kwargs)
892
893 # Update tensor_dict.
~/work/tf/lib/python3.7/site-packages/tensorflow_core/python/keras/engine/base_layer.py in __call__(self, inputs, *args, **kwargs)
820 with base_layer_utils.autocast_context_manager(
821 self._compute_dtype):
--> 822 outputs = self.call(cast_inputs, *args, **kwargs)
823 self._handle_activity_regularization(inputs, outputs)
824 self._set_mask_metadata(inputs, outputs, input_masks)
~/work/tf/lib/python3.7/site-packages/tensorflow_core/python/keras/saving/saved_model/utils.py in return_outputs_and_add_losses(*args, **kwargs)
57 inputs = args[inputs_arg_index]
58 args = args[inputs_arg_index + 1:]
---> 59 outputs, losses = fn(inputs, *args, **kwargs)
60 layer.add_loss(losses, inputs)
61 return outputs
~/work/tf/lib/python3.7/site-packages/tensorflow_core/python/keras/saving/saved_model/utils.py in wrap_with_training_arg(*args, **kwargs)
111 training,
112 lambda: replace_training_and_call(True),
--> 113 lambda: replace_training_and_call(False))
114
115 # Create arg spec for decorated function. If 'training' is not defined in the
~/work/tf/lib/python3.7/site-packages/tensorflow_core/python/keras/utils/tf_utils.py in smart_cond(pred, true_fn, false_fn, name)
57 pred, true_fn=true_fn, false_fn=false_fn, name=name)
58 return smart_module.smart_cond(
---> 59 pred, true_fn=true_fn, false_fn=false_fn, name=name)
60
61
~/work/tf/lib/python3.7/site-packages/tensorflow_core/python/framework/smart_cond.py in smart_cond(pred, true_fn, false_fn, name)
52 if pred_value is not None:
53 if pred_value:
---> 54 return true_fn()
55 else:
56 return false_fn()
~/work/tf/lib/python3.7/site-packages/tensorflow_core/python/keras/saving/saved_model/utils.py in <lambda>()
110 return tf_utils.smart_cond(
111 training,
--> 112 lambda: replace_training_and_call(True),
113 lambda: replace_training_and_call(False))
114
~/work/tf/lib/python3.7/site-packages/tensorflow_core/python/keras/saving/saved_model/utils.py in replace_training_and_call(training)
106 def replace_training_and_call(training):
107 set_training_arg(training, training_arg_index, args, kwargs)
--> 108 return wrapped_call(*args, **kwargs)
109
110 return tf_utils.smart_cond(
~/work/tf/lib/python3.7/site-packages/tensorflow_core/python/eager/def_function.py in __call__(self, *args, **kwds)
566 xla_context.Exit()
567 else:
--> 568 result = self._call(*args, **kwds)
569
570 if tracing_count == self._get_tracing_count():
~/work/tf/lib/python3.7/site-packages/tensorflow_core/python/eager/def_function.py in _call(self, *args, **kwds)
604 # In this case we have not created variables on the first call. So we can
605 # run the first trace but we should fail if variables are created.
--> 606 results = self._stateful_fn(*args, **kwds)
607 if self._created_variables:
608 raise ValueError("Creating variables on a non-first call to a function"
~/work/tf/lib/python3.7/site-packages/tensorflow_core/python/eager/function.py in __call__(self, *args, **kwargs)
2361 with self._lock:
2362 graph_function, args, kwargs = self._maybe_define_function(args, kwargs)
-> 2363 return graph_function._filtered_call(args, kwargs) # pylint: disable=protected-access
2364
2365 #property
~/work/tf/lib/python3.7/site-packages/tensorflow_core/python/eager/function.py in _filtered_call(self, args, kwargs)
1609 if isinstance(t, (ops.Tensor,
1610 resource_variable_ops.BaseResourceVariable))),
-> 1611 self.captured_inputs)
1612
1613 def _call_flat(self, args, captured_inputs, cancellation_manager=None):
~/work/tf/lib/python3.7/site-packages/tensorflow_core/python/eager/function.py in _call_flat(self, args, captured_inputs, cancellation_manager)
1695 possible_gradient_type,
1696 executing_eagerly)
-> 1697 forward_function, args_with_tangents = forward_backward.forward()
1698 if executing_eagerly:
1699 flat_outputs = forward_function.call(
~/work/tf/lib/python3.7/site-packages/tensorflow_core/python/eager/function.py in forward(self)
1421 """Builds or retrieves a forward function for this call."""
1422 forward_function = self._functions.forward(
-> 1423 self._inference_args, self._input_tangents)
1424 return forward_function, self._inference_args + self._input_tangents
1425
~/work/tf/lib/python3.7/site-packages/tensorflow_core/python/eager/function.py in forward(self, inference_args, input_tangents)
1183 (self._forward, self._forward_graph, self._backward,
1184 self._forwardprop_output_indices, self._num_forwardprop_outputs) = (
-> 1185 self._forward_and_backward_functions(inference_args, input_tangents))
1186 return self._forward
1187
~/work/tf/lib/python3.7/site-packages/tensorflow_core/python/eager/function.py in _forward_and_backward_functions(self, inference_args, input_tangents)
1329 outputs = self._func_graph.outputs[:self._num_inference_outputs]
1330 return self._build_functions_for_outputs(
-> 1331 outputs, inference_args, input_tangents)
1332
1333
~/work/tf/lib/python3.7/site-packages/tensorflow_core/python/eager/function.py in _build_functions_for_outputs(self, outputs, inference_args, input_tangents)
888 self._func_graph.inputs,
889 grad_ys=gradients_wrt_outputs,
--> 890 src_graph=self._func_graph)
891
892 captures_from_forward = [
~/work/tf/lib/python3.7/site-packages/tensorflow_core/python/ops/gradients_util.py in _GradientsHelper(ys, xs, grad_ys, name, colocate_gradients_with_ops, gate_gradients, aggregation_method, stop_gradients, unconnected_gradients, src_graph)
667 # functions.
668 in_grads = _MaybeCompile(grad_scope, op, func_call,
--> 669 lambda: grad_fn(op, *out_grads))
670 else:
671 # For function call ops, we add a 'SymbolicGradient'
~/work/tf/lib/python3.7/site-packages/tensorflow_core/python/ops/gradients_util.py in _MaybeCompile(scope, op, func, grad_fn)
334 xla_scope = op.get_attr("_XlaScope").decode()
335 except ValueError:
--> 336 return grad_fn() # Exit early
337
338 if not xla_compile:
~/work/tf/lib/python3.7/site-packages/tensorflow_core/python/ops/gradients_util.py in <lambda>()
667 # functions.
668 in_grads = _MaybeCompile(grad_scope, op, func_call,
--> 669 lambda: grad_fn(op, *out_grads))
670 else:
671 # For function call ops, we add a 'SymbolicGradient'
~/work/tf/lib/python3.7/site-packages/tensorflow_core/python/eager/function.py in _rewrite_forward_and_call_backward(self, op, *doutputs)
705 def _rewrite_forward_and_call_backward(self, op, *doutputs):
706 """Add outputs to the forward call and feed them to the grad function."""
--> 707 forward_function, backwards_function = self.forward_backward(len(doutputs))
708 if not backwards_function.outputs:
709 return backwards_function.structured_outputs
~/work/tf/lib/python3.7/site-packages/tensorflow_core/python/eager/function.py in forward_backward(self, num_doutputs)
614 if forward_backward is not None:
615 return forward_backward
--> 616 forward, backward = self._construct_forward_backward(num_doutputs)
617 self._cached_function_pairs[num_doutputs] = (forward, backward)
618 return forward, backward
~/work/tf/lib/python3.7/site-packages/tensorflow_core/python/eager/function.py in _construct_forward_backward(self, num_doutputs)
662 args=[], kwargs={},
663 signature=signature,
--> 664 func_graph=backwards_graph)
665 backwards_graph_captures = backwards_graph.external_captures
666 captures_from_forward = [
~/work/tf/lib/python3.7/site-packages/tensorflow_core/python/framework/func_graph.py in func_graph_from_py_func(name, python_func, args, kwargs, signature, func_graph, autograph, autograph_options, add_control_dependencies, arg_names, op_return_value, collections, capture_by_value, override_flat_arg_shapes)
976 converted_func)
977
--> 978 func_outputs = python_func(*func_args, **func_kwargs)
979
980 # invariant: `func_outputs` contains only Tensors, CompositeTensors,
~/work/tf/lib/python3.7/site-packages/tensorflow_core/python/eager/function.py in _backprop_function(*grad_ys)
652 self._func_graph.inputs,
653 grad_ys=grad_ys,
--> 654 src_graph=self._func_graph)
655
656 with self._func_graph.as_default():
~/work/tf/lib/python3.7/site-packages/tensorflow_core/python/ops/gradients_util.py in _GradientsHelper(ys, xs, grad_ys, name, colocate_gradients_with_ops, gate_gradients, aggregation_method, stop_gradients, unconnected_gradients, src_graph)
667 # functions.
668 in_grads = _MaybeCompile(grad_scope, op, func_call,
--> 669 lambda: grad_fn(op, *out_grads))
670 else:
671 # For function call ops, we add a 'SymbolicGradient'
~/work/tf/lib/python3.7/site-packages/tensorflow_core/python/ops/gradients_util.py in _MaybeCompile(scope, op, func, grad_fn)
334 xla_scope = op.get_attr("_XlaScope").decode()
335 except ValueError:
--> 336 return grad_fn() # Exit early
337
338 if not xla_compile:
~/work/tf/lib/python3.7/site-packages/tensorflow_core/python/ops/gradients_util.py in <lambda>()
667 # functions.
668 in_grads = _MaybeCompile(grad_scope, op, func_call,
--> 669 lambda: grad_fn(op, *out_grads))
670 else:
671 # For function call ops, we add a 'SymbolicGradient'
~/work/tf/lib/python3.7/site-packages/tensorflow_core/python/eager/function.py in _rewrite_forward_and_call_backward(self, op, *doutputs)
705 def _rewrite_forward_and_call_backward(self, op, *doutputs):
706 """Add outputs to the forward call and feed them to the grad function."""
--> 707 forward_function, backwards_function = self.forward_backward(len(doutputs))
708 if not backwards_function.outputs:
709 return backwards_function.structured_outputs
~/work/tf/lib/python3.7/site-packages/tensorflow_core/python/eager/function.py in forward_backward(self, num_doutputs)
614 if forward_backward is not None:
615 return forward_backward
--> 616 forward, backward = self._construct_forward_backward(num_doutputs)
617 self._cached_function_pairs[num_doutputs] = (forward, backward)
618 return forward, backward
~/work/tf/lib/python3.7/site-packages/tensorflow_core/python/eager/function.py in _construct_forward_backward(self, num_doutputs)
662 args=[], kwargs={},
663 signature=signature,
--> 664 func_graph=backwards_graph)
665 backwards_graph_captures = backwards_graph.external_captures
666 captures_from_forward = [
~/work/tf/lib/python3.7/site-packages/tensorflow_core/python/framework/func_graph.py in func_graph_from_py_func(name, python_func, args, kwargs, signature, func_graph, autograph, autograph_options, add_control_dependencies, arg_names, op_return_value, collections, capture_by_value, override_flat_arg_shapes)
976 converted_func)
977
--> 978 func_outputs = python_func(*func_args, **func_kwargs)
979
980 # invariant: `func_outputs` contains only Tensors, CompositeTensors,
~/work/tf/lib/python3.7/site-packages/tensorflow_core/python/eager/function.py in _backprop_function(*grad_ys)
652 self._func_graph.inputs,
653 grad_ys=grad_ys,
--> 654 src_graph=self._func_graph)
655
656 with self._func_graph.as_default():
~/work/tf/lib/python3.7/site-packages/tensorflow_core/python/ops/gradients_util.py in _GradientsHelper(ys, xs, grad_ys, name, colocate_gradients_with_ops, gate_gradients, aggregation_method, stop_gradients, unconnected_gradients, src_graph)
621 raise LookupError(
622 "No gradient defined for operation '%s' (op type: %s)" %
--> 623 (op.name, op.type))
624 if loop_state:
625 loop_state.EnterGradWhileContext(op, before=False)
LookupError: No gradient defined for operation 'while' (op type: While)

I have run into that in the past. The following is how I save/load keras models.
Note that the architecture and the weights are separate.
#saving model
open('model_weights.h5', 'w').write(model.to_json())
model.save_weights('architecture.json', overwrite=True)
from tensorflow.keras.models import model_from_json
#loading model
model = model_from_json(open('architecture.json').read())
model.load_weights('model_weights.h5')

Trying to use tensorflow.io.gfile.GFile and tensorflow.dataset.map, getting "Expected binary or unicode string, got Tensor dtype=string

I'm quite new to using Tensorflow, and imagine someone will quickly tell me I'm doing something stupid so here goes.
I'm working with the MSTAR dataset and trying to get it read in. The files have a very strange format, but suffice it to say that if eager execution is on the following code reads and displays an image from the dataset.
import tensorflow as tf
import matplotlib.pyplot as plt
tf.enable_eager_execution()
img1Path='HB15000.018'
img2Path='HB15001.018'
def pathToImgTF(path):
with tf.io.gfile.GFile(path,'rb') as filePath:
step1=filePath.readlines()
step2=[x.strip(b'\n') for x in step1]
for x in step2:
if b'PhoenixHeaderLength' in x:
line=x.strip().split(b'=')
PhoenixHeaderLength=int(line[1])
elif b'native_header_length' in x:
line=x.strip().split(b'=')
native_header_length=int(line[1])
elif b'NumberOfColumns' in x:
line=x.strip().split(b'=')
NumberOfColumns=int(line[1])
elif b'NumberOfRows' in x:
line=x.strip().split(b'=')
NumberOfRows=int(line[1])
filePath.seek(PhoenixHeaderLength+native_header_length)
step3=tf.decode_raw(filePath.read(),out_type=tf.float32,little_endian=False)
depth_major=tf.reshape(step3,[2,NumberOfRows,NumberOfColumns])
image=tf.transpose(depth_major,[1,2,0])[:,:,0] #Cut off phase for now
return image
img=pathToImgTF(imgPath)
plt.imshow(img,cmap='gray')
I would like to use tf.dataset.from_tensor_slices, but it appears that isn't an option because the following code:
ds=tf.data.Dataset.from_tensor_slices([img1Path,img2Path])
ds=ds.map(pathToImgTF)
Gives the error "TypeError: Expected binary or unicode string, got tf.Tensor 'args_0:0' shape=() dtype=string"
The traceback looks to me like it's breaking specifically on 'filePath.readlines()', any help would be greatly appreciated.
Full error output:
--------------------------------------------------------------------------- TypeError Traceback (most recent call
last) ipython-input-6-e12909fb73cd in module
1 ds=tf.data.Dataset.from_tensor_slices([img1Path,img2Path])
----> 2 ds=ds.map(pathToImgTF)
~\AppData\Local\Continuum\anaconda3\lib\site-packages\tensorflow\python\data\ops\dataset_ops.py
in map(self, map_func, num_parallel_calls) 1770 if
num_parallel_calls is None: 1771 return DatasetV1Adapter(
-> 1772 MapDataset(self, map_func, preserve_cardinality=False)) 1773 else: 1774 return
DatasetV1Adapter(
~\AppData\Local\Continuum\anaconda3\lib\site-packages\tensorflow\python\data\ops\dataset_ops.py
in init(self, input_dataset, map_func, use_inter_op_parallelism,
preserve_cardinality, use_legacy_function) 3188
self._transformation_name(), 3189 dataset=input_dataset,
-> 3190 use_legacy_function=use_legacy_function) 3191 variant_tensor = gen_dataset_ops.map_dataset( 3192
input_dataset._variant_tensor, # pylint: disable=protected-access
~\AppData\Local\Continuum\anaconda3\lib\site-packages\tensorflow\python\data\ops\dataset_ops.py
in init(self, func, transformation_name, dataset, input_classes,
input_shapes, input_types, input_structure, add_to_graph,
use_legacy_function, defun_kwargs) 2553 resource_tracker =
tracking.ResourceTracker() 2554 with
tracking.resource_tracker_scope(resource_tracker):
-> 2555 self._function = wrapper_fn._get_concrete_function_internal() 2556 if
add_to_graph: 2557
self._function.add_to_graph(ops.get_default_graph())
~\AppData\Local\Continuum\anaconda3\lib\site-packages\tensorflow\python\eager\function.py
in _get_concrete_function_internal(self, *args, **kwargs) 1353
"""Bypasses error checking when getting a graph function.""" 1354
graph_function =
self._get_concrete_function_internal_garbage_collected(
-> 1355 *args, **kwargs) 1356 # We're returning this concrete function to someone, and they may keep a 1357 #
reference to the FuncGraph without keeping a reference to the
~\AppData\Local\Continuum\anaconda3\lib\site-packages\tensorflow\python\eager\function.py
in _get_concrete_function_internal_garbage_collected(self, *args,
**kwargs) 1347 if self.input_signature: 1348 args, kwargs = None, None
-> 1349 graph_function, _, _ = self._maybe_define_function(args, kwargs) 1350 return graph_function 1351
~\AppData\Local\Continuum\anaconda3\lib\site-packages\tensorflow\python\eager\function.py
in _maybe_define_function(self, args, kwargs) 1650
graph_function = self._function_cache.primary.get(cache_key, None)
1651 if graph_function is None:
-> 1652 graph_function = self._create_graph_function(args, kwargs) 1653 self._function_cache.primary[cache_key] =
graph_function 1654 return graph_function, args, kwargs
~\AppData\Local\Continuum\anaconda3\lib\site-packages\tensorflow\python\eager\function.py
in _create_graph_function(self, args, kwargs,
override_flat_arg_shapes) 1543 arg_names=arg_names,
1544 override_flat_arg_shapes=override_flat_arg_shapes,
-> 1545 capture_by_value=self._capture_by_value), 1546 self._function_attributes) 1547
~\AppData\Local\Continuum\anaconda3\lib\site-packages\tensorflow\python\framework\func_graph.py
in func_graph_from_py_func(name, python_func, args, kwargs, signature,
func_graph, autograph, autograph_options, add_control_dependencies,
arg_names, op_return_value, collections, capture_by_value,
override_flat_arg_shapes)
713 converted_func)
714
--> 715 func_outputs = python_func(*func_args, **func_kwargs)
716
717 # invariant: func_outputs contains only Tensors, CompositeTensors,
~\AppData\Local\Continuum\anaconda3\lib\site-packages\tensorflow\python\data\ops\dataset_ops.py
in wrapper_fn(*args) 2547 attributes=defun_kwargs)
2548 def wrapper_fn(*args): # pylint: disable=missing-docstring
-> 2549 ret = _wrapper_helper(*args) 2550 ret = self._output_structure._to_tensor_list(ret) 2551 return
[ops.convert_to_tensor(t) for t in ret]
~\AppData\Local\Continuum\anaconda3\lib\site-packages\tensorflow\python\data\ops\dataset_ops.py
in _wrapper_helper(*args) 2487 nested_args = (nested_args,)
2488
-> 2489 ret = func(*nested_args) 2490 # If func returns a list of tensors, nest.flatten() and 2491 #
ops.convert_to_tensor() would conspire to attempt to stack
in pathToImgTF(path)
9 def pathToImgTF(path):
10 with tf.io.gfile.GFile(path,'rb') as filePath:
---> 11 step1=filePath.readlines()
12 step2=[x.strip(b'\n') for x in step1]
13 for x in step2:
~\AppData\Local\Continuum\anaconda3\lib\site-packages\tensorflow\python\lib\io\file_io.py
in readlines(self)
181 def readlines(self):
182 """Returns all lines from the file in a list."""
--> 183 self._preread_check()
184 lines = []
185 while True:
~\AppData\Local\Continuum\anaconda3\lib\site-packages\tensorflow\python\lib\io\file_io.py
in _preread_check(self)
82 "File isn't open for reading")
83 self._read_buf = pywrap_tensorflow.CreateBufferedInputStream(
---> 84 compat.as_bytes(self.__name), 1024 * 512)
85
86 def _prewrite_check(self):
~\AppData\Local\Continuum\anaconda3\lib\site-packages\tensorflow\python\util\compat.py
in as_bytes(bytes_or_text, encoding)
63 else:
64 raise TypeError('Expected binary or unicode string, got %r' %
---> 65 (bytes_or_text,))
66
67
TypeError: Expected binary or unicode string, got tf.Tensor
'args_0:0' shape=() dtype=string

I am not sure what kind of data do imgPath1 and imgPath2 have. Anyways, they must be of tensor type with predefined data type. These two consists the actual data.
If that is the case, then change the square brackets to circular brackets as follows-
ds=tf.data.Dataset.from_tensor_slices((imgPath1,imgPath2))
If this still throwing the same error, then please give information where exactly it is throwing the error.

TypeError: __init__() missing 2 required positional arguments: 'message' and 'code'

I have tried to run the exact same code (on Neural Machine Translation with Attention) on my local machine as here:
https://colab.research.google.com/github/tensorflow/tensorflow/blob/master/tensorflow/contrib/eager/python/examples/nmt_with_attention/nmt_with_attention.ipynb
I am getting the following error when training the model and trying to save checkpoints:
Epoch 1 Batch 0 Loss 0.3238
---------------------------------------------------------------------------
_FallbackException Traceback (most recent call last)
C:\Anaconda3\lib\site-packages\tensorflow\python\ops\gen_io_ops.py in save_v2(prefix, tensor_names, shape_and_slices, tensors, name)
1809 _ctx._post_execution_callbacks, prefix, tensor_names,
-> 1810 shape_and_slices, tensors)
1811 return _result
_FallbackException: This function does not handle the case of the path where all inputs are not already EagerTensors.
During handling of the above exception, another exception occurred:
TypeError Traceback (most recent call last)
<ipython-input-28-bb3738bfb3d8> in <module>()
43 # saving (checkpoint) the model every 2 epochs
44 if epoch % 2 == 0:
---> 45 checkpoint.save(file_prefix = checkpoint_prefix)
46
47 print('Epoch {} Loss {:.4f}'.format(epoch + 1,
C:\Anaconda3\lib\site-packages\tensorflow\python\training\checkpointable\util.py in save(self, file_prefix, session)
1485 file_prefix=file_prefix,
1486 checkpoint_number=self.save_counter,
-> 1487 session=session)
1488
1489 def restore(self, save_path):
C:\Anaconda3\lib\site-packages\tensorflow\python\training\checkpointable\util.py in save(self, file_prefix, checkpoint_number, session)
1184 save_path=file_prefix,
1185 write_meta_graph=False,
-> 1186 global_step=checkpoint_number)
1187 return save_path
1188
C:\Anaconda3\lib\site-packages\tensorflow\python\training\saver.py in save(self, sess, save_path, global_step, latest_filename, meta_graph_suffix, write_meta_graph, write_state, strip_default_attrs)
1613 if context.executing_eagerly():
1614 self._build_eager(
-> 1615 checkpoint_file, build_save=True, build_restore=False)
1616 model_checkpoint_path = self.saver_def.save_tensor_name
1617 else:
C:\Anaconda3\lib\site-packages\tensorflow\python\training\saver.py in _build_eager(self, checkpoint_path, build_save, build_restore)
1295 def _build_eager(self, checkpoint_path, build_save, build_restore):
1296 self._build(
-> 1297 checkpoint_path, build_save=build_save, build_restore=build_restore)
1298
1299 def _build(self, checkpoint_path, build_save, build_restore):
C:\Anaconda3\lib\site-packages\tensorflow\python\training\saver.py in _build(self, checkpoint_path, build_save, build_restore)
1328 restore_sequentially=self._restore_sequentially,
1329 filename=checkpoint_path,
-> 1330 build_save=build_save, build_restore=build_restore)
1331 elif self.saver_def and self._name:
1332 # Since self._name is used as a name_scope by builder(), we are
C:\Anaconda3\lib\site-packages\tensorflow\python\training\saver.py in _build_internal(self, names_to_saveables, reshape, sharded, max_to_keep, keep_checkpoint_every_n_hours, name, restore_sequentially, filename, build_save, build_restore)
773 else:
774 if build_save:
--> 775 save_tensor = self._AddSaveOps(filename_tensor, saveables)
776 if build_restore:
777 restore_op = self._AddRestoreOps(filename_tensor, saveables,
C:\Anaconda3\lib\site-packages\tensorflow\python\training\saver.py in _AddSaveOps(self, filename_tensor, saveables)
273 A tensor with the filename used to save.
274 """
--> 275 save = self.save_op(filename_tensor, saveables)
276 return control_flow_ops.with_dependencies([save], filename_tensor)
277
C:\Anaconda3\lib\site-packages\tensorflow\python\training\saver.py in save_op(self, filename_tensor, saveables)
191 # of a V2 checkpoint: e.g. "/fs/train/ckpt-<step>/tmp/worker<i>-<step>".
192 return io_ops.save_v2(filename_tensor, tensor_names, tensor_slices,
--> 193 tensors)
194 else:
195 raise RuntimeError("Unexpected write_version: " + self._write_version)
C:\Anaconda3\lib\site-packages\tensorflow\python\ops\gen_io_ops.py in save_v2(prefix, tensor_names, shape_and_slices, tensors, name)
1813 return save_v2_eager_fallback(
1814 prefix, tensor_names, shape_and_slices, tensors, name=name,
-> 1815 ctx=_ctx)
1816 except _core._NotOkStatusException as e:
1817 if name is not None:
C:\Anaconda3\lib\site-packages\tensorflow\python\ops\gen_io_ops.py in save_v2_eager_fallback(prefix, tensor_names, shape_and_slices, tensors, name, ctx)
1834 _attrs = ("dtypes", _attr_dtypes)
1835 _result = _execute.execute(b"SaveV2", 0, inputs=_inputs_flat, attrs=_attrs,
-> 1836 ctx=_ctx, name=name)
1837 _result = None
1838 return _result
C:\Anaconda3\lib\site-packages\tensorflow\python\eager\execute.py in quick_execute(op_name, num_outputs, inputs, attrs, ctx, name)
58 tensors = pywrap_tensorflow.TFE_Py_Execute(ctx._handle, device_name,
59 op_name, inputs, attrs,
---> 60 num_outputs)
61 except core._NotOkStatusException as e:
62 if name is not None:
TypeError: __init__() missing 2 required positional arguments: 'message' and 'code'