I want to run python program using pytorch with my own dataset.I come across with the error:
Traceback (most recent call last):
File "C:\Users\hp\Downloads\efficient_densenet_pytorch-master\demoEmotion.py", line 345, in
fire.Fire(demo)
File "C:\Users\hp\Anaconda3\envs\tf-gpu\lib\site-packages\fire\core.py", line 138, in Fire
component_trace = _Fire(component, args, parsed_flag_args, context, name)
File "C:\Users\hp\Anaconda3\envs\tf-gpu\lib\site-packages\fire\core.py", line 468, in _Fire
target=component.name)
File "C:\Users\hp\Anaconda3\envs\tf-gpu\lib\site-packages\fire\core.py", line 672, in _CallAndUpdateTrace
component = fn(*varargs, **kwargs)
File "C:\Users\hp\Downloads\efficient_densenet_pytorch-master\demoEmotion.py", line 323, in demo
n_epochs=n_epochs, batch_size=batch_size, seed=seed)
File "C:\Users\hp\Downloads\efficient_densenet_pytorch-master\demoEmotion.py", line 202, in train
n_epochs=n_epochs,
File "C:\Users\hp\Downloads\efficient_densenet_pytorch-master\demoEmotion.py", line 83, in train_epoch
output = model(input)
File "C:\Users\hp\Anaconda3\envs\tf-gpu\lib\site-packages\torch\nn\modules\module.py", line 555, in call
result = self.forward(*input, **kwargs)
File "C:\Users\hp\Downloads\efficient_densenet_pytorch-master\models\densenet.py", line 151, in forward
features = self.features(x)
File "C:\Users\hp\Anaconda3\envs\tf-gpu\lib\site-packages\torch\nn\modules\module.py", line 555, in call
result = self.forward(*input, **kwargs)
File "C:\Users\hp\Anaconda3\envs\tf-gpu\lib\site-packages\torch\nn\modules\container.py", line 100, in forward
input = module(input)
File "C:\Users\hp\Anaconda3\envs\tf-gpu\lib\site-packages\torch\nn\modules\module.py", line 555, in call
result = self.forward(*input, **kwargs)
File "C:\Users\hp\Anaconda3\envs\tf-gpu\lib\site-packages\torch\nn\modules\container.py", line 100, in forward
input = module(input)
File "C:\Users\hp\Anaconda3\envs\tf-gpu\lib\site-packages\torch\nn\modules\module.py", line 555, in call
result = self.forward(*input, **kwargs)
File "C:\Users\hp\Anaconda3\envs\tf-gpu\lib\site-packages\torch\nn\modules\pooling.py", line 557, in forward
self.padding, self.ceil_mode, self.count_include_pad, self.divisor_override)
RuntimeError: Given input size: (150x1x1). Calculated output size: (150x0x0). Output size is too small
Please guide me how to sove this problem.Thanks in advance!
Please check the input size of the image. It has to be exactly the same as specified in the model. You can use padding if the image is smaller.
Related
I am trying to upgrade a code from tensorflow 1.x to tensorflow 2.x, since I need to use it with python>=3.8. The code is from someone else who is not updating it, you can find it here if needed: https://github.com/NetManAIOps/donut.
I have used the tools provided by tensorflow to upgrade it (i.e. running tf_upgrade_v2 in each file).
Already for a while I have been stuck with an error that I cannot track down.
File "test_model.py", line 227, in <module>
get_anomaly_score(test_files, KPI=args.KPI)
File "test_model.py", line 79, in get_anomaly_score
test_score = dm.predictor.get_score(test_values, test_missing)
File "/home/miguel/gitlab_projects/test/donut_haowen/donut/prediction.py", line 138, in get_score
b_r = sess.run(self._get_score(), feed_dict=feed_dict)
File "/home/miguel/gitlab_projects/test/donut_haowen/donut/prediction.py", line 64, in _get_score
self._score = self.model.get_score(
File "/home/miguel/gitlab_projects/test/donut_haowen/donut/model.py", line 197, in get_score
x_r = iterative_masked_reconstruct(
File "/home/miguel/gitlab_projects/test/donut_haowen/donut/reconstruction.py", line 101, in iterative_masked_reconstruct
x_r, _ = tf.while_loop(
File "/home/miguel/anaconda3/envs/test/lib/python3.8/site-packages/tensorflow/python/util/deprecation.py", line 629, in new_func
return func(*args, **kwargs)
File "/home/miguel/anaconda3/envs/test/lib/python3.8/site-packages/tensorflow/python/ops/control_flow_ops.py", line 2516, in while_loop_v2
return while_loop(
File "/home/miguel/anaconda3/envs/test/lib/python3.8/site-packages/tensorflow/python/ops/control_flow_ops.py", line 2716, in while_loop
return while_v2.while_loop(
File "/home/miguel/anaconda3/envs/test/lib/python3.8/site-packages/tensorflow/python/ops/while_v2.py", line 222, in while_loop
body_graph = func_graph_module.func_graph_from_py_func(
File "/home/miguel/anaconda3/envs/test/lib/python3.8/site-packages/tensorflow/python/framework/func_graph.py", line 1283, in func_graph_from_py_func
func_outputs = python_func(*func_args, **func_kwargs)
File "/home/miguel/anaconda3/envs/test/lib/python3.8/site-packages/tensorflow/python/ops/while_v2.py", line 200, in wrapped_body
outputs = body(
File "/home/miguel/gitlab_projects/test/donut_haowen/donut/reconstruction.py", line 103, in <lambda>
body=lambda x_i, i: (masked_reconstruct(reconstruct, x_i, mask), i + 1),
File "/home/miguel/gitlab_projects/test/donut_haowen/donut/reconstruction.py", line 65, in masked_reconstruct
r_x = reconstruct(x)
File "/home/miguel/gitlab_projects/test/src/tfsnippet/tfsnippet/modules/auto_encoders/vae.py", line 469, in reconstruct
model = self.model(z=q_net['z'], n_z=n_z, n_x=n_x)
File "/home/miguel/gitlab_projects/test/src/tfsnippet/tfsnippet/utils/reuse.py", line 179, in wrapper
return method(*args, **kwargs)
File "/home/miguel/gitlab_projects/test/src/tfsnippet/tfsnippet/modules/auto_encoders/vae.py", line 314, in model
x_params = self.h_for_p_x(z)
File "/home/miguel/gitlab_projects/test/src/tfsnippet/tfsnippet/modules/base.py", line 89, in __call__
return self._forward(inputs, **kwargs)
File "/home/miguel/gitlab_projects/test/src/tfsnippet/tfsnippet/modules/container/lambda_.py", line 47, in _forward
return self._factory(inputs, **kwargs)
File "/home/miguel/gitlab_projects/test/donut_haowen/donut/model.py", line 24, in wrap_params_net
h = h_for_dist(inputs)
File "/home/miguel/gitlab_projects/test/src/tfsnippet/tfsnippet/modules/base.py", line 89, in __call__
return self._forward(inputs, **kwargs)
File "/home/miguel/gitlab_projects/test/src/tfsnippet/tfsnippet/modules/container/sequential.py", line 76, in _forward
outputs = c(outputs, **kwargs)
File "/home/miguel/anaconda3/envs/test/lib/python3.8/site-packages/keras/utils/traceback_utils.py", line 70, in error_handler
raise e.with_traceback(filtered_tb) from None
File "/home/miguel/anaconda3/envs/test/lib/python3.8/site-packages/keras/engine/base_layer.py", line 1140, in __call__
if self._saved_model_inputs_spec is None:
AttributeError: 'Dense' object has no attribute '_saved_model_inputs_spec'
This occurs when I call the predictor.py file in the donut package. This will call another package called TFsnippet, which I also upgraded, and this will finally call the keras package.
Since it is not my code that I am using I have trouble to write any kind of minimum reproducible example.
I am looking for any idea of where could the issue be, and what parts of the code should I maybe check. At the moment I do not have any idea of why or how this error occurs.
Please, tell me if I should add something.
I got an error message when I used pytorch-lightning Trainer module:
File "run.py", line 105, in <module>
runner.fit(experiment)
File "/opt/conda/lib/python3.7/site-packages/pytorch_lightning/trainer/trainer.py", line 769, in fit
self._fit_impl, model, train_dataloaders, val_dataloaders, datamodule, ckpt_path
File "/opt/conda/lib/python3.7/site-packages/pytorch_lightning/trainer/trainer.py", line 721, in _call_and_handle_interrupt
return trainer_fn(*args, **kwargs)
File "/opt/conda/lib/python3.7/site-packages/pytorch_lightning/trainer/trainer.py", line 809, in _fit_impl
results = self._run(model, ckpt_path=self.ckpt_path)
File "/opt/conda/lib/python3.7/site-packages/pytorch_lightning/trainer/trainer.py", line 1234, in _run
results = self._run_stage()
File "/opt/conda/lib/python3.7/site-packages/pytorch_lightning/trainer/trainer.py", line 1321, in _run_stage
return self._run_train()
File "/opt/conda/lib/python3.7/site-packages/pytorch_lightning/trainer/trainer.py", line 1351, in _run_train
self.fit_loop.run()
File "/opt/conda/lib/python3.7/site-packages/pytorch_lightning/loops/base.py", line 204, in run
self.advance(*args, **kwargs)
File "/opt/conda/lib/python3.7/site-packages/pytorch_lightning/loops/fit_loop.py", line 269, in advance
self._outputs = self.epoch_loop.run(self._data_fetcher)
File "/opt/conda/lib/python3.7/site-packages/pytorch_lightning/loops/base.py", line 204, in run
self.advance(*args, **kwargs)
File "/opt/conda/lib/python3.7/site-packages/pytorch_lightning/loops/epoch/training_epoch_loop.py", line 208, in advance
batch_output = self.batch_loop.run(batch, batch_idx)
File "/opt/conda/lib/python3.7/site-packages/pytorch_lightning/loops/base.py", line 204, in run
self.advance(*args, **kwargs)
File "/opt/conda/lib/python3.7/site-packages/pytorch_lightning/loops/batch/training_batch_loop.py", line 88, in advance
outputs = self.optimizer_loop.run(split_batch, optimizers, batch_idx)
File "/opt/conda/lib/python3.7/site-packages/pytorch_lightning/loops/base.py", line 204, in run
self.advance(*args, **kwargs)
File "/opt/conda/lib/python3.7/site-packages/pytorch_lightning/loops/optimization/optimizer_loop.py", line 207, in advance
self.optimizer_idx,
File "/opt/conda/lib/python3.7/site-packages/pytorch_lightning/loops/optimization/optimizer_loop.py", line 258, in _run_optimization
result = closure.consume_result()
File "/opt/conda/lib/python3.7/site-packages/pytorch_lightning/loops/optimization/closure.py", line 52, in consume_result
"The closure hasn't been executed."
pytorch_lightning.utilities.exceptions.MisconfigurationException: The closure hasn't been executed. HINT: did you call `optimizer_closure()` in your `optimizer_step` hook? It could also happen because the `optimizer.step(optimizer_closure)` call did not execute it internally.
I read the Pytorch Lightning 1.6.1 documentation. It seems that under the hood, the Lightning Trainer handles the training loop details for me. I don't know how to handle the "optimizer.step()" by myself...
The following is my code of calling Trainer:
runner = Trainer(checkpoint_callback=checkpoint_callback,
resume_from_checkpoint=model_path,
logger=tt_logger,
log_every_n_steps=100,
weights_summary='full',
# early_stop_callback = False,
**config['trainer_params'])
And I defined the following function in my model:
def forward(self, input: Tensor, **kwargs) -> Tensor
def training_step(self, batch, batch_idx, optimizer_idx)
def optimizer_step(self, epoch, batch_idx, optimizer, optimizer_idx, def optimizer_closure, on_tpu, using_native_amp, using_lbfgs)
on_load_checkpoint(self, checkpoint)
def training_epoch_end(self, outputs)
def configure_optimizers(self)
def train_dataloader(self)
def data_transforms(self)
This is my first time asking a question here. If there is something wrong, please forgive me. Thank you all for your help!
here is where i load my model:
_model = r"C:\Users\evead\Desktop\spacy_model_config_03_15\model-best"
nlp = spacy.load(_model)
txt = "Below are my data loader and neural net. I have also included the output of my data loader when I retrieve a batch of data"
doc = nlp(txt)
here is the error I got
but I got the following error:
Exception has occurred: RuntimeError
input must have 3 dimensions, got 2
File "C:\Users\evead\Desktop\Issam\PIXITREND-CLASSIFIER\Annotation-Training\src\debug_entity.py", line 46, in <module>
doc = nlp(txt)
here is my complete error:
(env) PS C:\Users\evead\Desktop\Issam\PIXITREND-CLASSIFIER> python -m spacy evaluate C:\Users\evead\Desktop\spacy_model_config_03_15\model-best C:\Users\evead\Desktop\batchdata-3-15\train.spacy --output C:\Users\evead\Desktop\batchdata-3-15\res.json
Using CPU
C:\Users\evead\Desktop\Issam\PIXITREND-CLASSIFIER\Annotation-Training\env\lib\site-packages\torch\autocast_mode.py:141: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling
warnings.warn('User provided device_type of \'cuda\', but CUDA is not available. Disabling')
Traceback (most recent call last):
File "C:\Users\evead\AppData\Local\Programs\Python\Python39\lib\runpy.py", line 197, in _run_module_as_main
return _run_code(code, main_globals, None,
File "C:\Users\evead\AppData\Local\Programs\Python\Python39\lib\runpy.py", line 87, in _run_code
exec(code, run_globals)
File "C:\Users\evead\Desktop\Issam\PIXITREND-CLASSIFIER\Annotation-Training\env\lib\site-packages\spacy\__main__.py", line 4, in <module>
setup_cli()
File "C:\Users\evead\Desktop\Issam\PIXITREND-CLASSIFIER\Annotation-Training\env\lib\site-packages\spacy\cli\_util.py", line 71, in setup_cli
command(prog_name=COMMAND)
File "C:\Users\evead\Desktop\Issam\PIXITREND-CLASSIFIER\Annotation-Training\env\lib\site-packages\click\core.py", line 1128, in __call__
return self.main(*args, **kwargs)
File "C:\Users\evead\Desktop\Issam\PIXITREND-CLASSIFIER\Annotation-Training\env\lib\site-packages\click\core.py", line 1053, in main
rv = self.invoke(ctx)
File "C:\Users\evead\Desktop\Issam\PIXITREND-CLASSIFIER\Annotation-Training\env\lib\site-packages\click\core.py", line 1659, in invoke
return _process_result(sub_ctx.command.invoke(sub_ctx))
File "C:\Users\evead\Desktop\Issam\PIXITREND-CLASSIFIER\Annotation-Training\env\lib\site-packages\click\core.py", line 1395, in invoke
return ctx.invoke(self.callback, **ctx.params)
File "C:\Users\evead\Desktop\Issam\PIXITREND-CLASSIFIER\Annotation-Training\env\lib\site-packages\click\core.py", line 754, in invoke
return __callback(*args, **kwargs)
File "C:\Users\evead\Desktop\Issam\PIXITREND-CLASSIFIER\Annotation-Training\env\lib\site-packages\typer\main.py", line 500, in wrapper
return callback(**use_params) # type: ignore
File "C:\Users\evead\Desktop\Issam\PIXITREND-CLASSIFIER\Annotation-Training\env\lib\site-packages\spacy\cli\evaluate.py", line 42, in evaluate_cli
evaluate(
File "C:\Users\evead\Desktop\Issam\PIXITREND-CLASSIFIER\Annotation-Training\env\lib\site-packages\spacy\cli\evaluate.py", line 78, in evaluate
scores = nlp.evaluate(dev_dataset)
File "C:\Users\evead\Desktop\Issam\PIXITREND-CLASSIFIER\Annotation-Training\env\lib\site-packages\spacy\language.py", line 1415, in evaluate
for eg, doc in zip(examples, docs):
File "C:\Users\evead\Desktop\Issam\PIXITREND-CLASSIFIER\Annotation-Training\env\lib\site-packages\spacy\language.py", line 1575, in pipe
for doc in docs:
File "C:\Users\evead\Desktop\Issam\PIXITREND-CLASSIFIER\Annotation-Training\env\lib\site-packages\spacy\util.py", line 1598, in _pipe
yield from proc.pipe(docs, **kwargs)
File "spacy\pipeline\transition_parser.pyx", line 230, in pipe
File "C:\Users\evead\Desktop\Issam\PIXITREND-CLASSIFIER\Annotation-Training\env\lib\site-packages\spacy\util.py", line 1547, in minibatch
batch = list(itertools.islice(items, int(batch_size)))
File "C:\Users\evead\Desktop\Issam\PIXITREND-CLASSIFIER\Annotation-Training\env\lib\site-packages\spacy\util.py", line 1598, in _pipe
yield from proc.pipe(docs, **kwargs)
File "spacy\pipeline\trainable_pipe.pyx", line 79, in pipe
File "C:\Users\evead\Desktop\Issam\PIXITREND-CLASSIFIER\Annotation-Training\env\lib\site-packages\spacy\util.py", line 1617, in raise_error
raise e
File "spacy\pipeline\trainable_pipe.pyx", line 75, in spacy.pipeline.trainable_pipe.TrainablePipe.pipe
File "C:\Users\evead\Desktop\Issam\PIXITREND-CLASSIFIER\Annotation-Training\env\lib\site-packages\spacy\pipeline\tok2vec.py", line 125, in predict
tokvecs = self.model.predict(docs)
File "C:\Users\evead\Desktop\Issam\PIXITREND-CLASSIFIER\Annotation-Training\env\lib\site-packages\thinc\model.py", line 315, in predict
return self._func(self, X, is_train=False)[0]
File "C:\Users\evead\Desktop\Issam\PIXITREND-CLASSIFIER\Annotation-Training\env\lib\site-packages\thinc\layers\chain.py", line 54, in forward
Y, inc_layer_grad = layer(X, is_train=is_train)
File "C:\Users\evead\Desktop\Issam\PIXITREND-CLASSIFIER\Annotation-Training\env\lib\site-packages\thinc\model.py", line 291, in __call__
return self._func(self, X, is_train=is_train)
File "C:\Users\evead\Desktop\Issam\PIXITREND-CLASSIFIER\Annotation-Training\env\lib\site-packages\thinc\layers\with_array.py", line 40, in forward
return _list_forward(cast(Model[List2d, List2d], model), Xseq, is_train)
File "C:\Users\evead\Desktop\Issam\PIXITREND-CLASSIFIER\Annotation-Training\env\lib\site-packages\thinc\layers\with_array.py", line 76, in _list_forward
Yf, get_dXf = layer(Xf, is_train)
File "C:\Users\evead\Desktop\Issam\PIXITREND-CLASSIFIER\Annotation-Training\env\lib\site-packages\thinc\model.py", line 291, in __call__
return self._func(self, X, is_train=is_train)
File "C:\Users\evead\Desktop\Issam\PIXITREND-CLASSIFIER\Annotation-Training\env\lib\site-packages\thinc\layers\with_padded.py", line 36, in forward
Y, backprop = _array_forward(layer, cast(Floats3d, Xseq), is_train)
File "C:\Users\evead\Desktop\Issam\PIXITREND-CLASSIFIER\Annotation-Training\env\lib\site-packages\thinc\layers\with_padded.py", line 76, in _array_forward
Yp, get_dXp = layer(Xp, is_train)
File "C:\Users\evead\Desktop\Issam\PIXITREND-CLASSIFIER\Annotation-Training\env\lib\site-packages\thinc\model.py", line 291, in __call__
return self._func(self, X, is_train=is_train)
File "C:\Users\evead\Desktop\Issam\PIXITREND-CLASSIFIER\Annotation-Training\env\lib\site-packages\thinc\layers\with_padded.py", line 30, in forward
Y, backprop = layer(Xseq, is_train)
File "C:\Users\evead\Desktop\Issam\PIXITREND-CLASSIFIER\Annotation-Training\env\lib\site-packages\thinc\model.py", line 291, in __call__
return self._func(self, X, is_train=is_train)
File "C:\Users\evead\Desktop\Issam\PIXITREND-CLASSIFIER\Annotation-Training\env\lib\site-packages\thinc\layers\pytorchwrapper.py", line 134, in forward
Ytorch, torch_backprop = model.shims[0](Xtorch, is_train)
File "C:\Users\evead\Desktop\Issam\PIXITREND-CLASSIFIER\Annotation-Training\env\lib\site-packages\thinc\shims\pytorch.py", line 56, in __call__
return self.predict(inputs), lambda a: ...
File "C:\Users\evead\Desktop\Issam\PIXITREND-CLASSIFIER\Annotation-Training\env\lib\site-packages\thinc\shims\pytorch.py", line 66, in predict
outputs = self._model(*inputs.args, **inputs.kwargs)
File "C:\Users\evead\Desktop\Issam\PIXITREND-CLASSIFIER\Annotation-Training\env\lib\site-packages\torch\nn\modules\module.py", line 1102, in _call_impl
return forward_call(*input, **kwargs)
File "C:\Users\evead\Desktop\Issam\PIXITREND-CLASSIFIER\Annotation-Training\env\lib\site-packages\torch\nn\modules\rnn.py", line 689, in forward
self.check_forward_args(input, hx, batch_sizes)
File "C:\Users\evead\Desktop\Issam\PIXITREND-CLASSIFIER\Annotation-Training\env\lib\site-packages\torch\nn\modules\rnn.py", line 632, in check_forward_args
self.check_input(input, batch_sizes)
File "C:\Users\evead\Desktop\Issam\PIXITREND-CLASSIFIER\Annotation-Training\env\lib\site-packages\torch\nn\modules\rnn.py", line 201, in check_input
raise RuntimeError(
RuntimeError: input must have 3 dimensions, got 2
I'm trying to get the predictions inside the function on_epoch_end of keras' Callback.
At the moment, to get the predictions, I execute self.model.predict with batch_size of 2, but at the 3rd epochs I get this error:
RuntimeError: Dst tensor is not initialized in Tensorflow
Reading on the web, I notice that this error appears when the GPU goes out of memory. In my case, reading the stack trace, this error is triggered by self.model.predict inside on_epoch_end, it says:
File "mlp_keras.py", line 20, in on_epoch_end predictions =
self.model.predict(self.dataset)
This is the full stack trace:
Traceback (most recent call last):
File "mlp_keras.py", line 150, in <module>
callbacks=[KendallTauHistory(training_dataset, training_dataset_labels, groups_id_count)])
File "/usr/home/studenti/sp171412/word_ordering/mlp/env/lib/python2.7/site-packages/tensorflow_core/python/keras/engine/training.py", line 819, in fit
use_multiprocessing=use_multiprocessing)
File "/usr/home/studenti/sp171412/word_ordering/mlp/env/lib/python2.7/site-packages/tensorflow_core/python/keras/engine/training_v2.py", line 397, in fit
prefix='val_')
File "/usr/lib64/python2.7/contextlib.py", line 24, in __exit__
self.gen.next()
File "/usr/home/studenti/sp171412/word_ordering/mlp/env/lib/python2.7/site-packages/tensorflow_core/python/keras/engine/training_v2.py", line 771, in on_epoch
self.callbacks.on_epoch_end(epoch, epoch_logs)
File "/usr/home/studenti/sp171412/word_ordering/mlp/env/lib/python2.7/site-packages/tensorflow_core/python/keras/callbacks.py", line 302, in on_epoch_end
callback.on_epoch_end(epoch, logs)
File "mlp_keras.py", line 20, in on_epoch_end
predictions = self.model.predict(self.dataset)
File "/usr/home/studenti/sp171412/word_ordering/mlp/env/lib/python2.7/site-packages/tensorflow_core/python/keras/engine/training.py", line 1013, in predict
use_multiprocessing=use_multiprocessing)
File "/usr/home/studenti/sp171412/word_ordering/mlp/env/lib/python2.7/site-packages/tensorflow_core/python/keras/engine/training_v2.py", line 498, in predict
workers=workers, use_multiprocessing=use_multiprocessing, **kwargs)
File "/usr/home/studenti/sp171412/word_ordering/mlp/env/lib/python2.7/site-packages/tensorflow_core/python/keras/engine/training_v2.py", line 426, in _model_iteration
use_multiprocessing=use_multiprocessing)
File "/usr/home/studenti/sp171412/word_ordering/mlp/env/lib/python2.7/site-packages/tensorflow_core/python/keras/engine/training_v2.py", line 706, in _process_inputs
use_multiprocessing=use_multiprocessing)
File "/usr/home/studenti/sp171412/word_ordering/mlp/env/lib/python2.7/site-packages/tensorflow_core/python/keras/engine/data_adapter.py", line 357, in __init__
dataset = self.slice_inputs(indices_dataset, inputs)
File "/usr/home/studenti/sp171412/word_ordering/mlp/env/lib/python2.7/site-packages/tensorflow_core/python/keras/engine/data_adapter.py", line 383, in slice_inputs
dataset_ops.DatasetV2.from_tensors(inputs).repeat()
File "/usr/home/studenti/sp171412/word_ordering/mlp/env/lib/python2.7/site-packages/tensorflow_core/python/data/ops/dataset_ops.py", line 566, in from_tensors
return TensorDataset(tensors)
File "/usr/home/studenti/sp171412/word_ordering/mlp/env/lib/python2.7/site-packages/tensorflow_core/python/data/ops/dataset_ops.py", line 2765, in __init__
element = structure.normalize_element(element)
File "/usr/home/studenti/sp171412/word_ordering/mlp/env/lib/python2.7/site-packages/tensorflow_core/python/data/util/structure.py", line 113, in normalize_element
ops.convert_to_tensor(t, name="component_%d" % i))
File "/usr/home/studenti/sp171412/word_ordering/mlp/env/lib/python2.7/site-packages/tensorflow_core/python/framework/ops.py", line 1314, in convert_to_tensor
ret = conversion_func(value, dtype=dtype, name=name, as_ref=as_ref)
File "/usr/home/studenti/sp171412/word_ordering/mlp/env/lib/python2.7/site-packages/tensorflow_core/python/framework/tensor_conversion_registry.py", line 52, in _default_conversion_function
return constant_op.constant(value, dtype, name=name)
File "/usr/home/studenti/sp171412/word_ordering/mlp/env/lib/python2.7/site-packages/tensorflow_core/python/framework/constant_op.py", line 258, in constant
allow_broadcast=True)
File "/usr/home/studenti/sp171412/word_ordering/mlp/env/lib/python2.7/site-packages/tensorflow_core/python/framework/constant_op.py", line 266, in _constant_impl
t = convert_to_eager_tensor(value, ctx, dtype)
File "/usr/home/studenti/sp171412/word_ordering/mlp/env/lib/python2.7/site-packages/tensorflow_core/python/framework/constant_op.py", line 96, in convert_to_eager_tensor
return ops.EagerTensor(value, ctx.device_name, dtype)
RuntimeError: Dst tensor is not initialized.
My question is: is there a way to get the predictions without performing predict inside on_epoch_end? Thanks in advance.
Alright, after seeing your last comment, what you could do:
epochs = 100
for epoch in range(epochs):
model.fit(x_train, y_train)
y_predict = model.predict(x_test)
I'm trying to fine tune resnet50 in half precision mode without success. It seems there are parts of model which are not compatible with float16. Here is my code:
dtype='float16'
K.set_floatx(dtype)
K.set_epsilon(1e-4)
model = Sequential()
model.add(ResNet50(weights='imagenet', include_top=False, pooling='avg'))
and I get this error:
Traceback (most recent call last):
File "train_resnet.py", line 40, in <module>
model.add(ResNet50(weights='imagenet', include_top=False, pooling='avg'))
File "/usr/local/lib/python3.6/dist-packages/keras/applications/__init__.py", line 28, in wrapper
return base_fun(*args, **kwargs)
File "/usr/local/lib/python3.6/dist-packages/keras/applications/resnet50.py", line 11, in ResNet50
return resnet50.ResNet50(*args, **kwargs)
File "/usr/local/lib/python3.6/dist-packages/keras_applications/resnet50.py", line 231, in ResNet50
x = layers.BatchNormalization(axis=bn_axis, name='bn_conv1')(x)
File "/usr/local/lib/python3.6/dist-packages/keras/engine/base_layer.py", line 457, in __call__
output = self.call(inputs, **kwargs)
File "/usr/local/lib/python3.6/dist-packages/keras/layers/normalization.py", line 185, in call
epsilon=self.epsilon)
File "/usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py", line 1864, in normalize_batch_in_training
epsilon=epsilon)
File "/usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py", line 1839, in _fused_normalize_batch_in_training
data_format=tf_data_format)
File "/usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/nn_impl.py", line 1329, in fused_batch_norm
name=name)
File "/usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/gen_nn_ops.py", line 4488, in fused_batch_norm_v2
name=name)
File "/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/op_def_library.py", line 626, in _apply_op_helper
param_name=input_name)
File "/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/op_def_library.py", line 60, in _SatisfiesTypeConstraint
", ".join(dtypes.as_dtype(x).name for x in allowed_list)))
TypeError: Value passed to parameter 'scale' has DataType float16 not in list of allowed values: float32
This was a reported bug and upgrading to Keras==2.2.5 solved the issue.