I'm trying to record some values with tensorboard writer with the following lines. If I leave the commented section commented, then the code runs fine. But when I uncomment it, I get an error. Any advice would be appreciated.
if (tb_writer != None):
for key, value in qps.items():
tb_writer.add_scalar("QPv2_" + mode + "_Threshold_" + str(key), value, global_step)
# for annotation_idx in range(1, args.num_classes-1):
# annotation_class = annotation_classes[annotation_idx]
# for threshold_idx in range(len(annotation_thresholds)):
# threshold = annotation_thresholds[threshold_idx]
# tb_writer.add_scalar(annotation_class + "_Threshold_" + str(threshold) + "_Precision", str(precision[threshold_idx][annotation_idx].item()), global_step)
# tb_writer.add_scalar(annotation_class + "_Threshold_" + str(threshold) + "_Recall", str(recall[threshold_idx][annotation_idx].item()), global_step)
Traceback (most recent call last):
File "train_transformer_v5.py", line 782, in <module>
main()
File "train_transformer_v5.py", line 763, in main
global_step, tr_loss, metrics_result = trainer.train(args)
File "train_transformer_v5.py", line 451, in train
tb_writer=self.tb_writer, global_step=self.global_step)
File "/home/xianx/model/metrics_v5.py", line 606, in eval_metrics
tb_writer.add_scalar(annotation_class + "_Threshold_" + str(threshold) + "_Precision", str(precision[threshold_idx][annotation_idx].item()), global_step)
File "/opt/conda/lib/python3.6/site-packages/torch/utils/tensorboard/writer.py", line 348, in add_scalar
scalar_value = workspace.FetchBlob(scalar_value)
File "/opt/conda/lib/python3.6/site-packages/caffe2/python/workspace.py", line 379, in FetchBlob
result = C.fetch_blob(StringifyBlobName(name))
RuntimeError: [enforce fail at pybind_state.cc:221] ws->HasBlob(name). Can't find blob: 0.7158585786819458
My issue was that I was using .add_scalar(...) while converting my value to a string. Removing that conversion fixed the problem.
Related
Currently working on a portfolio code, and I am trying to print an array but I keep getting hit with
w = np.random.random((1000, len(symbols)))
w = (w.T / w.sum(axis=1)).T
print(w[:5])
Traceback (most recent call last):
File "<ipython-input-23-246da7acc0b7>", line 3, in <module>
print(w[:5])
File "C:\Users\godso\anaconda3\lib\site-packages\numpy\core\arrayprint.py", line 1506, in _array_str_implementation
return array2string(a, max_line_width, precision, suppress_small, ' ', "")
File "C:\Users\godso\anaconda3\lib\site-packages\numpy\core\arrayprint.py", line 712, in array2string
return _array2string(a, options, separator, prefix)
File "C:\Users\godso\anaconda3\lib\site-packages\numpy\core\arrayprint.py", line 484, in wrapper
return f(self, *args, **kwargs)
File "C:\Users\godso\anaconda3\lib\site-packages\numpy\core\arrayprint.py", line 510, in _array2string
format_function = _get_format_function(data, **options)
File "C:\Users\godso\anaconda3\lib\site-packages\numpy\core\arrayprint.py", line 431, in _get_format_function
formatdict = _get_formatdict(data, **options)
File "C:\Users\godso\anaconda3\lib\site-packages\numpy\core\arrayprint.py", line 403, in _get_formatdict
fkeys = [k for k in formatter.keys() if formatter[k] is not None]
AttributeError: 'set' object has no attribute 'keys'
I have the symbols defined and all I am trying to do is the print the code below, note I have way more code but I didn't include it because everything worked fine up until now.
w = np.random.random((1000, len(symbols)))
w = (w.T / w.sum(axis=1)).T
print(w[:5])
I am trying to run the following strategy:
def max_n(array, n):
return np.argpartition(array, -n)[-n:]
class CrossSectionalMR(bt.Strategy):
params = (
('num_positions', 100),
)
def __init__(self, temp):
self.inds = {}
for d in self.datas:
self.inds[d] = {}
self.inds[d]["pct"] = bt.indicators.PercentChange(d.close, period=1)
def prenext(self):
self.next()
def next(self):
available = list(filter(lambda d: len(d), self.datas)) # only look at data that existed yesterday
rets = np.zeros(len(available))
for i, d in enumerate(available):
rets[i] = self.inds[d]['pct'][0]
market_ret = np.mean(rets)
weights = -(rets - market_ret)
max_weights_index = max_n(np.abs(weights), self.params.num_positions)
max_weights = weights[max_weights_index]
weights = weights / np.sum(np.abs(max_weights))
for i, d in enumerate(available):
if i in max_weights_index:
self.order_target_percent(d, target=weights[i])
else:
self.order_target_percent(d, 0)
The full error is:
Traceback (most recent call last):
File "/home/poblivsig/Software/pycharm-2020.3.1/plugins/python/helpers/pydev/pydevd.py", line 1477, in _exec
pydev_imports.execfile(file, globals, locals) # execute the script
File "/home/poblivsig/Software/pycharm-2020.3.1/plugins/python/helpers/pydev/_pydev_imps/_pydev_execfile.py", line 18, in execfile
exec(compile(contents+"\n", file, 'exec'), glob, loc)
File "/home/poblivsig/Dropbox/meanrev/main.py", line 190, in <module>
dd, cagr, sharpe = backtest(datas, CrossSectionalMR, plot=True, num_positions=100)
File "/home/poblivsig/Dropbox/meanrev/main.py", line 181, in backtest
results = cerebro.run()
File "/home/poblivsig/Dropbox/meanrev/venv/lib/python3.8/site-packages/backtrader/cerebro.py", line 1127, in run
runstrat = self.runstrategies(iterstrat)
File "/home/poblivsig/Dropbox/meanrev/venv/lib/python3.8/site-packages/backtrader/cerebro.py", line 1293, in runstrategies
self._runonce(runstrats)
File "/home/poblivsig/Dropbox/meanrev/venv/lib/python3.8/site-packages/backtrader/cerebro.py", line 1652, in _runonce
strat._once()
File "/home/poblivsig/Dropbox/meanrev/venv/lib/python3.8/site-packages/backtrader/lineiterator.py", line 297, in _once
indicator._once()
File "/home/poblivsig/Dropbox/meanrev/venv/lib/python3.8/site-packages/backtrader/lineiterator.py", line 297, in _once
indicator._once()
File "/home/poblivsig/Dropbox/meanrev/venv/lib/python3.8/site-packages/backtrader/linebuffer.py", line 630, in _once
self.oncestart(self._minperiod - 1, self._minperiod)
File "/home/poblivsig/Dropbox/meanrev/venv/lib/python3.8/site-packages/backtrader/lineroot.py", line 165, in oncestart
self.once(start, end)
File "/home/poblivsig/Dropbox/meanrev/venv/lib/python3.8/site-packages/backtrader/linebuffer.py", line 672, in once
dst[i] = src[i + ago]
IndexError: array assignment index out of range
python-BaseExceptio
Any help would be greatly appreciated.
I grab the data from Yahoo and store it in csv files which are then loaded up and added to Cerebro. Sometimes, the code cannot get the full list of the SPY, but I don't think that is the problem here.
I'm trying to run this code: https://github.com/aitorzip/PyTorch-CycleGAN
I modified only the dataloader and transforms to be compatible with my data.
When trying to run it I get this error:
Traceback (most recent call last):
File "models/CycleGANs/train",
line 150, in
loss_D_A.backward()
File "/opt/conda/lib/python3.8/site-packages/torch/tensor.py", line 221, in
backward
torch.autograd.backward(self, gradient, retain_graph, create_graph)
File
"/opt/conda/lib/python3.8/site-packages/torch/autograd/init.py",
line 130, in backward
Variable._execution_engine.run_backward(
RuntimeError: Trying to backward through the graph a second time, but the saved intermediate
results have already been freed. Specify retain_graph=True when
calling backward the first time.
This is the train loop up to the point of error:
for epoch in range(opt.epoch, opt.n_epochs):
for i, batch in enumerate(dataloader):
# Set model input
real_A = Variable(input_A.copy_(batch['A']))
real_B = Variable(input_B.copy_(batch['B']))
##### Generators A2B and B2A #####
optimizer_G.zero_grad()
# Identity loss
# G_A2B(B) should equal B if real B is fed
same_B = netG_A2B(real_B)
loss_identity_B = criterion_identity(same_B, real_B)*5.0
# G_B2A(A) should equal A if real A is fed
same_A = netG_B2A(real_A)
loss_identity_A = criterion_identity(same_A, real_A)*5.0
# GAN loss
fake_B = netG_A2B(real_A)
pred_fake = netD_B(fake_B)
loss_GAN_A2B = criterion_GAN(pred_fake, target_real)
fake_A = netG_B2A(real_B)
pred_fake = netD_A(fake_A)
loss_GAN_B2A = criterion_GAN(pred_fake, target_real)
# Cycle loss
# TODO: cycle loss doesn't allow for multimodality. I leave it for now but needs to be thrown out later
recovered_A = netG_B2A(fake_B)
loss_cycle_ABA = criterion_cycle(recovered_A, real_A)*10.0
recovered_B = netG_A2B(fake_A)
loss_cycle_BAB = criterion_cycle(recovered_B, real_B)*10.0
# Total loss
loss_G = loss_identity_A + loss_identity_B + loss_GAN_A2B + loss_GAN_B2A + loss_cycle_ABA + loss_cycle_BAB
loss_G.backward()
optimizer_G.step()
##### Discriminator A #####
optimizer_D_A.zero_grad()
# Real loss
pred_real = netD_A(real_A)
loss_D_real = criterion_GAN(pred_real, target_real)
# Fake loss
fake_A = fake_A_buffer.push_and_pop(fake_A)
pred_fale = netD_A(fake_A.detach())
loss_D_fake = criterion_GAN(pred_fake, target_fake)
# Total loss
loss_D_A = (loss_D_real + loss_D_fake)*0.5
loss_D_A.backward()
I am not familiar at all what it means. My guess is it's something to do with fake_A_buffer. It's just a fake_A_buffer = ReplayBuffer()
class ReplayBuffer():
def __init__(self, max_size=50):
assert (max_size > 0), 'Empty buffer or trying to create a black hole. Be careful.'
self.max_size = max_size
self.data = []
def push_and_pop(self, data):
to_return = []
for element in data.data:
element = torch.unsqueeze(element, 0)
if len(self.data) < self.max_size:
self.data.append(element)
to_return.append(element)
else:
if random.uniform(0,1) > 0.5:
i = random.randint(0, self.max_size-1)
to_return.append(self.data[i].clone())
self.data[i] = element
else:
to_return.append(element)
return Variable(torch.cat(to_return))
Error after setting `loss_G.backward(retain_graph=True)
Traceback (most recent call last): File "models/CycleGANs/train",
line 150, in
loss_D_A.backward() File "/opt/conda/lib/python3.8/site-packages/torch/tensor.py", line 221, in
backward
torch.autograd.backward(self, gradient, retain_graph, create_graph) File
"/opt/conda/lib/python3.8/site-packages/torch/autograd/init.py",
line 130, in backward
Variable._execution_engine.run_backward( RuntimeError: one of the variables needed for gradient computation has been modified by an
inplace operation: [torch.FloatTensor [3, 64, 7, 7]] is at version 2;
expected version 1 instead. Hint: enable anomaly detection to find the
operation that failed to compute its gradient, with
torch.autograd.set_detect_anomaly(True).
And after setting torch.autograd.set_detect_anomaly(True)
/opt/conda/lib/python3.8/site-packages/torch/autograd/init.py:130:
UserWarning: Error detected in MkldnnConvolutionBackward. Traceback of
forward call that caused the error:
File "models/CycleGANs/train",
line 115, in
fake_B = netG_A2B(real_A)
File "/opt/conda/lib/python3.8/site-packages/torch/nn/modules/module.py",
line 727, in _call_impl
result = self.forward(*input, **kwargs)
File "/home/Histology-Style-Transfer-Research/models/CycleGANs/models.py",
line 67, in forward
return self.model(x)
File "/opt/conda/lib/python3.8/site-packages/torch/nn/modules/module.py",
line 727, in _call_impl
result = self.forward(*input, **kwargs)
File "/opt/conda/lib/python3.8/site-packages/torch/nn/modules/container.py",
line 117, in forward
input = module(input)
File "/opt/conda/lib/python3.8/site-packages/torch/nn/modules/module.py",
line 727, in _call_impl
result = self.forward(*input, **kwargs)
File "/home/Histology-Style-Transfer-Research/models/CycleGANs/models.py",
line 19, in forward
return x + self.conv_block(x)
File "/opt/conda/lib/python3.8/site-packages/torch/nn/modules/module.py",
line 727, in _call_impl
result = self.forward(*input, **kwargs)
File "/opt/conda/lib/python3.8/site-packages/torch/nn/modules/container.py",
line 117, in forward
input = module(input)
File "/opt/conda/lib/python3.8/site-packages/torch/nn/modules/module.py",
line 727, in _call_impl
result = self.forward(*input, **kwargs)
File "/opt/conda/lib/python3.8/site-packages/torch/nn/modules/conv.py",
line 423, in forward
return self._conv_forward(input, self.weight)
File "/opt/conda/lib/python3.8/site-packages/torch/nn/modules/conv.py",
line 419, in _conv_forward
return F.conv2d(input, weight, self.bias, self.stride, (Triggered internally at
/opt/conda/conda-bld/pytorch_1603729096996/work/torch/csrc/autograd/python_anomaly_mode.cpp:104.)
Variable._execution_engine.run_backward(
Traceback (most recent call
last): File "models/CycleGANs/train", line 133, in
loss_G.backward(retain_graph=True)
File "/opt/conda/lib/python3.8/site-packages/torch/tensor.py", line 221, in
backward
torch.autograd.backward(self, gradient, retain_graph, create_graph)
File
"/opt/conda/lib/python3.8/site-packages/torch/autograd/init.py",
line 130, in backward
Variable._execution_engine.run_backward( RuntimeError: Function 'MkldnnConvolutionBackward' returned nan values in its 2th output.
loss_G.backward() should be loss_G.backward(retain_graph=True) this is because when you use backward normally it doesn't record the operations it performs in the backward pass, retain_graph=True is telling to do so.
I have a deep network using Keras and I need to apply cropping on the output of one layer and then send to the next layer. for this aim, I write the following code as a lambda layer:
def cropping_fillzero(img, rate=0.6): # Q = percentage
residual_shape = img.get_shape().as_list()
h, w = residual_shape[1:3]
blacked_pixels = int((rate) * (h*w))
ratio = int(np.floor(np.sqrt(blacked_pixels)))
# width = int(np.ceil(np.sqrt(blacked_pixels)))
x = np.random.randint(0, w - ratio)
y = np.random.randint(0, h - ratio)
cropingImg = img[y:y+ratio, x:x+ratio].assign(tf.zeros([ratio, ratio]))
return cropingImg
decoded_noise = layers.Lambda(cropping_fillzero, arguments={'rate':residual_cropRate}, name='residual_cropout_attack')(bncv11)
but it produces the following error and I do not know why?!
Traceback (most recent call last):
File "", line 1, in
runfile('E:/code_v28-7-19/CIFAR_los4x4convolvedw_5_cropAttack_RandomSize_Pad.py',
wdir='E:/code_v28-7-19')
File
"D:\software\Anaconda3\envs\py36\lib\site-packages\spyder_kernels\customize\spydercustomize.py",
line 704, in runfile
execfile(filename, namespace)
File
"D:\software\Anaconda3\envs\py36\lib\site-packages\spyder_kernels\customize\spydercustomize.py",
line 108, in execfile
exec(compile(f.read(), filename, 'exec'), namespace)
File
"E:/code_v28-7-19/CIFAR_los4x4convolvedw_5_cropAttack_RandomSize_Pad.py",
line 143, in
decoded_noise = layers.Lambda(cropping_fillzero, arguments={'rate':residual_cropRate},
name='residual_cropout_attack')(bncv11)
File
"D:\software\Anaconda3\envs\py36\lib\site-packages\keras\engine\base_layer.py",
line 457, in call
output = self.call(inputs, **kwargs)
File
"D:\software\Anaconda3\envs\py36\lib\site-packages\keras\layers\core.py",
line 687, in call
return self.function(inputs, **arguments)
File
"E:/code_v28-7-19/CIFAR_los4x4convolvedw_5_cropAttack_RandomSize_Pad.py",
line 48, in cropping_fillzero
cropingImg = img[y:y+ratio, x:x+ratio].assign(tf.zeros([ratio, ratio]))
File
"D:\software\Anaconda3\envs\py36\lib\site-packages\tensorflow\python\ops\array_ops.py",
line 700, in assign
raise ValueError("Sliced assignment is only supported for variables")
ValueError: Sliced assignment is only supported for variables
could you please tell me how can I solve this error?
Thank you
h, w = residual_shape[1:3]
I'm not entirely sure what you're trying to do here, but Python interprets this as 'return between the 2nd element and the 4th'. Maybe you mean residual_shape[1], residual_shape[3]?
cropingImg = img[y:y+ratio, x:x+ratio].assign(tf.zeros([ratio, ratio]))
You are trying to use slice notation on a tensor. Slice notation may only be used on variables, as the error message says. In order to get the actual variable, try loading the previous layer output using tf.identity():
self.output = your_layer
self.output = tf.identity(self.output, name='output')
output = graph.get_tensor_by_name('output:0')
I have a TensorBoard embedding that is created and saved like this from a gensim Doc2Vec model:
embedding = model.docvecs.vectors_docs
tf.reset_default_graph()
sess = tf.InteractiveSession()
X = tf.Variable([0.0], name='embedding')
place = tf.placeholder(tf.float32, shape=embedding.shape)
set_x = tf.assign(X, place, validate_shape=False)
sess.run(tf.global_variables_initializer())
sess.run(set_x, feed_dict={place: embedding})
summary_writer = tf.summary.FileWriter('log', sess.graph)
config = projector.ProjectorConfig()
embedding_conf = config.embeddings.add()
embedding_conf.tensor_name = 'embedding:0'
embedding_conf.metadata_path = os.path.join('log','metadata.tsv')
projector.visualize_embeddings(summary_writer, config)
saver = tf.train.Saver([X])
saver.save(sess, os.path.join('log', 'model.ckpt'), 1)
Results in this error:
TensorBoard 1.5.1 at http://COMPUTER_NAME:6006 (Press CTRL+C to quit)
E0222 17:15:20.231085 Thread-1 _internal.py:88] Error on request:
Traceback (most recent call last):
File "c:\users\user\_installed\anaconda\envs\docsim\lib\site-packages\werkzeug
\serving.py", line 270, in run_wsgi
execute(self.server.app)
File "c:\users\user\_installed\anaconda\envs\docsim\lib\site-packages\werkzeug
\serving.py", line 258, in execute
application_iter = app(environ, start_response)
File "c:\users\user\_installed\anaconda\envs\docsim\lib\site-packages\tensorbo
ard\backend\application.py", line 271, in __call__
return self.data_applications[clean_path](environ, start_response)
File "c:\users\user\_installed\anaconda\envs\docsim\lib\site-packages\werkzeug
\wrappers.py", line 308, in application
resp = f(*args[:-2] + (request,))
File "c:\users\user\_installed\anaconda\envs\docsim\lib\site-packages\tensorbo
ard\plugins\projector\projector_plugin.py", line 514, in _serve_metadata
for line in f:
File "c:\users\user\_installed\anaconda\envs\docsim\lib\site-packages\tensorfl
ow\python\lib\io\file_io.py", line 214, in __next__
return self.next()
File "c:\users\user\_installed\anaconda\envs\docsim\lib\site-packages\tensorfl
ow\python\lib\io\file_io.py", line 208, in next
retval = self.readline()
File "c:\users\user\_installed\anaconda\envs\docsim\lib\site-packages\tensorfl
ow\python\lib\io\file_io.py", line 178, in readline
return self._prepare_value(self._read_buf.ReadLineAsString())
File "c:\users\user\_installed\anaconda\envs\docsim\lib\site-packages\tensorfl
ow\python\lib\io\file_io.py", line 94, in _prepare_value
return compat.as_str_any(val)
File "c:\users\user\_installed\anaconda\envs\docsim\lib\site-packages\tensorfl
ow\python\util\compat.py", line 106, in as_str_any
return as_str(value)
File "c:\users\user\_installed\anaconda\envs\docsim\lib\site-packages\tensorfl
ow\python\util\compat.py", line 84, in as_text
return bytes_or_text.decode(encoding)
UnicodeDecodeError: 'utf-8' codec can't decode byte 0x92 in position 84: invalid
start byte
When I remove the embedding_conf.metadata_path = os.path.join('log','metadata.tsv') line, no error occurs. In addition, I can then select the very same metadata file that TensorFlow was attempting to bind to the embedding when the error occurred.
Why is this error occurring?
Is it possible that the data inside your metadata.tsv file isn't UTF-8 encoded?