I am trying to make a network using 3x64x64 image at pytorch environment, and it seems that I succeeded in training my network and save it. The network looks like :
class LC_small(nn.Module):
def __init__(self,c_in,c_out = 256):
super(LC_small,self).__init__()
self.conv1 = conv(c_in,64,k=3,stride=1,pad=1)
self.conv2 = conv(64, 128, k=3, stride=2, pad=1)
self.conv3 = conv(128, 128, k=3, stride=1, pad=1)
self.conv4 = conv(128, 128, k=3, stride=2, pad=1)
self.conv5 = conv(128, 128, k=3, stride=1, pad=1)
self.conv6 = conv(128, 256, k=3, stride=2, pad=1)
self.conv7 = conv(256, 256, k=3, stride=1, pad=1)# int(h/8 x w/8 x 256)
self.flat = dense(int(w_rsz/8)*int(h_rsz/8)*256,256)
self.dense1 = dense(256,128,False)
self.dense2 = dense(128,3,False)
def forward(self, input):
out = self.conv1(input)
out = self.conv2(out)
out = self.conv3(out)
out = self.conv4(out)
out = self.conv5(out)
out = self.conv6(out)
out = self.conv7(out)
out = out.view(out.size(0),-1)
out = self.flat(out)
out = self.dense1(out)
out = self.dense2(out)
# print(out.shape)
normal = torch.nn.functional.normalize(out, 2, 1)
return normal
And I saved my model while Training :
for epoch in range(10):
# continue # 현재 Training 됐다고 가정하고
total_loss = 0
route_param = open(route_diffuse+'/netparam.txt','w')
for param in lcnet.state_dict():
route_param.write(str(param)+'\t'+str(lcnet.state_dict()[param].size())+'\n')
for i,data in enumerate(load_LC,0):
input, gtval = data[0].to(dev),data[1].to(dev)
opt.zero_grad()
output = lcnet(input)
loss = crit(output,gtval)
loss.backward()
opt.step()
total_loss +=loss.item()
if i%10 == 9:
print(epoch,i,total_loss/10)
torch.save(lcnet,route_save)
total_loss = 0
However, by the time I try to load the network I made I saw a error message as following :
Traceback (most recent call last):
File "E:/DLPrj/venv/torch_practice.py", line 324, in <module>
ipl,npl = getseqi_np(sq_t,lcnet) # data : 8 x 6 x w x h
File "E:/DLPrj/venv/torch_practice.py", line 133, in getseqi_np
l1 = net_lc(torch.from_numpy(i1r))
File "E:\DLPrj\venv\lib\site-packages\torch\nn\modules\module.py", line 541, in __call__
result = self.forward(*input, **kwargs)
File "E:/DLPrj/venv/torch_practice.py", line 216, in forward
out = self.conv1(input)
File "E:\DLPrj\venv\lib\site-packages\torch\nn\modules\module.py", line 541, in __call__
result = self.forward(*input, **kwargs)
File "E:\DLPrj\venv\lib\site-packages\torch\nn\modules\container.py", line 92, in forward
input = module(input)
File "E:\DLPrj\venv\lib\site-packages\torch\nn\modules\module.py", line 541, in __call__
result = self.forward(*input, **kwargs)
File "E:\DLPrj\venv\lib\site-packages\torch\nn\modules\conv.py", line 345, in forward
return self.conv2d_forward(input, self.weight)
File "E:\DLPrj\venv\lib\site-packages\torch\nn\modules\conv.py", line 342, in conv2d_forward
self.padding, self.dilation, self.groups)
RuntimeError: Expected 4-dimensional input for 4-dimensional weight 64 3 3 3, but got 3-dimensional input of size [64, 64, 3] instead
After this error pycharm freezes and I cannot re-run this code until I restart the pycharm.
When I train my network, I also get some warning messages :
E:\DLPrj\venv\lib\site-packages\torch\serialization.py:292: UserWarning: Couldn't retrieve source code for container of type LC_small. It won't be checked for correctness upon loading.
"type " + obj.__name__ + ". It won't be checked "
E:\DLPrj\venv\lib\site-packages\torch\serialization.py:292: UserWarning: Couldn't retrieve source code for container of type Sequential. It won't be checked for correctness upon loading.
"type " + obj.__name__ + ". It won't be checked "
E:\DLPrj\venv\lib\site-packages\torch\serialization.py:292: UserWarning: Couldn't retrieve source code for container of type Conv2d. It won't be checked for correctness upon loading.
"type " + obj.__name__ + ". It won't be checked "
E:\DLPrj\venv\lib\site-packages\torch\serialization.py:292: UserWarning: Couldn't retrieve source code for container of type BatchNorm2d. It won't be checked for correctness upon loading.
"type " + obj.__name__ + ". It won't be checked "
E:\DLPrj\venv\lib\site-packages\torch\serialization.py:292: UserWarning: Couldn't retrieve source code for container of type LeakyReLU. It won't be checked for correctness upon loading.
"type " + obj.__name__ + ". It won't be checked "
E:\DLPrj\venv\lib\site-packages\torch\serialization.py:292: UserWarning: Couldn't retrieve source code for container of type Linear. It won't be checked for correctness upon loading.
"type " + obj.__name__ + ". It won't be checked "
I cannot understand why the input size the network takes suddenly changes, or why it saved my network wrong. Please check my problem and thank you very much.
So your first error message is because torch.from_numpy(i1r) has the wrong shape. You need to do
np.expand_dims(i1r.transpose(2,0,1), axis=0)
Then it'll get processed correctly. This is because it expects a batch dimension and you aren't providing one along with the channels being in the first dimension not the last.
As for your second error message it's probably because you incorrectly defined conv, and dense so it messes up when saving the model.
Related
I am trying to immplement the VGG16 network using Tensorflow.
to test the model, i want to classify a dataset of images.
i sarted by creating a train_data with tensorflow.data.Dataset:
<TensorSliceDataset element_spec=(TensorSpec(shape=(215, 160, 3), dtype=tf.float64, name=None), TensorSpec(shape=(20,), dtype=tf.float64, name=None))>
(their is 20 ouput classes)
created a custom 2d Conv Layer:
`class CustomConv2d(Layer):
def __init__(self,filters,kernel_size,padding,name):
super(CustomConv2d,self).__init__()
self.conv = Conv2D(filters=filters,
kernel_size=kernel_size,
activation='relu',
padding =padding,
name=name,
)
self.batchN = BatchNormalization()
def call(self,x,training=True):
output = self.conv(x)
output = self.batchN(output)
return output`
Created a Sub Model Class
`class VGG16(Model):
def __init__(self,input_shape,NUM_OF_CLASSES=20,dropout_parameters=0.5):
super(VGG16,self).__init__()
self.dropout = Dropout(dropout_parameters)
### First Conv Block
self.conv_11 = Conv2D(filters=53,
kernel_size=(3,3),
activation='relu',
padding ='same',
name='conv11',
input_shape=input_shape
)
self.conv_12 = CustomConv2d(64,(3,3),padding='same',name='conv_12')
self.maxpool = MaxPool2D(pool_size=(2,2),padding='same')`
## Second Conv Block
self.conv21 = CustomConv2d(64,(3,3),padding='same',name='conv_21')
self.conv22 = CustomConv2d(64,(3,3),padding='same',name='conv_22')
## Third Conv Block
self.conv31 = CustomConv2d(256,(3,3),padding='same',name='conv_31')
self.conv32 = CustomConv2d(256,(3,3),padding='same',name='conv_32')
self.conv33 = CustomConv2d(256,(3,3),padding='same',name='conv_33')
## Fourth Conv Block
self.conv41 = CustomConv2d(512,(3,3),padding='same',name='conv_41')
self.conv42 = CustomConv2d(512,(3,3),padding='same',name='conv_42')
self.conv43 = CustomConv2d(512,(3,3),padding='same',name='conv_43')
## Fifth CConv Block
self.conv51 = CustomConv2d(512,(3,3),padding='same',name='conv_51')
self.conv52 = CustomConv2d(512,(3,3),padding='same',name='conv_52')
self.conv53 = CustomConv2d(512,(3,3),padding='same',name='conv_53')
#####
self.flatten = Flatten()
self.dense1 = Dense(1024,activation='relu',name='Dense_1')
self.dense2 = Dense(512,activation='relu',name='Dense_2')
self.dense3 = Dense(NUM_OF_CLASSES,activation='softmax',name='Dense_3')
def call(self,x,training=True):
x = self.maxpool(self.conv_12(self.conv_11(x)))
x = self.maxpool(self.conv22(self.conv21(x)))
x = self.maxpool(self.conv33(self.conv32(self.conv31(x))))
x = self.maxpool(self.conv43(self.conv42(self.conv41(x))))
x = self.maxpool(self.conv53(self.conv52(self.conv51(x))))
x = self.flatten(x)
x = self.dense3(self.dense2(self.dense1(x)))
return x
model = VGG16((215,160,3,1))
`
by the way i dont know why i have to put 1 at the end of shape
After Compiling the model
when i try to fit the data to the model i have this error:
`ValueError: in user code:
` File "/usr/local/lib/python3.8/dist-packages/keras/engine/training.py", line 1021, in train_function *
return step_function(self, iterator)
File "/usr/local/lib/python3.8/dist-packages/keras/engine/training.py", line 1010, in step_function **
outputs = model.distribute_strategy.run(run_step, args=(data,))
File "/usr/local/lib/python3.8/dist-packages/keras/engine/training.py", line 1000, in run_step **
outputs = model.train_step(data)
File "/usr/local/lib/python3.8/dist-packages/keras/engine/training.py", line 859, in train_step
y_pred = self(x, training=True)
File "/usr/local/lib/python3.8/dist-packages/keras/utils/traceback_utils.py", line 67, in error_handler
raise e.with_traceback(filtered_tb) from None
ValueError: Exception encountered when calling layer "vgg16_2" (type VGG16).
in user code:
File "/tmp/ipykernel_49926/980605695.py", line 38, in call *
x = self.maxpool(self.conv_12(self.conv_11(x)))
File "/usr/local/lib/python3.8/dist-packages/keras/utils/traceback_utils.py", line 67, in error_handler **
raise e.with_traceback(filtered_tb) from None
File "/usr/local/lib/python3.8/dist-packages/keras/engine/input_spec.py", line 228, in assert_input_compatibility
raise ValueError(f'Input {input_index} of layer "{layer_name}" '
ValueError: Input 0 of layer "conv11" is incompatible with the layer: expected min_ndim=4, found ndim=3. Full shape received: (215, 160, 3)
Call arguments received:
• x=tf.Tensor(shape=(215, 160, 3), dtype=float32)
• training=True``
Tought maybe their is problem in the shape of the Dataset
Edit:
the problem was solved after spliting the data into batches with:
trainDataset = trainDataset.shuffle(buffer_size=20).prefetch(buffer_size=15).batch(32)
but now the training is very slow. is it normal to take about 5 sec on each image of the batch?
It should be
model = VGG16((215,160,3))
Because conv2d input shape parameter take 3 values. (Height,width,channel)
you can input model like model(np.ones((1,215,160,3))) where 1 is the batch size
I am making a neural network to predict audio data (to learn more about how neural networks function and how to use tensorflow), and everything is going pretty smoothly so far, with one exception. I've looked around quite a bit to solve this issue and haven't been able to find anything specific enough to help me. I set up the dataset and model and those work fine, but for some reason when I try to train the model, it gives me a type error, even though all of the values in the dataset are 32 bit floats. It'd be much appreciated if someone could answer this for me, or at least push in the right direction to figuring this out. Code and console outputs are below. (BTW all values in dataset are between 0 and 1, I don't know if that's relevant but I thought I'd add that in)
EDIT: I've included the AudioHandler class as well, which you can use to reproduce the error. get_audio_array or get_audio_arrays can be used to convert a single mp3 or a directory of mp3s into array(s) of the audio data. You can also use dataset_from_arrays to generate a dataset from the audio arrays created with dataset_from_arrays.
from AudioHandler import AudioHandler
import os
seq_length = 22050
BATCH_SIZE = 64
BUFFER_SIZE = 10000
audio_arrays = AudioHandler.get_audio_arrays("AudioDataset", normalized=True)
dataset = AudioHandler.dataset_from_arrays(audio_arrays, seq_length, BATCH_SIZE, buffer_size=BUFFER_SIZE)
print(dataset)
rnn_units = 256
def build_model(rnn_units, batch_size):
model = tf.keras.Sequential([
tf.keras.layers.InputLayer(batch_input_shape=(batch_size, None, 2)),
tf.keras.layers.GRU(rnn_units, return_sequences=True, stateful=True),
tf.keras.layers.Dense(2)
])
return model
model = build_model(rnn_units, BATCH_SIZE)
model.summary()
model.compile(optimizer='adam', loss=tf.keras.losses.MeanSquaredError)
EPOCHS = 10
history = model.fit(dataset, epochs=EPOCHS)
from pydub import AudioSegment
import numpy as np
from pathlib import Path
from tensorflow import data
import os
class AudioHandler:
#staticmethod
def print_audio_info(file_name):
audio_segment = AudioSegment.from_file(file_name)
print("Information of '" + file_name + "':")
print("Sample rate: " + str(audio_segment.frame_rate) + "kHz")
# Multiply frame_width by 8 to get bits, since it is given in bytes
print("Sample width: " + str(audio_segment.frame_width * 8) + " bits per sample (" + str(
int(audio_segment.frame_width * 8 / audio_segment.channels)) + " bits per channel)")
print("Channels: " + str(audio_segment.channels))
#staticmethod
def get_audio_array(file_name, normalized=True):
audio_segment = AudioSegment.from_file(file_name)
# Get bytestring of raw audio data
raw_audio_bytestring = audio_segment.raw_data
# Adjust sample width to accommodate multiple channels in each sample
sample_width = audio_segment.frame_width / audio_segment.channels
# Convert bytestring to numpy array
if sample_width == 1:
raw_audio = np.array(np.frombuffer(raw_audio_bytestring, dtype=np.int8))
elif sample_width == 2:
raw_audio = np.array(np.frombuffer(raw_audio_bytestring, dtype=np.int16))
elif sample_width == 4:
raw_audio = np.array(np.frombuffer(raw_audio_bytestring, dtype=np.int32))
else:
raw_audio = np.array(np.frombuffer(raw_audio_bytestring, dtype=np.int16))
# Normalize the audio data
if normalized:
# Cast the audio data as 32 bit floats
raw_audio = raw_audio.astype(dtype=np.float32)
# Make all values positive
raw_audio += np.power(2, 8*sample_width)/2
# Normalize all values between 0 and 1
raw_audio *= 1/np.power(2, 8*sample_width)
# Reshape the array to accommodate multiple channels
if audio_segment.channels > 1:
raw_audio = raw_audio.reshape((-1, audio_segment.channels))
return raw_audio
#staticmethod
# Return an array of all audio files in directory, as arrays of audio data
def get_audio_arrays(directory, filetype='mp3', normalized=True):
file_count_total = len([name for name in os.listdir(directory) if os.path.isfile(os.path.join(directory, name))]) - 1
audio_arrays = []
# Iterate through all audio files
pathlist = Path(directory).glob('**/*.' + filetype)
# Keep track of progress
file_count = 0
print("Loading audio files... 0%")
for path in pathlist:
path_string = str(path)
audio_array = AudioHandler.get_audio_array(path_string, normalized=normalized)
audio_arrays.append(audio_array)
# Update Progress
file_count += 1
print('Loading audio files... ' + str(int(file_count/file_count_total*100)) + '%')
return audio_arrays
#staticmethod
def export_to_file(audio_data_array, file_name, normalized=True, file_type="mp3", bitrate="256k"):
if normalized:
audio_data_array *= np.power(2, 16)
audio_data_array -= np.power(2, 16)/2
audio_data_array = audio_data_array.astype(np.int16)
audio_data_array = audio_data_array.reshape((1, -1))[0]
raw_audio = audio_data_array.tostring()
audio_segment = AudioSegment(data=raw_audio, sample_width=2, frame_rate=44100, channels=2)
audio_segment.export(file_name, format=file_type, bitrate=bitrate)
# Splits a sequence into input values and target values
#staticmethod
def __split_input_target(chunk):
input_audio = chunk[:-1]
target_audio = chunk[1:]
return input_audio, target_audio
#staticmethod
def dataset_from_arrays(audio_arrays, sequence_length, batch_size, buffer_size=10000):
# Create main data set, starting with first audio array
dataset = data.Dataset.from_tensor_slices(audio_arrays[0])
dataset = dataset.batch(sequence_length + 1, drop_remainder=True)
# Split each audio array into sequences individually,
# then concatenate each individual data set with the main data set
for i in range(1, len(audio_arrays)):
audio_data = audio_arrays[i]
tensor_slices = data.Dataset.from_tensor_slices(audio_data)
audio_dataset = tensor_slices.batch(sequence_length + 1, drop_remainder=True)
dataset.concatenate(audio_dataset)
dataset = dataset.map(AudioHandler.__split_input_target)
dataset = dataset.shuffle(buffer_size).batch(batch_size, drop_remainder=True)
return dataset
Loading audio files... 0%
Loading audio files... 25%
Loading audio files... 50%
Loading audio files... 75%
Loading audio files... 100%
2020-06-21 00:20:10.796993: I tensorflow/core/platform/cpu_feature_guard.cc:143] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX2 FMA
2020-06-21 00:20:10.811357: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x7fddb7b23fd0 initialized for platform Host (this does not guarantee that XLA will be used). Devices:
2020-06-21 00:20:10.811368: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Host, Default Version
<BatchDataset shapes: ((64, 22050, 2), (64, 22050, 2)), types: (tf.float32, tf.float32)>
Model: "sequential"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
gru (GRU) (64, None, 256) 199680
_________________________________________________________________
dense (Dense) (64, None, 2) 514
=================================================================
Total params: 200,194
Trainable params: 200,194
Non-trainable params: 0
_________________________________________________________________
Epoch 1/10
Traceback (most recent call last):
File "/Users/anonteau/Desktop/Development/Python/Lo-FiGenerator/RNN.py", line 57, in <module>
history = model.fit(dataset, epochs=EPOCHS)
File "/Users/anonteau/Desktop/Development/Python/Lo-FiGenerator/venv/lib/python3.7/site-packages/tensorflow/python/keras/engine/training.py", line 66, in _method_wrapper
return method(self, *args, **kwargs)
File "/Users/anonteau/Desktop/Development/Python/Lo-FiGenerator/venv/lib/python3.7/site-packages/tensorflow/python/keras/engine/training.py", line 848, in fit
tmp_logs = train_function(iterator)
File "/Users/anonteau/Desktop/Development/Python/Lo-FiGenerator/venv/lib/python3.7/site-packages/tensorflow/python/eager/def_function.py", line 580, in __call__
result = self._call(*args, **kwds)
File "/Users/anonteau/Desktop/Development/Python/Lo-FiGenerator/venv/lib/python3.7/site-packages/tensorflow/python/eager/def_function.py", line 627, in _call
self._initialize(args, kwds, add_initializers_to=initializers)
File "/Users/anonteau/Desktop/Development/Python/Lo-FiGenerator/venv/lib/python3.7/site-packages/tensorflow/python/eager/def_function.py", line 506, in _initialize
*args, **kwds))
File "/Users/anonteau/Desktop/Development/Python/Lo-FiGenerator/venv/lib/python3.7/site-packages/tensorflow/python/eager/function.py", line 2446, in _get_concrete_function_internal_garbage_collected
graph_function, _, _ = self._maybe_define_function(args, kwargs)
File "/Users/anonteau/Desktop/Development/Python/Lo-FiGenerator/venv/lib/python3.7/site-packages/tensorflow/python/eager/function.py", line 2777, in _maybe_define_function
graph_function = self._create_graph_function(args, kwargs)
File "/Users/anonteau/Desktop/Development/Python/Lo-FiGenerator/venv/lib/python3.7/site-packages/tensorflow/python/eager/function.py", line 2667, in _create_graph_function
capture_by_value=self._capture_by_value),
File "/Users/anonteau/Desktop/Development/Python/Lo-FiGenerator/venv/lib/python3.7/site-packages/tensorflow/python/framework/func_graph.py", line 981, in func_graph_from_py_func
func_outputs = python_func(*func_args, **func_kwargs)
File "/Users/anonteau/Desktop/Development/Python/Lo-FiGenerator/venv/lib/python3.7/site-packages/tensorflow/python/eager/def_function.py", line 441, in wrapped_fn
return weak_wrapped_fn().__wrapped__(*args, **kwds)
File "/Users/anonteau/Desktop/Development/Python/Lo-FiGenerator/venv/lib/python3.7/site-packages/tensorflow/python/framework/func_graph.py", line 968, in wrapper
raise e.ag_error_metadata.to_exception(e)
TypeError: in user code:
/Users/anonteau/Desktop/Development/Python/Lo-FiGenerator/venv/lib/python3.7/site-packages/tensorflow/python/keras/engine/training.py:571 train_function *
outputs = self.distribute_strategy.run(
/Users/anonteau/Desktop/Development/Python/Lo-FiGenerator/venv/lib/python3.7/site-packages/tensorflow/python/distribute/distribute_lib.py:951 run **
return self._extended.call_for_each_replica(fn, args=args, kwargs=kwargs)
/Users/anonteau/Desktop/Development/Python/Lo-FiGenerator/venv/lib/python3.7/site-packages/tensorflow/python/distribute/distribute_lib.py:2290 call_for_each_replica
return self._call_for_each_replica(fn, args, kwargs)
/Users/anonteau/Desktop/Development/Python/Lo-FiGenerator/venv/lib/python3.7/site-packages/tensorflow/python/distribute/distribute_lib.py:2649 _call_for_each_replica
return fn(*args, **kwargs)
/Users/anonteau/Desktop/Development/Python/Lo-FiGenerator/venv/lib/python3.7/site-packages/tensorflow/python/keras/engine/training.py:533 train_step **
y, y_pred, sample_weight, regularization_losses=self.losses)
/Users/anonteau/Desktop/Development/Python/Lo-FiGenerator/venv/lib/python3.7/site-packages/tensorflow/python/keras/engine/compile_utils.py:205 __call__
loss_value = loss_obj(y_t, y_p, sample_weight=sw)
/Users/anonteau/Desktop/Development/Python/Lo-FiGenerator/venv/lib/python3.7/site-packages/tensorflow/python/keras/losses.py:143 __call__
losses = self.call(y_true, y_pred)
/Users/anonteau/Desktop/Development/Python/Lo-FiGenerator/venv/lib/python3.7/site-packages/tensorflow/python/keras/losses.py:246 call
return self.fn(y_true, y_pred, **self._fn_kwargs)
/Users/anonteau/Desktop/Development/Python/Lo-FiGenerator/venv/lib/python3.7/site-packages/tensorflow/python/keras/losses.py:313 __init__
mean_squared_error, name=name, reduction=reduction)
/Users/anonteau/Desktop/Development/Python/Lo-FiGenerator/venv/lib/python3.7/site-packages/tensorflow/python/keras/losses.py:229 __init__
super(LossFunctionWrapper, self).__init__(reduction=reduction, name=name)
/Users/anonteau/Desktop/Development/Python/Lo-FiGenerator/venv/lib/python3.7/site-packages/tensorflow/python/keras/losses.py:94 __init__
losses_utils.ReductionV2.validate(reduction)
/Users/anonteau/Desktop/Development/Python/Lo-FiGenerator/venv/lib/python3.7/site-packages/tensorflow/python/ops/losses/loss_reduction.py:67 validate
if key not in cls.all():
/Users/anonteau/Desktop/Development/Python/Lo-FiGenerator/venv/lib/python3.7/site-packages/tensorflow/python/ops/math_ops.py:1491 tensor_equals
return gen_math_ops.equal(self, other, incompatible_shape_error=False)
/Users/anonteau/Desktop/Development/Python/Lo-FiGenerator/venv/lib/python3.7/site-packages/tensorflow/python/ops/gen_math_ops.py:3224 equal
name=name)
/Users/anonteau/Desktop/Development/Python/Lo-FiGenerator/venv/lib/python3.7/site-packages/tensorflow/python/framework/op_def_library.py:479 _apply_op_helper
repr(values), type(values).__name__, err))
TypeError: Expected float32 passed to parameter 'y' of op 'Equal', got 'auto' of type 'str' instead. Error: Expected float32, got 'auto' of type 'str' instead.```
Try changing
model.compile(optimizer='adam', loss=tf.keras.losses.MeanSquaredError)
to
model.compile(optimizer='adam', loss=tf.keras.losses.MeanSquaredError())
I was compiling my model:
model.compile(optimizer='Adam', loss=tf.losses.CosineSimilarity,
metrics= ['accuracy'])
Then I got the same error. I changed my code as follow:
model.compile(optimizer='Adam', loss= tf.losses.CosineSimilarity(),
metrics= ['accuracy'])
It worked.
I had a similar issue while I was compiling my model with the following code:
model.compile(loss=tf.keras.losses.MeanSquaredError, optimizer='Adam')
I changed it to the following and it worked. Thanks to https://stackoverflow.com/users/7349864/sourcerer
model.compile(loss=tf.keras.losses.MeanSquaredError(), optimizer='Adam')
There is also a type of error with a solution
x = self.input1(x)
You don't need this sentence
, you can pass input directly into LSTM without predefining the input shape
It is recommended that you browse
import tensorflow as tf
class MyModel(tf.keras.Model):
def __init__(self):
super(MyModel, self).__init__()
self.dense1 = tf.keras.layers.Dense(4, activation=tf.nn.relu)
self.dense2 = tf.keras.layers.Dense(5, activation=tf.nn.softmax)
def call(self, inputs):
x = self.dense1(inputs)
return self.dense2(x)
model = MyModel()
I was compiling my model:
model.compile(loss=tf.keras.losses.SparseCategoricalCrossentropy(),
optimizer='adam', metrics=['accuracy'])
Then I got the same error. I changed my code as follow:
model.compile(loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),
optimizer='adam', metrics=['accuracy'])
Then it worked.
The from_logits=True attribute inform the loss function that the output values generated by the model are not normalized, a.k.a. logits. In other words, the softmax function has not been applied on them to produce a probability distribution. Therefore, the output layer in this case does not have a softmax activation function
I'm trying to train a network using NiftyNet with my own data (CT images and their corresponding labels). I designed the Net class shortly following some other training with similar sample data, all NiftyNet documentation I could find and parameters of my own data adjusted. But I keep getting this error:
"TypeError: init() got an unexpected keyword argument 'w_initializer'".
I've tried every change I could think of in my config.ini, Net class, etc. But I can't make it work nor find the reason. Can anyone help with this error? Or maybe share some guidelines to train my own network from the beginning so I can at least try to start an alternative from zero and see if I find a way out?
Training command:
! net_segment train -c /home/niftynet/extensions/dense_vnet_TC/config.ini --name dense_vnet_TC.net_TC.MyNet
Some values in config.ini:
[NETWORK]
name = dense_vnet
batch_size = 6
volume_padding_size = 0
window_sampling = resize
[TRAINING]
sample_per_volume = 1
lr = 0.001
loss_type = dense_vnet_TC.dice_hinge.dice
starting_iter = 0
save_every_n = 1000
max_iter = 3001
[INFERENCE]
border = (0, 0, 0)
inference_iter = 3000
output_interp_order = 0
spatial_window_size = (512, 512, 40)
save_seg_dir = ./segmentation_output/
############################ Custom configuration
[SEGMENTATION]
image = ct
label = label
label_normalisation = False
output_prob = False
num_classes = 2
Basics of Net class:
from niftynet.network.base_net import BaseNet
class MyNet(BaseNet):
def __init__(self, num_classes, name='MyNet'):
super(MyNet, self).__init__(num_classes=num_classes, acti_func=acti_func, name=name)
# network specific property
self.hidden_features = 10
def layer_op(self, images, is_training):
# create layer instances
conv_1 = ConvolutionalLayer(self.hidden_features, kernel_size=3, name='conv_input')
conv_2 = ConvolutionalLayer(self.num_classes, kernel_size=1, acti_func=None, name='conv_output')
# apply layer instances
flow = conv_1(images, is_training)
flow = conv_2(flow, is_training)
return flow
End of output, after doing some of the processing as expected:
Traceback (most recent call last): File
"/home/niftynet/bin/net_segment", line 10, in
sys.exit(main()) File "/home/niftynet/lib/python3.6/site- packages/niftynet/init.py",
line 142, in main
app_driver.run(app_driver.app) File "/home/niftynet/lib/python3.6/site-packages/niftynet/engine/application_driver.py",
line 189, in run
is_training_action=self.is_training_action) File "/home/niftynet/lib/python3.6/site- packages/niftynet/engine/application_driver.py",
line 258, in create_graph
application.initialise_network() File "/home/niftynet/lib/python3.6/site-packages/niftynet/application/segmentation_application.py",
line 280, in initialise_network
acti_func=self.net_param.activation_function) TypeError: init() got an unexpected keyword argument 'w_initializer'
I think you need to change this line (based on a similar problem I had):
super(MyNet, self).__init__(num_classes=num_classes, acti_func=acti_func, name=name)
for (just add w_regularizer) :
super(MyNet, self).__init__(num_classes=num_classes, w_regularizer=w_regularizer, acti_func=acti_func, name=name)
if not try also to add it here :
def __init__(self, num_classes, w_regularizer=w_regularizer, name='MyNet'):
I hope it helps.
I am trying to train a neural network for it to find lines in images, the images are in bmp file format and grey-scale. After the network training phase has begun the program stops and outputs the corresponding error message.
This code is based entirely on 'Image Segmentation with tf.keras' by Raymond Yuan (https://ej.uz/hk9s) with changes in input pipeline, yet the model itself is exactly the same. I have tried redefining the shape of input to something else, yet every time I change something, the error message changes, but always happens at the same place.
#%%
import numpy as np
import matplotlib as mpl
import tensorflow as tf
from tensorflow.python.keras import layers
from tensorflow.python.keras import losses
from tensorflow.python.keras import models
#%%
#Defining paths to all the images
x_train_filenames = []
y_train_filenames = []
x_eval_filenames = []
y_eval_filenames = []
for x in range(250):
x_train_filenames.append(r'Train/Images/gen_{}_.bmp'.format(x))
y_train_filenames.append(r'Train/Labels/gen_{}_seg_.bmp'.format(x))
x_eval_filenames.append(r'Evaluate/Images/gen_{}_.bmp'.format(x))
y_eval_filenames.append(r'Evaluate/Labels/gen_{}_seg_.bmp'.format(x))
num_train_examples = len(x_train_filenames)
num_eval_examples = len(x_eval_filenames)
#%%
#Creating dataset from all of the pathnames.
img_shape = (3296, 3008, 1)
batch_size = 3
epochs = 5
threads = 5
def _process_pathnames(img_name, lbl_name):
img_str = tf.read_file(img_name)
img = tf.image.decode_bmp(img_str)
lbl_str = tf.read_file(lbl_name)
lbl = tf.image.decode_bmp(lbl_str)
return img, lbl
training_dataset = tf.data.Dataset.from_tensor_slices((x_train_filenames, y_train_filenames))
training_dataset = training_dataset.map(_process_pathnames, num_parallel_calls=threads)
training_dataset = training_dataset.shuffle(num_train_examples)
training_dataset = training_dataset.repeat().batch(batch_size)
evaluation_dataset = tf.data.Dataset.from_tensor_slices(((x_eval_filenames, y_eval_filenames)))
evaluation_dataset = evaluation_dataset.map(_process_pathnames, num_parallel_calls=threads)
evaluation_dataset = evaluation_dataset.shuffle(num_eval_examples)
evaluation_dataset = evaluation_dataset.repeat().batch(batch_size)
#%%
#Deining model
def conv_block(input_tensor, num_filters):
encoder = layers.Conv2D(num_filters, (3, 3), padding='same')(input_tensor)
encoder = layers.BatchNormalization()(encoder)
encoder = layers.Activation('relu')(encoder)
encoder = layers.Conv2D(num_filters, (3, 3), padding='same')(encoder)
encoder = layers.BatchNormalization()(encoder)
encoder = layers.Activation('relu')(encoder)
return encoder
def encoder_block(input_tensor, num_filters):
encoder = conv_block(input_tensor, num_filters)
encoder_pool = layers.MaxPooling2D((2, 2), strides=(2, 2))(encoder)
return encoder_pool, encoder
def decoder_block(input_tensor, concat_tensor, num_filters):
decoder = layers.Conv2DTranspose(num_filters, (2, 2), strides=(2, 2), padding='same')(input_tensor)
decoder = layers.concatenate([concat_tensor, decoder], axis=-1)
decoder = layers.BatchNormalization()(decoder)
decoder = layers.Activation('relu')(decoder)
decoder = layers.Conv2D(num_filters, (3, 3), padding='same')(decoder)
decoder = layers.BatchNormalization()(decoder)
decoder = layers.Activation('relu')(decoder)
decoder = layers.Conv2D(num_filters, (3, 3), padding='same')(decoder)
decoder = layers.BatchNormalization()(decoder)
decoder = layers.Activation('relu')(decoder)
return decoder
inputs = layers.Input(shape=img_shape)
encoder0_pool, encoder0 = encoder_block(inputs, 32)
encoder1_pool, encoder1 = encoder_block(encoder0_pool, 64)
encoder2_pool, encoder2 = encoder_block(encoder1_pool, 128)
encoder3_pool, encoder3 = encoder_block(encoder2_pool, 256)
encoder4_pool, encoder4 = encoder_block(encoder3_pool, 512)
center = conv_block(encoder4_pool, 1024)
decoder4 = decoder_block(center, encoder4, 512)
decoder3 = decoder_block(decoder4, encoder3, 256)
decoder2 = decoder_block(decoder3, encoder2, 128)
decoder1 = decoder_block(decoder2, encoder1, 64)
decoder0 = decoder_block(decoder1, encoder0, 32)
outputs = layers.Conv2D(1, (1, 1), activation='sigmoid')(decoder0)
model = models.Model(inputs=[inputs], outputs=[outputs])
#%%
#Defining custom loss functions
def dice_coeff(y_true, y_pred):
smooth = 1.
# Flatten
y_true_f = tf.reshape(y_true, [-1])
y_pred_f = tf.reshape(y_pred, [-1])
intersection = tf.reduce_sum(y_true_f * y_pred_f)
score = (2. * intersection + smooth) / (tf.reduce_sum(y_true_f) + tf.reduce_sum(y_pred_f) + smooth)
return score
def dice_loss(y_true, y_pred):
loss = 1 - dice_coeff(y_true, y_pred)
return loss
def bce_dice_loss(y_true, y_pred):
loss = losses.binary_crossentropy(y_true, y_pred) + dice_loss(y_true, y_pred)
return loss
model.compile(optimizer='adam', loss=bce_dice_loss, metrics=[dice_loss])
save_model_path = '/tmp/weights.hdf5'
cp = tf.keras.callbacks.ModelCheckpoint(filepath=save_model_path, monitor='val_dice_loss', save_best_only=True, verbose=1)
#%%
#Training the model
history = model.fit(training_dataset,
steps_per_epoch=int(np.ceil(num_train_examples / float(batch_size))),
epochs=epochs,
validation_data=evaluation_dataset,
validation_steps=int(np.ceil(num_eval_examples / float(batch_size))),
callbacks=[cp])
Complete error message:
Traceback (most recent call last):
File "<ipython-input-19-f1dcac0996cd>", line 1, in <module>
runfile('//upb.lv/usr/profiles/Peteris.Zvejnieks/Desktop/Tests/Train data/A thing_retry.py', wdir='//upb.lv/usr/profiles/Peteris.Zvejnieks/Desktop/Tests/Train data')
File "C:\ProgramData\Anaconda3\lib\site-packages\spyder_kernels\customize\spydercustomize.py", line 786, in runfile
execfile(filename, namespace)
File "C:\ProgramData\Anaconda3\lib\site-packages\spyder_kernels\customize\spydercustomize.py", line 110, in execfile
exec(compile(f.read(), filename, 'exec'), namespace)
File "//upb.lv/usr/profiles/Peteris.Zvejnieks/Desktop/Tests/Train data/A thing_retry.py", line 159, in <module>
callbacks=[cp])
File "C:\ProgramData\Anaconda3\lib\site-packages\tensorflow\python\keras\engine\training.py", line 880, in fit
validation_steps=validation_steps)
File "C:\ProgramData\Anaconda3\lib\site-packages\tensorflow\python\keras\engine\training_arrays.py", line 266, in model_iteration
batch_outs = f(actual_inputs)
File "C:\ProgramData\Anaconda3\lib\site-packages\tensorflow\python\keras\backend.py", line 3076, in __call__
run_metadata=self.run_metadata)
File "C:\ProgramData\Anaconda3\lib\site-packages\tensorflow\python\client\session.py", line 1439, in __call__
run_metadata_ptr)
File "C:\ProgramData\Anaconda3\lib\site-packages\tensorflow\python\framework\errors_impl.py", line 528, in __exit__
c_api.TF_GetCode(self.status.status))
InvalidArgumentError: Number of channels must be 1, 3 or 4, was 0
[[{{node DecodeBmp_1}}]]
[[{{node IteratorGetNext_13}}]]
It appears that tensorflow.image.decode_bmp(...) is faulty, so I switched my project entirely to png, and that seems to work just fine.
As an added bonus I reduced the size of my data significantly, yet did`t loose any precious detail. However, now I understand, that what I am asking from my computer is extremely resource demanding.
I had the same exact situation but with the following error message:
InvalidArgumentError: `channels` must be 0, 3 or 4 for BMP, but got 1
[[{{node decode_image/DecodeImage}}]] [Op:IteratorGetNext]
Had to change 14k images from bmp to jpg.
But then it worked
I am building a neural machine translator, and I have to use two different LSTM cells (one for the encoder, and one for the decode).
The two cells have differents shapes:
the encoder (first one) is fed with the token of the input sentence and produces a state vector
the decoder (second one) is fed with the previous state vector, and the tokens generated by itself
I writed this in Tensorflow, and when I run the script, I got the following error (raised during the decoder phase):
outputs, states = tf.nn.rnn(cell_backward, inputs, initial_state=initial_state)
File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/rnn.py", line 158, in rnn
(output, state) = call_cell()
File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/rnn.py", line 145, in <lambda>
call_cell = lambda: cell(input_, state)
File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/rnn_cell.py", line 520, in __call__
dtype, self._num_unit_shards)
File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/rnn_cell.py", line 357, in _get_concat_variable
sharded_variable = _get_sharded_variable(name, shape, dtype, num_shards)
File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/rnn_cell.py", line 387, in _get_sharded_variable
dtype=dtype))
File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/variable_scope.py", line 732, in get_variable
partitioner=partitioner, validate_shape=validate_shape)
File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/variable_scope.py", line 596, in get_variable
partitioner=partitioner, validate_shape=validate_shape)
File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/variable_scope.py", line 161, in get_variable
caching_device=caching_device, validate_shape=validate_shape)
File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/variable_scope.py", line 437, in _get_single_variable
name, "".join(traceback.format_list(tb))))
ValueError: Variable backward/RNN/LSTMCell/W_0 already exists, disallowed. Did you mean to set reuse=True in VarScope? Originally defined at:
File "/home/alexis/Documents/NMT/NMT.py", line 88, in dense_to_vector_state
outputs, states = tf.nn.rnn(cell_forward, inputs, initial_state=initial_state)
How can I explicitly specify that I want to create a totally new LSTM cell ?
Thanks in advance !
Alexis
Use variable scopes
with tf.variable_scope('enc'):
cell_enc = LSTMCell(hidden_size)
with tf.variable_scope('dec'):
cell_dec = LSTMCell(hidden_size)
I am trying to do machine translation. Here is my encoder and decoder. You just need to use different variable scopes for each rnn. Rather than using the MultiRNNCell cell for the encoder I unroll each layer manually which lets me alternate directions between layers. See how each layer gets its own scope.
with tf.variable_scope('encoder'):
rnn_cell = tf.nn.rnn_cell.LSTMCell(512, num_proj = 256, state_is_tuple = True)
for level in range(3):
with tf.variable_scope('level_%d' % level) as scope:
state = [tf.zeros((BATCH_SIZE, sz)) for sz in rnn_cell.state_size]
for t in range(TIME_STEPS) if level % 2 else reversed(range(TIME_STEPS)):
y[t], state = rnn_cell(y[t], state)
scope.reuse_variables()
with tf.variable_scope('decoder') as scope:
rnn_cell = tf.nn.rnn_cell.MultiRNNCell \
([
tf.nn.rnn_cell.LSTMCell(512, num_proj = 256, state_is_tuple = True),
tf.nn.rnn_cell.LSTMCell(512, num_proj = WORD_VEC_SIZE, state_is_tuple = True)
], state_is_tuple = True)
state = [[tf.zeros((BATCH_SIZE, sz)) for sz in sz_outer] for sz_outer in rnn_cell.state_size]
W_soft = tf.get_variable('W_soft', shape = (NWORDS, WORD_VEC_SIZE), initializer = tf.truncated_normal_initializer(0.0, 1 / np.sqrt(WORD_VEC_SIZE)))
b_soft = tf.get_variable('b_soft', shape = (NWORDS,), initializer = tf.truncated_normal_initializer(0.0, 0.01))
cost = 0
output = [None] * TIME_STEPS
for t in range(TIME_STEPS):
if t:
last = y_[t - 1] if TRAINING else y[t - 1]
else:
last = tf.zeros((BATCH_SIZE, WORD_VEC_SIZE))
y[t] = tf.concat(1, (y[t], last))
y[t], state = rnn_cell(y[t], state)
cost += tf.reduce_mean(tf.nn.sampled_softmax_loss(W_soft, b_soft, y[t], target_output[:, t : t + 1], 1000, NWORDS))
output[t] = tf.reshape(tf.nn.softmax(tf.matmul(y[t], W_soft, transpose_b = True) + b_soft), (BATCH_SIZE, 1, NWORDS))
scope.reuse_variables()
output = tf.concat(1, output)
cost /= TIME_STEPS