Related
Due to the size of my data, I have my training, validation and test data fed via a generator as below:
def sequence_generator(data_type):
data_type = data_type.decode('ascii')
sequence_folder = f"{model_attributes['sequences_path']}/{data_type}"
for numpy_file in random.sample(os.listdir(sequence_folder),len(os.listdir(sequence_folder))):
if numpy_file.endswith(".npz"):
with np.load(f"{sequence_folder}/{numpy_file}") as numpy_data:
sequences = numpy_data['x']
labels = numpy_data['y']
labels = tf.one_hot(labels, 3)
for X,y in random.sample(list(zip(sequences,labels)),len(sequences)):
yield X,y
train_dataset = tf.data.Dataset.from_generator(generator=sequence_generator,args= ['train'], output_types = (tf.float32, tf.float32))# output_shapes = ((train_shape), (3,)))
train_dataset = train_dataset.cache().batch(BATCH).prefetch(tf.data.AUTOTUNE)
history = model.fit(train_dataset, epochs=EPOCHS, validation_data=validation_dataset, callbacks=[tensorboard,checkpoint,earlystop,lr_callback])
From my previous questions accepted answer, I want to be able to apply weighting to 2 of the 3 classes. The accepted answers code is:
sample_weight = np.ones(shape=(len(train_y),))
sample_weight[(train_y[:, 1] == 1) | (train_y[:, 2] == 1)] = 1.5
model = get_model()
model.fit(
train_x,
train_y,
sample_weight=sample_weight,
....,
...,
..,
)
But given that my training data is behind the generator code I am using, I am scratching my head on how best to apply the weighting and with minimal overhead
EDIT
I added the following code as recommended
class_weight = {0: 1.,
1: 2.,
2: 2.}
history = model.fit(train_dataset, epochs=EPOCHS, validation_data=validation_dataset,class_weight=class_weight, callbacks=[tensorboard,checkpoint,earlystop,lr_callback])
but I get this error
File "Train.py", line 1993, in build_and_train_model
history = model.fit(train_dataset, epochs=EPOCHS, validation_data=validation_dataset,class_weight=class_weight, callbacks=[tensorboard,checkpoint,earlystop,lr_callback])
File "Location\\AppData\Local\Programs\Python\Python36\lib\site-packages\tensorflow\python\keras\engine\training.py", line 1147, in fit
steps_per_execution=self._steps_per_execution)
File "Location\\AppData\Local\Programs\Python\Python36\lib\site-packages\tensorflow\python\keras\engine\data_adapter.py", line 1364, in get_data_handler
return DataHandler(*args, **kwargs)
File "Location\\AppData\Local\Programs\Python\Python36\lib\site-packages\tensorflow\python\keras\engine\data_adapter.py", line 1175, in __init__
class_weight, distribute)
File "Location\\AppData\Local\Programs\Python\Python36\lib\site-packages\tensorflow\python\keras\engine\data_adapter.py", line 1186, in _configure_dataset_and_inferred_steps
dataset = dataset.map(_make_class_weight_map_fn(class_weight))
File "Location\\AppData\Local\Programs\Python\Python36\lib\site-packages\tensorflow\python\data\ops\dataset_ops.py", line 1925, in map
return MapDataset(self, map_func, preserve_cardinality=True)
File "Location\\AppData\Local\Programs\Python\Python36\lib\site-packages\tensorflow\python\data\ops\dataset_ops.py", line 4487, in __init__
use_legacy_function=use_legacy_function)
File "Location\\AppData\Local\Programs\Python\Python36\lib\site-packages\tensorflow\python\data\ops\dataset_ops.py", line 3712, in __init__
self._function = fn_factory()
File "Location\\AppData\Local\Programs\Python\Python36\lib\site-packages\tensorflow\python\eager\function.py", line 3135, in get_concrete_function
*args, **kwargs)
File "Location\\AppData\Local\Programs\Python\Python36\lib\site-packages\tensorflow\python\eager\function.py", line 3100, in _get_concrete_function_garbage_collected
graph_function, _ = self._maybe_define_function(args, kwargs)
File "Location\\AppData\Local\Programs\Python\Python36\lib\site-packages\tensorflow\python\eager\function.py", line 3444, in _maybe_define_function
graph_function = self._create_graph_function(args, kwargs)
File "Location\\AppData\Local\Programs\Python\Python36\lib\site-packages\tensorflow\python\eager\function.py", line 3289, in _create_graph_function
capture_by_value=self._capture_by_value),
File "Location\\AppData\Local\Programs\Python\Python36\lib\site-packages\tensorflow\python\framework\func_graph.py", line 999, in func_graph_from_py_func
func_outputs = python_func(*func_args, **func_kwargs)
File "Location\\AppData\Local\Programs\Python\Python36\lib\site-packages\tensorflow\python\data\ops\dataset_ops.py", line 3687, in wrapped_fn
ret = wrapper_helper(*args)
File "Location\\AppData\Local\Programs\Python\Python36\lib\site-packages\tensorflow\python\data\ops\dataset_ops.py", line 3617, in wrapper_helper
ret = autograph.tf_convert(self._func, ag_ctx)(*nested_args)
File "Location\\AppData\Local\Programs\Python\Python36\lib\site-packages\tensorflow\python\autograph\impl\api.py", line 692, in wrapper
return converted_call(f, args, kwargs, options=options)
File "Location\\AppData\Local\Programs\Python\Python36\lib\site-packages\tensorflow\python\autograph\impl\api.py", line 382, in converted_call
return _call_unconverted(f, args, kwargs, options)
File "Location\\AppData\Local\Programs\Python\Python36\lib\site-packages\tensorflow\python\autograph\impl\api.py", line 463, in _call_unconverted
return f(*args, **kwargs)
File "Location\\AppData\Local\Programs\Python\Python36\lib\site-packages\tensorflow\python\keras\engine\data_adapter.py", line 1400, in _class_weights_map_fn
if y.shape.rank > 2:
TypeError: '>' not supported between instances of 'NoneType' and 'int
Not sure what is being referred to as NoneType, from the research I read I get the inclination that it has something to do with one_hotting maybe?
This looks a great moment to use class_weight instead of sample_weight:
Optional dictionary mapping class indices (integers) to a weight (float) value, used for weighting the loss function (during training only). This can be useful to tell the model to "pay more attention" to samples from an under-represented class.
I ended up doing this by modifying the generator:
train_dataset = tf.data.Dataset.from_generator(generator=sequence_generator,args= ['train'], output_types = (tf.float32, tf.float32, tf.float32))# output_shapes = ((train_shape), (3,)))
def sequence_generator(data_type):
data_type = data_type.decode('ascii')
sequence_folder = f"{model_attributes['sequences_path']}/{data_type}"
for numpy_file in random.sample(os.listdir(sequence_folder),len(os.listdir(sequence_folder))):
if numpy_file.endswith(".npz"):
with np.load(f"{sequence_folder}/{numpy_file}") as numpy_data:
sequences = numpy_data['x']
labels = numpy_data['y']
class_weight = {0: 1, 1: 2 ,2: 2}
labels = tf.one_hot(labels, 3)
sample_weights = np.ones(shape=(len(labels),))
sample_weights[(labels[:, 1] == 1) | (labels[:, 2] == 1) ] = 2
for X,y,sample_weight in random.sample(list(zip(sequences,labels,sample_weights)),len(sequences)):
yield X,y,sample_weight
The class_weights method did not seem to work. This article seemed to shed some light on it
I am trying to implement an autoencoder using a datagenerator, the autoencoder works fine if I implement it directly with X_train, but when I try to use the data generator always returns that error.
#!/usr/bin/env python
# coding: utf-8
import zipfile
import pandas as pd
import tensorflow as tf
from tensorflow.keras.models import Sequential, Model
from tensorflow.keras.layers import Dense, Conv2D, Flatten, Activation, MaxPooling2D, Reshape, Input, Dropout
from tensorflow.keras.layers import BatchNormalization, Lambda, Conv2DTranspose, Add
from tensorflow.keras import regularizers
from tensorflow.keras import initializers
from tensorflow.keras import constraints
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.utils import Sequence
import random
import numpy as np
from sklearn.preprocessing import MinMaxScaler
from matplotlib import pyplot as plt
# descargamos el conjunto de datos
train_zip_path = tf.keras.utils.get_file(
fname='train_femto.zip',
origin='https://hdvirtual.us.es/discovirt/index.php/s/z75ekAqY3tt3aKy/download',
cache_subdir='datasets', extract=False,
)
WINDOW_SIZE = 500
HEADERS = ['Hour', 'Minute', 'Second', 'Microsecond', 'H-acc', 'V-acc']
FEATURES = ['H-acc', 'V-acc']
TEST_RULS = {
'Bearing1_3': 5730,
'Bearing1_4': 339,
'Bearing1_5': 1610,
'Bearing1_6': 1460,
'Bearing1_7': 7570,
'Bearing2_3': 7530,
'Bearing2_4': 1390,
'Bearing2_5': 3090,
'Bearing2_6': 1290,
'Bearing2_7': 580,
'Bearing3_3': 820
}
def read_femto_file(z, file_path, bearing):
with z.open(file_path) as f:
X_aux = pd.read_csv(f, names=HEADERS, delimiter=",")
X_aux['Bearing'] = bearing.split('/')[-1][-3:]
del X_aux['Hour']
del X_aux['Minute']
del X_aux['Second']
del X_aux['Microsecond']
return X_aux
def read_femto_dataset(file_path, RULS=None):
datasets = []
with zipfile.ZipFile(file_path) as z:
files = z.namelist()
dirs = sorted(set(['/'.join(f.split('/')[:-1]) for f in files if len(f.split('/')) > 2]))
ds = []
for bearing in dirs:
bearing_name = bearing.split('/')[-1]
print("Reading", bearing_name)
bearing_files = sorted([f for f in files if bearing in f and 'acc' in f])
for i, bearing_file in enumerate(bearing_files):
ds.append(read_femto_file(z, bearing_file, bearing))
X = pd.concat(ds, axis=0)
X = X.reset_index(drop=True)
# compute RUL
X['RUL'] = (X.index / 256).astype('int')[::-1].values
if RULS is not None:
X['RUL'] += RULS[bearing.split('/')[-1]]
X.RUL = X.RUL.astype('int32')
datasets.append(X)
X = pd.concat(datasets, axis=0)
X['H-acc'] = X['H-acc'].astype('float32')
X['V-acc'] = X['V-acc'].astype('float32') #errata, H-acc en vez de V-acc
X['RUL'] = X['RUL'].astype('int32')
return X
# leemos el conjunto de entrenamiento
X_train = read_femto_dataset(train_zip_path)
# truncamos el RUL y lo normalzamos entre 100 y 0
X_train['RUL'] = X_train.RUL.clip(0, 10000) / 100
# dividimos entre entrenamiento y validación
X_val = X_train[X_train.Bearing.isin(['1_1', '2_1', '3_1'])]
X_train = X_train[X_train.Bearing.isin(['1_2', '2_2', '3_2'])]
# normalizamos los sensores
min_max = {}
for feature in FEATURES:
# calculamos los parámetros de la normalización con el train
min_max[feature] = {}
min_max[feature]['min'] = X_train[feature].min()
min_max[feature]['max'] = X_train[feature].max()
# normalizamos ambos datasets
X_train[feature] = ((X_train[feature] - min_max[feature]['min']) /
(min_max[feature]['max'] - min_max[feature]['min']))
X_val[feature] = ((X_val[feature] - min_max[feature]['min']) /
(min_max[feature]['max'] - min_max[feature]['min']))
print("Feature %s: train(%f, %f), val(%f, %f)" % (feature, X_train[feature].min(),
X_train[feature].max(), X_val[feature].min(),
X_val[feature].max()))
# generador de datos
class DataGenerator(Sequence):
def __init__(self, X, attributes, window_size=10, batch_size=32,
epoch_len_reducer=100, add_extra_channel=False,
return_label=True, y_key='Y', unit_key='id'):
self.batch_size = batch_size
self.return_label = return_label
self.window_size = window_size
self.attributes = attributes
self.epoch_len_reducer = epoch_len_reducer
self._X = {}
self._Y = {}
self._ids = X[unit_key].unique()
self.add_extra_channel = add_extra_channel
for _id in self._ids:
self._X[_id] = X.loc[(X[unit_key]==_id), self.attributes].values
self._Y[_id] = X.loc[(X[unit_key]==_id), y_key].values
self.__len = int((X.groupby(unit_key).size() - self.window_size).sum() /
self.batch_size)
del X
def __len__(self):
return int(self.__len / self.epoch_len_reducer)
def __getitem__(self, index):
X = self._X
_X = []
_y = []
for _ in range(self.batch_size):
sid = random.choice(self._ids)
unit = self._X[sid]
nrows = unit.shape[0]
cut = random.randint(0, nrows - self.window_size)
s = unit[cut: cut + self.window_size].T
y =self._Y[sid][cut + self.window_size-1]
_X.append(s)
_y.append(y)
_X = np.array(_X)
if self.add_extra_channel:
_X = _X.reshape(_X.shape + (1,))
if self.return_label:
return _X, np.array(_y).reshape((self.batch_size, 1))
else:
return _X, _X
def on_epoch_end(self):
pass
# generators
train_gen = DataGenerator(X_train, attributes=FEATURES, window_size=WINDOW_SIZE, batch_size=256,
add_extra_channel=True, return_label=True, y_key='RUL', unit_key='Bearing')
val_gen = DataGenerator(X_val, attributes=FEATURES, window_size=WINDOW_SIZE, batch_size=256,
add_extra_channel=True, return_label=True, y_key='RUL',
unit_key='Bearing', epoch_len_reducer=1000)
#Autoencoder
class AutoEncoder(Model):
def __init__(self):
super(AutoEncoder, self).__init__()
self.encoder = tf.keras.Sequential([
tf.keras.layers.Dense(64, activation='relu'),
tf.keras.layers.Dense(32, activation='relu'),
tf.keras.layers.Dense(16, activation='relu'),
tf.keras.layers.Dense(8, activation='relu')])
self.decoder = tf.keras.Sequential([
tf.keras.layers.Dense(16, activation='relu'),
tf.keras.layers.Dense(32, activation='relu'),
tf.keras.layers.Dense(64, activation='relu'),
tf.keras.layers.Dense(2, activation='sigmoid')
])
def call(self, x):
encoded = self.encoder(x)
decoded = self.decoder(encoded)
return decoded
definition of the autoencoder
model = AutoEncoder()
early_stopping = tf.keras.callbacks.EarlyStopping(monitor = 'val_loss', patience = 2, mode='min')
model.compile(optimizer = 'adam', loss='mae')
history = model.fit_generator(train_gen,
validation_data = val_gen,
epochs = 10)
After this is where I get the error,I have tried using different parameters but the result is always the same:
Epoch 1/10
/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:3: UserWarning: `Model.fit_generator` is deprecated and will be removed in a future version. Please use `Model.fit`, which supports generators.
This is separate from the ipykernel package so we can avoid doing imports until
---------------------------------------------------------------------------
InvalidArgumentError Traceback (most recent call last)
<ipython-input-12-ee4502a47627> in <module>()
1 history = model.fit_generator(train_gen,
2 validation_data = val_gen,
----> 3 epochs = 10)
2 frames
/usr/local/lib/python3.7/dist-packages/tensorflow/python/eager/execute.py in quick_execute(op_name, num_outputs, inputs, attrs, ctx, name)
53 ctx.ensure_initialized()
54 tensors = pywrap_tfe.TFE_Py_Execute(ctx._handle, device_name, op_name,
---> 55 inputs, attrs, num_outputs)
56 except core._NotOkStatusException as e:
57 if name is not None:
InvalidArgumentError: Graph execution error:
Detected at node 'mean_absolute_error/sub' defined at (most recent call last):
File "/usr/lib/python3.7/runpy.py", line 193, in _run_module_as_main
"__main__", mod_spec)
File "/usr/lib/python3.7/runpy.py", line 85, in _run_code
exec(code, run_globals)
File "/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py", line 16, in <module>
app.launch_new_instance()
File "/usr/local/lib/python3.7/dist-packages/traitlets/config/application.py", line 846, in launch_instance
app.start()
File "/usr/local/lib/python3.7/dist-packages/ipykernel/kernelapp.py", line 499, in start
self.io_loop.start()
File "/usr/local/lib/python3.7/dist-packages/tornado/platform/asyncio.py", line 132, in start
self.asyncio_loop.run_forever()
File "/usr/lib/python3.7/asyncio/base_events.py", line 541, in run_forever
self._run_once()
File "/usr/lib/python3.7/asyncio/base_events.py", line 1786, in _run_once
handle._run()
File "/usr/lib/python3.7/asyncio/events.py", line 88, in _run
self._context.run(self._callback, *self._args)
File "/usr/local/lib/python3.7/dist-packages/tornado/platform/asyncio.py", line 122, in _handle_events
handler_func(fileobj, events)
File "/usr/local/lib/python3.7/dist-packages/tornado/stack_context.py", line 300, in null_wrapper
return fn(*args, **kwargs)
File "/usr/local/lib/python3.7/dist-packages/zmq/eventloop/zmqstream.py", line 452, in _handle_events
self._handle_recv()
File "/usr/local/lib/python3.7/dist-packages/zmq/eventloop/zmqstream.py", line 481, in _handle_recv
self._run_callback(callback, msg)
File "/usr/local/lib/python3.7/dist-packages/zmq/eventloop/zmqstream.py", line 431, in _run_callback
callback(*args, **kwargs)
File "/usr/local/lib/python3.7/dist-packages/tornado/stack_context.py", line 300, in null_wrapper
return fn(*args, **kwargs)
File "/usr/local/lib/python3.7/dist-packages/ipykernel/kernelbase.py", line 283, in dispatcher
return self.dispatch_shell(stream, msg)
File "/usr/local/lib/python3.7/dist-packages/ipykernel/kernelbase.py", line 233, in dispatch_shell
handler(stream, idents, msg)
File "/usr/local/lib/python3.7/dist-packages/ipykernel/kernelbase.py", line 399, in execute_request
user_expressions, allow_stdin)
File "/usr/local/lib/python3.7/dist-packages/ipykernel/ipkernel.py", line 208, in do_execute
res = shell.run_cell(code, store_history=store_history, silent=silent)
File "/usr/local/lib/python3.7/dist-packages/ipykernel/zmqshell.py", line 537, in run_cell
return super(ZMQInteractiveShell, self).run_cell(*args, **kwargs)
File "/usr/local/lib/python3.7/dist-packages/IPython/core/interactiveshell.py", line 2718, in run_cell
interactivity=interactivity, compiler=compiler, result=result)
File "/usr/local/lib/python3.7/dist-packages/IPython/core/interactiveshell.py", line 2822, in run_ast_nodes
if self.run_code(code, result):
File "/usr/local/lib/python3.7/dist-packages/IPython/core/interactiveshell.py", line 2882, in run_code
exec(code_obj, self.user_global_ns, self.user_ns)
File "<ipython-input-7-342600d44df8>", line 3, in <module>
epochs = 10)
File "/usr/local/lib/python3.7/dist-packages/keras/engine/training.py", line 2223, in fit_generator
initial_epoch=initial_epoch)
File "/usr/local/lib/python3.7/dist-packages/keras/utils/traceback_utils.py", line 64, in error_handler
return fn(*args, **kwargs)
File "/usr/local/lib/python3.7/dist-packages/keras/engine/training.py", line 1384, in fit
tmp_logs = self.train_function(iterator)
File "/usr/local/lib/python3.7/dist-packages/keras/engine/training.py", line 1021, in train_function
return step_function(self, iterator)
File "/usr/local/lib/python3.7/dist-packages/keras/engine/training.py", line 1010, in step_function
outputs = model.distribute_strategy.run(run_step, args=(data,))
File "/usr/local/lib/python3.7/dist-packages/keras/engine/training.py", line 1000, in run_step
outputs = model.train_step(data)
File "/usr/local/lib/python3.7/dist-packages/keras/engine/training.py", line 860, in train_step
loss = self.compute_loss(x, y, y_pred, sample_weight)
File "/usr/local/lib/python3.7/dist-packages/keras/engine/training.py", line 919, in compute_loss
y, y_pred, sample_weight, regularization_losses=self.losses)
File "/usr/local/lib/python3.7/dist-packages/keras/engine/compile_utils.py", line 201, in __call__
loss_value = loss_obj(y_t, y_p, sample_weight=sw)
File "/usr/local/lib/python3.7/dist-packages/keras/losses.py", line 141, in __call__
losses = call_fn(y_true, y_pred)
File "/usr/local/lib/python3.7/dist-packages/keras/losses.py", line 245, in call
return ag_fn(y_true, y_pred, **self._fn_kwargs)
File "/usr/local/lib/python3.7/dist-packages/keras/losses.py", line 1457, in mean_absolute_error
return backend.mean(tf.abs(y_pred - y_true), axis=-1)
Node: 'mean_absolute_error/sub'
required broadcastable shapes
[[{{node mean_absolute_error/sub}}]] [Op:__inference_train_function_1827]
I am trying to load my saved keras model
model= tf.keras.models.load_model("my_model.h5",
custom_objects=None,
compile=True)
model.summary()
and getting the following error
Traceback (most recent call last):
File "C:\Users\admin\Desktop\phd python
projects\tensorflow_img_class\src\tensorflow ui.py", line 45, in <module>
compile=True
File "C:\Python37\lib\site-packages\tensorflow\python\keras\saving\save.py", line 146, in load_model
return hdf5_format.load_model_from_hdf5(filepath, custom_objects, compile)
File "C:\Python37\lib\site-packages\tensorflow\python\keras\saving\hdf5_format.py", line 212, in load_model_from_hdf5
custom_objects=custom_objects)
File "C:\Python37\lib\site-packages\tensorflow\python\keras\saving\model_config.py", line 55, in model_from_config
return deserialize(config, custom_objects=custom_objects)
File "C:\Python37\lib\site-packages\tensorflow\python\keras\layers\serialization.py", line 89, in deserialize
printable_module_name='layer')
File "C:\Python37\lib\site-packages\tensorflow\python\keras\utils\generic_utils.py", line 192, in deserialize_keras_object
list(custom_objects.items())))
File "C:\Python37\lib\site-packages\tensorflow\python\keras\engine\sequential.py", line 353, in from_config
model.add(layer)
File "C:\Python37\lib\site-packages\tensorflow\python\training\tracking\base.py", line 460, in _method_wrapper
result = method(self, *args, **kwargs)
File "C:\Python37\lib\site-packages\tensorflow\python\keras\engine\sequential.py", line 174, in add
layer(x)
File "C:\Python37\lib\site-packages\tensorflow\python\keras\engine\base_layer.py", line 632, in __call__
outputs = call_fn(inputs, *args, **kwargs)
File "C:\Python37\lib\site-packages\tensorflow\python\keras\layers\core.py", line 782, in call
return self.function(inputs, **arguments)
File "C:/Users/admin/Desktop/phd python projects/tensorflow_img_class/src/tensorflow_img_class.py", line 35, in feature_extractor
feature_extractor_module = hub.Module(feature_extractor_url)
NameError: name 'feature_extractor_url' is not defined
More details are on this question. I have opened this post by following a suggestion from my previous linked post.
Code for this model is
image_generator = tf.compat.v1.keras.preprocessing.image.ImageDataGenerator(rescale=1/255)
data_root = tf.compat.v1.keras.utils.get_file('Annotated_Image_Classes', 'https://github.com/PawanKaur/Viz-Image-Classification/tree/master/Annotated%20Image%20Classes.tqz',
untar=True)
feature_extractor_url = "https://tfhub.dev/google/imagenet/mobilenet_v2_100_224/feature_vector/2"
def feature_extractor(x):
feature_extractor_module = hub.Module(feature_extractor_url)
return feature_extractor_module(x)
IMAGE_SIZE = hub.get_expected_image_size(hub.Module(feature_extractor_url))
image_data = image_generator.flow_from_directory(str(data_root), target_size=IMAGE_SIZE)
for image_batch,label_batch in image_data:
print("Image batch shape: ", image_batch.shape)
print("Label batch shape: ", label_batch.shape)
break
features_extractor_layer = layers.Lambda(feature_extractor, input_shape=IMAGE_SIZE+[3])
features_extractor_layer.trainable = False
model = tf.keras.Sequential([
features_extractor_layer,
layers.Dense(image_data.num_classes, activation='softmax')
])
model.summary()
sess = tf.compat.v1.keras.backend.get_session()
init = tf.compat.v1.global_variables_initializer()
sess.run(init)
result = model.predict(image_batch)
result.shape
model.compile(
optimizer=tf.train.AdamOptimizer(),
loss='categorical_crossentropy',
metrics=['accuracy'])
class CollectBatchStats(tf.keras.callbacks.Callback):
def __init__(self):
self.batch_losses = []
self.batch_acc = []
def on_batch_end(self, batch, logs=None):
self.batch_losses.append(logs['loss'])
self.batch_acc.append(logs['acc'])
steps_per_epoch = image_data.samples//image_data.batch_size
batch_stats = CollectBatchStats()
model.fit((item for item in image_data), epochs=18,
steps_per_epoch=steps_per_epoch,
callbacks = [batch_stats])
model.save('my_model.h5')
Basically I have created this model by following transfer learning instructions from here. I am modelling this to run on my image data. After that I just need to open and view this pre-trained and saved model in another program but so far I am unable to do so. Any help will be appreciable.
just add
feature_extractor_url = "https://tfhub.dev/google/imagenet/mobilenet_v2_100_224/feature_vector/2"
in your loading model script after import statments
I am trying to set up a simple convolutional network in Tensorflow. I'll try to keep the amount of code at a minimum level. Here's the class:
class ConvNet(object):
def __init__(self, input, labels, dataset):
self.input = input
self.true_labels = labels
#'dataset' is an instance of a class that
#I am using to read the training images
self.data = dataset
self.build()
self._optimize = None
self.sess = tf.Session()
self.sess.run(tf.global_variables_initializer())
def build(self):
self.conv_layers = []
# create 3 conv layers using tf.nn.conv2d
# and append them to self.conv_layers
# create flattening layer using tf.nn.reshape
self.fc_layers = []
# create 2 fully connected layers using tf.matmul
# and append them to self.fc_layers
#property
def optimize(self):
"""Return the optimize operation."""
if not self._optimize:
cross_entropy = tf.nn.softmax_cross_entropy_with_logits(logits = self.fc_layers[-1],
labels = self.true_labels)
cost = tf.reduce_mean(cross_entropy)
self._optimize = tf.train.AdamOptimizer(learning_rate=1e-4).minimize(cost)
return self._optimize
def train(self, num_epochs=1, batch_size=16):
epochs_done = 0
while not epochs_done == num_epochs:
# get next batch from training data
x_batch, y_true_batch, _, _ = self.data.train.next_batch(batch_size)
feed_dict_train = {self.input: x_batch, self.true_labels: y_true_batch}
#even if I initialize variables here, same error occurs
#self.sess.run(tf.global_variables_initializer())
self.sess.run(self.optimize, feed_dict=feed_dict_train)
if self.data.train.epochs_done > epochs_done:
#print stuff...
epochs_done += 1
self.sess.close()
I was trying to run the optimize operation to train the network on a simple dataset (e.g. images of cats vs dogs). Here is the main:
if __name__ == "__main__":
classes = ['dogs', 'cats']
num_classes = len(classes)
batch_size = 16
img_size = 128
num_channels = 3
#read training data and resize images to 128 x 128 pixels
data = dataset.read_train_sets(img_size, classes)
x = tf.placeholder(tf.float32, shape = [None, img_size, img_size, num_channels], name='x')
y_true = tf.placeholder(tf.float32, shape = [None, num_classes], name = 'y_true')
net = ConvNet(x, y_true, data)
net.train()
However I keep getting the following error (it's actually much longer than this, I can post it if necessary):
Caused by op 'beta1_power/read', defined at:
File "<string>", line 1, in <module>
File "/usr/lib/python3.5/idlelib/run.py", line 124, in main
ret = method(*args, **kwargs)
File "/usr/lib/python3.5/idlelib/run.py", line 351, in runcode
exec(code, self.locals)
File "/home/gian/face_recognition/model.py", line 366, in <module>
net.train()
File "/home/gian/face_recognition/model.py", line 309, in train
self.sess.run(self.optimize, feed_dict=feed_dict_train)
File "/home/gian/face_recognition/model.py", line 269, in optimize
self._optimize = tf.train.AdamOptimizer(learning_rate=1e-4).minimize(cost)
File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/training/optimizer.py", line 409, in minimize
name=name)
File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/training/optimizer.py", line 552, in apply_gradients
self._create_slots([_get_variable_for(v) for v in var_list])
File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/training/adam.py", line 124, in _create_slots
colocate_with=first_var)
File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/training/optimizer.py", line 665, in _create_non_slot_variable
v = variable_scope.variable(initial_value, name=name, trainable=False)
File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/ops/variable_scope.py", line 2157, in variable
use_resource=use_resource)
File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/ops/variable_scope.py", line 2147, in <lambda>
previous_getter = lambda **kwargs: default_variable_creator(None, **kwargs)
File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/ops/variable_scope.py", line 2130, in default_variable_creator
constraint=constraint)
File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/ops/variables.py", line 235, in __init__
constraint=constraint)
File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/ops/variables.py", line 391, in _init_from_args
self._snapshot = array_ops.identity(self._variable, name="read")
File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/ops/array_ops.py", line 142, in identity
return gen_array_ops.identity(input, name=name)
File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/ops/gen_array_ops.py", line 3053, in identity
"Identity", input=input, name=name)
File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/framework/op_def_library.py", line 787, in _apply_op_helper
op_def=op_def)
File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/framework/ops.py", line 3290, in create_op
op_def=op_def)
File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/framework/ops.py", line 1654, in __init__
self._traceback = self._graph._extract_stack() # pylint: disable=protected-access
FailedPreconditionError (see above for traceback): Attempting to use uninitialized value beta1_power
[[Node: beta1_power/read = Identity[T=DT_FLOAT, _class=["loc:#Variable"], _device="/job:localhost/replica:0/task:0/device:GPU:0"](beta1_power)]]
Can somebody help me figure out where the problem is? Thanks a lot
This error generally occurs when you haven't initialized the optimizer.
So just add self.optimize before you initialize all the global variables.
Your code should look like this.
def __init__(self, input, labels, dataset):
self.input = input
self.true_labels = labels
#'dataset' is an instance of a class that
#I am using to read the training images
self.data = dataset
self.build()
self._optimize = None
self.sess = tf.Session()
self.optimize()
self.sess.run(tf.global_variables_initializer())
I am trying to build this LSTM network and I ran into this error constantly. I looked it up on Google and am still not sure what is going on. I have tried adding this:with tf.variable_scope("cell1"). Still not working. Any help is greatly appreciated.
Here is the code of the main LSTM structure
def lstm_structure(self):
train_dataset, train_labels, valid_dataset, valid_labels, test_dataset, test_labels = self.data_preprocessing(
'SSE_Composite_Index.csv')
graph=tf.Graph()
with graph.as_default():
'''Placeholders'''
X=tf.placeholder(tf.float32,shape=[None,self.num_steps,self.input_size])
y=tf.placeholder(tf.float32,shape=[None,self.num_classes])
valid=tf.constant(valid_dataset)
test=tf.constant(test_dataset)
'''Weights'''
weights={'in':tf.Variable(tf.random_normal([self.input_size,self.num_neurons])),'out':tf.Variable(tf.random_normal([self.num_neurons,self.num_classes]))}
'''Biases'''
biases={'in':tf.Variable(tf.zeros(shape=[self.num_neurons,])),'out':tf.Variable(tf.zeros(shape=[self.num_classes,]))}
def lstm(X, weights, biases, reuse=True
):
'''from input to cell'''
with tf.variable_scope("foo") as f:
if reuse:
f.reuse_variables()
X = tf.reshape(X, shape=[-1, self.input_size])
X_in = tf.matmul(X, weights['in']) + biases['in']
X_in = tf.reshape(X_in, shape=[-1, self.num_steps, self.num_neurons])
'''cell'''
cell_ = tf.contrib.rnn.BasicLSTMCell(self.num_neurons, forget_bias=1, state_is_tuple=True)
_init_state = cell_.zero_state(self.batch_size, dtype=tf.float32)
outputs, states = tf.nn.dynamic_rnn(cell_, X_in, initial_state=_init_state, time_major=False, dtype=tf.float32)
'''from cell to output'''
results = tf.matmul(states[1], weights['out']) + biases['out']
return results
logits=lstm(X,weights,biases,False)
loss=tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=logits,labels=y))
optimizer = tf.train.AdamOptimizer(learning_rate=self.learning_rate).minimize(loss)
train_prediction=tf.nn.softmax(logits=logits)
valid_prediction=tf.nn.softmax(lstm(valid,weights,biases,True))
test_prediction=tf.nn.softmax(lstm(test,weights,biases,True))
with tf.Session(graph=graph) as session:
tf.initialize_all_variables().run()
print('Initialised')
loss_list = []
for step in range(self.num_epochs):
if step==0:
i = 0
while i < len(train_dataset):
start = i
end = i + self.batch_size
batch_data = train_dataset[start:end, :]
batch_label = train_labels[start:end, :]
i += self.batch_size
a, b, prediction = session.run([optimizer, loss, train_prediction],
feed_dict={X: batch_data, y: batch_label})
loss_list.append(b)
else:
i = 0
while i < len(train_dataset):
start = i
end = i + self.batch_size
batch_data = train_dataset[start:end, :]
batch_label = train_labels[start:end, :]
i += self.batch_size
a, b, prediction = session.run([optimizer, loss, train_prediction],
feed_dict={X: batch_data, y: batch_label})
loss_list.append(b)
if step % 10 == 0:
print('Step', step, 'Loss', b)
print('Training Accuracy', self.accuracy(prediction, batch_label), '%')
print('Validation Accuracy',
self.accuracy(valid_prediction.eval(), valid_labels), '%')
print('test Accuracy', self.accuracy(test_prediction.eval(), test_labels), '%')
print('Finished')
The error message that I am getting is:
Traceback (most recent call last):
File "C:/Users/LiXin/PycharmProjects/PythonProjects/LSTM_CLASSIFIER.py", line 114, in <module>
lstm.LSTM_structure()
File "C:/Users/LiXin/PycharmProjects/PythonProjects/LSTM_CLASSIFIER.py", line 87, in LSTM_structure
valid_prediction=tf.nn.softmax(LSTM(valid,weights,biases))
File "C:/Users/LiXin/PycharmProjects/PythonProjects/LSTM_CLASSIFIER.py", line 75, in LSTM
outputs,states=tf.nn.dynamic_rnn(cell_,X_in,initial_state=_init_state,time_major=False,dtype=tf.float32)
File "C:\Users\LiXin\PycharmProjects\PythonProjects\venv\lib\site-packages\tensorflow\python\ops\rnn.py", line 614, in dynamic_rnn
dtype=dtype)
File "C:\Users\LiXin\PycharmProjects\PythonProjects\venv\lib\site-packages\tensorflow\python\ops\rnn.py", line 777, in _dynamic_rnn_loop
swap_memory=swap_memory)
File "C:\Users\LiXin\PycharmProjects\PythonProjects\venv\lib\site-packages\tensorflow\python\ops\control_flow_ops.py", line 2816, in while_loop
result = loop_context.BuildLoop(cond, body, loop_vars, shape_invariants)
File "C:\Users\LiXin\PycharmProjects\PythonProjects\venv\lib\site-packages\tensorflow\python\ops\control_flow_ops.py", line 2640, in BuildLoop
pred, body, original_loop_vars, loop_vars, shape_invariants)
File "C:\Users\LiXin\PycharmProjects\PythonProjects\venv\lib\site-packages\tensorflow\python\ops\control_flow_ops.py", line 2590, in _BuildLoop
body_result = body(*packed_vars_for_body)
File "C:\Users\LiXin\PycharmProjects\PythonProjects\venv\lib\site-packages\tensorflow\python\ops\rnn.py", line 762, in _time_step
(output, new_state) = call_cell()
File "C:\Users\LiXin\PycharmProjects\PythonProjects\venv\lib\site-packages\tensorflow\python\ops\rnn.py", line 748, in <lambda>
call_cell = lambda: cell(input_t, state)
File "C:\Users\LiXin\PycharmProjects\PythonProjects\venv\lib\site-packages\tensorflow\python\ops\rnn_cell_impl.py", line 183, in __call__
return super(RNNCell, self).__call__(inputs, state)
File "C:\Users\LiXin\PycharmProjects\PythonProjects\venv\lib\site-packages\tensorflow\python\layers\base.py", line 575, in __call__
outputs = self.call(inputs, *args, **kwargs)
File "C:\Users\LiXin\PycharmProjects\PythonProjects\venv\lib\site-packages\tensorflow\python\ops\rnn_cell_impl.py", line 438, in call
self._linear = _Linear([inputs, h], 4 * self._num_units, True)
File "C:\Users\LiXin\PycharmProjects\PythonProjects\venv\lib\site-packages\tensorflow\python\ops\rnn_cell_impl.py", line 1171, in __init__
initializer=kernel_initializer)
File "C:\Users\LiXin\PycharmProjects\PythonProjects\venv\lib\site-packages\tensorflow\python\ops\variable_scope.py", line 1203, in get_variable
constraint=constraint)
File "C:\Users\LiXin\PycharmProjects\PythonProjects\venv\lib\site-packages\tensorflow\python\ops\variable_scope.py", line 1092, in get_variable
constraint=constraint)
File "C:\Users\LiXin\PycharmProjects\PythonProjects\venv\lib\site-packages\tensorflow\python\ops\variable_scope.py", line 417, in get_variable
return custom_getter(**custom_getter_kwargs)
File "C:\Users\LiXin\PycharmProjects\PythonProjects\venv\lib\site-packages\tensorflow\python\ops\rnn_cell_impl.py", line 186, in _rnn_get_variable
variable = getter(*args, **kwargs)
File "C:\Users\LiXin\PycharmProjects\PythonProjects\venv\lib\site-packages\tensorflow\python\ops\variable_scope.py", line 394, in _true_getter
use_resource=use_resource, constraint=constraint)
File "C:\Users\LiXin\PycharmProjects\PythonProjects\venv\lib\site-packages\tensorflow\python\ops\variable_scope.py", line 742, in _get_single_variable
name, "".join(traceback.format_list(tb))))
ValueError: Variable rnn/basic_lstm_cell/kernel already exists, disallowed. Did you mean to set reuse=True or reuse=tf.AUTO_REUSE in VarScope? Originally defined at:
File "C:\Users\LiXin\PycharmProjects\PythonProjects\venv\lib\site-packages\tensorflow\python\framework\ops.py", line 1470, in __init__
self._traceback = self._graph._extract_stack() # pylint: disable=protected-access
File "C:\Users\LiXin\PycharmProjects\PythonProjects\venv\lib\site-packages\tensorflow\python\framework\ops.py", line 2956, in create_op
op_def=op_def)
File "C:\Users\LiXin\PycharmProjects\PythonProjects\venv\lib\site-packages\tensorflow\python\framework\op_def_library.py", line 787, in _apply_op_helper
op_def=op_def)
The following code works, however.
def RNN_neural_network_model(x):
layer={'weights':tf.Variable(tf.random_normal([rnn_size,n_classes])),
'biases':tf.Variable(tf.random_normal([n_classes]))}
x=tf.transpose(x,[1,0,2])
x=tf.reshape(x,[-1,chunk_size])
x=tf.split(x,n_chunks,0)
lstm_cell=tf.contrib.rnn.BasicLSTMCell(rnn_size,state_is_tuple=True)
# stacked_lstm = tf.contrib.rnn.MultiRNNCell(
# [lstmcell(rnn_size) for _ in range(num_layers)])
outputs, states=rnn.static_rnn(lstm_cell,x,dtype=tf.float32)
output = tf.matmul(outputs[-1], layer['weights'])+ layer['biases']
return output
You should wrap your LSTM definition inside a variable scope and then reuse it for validation and testing. Try the following
def LSTM(X,weights,biases, reuse=reuse):
'''from input to cell'''
with tf.variable_scope("foo") as f:
if reuse:
f.reuse_variables()
X=tf.reshape(X,shape=[-1,self.input_size])
X_in=tf.matmul(X,weights['in'])+biases['in']
X_in=tf.reshape(X_in,shape=[-1,self.num_steps,self.num_neurons])
'''cell'''
cell_=tf.contrib.rnn.BasicLSTMCell(self.num_neurons,forget_bias=1,state_is_tuple=True)
_init_state=cell_.zero_state(self.batch_size,dtype=tf.float32)
outputs,states=tf.nn.dynamic_rnn(cell_,X_in,initial_state=_init_state,time_major=False,dtype=tf.float32)
'''from cell to output'''
results=tf.matmul(states[1],weights['out'])+biases['out']
return results
Change your training, testing code as below
logits=LSTM(X,weights,biases,False)
loss=tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=logits,labels=y))
optimizer = tf.train.AdamOptimizer(learning_rate=self.learning_rate).minimize(loss)
train_prediction=tf.nn.softmax(logits=logits)
valid_prediction=tf.nn.softmax(LSTM(valid,weights,biases, True))
test_prediction=tf.nn.softmax(LSTM(test,weights,biases, True))