I have written my own keras Model, and I'm trying to pass a Keras Generator as input to model.fit.
The problem is that I don't know how to process the generator, when I'm in the call method of MyModel. How do I access the x and y from the generator in order to pass them as inputs to my Encoder and Decoder Network, and also keep the generator working its magic, loading the batches each epoch ?
Ok so this MyModel class which inherits tf.keras.Model
class MyModel(tf.keras.Model):
def __init__(self):
super(MyModel, self).__init__()
self.enc = Encoder()
self.dec1 = Decoder1()
self.dec2 = Decoder2()
def __call__(self, data_generator, **kwargs):
################################################
? how do I acces x and y in order to pass them to the encoder and decoder ?
and also keep the generator proprieties
###############################################
x_train, y_train = data_generator # ?????????
#####################################
dec_inputs = tf.concat((tf.zeros_like(y_train[:, :1, :]), y_train[:, :-1, :]), 1)
dec_inputs = dec_inputs[:, :, -hp.n_mels:]
print("########ENC INPUTS #####")
#print(tf.shape(x_train))
print("######################")
print("#########DEC INPUTS #####")
#print(tf.shape(dec_inputs))
print("######################")
memory = self.enc(x_train)
y_hat = self.dec1(dec_inputs, memory)
#z_hat = self.dec2(y_hat)
return y_hat
And this is my generator function
class DataGenerator(keras.utils.Sequence):
def __init__(self, list_IDs, ID_dictionary, labels, batch_size=8, dim1=(32, 32, 32), dim2=(32, 32, 32),
n_channels=None, n_classes=None, shuffle=True):
'Initialization'
self.dim1 = dim1 # dimensiune X
self.dim2 = dim2 # dimensiune Y
self.batch_size = batch_size
self.ID_dictionary = ID_dictionary
self.labels = labels
self.list_IDs = list_IDs
self.n_channels = n_channels
self.n_classes = n_classes
self.shuffle = shuffle
self.on_epoch_end()
def __len__(self):
'Denotes the number of batches per epoch'
return int(np.floor(len(self.list_IDs) / self.batch_size))
# 3
def __getitem__(self, index):
'Generate one batch of data'
# Generate indexes of the batch
indexes = self.indexes[index * self.batch_size:(index + 1) * self.batch_size]
# Find list of IDs
list_IDs_temp = [self.list_IDs[k] for k in indexes]
# Generate data
x, y = self.__data_generation(list_IDs_temp)
return x, y
# 1
def on_epoch_end(self):
'Updates indexes after each epoch'
self.indexes = np.arange(len(self.list_IDs))
if self.shuffle == True:
np.random.shuffle(self.indexes)
# 2
def __data_generation(self, list_IDs_temp):
# Initialization
x = np.empty((self.batch_size, self.dim1))
y = np.empty((self.batch_size, *self.dim2), dtype=float)
# Generate data
for i, ID in enumerate(list_IDs_temp):
# Store sample
x[i, ] = self.ID_dictionary[ID]
# Store class
y[i] = self.labels[ID]
return x, y
And this is how I call MyModel in main
listID, dict1, dict2, text_shape, mel_shape = get_batch()
# dict1 has the inputs ( text ) and dict2 has the labels ( the mels )
training_generator = DataGenerator(listID, dict1, dict2, dim1=text_shape, dim2=mel_shape)
model = MyModel()
model.compile(
optimizer=keras.optimizers.Adam(),
metrics=["accuracy"],
)
#model.fit_generator(generator=training_generator, use_multiprocessing=True, workers=6)
model.fit(training_generator, epochs=2)
call method is called in model.fit, which expects one input to it i.e x_input, so you cannot expect generator as the input of the call method when using model.fit method. Please read tensorflow.org/guide/keras/custom_layers_and_models and tensorflow.org/tutorials/text/nmt_with_attention for greater understanding how things are working.
Edit 1: how can you pass two variables in call method
# pass list [x,y] to your call function instead of only x, we will club x and y into one variable
def __call__(self, inputs):
x = inputs[0]
y = inputs[1]
# now you can use x and y coming from your generator without changing much
# update your generator to return [x,y] and y
def generator
yield [x, y], y
# simply call model.fit like you were doing before
model.fit(generator)
Related
enter image description here
So we've translated this image into a code.
class pre_h(nn.Module):
def __init__(self, vocab_size, hidden_size, window_size):
super().__init__()
self.window_size = window_size
self.in_layer = nn.Embedding(vocab_size, hidden_size)
self.weight_list = []
def forward(self, contexts):
for i in range(2 * self.window_size):
weight = self.in_layer(contexts[:,i])
self.weight_list.append(weight)
h = sum(self.weight_list)
h = h / len(self.weight_list)
return h
class UnigramSampler:
def __init__(self, corpus, power, sample_size):
super().__init__()
self.sample_size = sample_size
self.vocab_size = None
self.word_p = None
counts = collections.Counter()
for word_id in corpus:
counts[word_id] += 1
vocab_size = len(counts)
self.vocab_size = vocab_size
self.word_p = np.zeros(vocab_size)
for i in range(vocab_size):
self.word_p[i] = counts[i]
self.word_p = np.power(self.word_p, power)
self.word_p /= np.sum(self.word_p)
def get_negative_sample(self, target):
batch_size = target.shape[0]
negative_sample = np.random.choice(self.vocab_size, size=(batch_size, self.sample_size), replace=True, p=self.word_p)
negative_sample = torch.tensor(negative_sample)
return negative_sample
class NegativeSampling(nn.Module):
def __init__(self, vocab_size, hidden_size, corpus, power = 0.75, sample_size = 5):
super().__init__()
self.sample_size = sample_size
self.sampler = UnigramSampler(corpus, power, sample_size)
self.embedding = nn.Embedding(vocab_size, hidden_size)
self.sigmoid = nn.Sigmoid()
self.loss_layer = nn.CrossEntropyLoss()
self.hidden_size = hidden_size
def forward(self, h, target):
loss_data = 0
w = self.embedding(target)
w = torch.squeeze(w).reshape(self.hidden_size, -1)
loss = h # w
loss = self.sigmoid(loss)
correct_label = torch.ones_like(loss)
loss = self.loss_layer(loss, correct_label)
loss_data += loss
negative_target = self.sampler.get_negative_sample(target)
for i in range(self.sample_size):
w = self.embedding(negative_target[:,i])
w = torch.squeeze(w).reshape(self.hidden_size, -1)
loss = h # w
loss = self.sigmoid(loss)
negative_label = torch.zeros_like(loss)
loss = self.loss_layer(loss, negative_label)
return loss_data
class sample_cbow(nn.Module):
def __init__(self, vocab_size, hidden_size, corpus, window_size, power = 0.75, sample_size = 5):
super().__init__()
self.model_1 = pre_h(vocab_size, hidden_size, window_size)
self.model_2 = NegativeSampling(vocab_size, hidden_size, corpus, power, sample_size)
def forward(self, contexts, target):
out = self.model_1(contexts)
out = self.model_2(out, target)
return out
vocab_size = len(id_to_word)
model = sample_cbow(vocab_size, 100, corpus, 1)
learning_rate = 0.001
optimizer = torch.optim.Adam(model.parameters(), lr = learning_rate) # 아담으로 옵티마이저 설정
i = 1
for contexts, target in dataloader:
print('===================================')
print(i,'experiment')
contexts.to(device)
target.to(device)
model.to(device)
output = model(contexts, target)
optimizer.zero_grad()
output.backward()
optimizer.step()
Here is the dataset I used. I'm leaving this just in case.
class Corpus_Dataset(Dataset):
def __init__(self, corpus, window_size = 1):
contexts, target = create_contexts_target(corpus, window_size)
self.contexts = contexts
self.target = target
def __len__(self):
return len(self.contexts)
def __getitem__(self, idx):
contexts = torch.tensor(self.contexts[idx], dtype = torch.long)
target = torch.tensor(self.target[idx], dtype = torch.long)
return contexts, target
dataset = Corpus_Dataset(corpus, window_size = 2)
dataloader = DataLoader(dataset, batch_size = 100)
RuntimeError: Trying to backward through the graph a second time (or directly access saved tensors after they have already been freed). Saved intermediate values of the graph are freed when you call .backward() or autograd.grad(). Specify retain_graph=True if you need to backward through the graph a second time or if you need to access saved tensors after calling backward
We know that with backward(), the graph is no longer formed.
But I've only used one backward(), and I get this error even though I didn't recycle it again.
If the backward() of the model was not executed from the beginning, it would have been suspected as a problem with the model, but only the first or second time the backward() runs normally and does not proceed any further.
Clone() or detach() were used for more diverse tensors, but this error was not resolved.
I searched and thought about what the problem was, but I can't think of a solution anymore.
I'm sorry for my poor English.
Given a simple multivariate time series problem
l = [ list(range(1000)),list(range(1000,2000)),list(range(2000,3000)), list(range(3000,4000)), list(range(4000,5000))]
df = pd.DataFrame(l).T
df.columns = ['feat_1', 'feat_2', 'feat_3', 'feat_4', 'feat_5']
target_sensor = 'feat_5'
If We want to predict the value of target_sensor in t+forecast_lead time steps
forecast_lead = 15
print('\nforecast_lead', forecast_lead)
target = f"{target_sensor}_TARGET{forecast_lead}"
features = list(df.columns.difference([target]))
df[target] = df[target_sensor].shift(-forecast_lead)
df = df.iloc[:-forecast_lead]
The input preparation is based on this torch.Dataset class:
class My_Dataset(Dataset):
def __init__(self, dataframe, target, features, sequence_length):
self.features = features #list of columns
self.target = target #str target col name
self.sequence_length = sequence_length #history we want to use
self.X = torch.tensor(dataframe[features].values).float() #to tensor
self.y = torch.tensor(dataframe[target].values).float() #to tensor
def __len__(self):
return self.X.shape[0]
def __getitem__(self, i):
if i >= self.sequence_length - 1:
i_start = i - self.sequence_length + 1
x = self.X[i_start:(i + 1), :]
else:
padding = self.X[0].repeat(self.sequence_length - i - 1, 1)
x = self.X[0:(i + 1), :]
x = torch.cat((padding, x), 0)
return x, self.y[i]
With the following Dataloader (batch_size =1 ) for simplicity.
train_loader = DataLoader(train_dataset, batch_size=1, shuffle=False,num_workers=1)
test_loader = DataLoader(test_dataset, batch_size=1, shuffle=False, num_workers=1)
The Model I am using is this:
class LSTM_Multivariate_Time_Series_Regression(nn.Module):
def __init__(self, num_features, hidden_size):
super().__init__()
self.num_feture= num_features # this is the number of features
self.hidden_size = hidden_size
self.num_layers = 1 # OR MORE THAN 1 HERE
self.lstm = nn.LSTM(input_size=num_features,
hidden_size=hidden_size,
batch_first=True,
num_layers=self.num_layers
)
self.linear = nn.Linear(in_features=self.hidden_size,
out_features=1)
In the forward pass if NUM_LAYER = 1
def forward(self, x):
lstm_output , (hn, cn) = self.lstm(x)
out = self.linear(hn[0]) # First dim of Hn is num_layers, which is set to 1 above.
In the forward pass, before passing through the Linear Layer IF SELF.NUM_LAYER > 1, I suppose following options are available
Use the last hidden state hn[-1]
Use the concatenation of all hidden state
#Docs WITH BATCH FIRST = TRUE : lstm_output tensor of shape: (BATCH_SIZE , SEQ_LENGHT, HIDDEN_SIZE)
#docs WITH BATCH FIRST h_n: tensor of shape (NUM_LAYER, BATCH_SIZE, HIDDEN_SIZE) containing the final hidden state for each element in the batch
.1
out = self.linear(hn[-1]).flatten()
return out
Is a correct solution to add final Linear layer to the lstm layer this way?
I am training a keras model with a data generator that reads the data in batches from a directory. This works great with model.fit(). But when using model.predict(), I would like to have both ypred and ytrue values returned.
Can I enable/modify model.predict() to do this (maybe with a custom callback)?
class DataGenerator(tf.keras.utils.Sequence):
def__init__(self, ids, batch_size=256):
self.batch_size=batch_size
self.ids = ids
def __len__(self):
return(self.ids)
def __getitem__(self, index):
X, y = np.load(f'data/{index}.npy', allow_pickle=True)
return X, y
def on_epoch_end(self):
'''Shuffle ids in each epoch'''
self.ids = np.random.choice(self.ids, len(self.ids), replace=False)
model = buildModel() #builds a multilayer perceptron
train_ids = np.arange(10000) #training data are in data/0.npy, data/1.npy, ... data/9999.npy
val_ids = np.arange(10000, 12000)
train_generator = DataGenerator(train_ids)
val_generator = DataGenerator(val_ids)
# Train model
history = model.fit(x=train_generator, epochs=100)
# Validate model (but I don't have ytrue)
ypred = model.predict(x=val_generator).reshape(-1)
# What I would like to achieve
(ypred, ytrue) = model.predict(x=val_generator, callbacks=[some_custom_callback])
# Or
ypred = model.predict(x=val_generator)
ytrue = some_fancy_method(val_generator)
This can be done by adding a method to your DataGenerator class that takes the fitted model as input, applies it to the generated data batches, and returns ytrue and ypred.
class DataGenerator(tf.keras.utils.Sequence):
def__init__(self, ids, batch_size=256):
self.batch_size=batch_size
self.ids = ids
def __len__(self):
return(self.ids)
def __getitem__(self, index):
X, y = self.load_data(index)
return X, y
def load_data(self, index):
X, y = np.load(f'data/{index}.npy', allow_pickle=True)
return X, y
def predict(self, model):
ytrue, ypred = [], []
for index in self.ids:
X, y = self.load_data(index)
pred = model.predict(X).reshape(-1)
ytrue.extend(y)
ypred.extend(pred)
return ytrue, ypred
def on_epoch_end(self):
'''Shuffle ids in each epoch'''
self.ids = np.random.choice(self.ids, len(self.ids), replace=False)
train_generator = DataGenerator(train_ids)
val_generator = DataGenerator(val_ids)
# Train model
history = model.fit(x=train_generator, epochs=100)
# Validate model
ypred, ytrue = val_generator.predict(model)
I am trying to run the following code (as given in Tensorflow documentation) to create windows of my data and then flatten the dataset of datasets.
window_size = 5
windows = range_ds.window(window_size, shift=1)
for sub_ds in windows.take(5):
print(sub_ds)
flat_windows = windows.flat_map(lambda x: x)
The problem is that flat_windows.cardinality().numpy() returns cardinality to be -2 which is creating problem for me during training. I tried looking for ways to set_cardinality of a dataset but couldn't find anything. I also tried other ways of flattening a dataset of datasets, but again no success.
Edit-1: The problem with the training is that the shape is unknown (at Linear and Dense layers) when I am training a subclass model (given below). The model trains well when I train the model eagerly (through tf.config.run_functions_eagerly(True)) but that is slow. Therefore I want the input data to be known for the model training.
Neural Network
class NeuralNetworkModel(tf.keras.Model):
def __init__(self):
super(NeuralNetworkModel, self).__init__()
self.encoder = Encoder()
def train_step(self, inputs):
X = inputs[0]
Y = inputs[1]
with tf.GradientTape() as tape:
enc_X = self.encoder(X)
enc_Y = self.encoder(Y)
# loss:
loss = tf.norm(enc_Y - enc_X, axis = [0, 1], ord = 'fro')
# Compute gradients
trainable_vars = self.encoder.trainable_variables
gradients = tape.gradient(loss, trainable_vars)
# Update weights
self.optimizer.apply_gradients(zip(gradients, trainable_vars))
# Compute our own metrics
loss_tracker.update_state(loss)
# Return a dict mapping metric names to current value.
# Note that it will include the loss (tracked in self.metrics).
return {"loss": loss_tracker.result()}
#property
def metrics(self):
# We list our `Metric` objects here so that `reset_states()` can be
# called automatically at the start of each epoch
# or at the start of `evaluate()`.
# If you don't implement this property, you have to call
# `reset_states()` yourself at the time of your choosing.
return [loss_tracker]
def test_step(self, inputs):
X = inputs[0]
Y = inputs[1]
Psi_X = self.encoder(X)
Psi_Y = self.encoder(Y)
# loss:
loss = tf.norm(Psi_Y - Psi_X, axis = [0, 1], ord = 'fro')
# Compute our own metrics
loss_tracker.update_state(loss)
# Return a dict mapping metric names to current value.
# Note that it will include the loss (tracked in self.metrics).
return {"loss": loss_tracker.result()}
class Encoder(tf.keras.Model):
def __init__(self):
super(Encoder, self).__init__(dtype = 'float64', name = 'Encoder')
self.input_layer = DenseLayer(128)
self.hidden_layer1 = DenseLayer(128)
self.hidden_layer2 = DenseLayer(64)
self.hidden_layer3 = DenseLayer(64)
self.output_layer = LinearLayer(64)
def call(self, input_data, training):
fx = self.input_layer(input_data)
fx = self.hidden_layer1(fx)
fx = self.hidden_layer2(fx)
fx = self.hidden_layer3(fx)
return self.output_layer(fx)
class LinearLayer(tf.keras.layers.Layer):
def __init__(self, units):
super(LinearLayer, self).__init__(dtype = 'float64')
self.units = units
def build(self, input_shape):
input_dim = input_shape[-1]
self.w = self.add_weight(shape = (input_dim, self.units),
initializer = "random_normal",
trainable = True)
self.b = self.add_weight(shape = (self.units,),
initializer = tf.zeros_initializer(),
trainable = True)
def call(self, inputs):
return tf.matmul(inputs, self.w) + self.b
class DenseLayer(tf.keras.layers.Layer):
def __init__(self, units):
super(DenseLayer, self).__init__(dtype = 'float64')
self.units = units
def build(self, input_shape):
input_dim = input_shape[-1]
self.w = self.add_weight(shape = (input_dim, self.units),
initializer = "random_normal",
trainable = True)
self.b = self.add_weight(shape = (self.units,),
initializer = tf.zeros_initializer(),
trainable = True)
def call(self, inputs):
x = tf.matmul(inputs, self.w) + self.b
return tf.nn.elu(x)
I was wondering about this as well. Turns out that -2 is tf.data.UNKNOWN_CARDINALITY (https://www.tensorflow.org/api_docs/python/tf/data#UNKNOWN_CARDINALITY), which represents that TF doesn't know how many elements the flat_map returns per item.
I just asked Windowing a TensorFlow dataset without losing cardinality information? to see if anyone knows a way to window datasets without losing cardinality.
I am using data generator in Keras to train a model with a large dataset. But I am getting the error Error when checking input: expected input_8 to have 4 dimensions, but got array with shape () ever time on the last batch of the first epoch. But I checked my dataset file, it doesn't have an empty array, so how did empty array come? I even tried printing the array as they were generated and few of them were shown empty. Here is my code for the data generator:
class data_generator(Sequence):
def __init__(self,data_file,type_data,batch_size,shuffle=True):
self.data_file = data_file
self.type_data = type_data
self.batch_size = batch_size
self.shuffle = shuffle
self.on_epoch_end()
def on_epoch_end(self):
if self.type_data == "train":
self.indices = np.arange(3450000)
else:
self.indices = np.arange(345000)
if self.shuffle:
np.random.shuffle(self.indices)
def __data__generation(self,indices):
return X,Y
def __len__(self):
if self.type_data == "train":
return int(np.ceil(10000 / float(self.batch_size)))
else:
return int(np.ceil(1000 / float(self.batch_size)))
def __getitem__(self,index):
#print(self.indices[(index)*self.batch_size], self.indices[(index+1)*self.batch_size])
X = np.array(HDF5Matrix(self.data_file, self.type_data + "_X", start = self.indices[index*self.batch_size], end = self.indices[(index+1)*self.batch_size]))
Y = np.array(HDF5Matrix(self.data_file, self.type_data + "_Y", start = self.indices[index*self.batch_size], end = self.indices[(index+1)*self.batch_size]))
#print(X.shape, Y.shape)
return X,Y
And here is my code for starting the fit generator:
train_generator = data_generator("drive/My Drive/Dataset/dataset.h5", "train", 20)
eval_generator = data_generator("drive/My Drive/Dataset/dataset.h5", "eval", 20)
model = create_model()
history = model.fit_generator(generator = train_generator,epochs = 100,validation_data=eval_generator,use_multiprocessing=False)
How do I solve this issue? Also is there any alternative for data generator for training on large datasets? The data generator is very buggy and gives lots of error.
The code had few mistakes. I changed it and now it is working, but still, don't know why exactly that error occurred. Here is the new code:
class data_generator(Sequence):
def __init__(self,data_file,type_data,batch_size,shuffle=True):
self.data_file = data_file
self.type_data = type_data
self.batch_size = batch_size
self.shuffle = shuffle
self.on_epoch_end()
def on_epoch_end(self):
if self.type_data == "train":
self.indices = np.arange(3450000)
else:
self.indices = np.arange(345000)
if self.shuffle:
np.random.shuffle(self.indices)
def __data__generation(self,indices):
X = []
Y = []
for index in indices:
X.append(np.array(HDF5Matrix(self.data_file, self.type_data + "_X", start = index, end = index + 1)[0]))
Y.append(np.array(HDF5Matrix(self.data_file, self.type_data + "_Y", start = index, end = index + 1)[0]))
X = np.array(X)
Y = np.array(Y)
return X,Y
def __len__(self):
if self.type_data == "train":
return int(np.ceil(3450000 / float(self.batch_size)))
else:
return int(np.ceil(345000 / float(self.batch_size)))
def __getitem__(self,index):
indices = self.indices[index*self.batch_size:(index+1)*self.batch_size]
X, Y = self.__data__generation(indices)
#print(X.shape, Y.shape, index)
return X,Y
Keras needs while true (infinite loop) to avoid StopIteration. But in a plain generator, after correct steps_per_epoch (sample_size//batch_size), the shape would be zero.