We Keep Coding

This model has not yet been built error on model.summary() - python

I've keras model defined as follow
class ConvLayer(Layer) :
def __init__(self, nf, ks=3, s=2, **kwargs):
self.nf = nf
self.grelu = GeneralReLU(leak=0.01)
self.conv = (Conv2D(filters = nf,
kernel_size = ks,
strides = s,
padding = "same",
use_bias = False,
activation = "linear"))
super(ConvLayer, self).__init__(**kwargs)
def rsub(self): return -self.grelu.sub
def set_sub(self, v): self.grelu.sub = -v
def conv_weights(self): return self.conv.weight[0]
def build(self, input_shape):
# No weight to train.
super(ConvLayer, self).build(input_shape) # Be sure to call this at the end
def compute_output_shape(self, input_shape):
output_shape = (input_shape[0],
input_shape[1]/2,
input_shape[2]/2,
self.nf)
return output_shape
def call(self, x):
return self.grelu(self.conv(x))
def __repr__(self):
return f'ConvLayer(nf={self.nf}, activation={self.grelu})'
class ConvModel(tf.keras.Model):
def __init__(self, nfs, input_shape, output_shape, use_bn=False, use_dp=False):
super(ConvModel, self).__init__(name='mlp')
self.use_bn = use_bn
self.use_dp = use_dp
self.num_classes = num_classes
# backbone layers
self.convs = [ConvLayer(nfs[0], s=1, input_shape=input_shape)]
self.convs += [ConvLayer(nf) for nf in nfs[1:]]
# classification layers
self.convs.append(AveragePooling2D())
self.convs.append(Dense(output_shape, activation='softmax'))
def call(self, inputs):
for layer in self.convs: inputs = layer(inputs)
return inputs
I'm able to compile this model without any issues
>>> model.compile(optimizer=tf.keras.optimizers.Adam(lr=lr),
loss='categorical_crossentropy',
metrics=['accuracy'])
But when I query the summary for this model, I see this error
>>> model = ConvModel(nfs, input_shape=(32, 32, 3), output_shape=num_classes)
>>> model.summary()
---------------------------------------------------------------------------
ValueError Traceback (most recent call last)
<ipython-input-220-5f15418b3570> in <module>()
----> 1 model.summary()
/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/network.py in summary(self, line_length, positions, print_fn)
1575 """
1576 if not self.built:
-> 1577 raise ValueError('This model has not yet been built. '
1578 'Build the model first by calling `build()` or calling '
1579 '`fit()` with some data, or specify '
ValueError: This model has not yet been built. Build the model first by calling `build()` or calling `fit()` with some data, or specify an `input_shape` argument in the first layer(s) for automatic build.
I'm providing input_shape for the first layer of my model, why is throwing this error?

The error says what to do:
This model has not yet been built. Build the model first by calling build()
model.build(input_shape) # `input_shape` is the shape of the input data
# e.g. input_shape = (None, 32, 32, 3)
model.summary()

There is a very big difference between keras subclassed model and other keras models (Sequential and Functional).
Sequential models and Functional models are datastructures that represent a DAG of layers. In simple words, Functional or Sequential model are static graphs of layers built by stacking one on top of each other like LEGO. So when you provide input_shape to first layer, these (Functional and Sequential) models can infer shape of all other layers and build a model. Then you can print input/output shapes using model.summary().
On the other hand, subclassed model is defined via the body (a call method) of Python code. For subclassed model, there is no graph of layers here. We cannot know how layers are connected to each other (because that's defined in the body of call, not as an explicit data structure), so we cannot infer input / output shapes. So for a subclass model, the input/output shape is unknown to us until it is first tested with proper data. In the compile() method, we will do a deferred compile and wait for a proper data. In order for it to infer shape of intermediate layers, we need to run with a proper data and then use model.summary(). Without running the model with a data, it will throw an error as you noticed. Please check GitHub gist for complete code.
The following is an example from Tensorflow website.
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras import layers
class ThreeLayerMLP(keras.Model):
def __init__(self, name=None):
super(ThreeLayerMLP, self).__init__(name=name)
self.dense_1 = layers.Dense(64, activation='relu', name='dense_1')
self.dense_2 = layers.Dense(64, activation='relu', name='dense_2')
self.pred_layer = layers.Dense(10, name='predictions')
def call(self, inputs):
x = self.dense_1(inputs)
x = self.dense_2(x)
return self.pred_layer(x)
def get_model():
return ThreeLayerMLP(name='3_layer_mlp')
model = get_model()
(x_train, y_train), (x_test, y_test) = keras.datasets.mnist.load_data()
x_train = x_train.reshape(60000, 784).astype('float32') / 255
x_test = x_test.reshape(10000, 784).astype('float32') / 255
model.compile(loss=keras.losses.SparseCategoricalCrossentropy(from_logits=True),
optimizer=keras.optimizers.RMSprop())
model.summary() # This will throw an error as follows
# ValueError: This model has not yet been built. Build the model first by calling `build()` or calling `fit()` with some data, or specify an `input_shape` argument in the first layer(s) for automatic build.
# Need to run with real data to infer shape of different layers
history = model.fit(x_train, y_train,
batch_size=64,
epochs=1)
model.summary()
Thanks!

Another method is to add the attribute input_shape() like this:
model = Sequential()
model.add(Bidirectional(LSTM(n_hidden,return_sequences=False, dropout=0.25,
recurrent_dropout=0.1),input_shape=(n_steps,dim_input)))

# X is a train dataset with features excluding a target variable
input_shape = X.shape
model.build(input_shape)
model.summary()

Make sure you create your model properly. A small typo mistake like the following code may also cause a problem:
model = Model(some-input, some-output, "model-name")
while the correct code should be:
model = Model(some-input, some-output, name="model-name")

If your Tensorflow, Keras version is 2.5.0 then just add Tensorflow when you import Keras package
Not this:
from tensorflow import keras
from keras.models import Sequential
import tensorflow as tf
Like this:
from tensorflow import keras
from tensorflow.keras.models import Sequential
import tensorflow as tf

Version issues of your Tensorflow, Keras, can be the reason for this.
Same problem I encountered during training for the LSTM model for regression.
Error:
ValueError: This model has not yet been built. Build the model first
by calling build() or by calling the model on a batch of data.
Earlier:
from tensorflow.keras.models import Sequential
from tensorflow.python.keras.models import Sequential
Corrected:
from keras.models import Sequential

I was also facing same error, so I have removed model.summary(). Then issue is resolved. As it arises if model of summary is defined before the model is built.
Here is the LINK for description which states that
Raises:
ValueError: if `summary()` is called before the model is built.**

Related

I had fine tuned a bert model in pytorch and saved its checkpoints via torch.save(model.state_dict(), 'model.pt')
Now When I want to reload the model, I have to explain whole network again and reload the weights and then push to the device.
Can anyone tell me how can I save the bert model directly and load directly to use in production/deployment?
Following is the training code and you can try running there in colab itself! After training completion, you will notice in file system we have a checkpoint file. But I want to save the model itself.
LINK TO COLAB NOTEBOOK FOR SAMPLE TRAINING
Following is the current inferencing code I written.
import torch
import torch.nn as nn
from transformers import AutoModel, BertTokenizerFast
import numpy as np
import json
tokenizer = BertTokenizerFast.from_pretrained('bert-base-uncased')
device = torch.device("cpu")
class BERT_Arch(nn.Module):
def __init__(self, bert):
super(BERT_Arch, self).__init__()
self.bert = bert
# dropout layer
self.dropout = nn.Dropout(0.1)
# relu activation function
self.relu = nn.ReLU()
# dense layer 1
self.fc1 = nn.Linear(768, 512)
# dense layer 2 (Output layer)
self.fc2 = nn.Linear(512, 2)
# softmax activation function
self.softmax = nn.LogSoftmax(dim=1)
# define the forward pass
def forward(self, sent_id, mask):
# pass the inputs to the model
_, cls_hs = self.bert(sent_id, attention_mask=mask, return_dict=False)
x = self.fc1(cls_hs)
x = self.relu(x)
x = self.dropout(x)
# output layer
x = self.fc2(x)
# apply softmax activation
x = self.softmax(x)
return x
bert = AutoModel.from_pretrained('bert-base-uncased')
model = BERT_Arch(bert)
path = './models/saved_weights_new_data.pt'
model.load_state_dict(torch.load(path, map_location=device))
model.to(device)
def inference(comment):
tokens_test = tokenizer.batch_encode_plus(
list([comment]),
max_length=75,
pad_to_max_length=True,
truncation=True,
return_token_type_ids=False
)
test_seq = torch.tensor(tokens_test['input_ids'])
test_mask = torch.tensor(tokens_test['attention_mask'])
predictions = model(test_seq.to(device), test_mask.to(device))
predictions = predictions.detach().cpu().numpy()
predictions = np.argmax(predictions, axis=1)
return predictions
I simply want to save a model from this notebook in a way such that I can use it for inferencing anywhere.

Just save your model using model.save_pretrained, here is an example:
model.save_pretrained("<path_to_dummy_folder>")
You can download the model from colab, save it on your gdrive or at any other location of your choice. While doing inference, you can just give path to this model (you may have to upload it) and start with inference.
To load the model
model = AutoModel.from_pretrained("<path_to_saved_pretrained_model>")
#Note: Instead of AutoModel class, you may use the task specific class as well.

I am working on a modified resnet, and want to insert dropout after activation layers.
I have tried the following but due to the model not being sequential, it did not work:
def add_dropouts(model, probability = 0.5):
print("Adding Dropouts")
updated_model = tf.keras.models.Sequential()
for layer in model.layers:
print("layer = ", layer)
updated_model.add(layer)
if isinstance(layer, tf.keras.layers.Activation):
updated_model.add(tf.keras.layers.Dropout(probability))
print("updated model Summary = ", updated_model.summary)
print("model Summary = ", model.summary)
model = updated_model
return model
base_model = tf.keras.applications.ResNet50V2(include_top=False, input_shape=input_img_shape, pooling='avg')
base_model = add_dropouts(base_model, probability = 0.5)
Then i tried my own version using the functional API, but this method doesn't work and returns a value error say Tensor doesn't have output.
prev_layer = base_model.layers[0]
for layer in base_model.layers:
next_layer = layer(prev_layer.output)
if isinstance(layer, tf.keras.layers.Activation):
next_layer = Dropout(0.5)(next_layer.output)
prev_layer = next_layer
Does anyone know how someone would add dropout layers into resnet or any other pretrained network?

So eventually i figured out how to do it; but its very hacky. Go to:
C:\ProgramData\Anaconda3\envs*your env name*\Lib\site-packages\tensorflow\python\keras\applications
Go to resnet.py. This will also change resnetv2 instances because it is based on the original resnet. Just Cntrl+F for activation,and where you see an activation layer(which is usually in the format x = Layer(x) building the model a layer at a time) then just add:
x = Dropout(prob)(x)
Here is an example:
if not preact:
x = layers.BatchNormalization(
axis=bn_axis, epsilon=1.001e-5, name='conv1_bn')(x)
x = layers.Activation('relu', name='conv1_relu')(x)#insert layer after each of these
x = layers.Dropout(prob)(x) # added dropout
Do this for all similar search results for 'activation'.
Then you will see the dropout added in your model summary.

I'm using Tensorflow 2 and I need to build a multi-input multi-output model, and my data is timeseries data, which does not have a consistent shape for its time dimension. I've tried many ways but none worked due to the inconsistent shape.
There are three data and one of them is used twice. They have the format of (number of files, None, 5), with the None dimension to be the inconsistent dimension.
Here's some testing codes that reproduce my issues, and I'm using a generator in this case, but feel free to change to whatever method. Could someone help me with this input pipeline?
import numpy as np
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras import layers
dummy_1 = [[[1.1,2,3,4,5],[2,3,4,5,6],[3,4,5,6,7]],
[[1.2,2,3,4,5],[2,3,4,5,6.8]],
[[1.3,2,3,4,5],[2,3,4,5,6],[3,4,5,6,7],[4,5,6,7,8.9]]]
dummy_2 = [[[1.1,2,3,4,5],[2,3,4,5,6]],
[[1.1,2,3,4,5],[2,3,4,5,6]],[3,4,5,6,7],
[[1.3,2,3,4,5],[2,3,4,5,6]]]
dummy_3 = [[[1.5,2,3,4,5],[2,3,4,5,6]],
[[1.6,2,3,4,5],[2,3,4,5,6]],[3,4,5,6,7],
[[1.7,2,3,4,5],[2,3,4,5,6]]]
def gen():
for i in range(len(dummy_1)):
yield(dummy_1[i],dummy_2[i],dummy_2[i],dummy_3[i])
def custom_loss(y_true, y_pred):
return tf.reduce_mean(tf.abs(y_pred - y_true))
class network():
def __init__(self):
input_1 = keras.Input(shape=(None,5))
input_2 = keras.Input(shape=(None,5))
output_1 = layers.Conv1DTranspose(16, 3, padding='same', activation='relu')(input_1)
output_2 = layers.Conv1DTranspose(16, 3, padding='same', activation='relu')(input_2)
self.model = keras.Model(inputs=[input_1, input_2],
outputs=[output_1, output_2])
# compile model
self.model.compile(optimizer=keras.optimizers.SGD(learning_rate=0.001),
loss={"mel_loss":custom_loss, "mag_loss":custom_loss})
def train(self):
self.dataset = tf.data.Dataset.from_generator(gen,
(tf.float32, tf.float32, tf.float32, tf.float32))
self.dataset.batch(32).repeat()
self.model.fit(self.dataset,epochs=3)
#self.model.fit([dummy_1, dummy_2],
# [dummy_2, dummy_3],
# epochs=3)
net = network()
net.train()

This is not possible currently for TF2, referring to https://github.com/tensorflow/tensorflow/issues/45112

I am trying to implement the Beholder plugin from Tensorboard into a simple CNN code (I am a beginner at Tensorflow), but I am not sure where to put the visualizer.update(session=session).
At the beginning I have:
from tensorboard.plugins.beholder import Beholder
LOG_DIRECTORY='/tmp/tensorflow_logs'
visualizer = Beholder(logdir=LOG_DIRECTORY)
I train my model like this:
model = Sequential()
model.add(Conv2D(32, (3, 3), input_shape=(253,27,3)))
.
.
.
model.compile(loss='binary_crossentropy',
optimizer='rmsprop',
metrics=['accuracy'])
Where should I put the visualizer.update(session=session) and what else should I put in my code, as for now it says No Beholder data was found. Thank you!

It would be appropriate to create a custom Keras callback, so that you can call visualizer.update(session=session) at the end of each epoch (or whenever you want). Here is an example showing how such callback could look like:
from tensorboard.plugins.beholder import Beholder
import tensorflow as tf
import keras.backend as K
import keras
LOG_DIRECTORY='/tmp/tensorflow_logs'
class BeholderCallback(keras.callbacks.Callback):
def __init__(self, tensor, logdir=LOG_DIRECTORY, sess=None):
self.visualizer = Beholder(logdir=logdir)
self.sess = sess
if sess is None:
self.sess = K.get_session()
self.tensor = tensor
def on_epoch_end(self, epoch, logs=None):
frame = self.sess.run(self.tensor) # depending on the tensor, this might require a feed_dict
self.visualizer.update(
session=self.sess,
frame=frame
)
Then, after defining your model, instantiate the callback and pass it to model.fit:
# Define your Keras model
# ...
# Prepare callback
sess = K.get_session()
beholder_callback = BeholderCallback(your_tensor, sess=sess)
# Fit data into model and pass callback to model.fit
model.fit(x=x_train,
y=y_train,
callbacks=[beholder_callback])
You could also use the argument arrays of visualizer.update in a similar way.

I want to create a simple neural network using Tensorflow and Keras.
When I try to instantiate a Model by subclassing the Model class
class TwoLayerFC(tf.keras.Model):
def __init__(self, hidden_size, num_classes):
super(TwoLayerFC, self).__init__()
self.fc1 = keras.layers.Dense(hidden_size,activation=tf.nn.relu)
self.fc2 = keras.layers.Dense(num_classes)
def call(self, x, training=None):
x = tf.layers.flatten(x)
x = self.fc1(x)
x = self.fc2(x)
return x
This is how I test the network
def test_TwoLayerFC():
tf.reset_default_graph()
input_size, hidden_size, num_classes = 50, 42, 10
model = TwoLayerFC(hidden_size, num_classes)
with tf.device(device):
x = tf.zeros((64, input_size))
scores = model(x)
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
scores_np = sess.run(scores)
print(scores_np.shape)
I get an error:
TypeError: init() takes at least 3 arguments (2 given)
I followed this tutorial, and it seems that there should be two parameters.

I read your code and I see a PyTorch model being created, including the mistake in the second Dense layer with two passed numbers.
Keras models should not follow the same logic of PyTorch models.
This model should be created like this:
input_tensor = Input(input_shape)
output_tensor = Flatten()(input_tensor)
output_tensor = Dense(hidden_size, activation='relu')(output_tensor)
output_tensor = Dense(num_classes)
model = keras.models.Model(input_tensor, output_tensor)
This model instance is ready to be compiled and trained:
model.compile(optimizer=..., loss = ..., metrics=[...])
model.fit(x_train, y_train, epochs=..., batch_size=..., ...)
There is no reason in Keras to subclass Model, unless you're a really advanced user trying some very unconventional things.
By the way, be careful not to mix tf.keras.anything with keras.anything. The first is a version of Keras maitained directly by tensorflow, while the second is original Keras. They're not exactly the same, tensorflow's version seems more buggy and mixing the two in the same code sounds like a bad idea.