We Keep Coding

How to save optimizer in Tensorflow 2.x

I am not using model.compile() function, and I am following 'Writing a training loop from scratch' (Therefore, simple 'model.save()' does not include my optimizer). Also, I create NN using a functional API in Tensorflow. So, I am able to save using 'model.save()', but it saves only NN model and does not save the state of my optimizer.
I think I was able to save an optimizer using pickle, but somehow I am no longer able to save it before
(it give me an error message (AttributeError: Can't pickle local object 'make_gradient_clipnorm_fn..'))
I can use 'get_config' and 'from_config' methods, but it does not save weights. There is another solution to save weights like here. I am looking for a better (and simple) solution.
(and actually this method does not work for me because I am not only training 'model.trainable_weights', but also other 'tf.Variable', which are not belong to my 'model'.)
Is there any easy and convenient way to save and load optimizer?
Thanks
colab: https://colab.research.google.com/drive/1Jn2lbMcaVURpKITL6qEQ-05XndVDoABQ?usp=sharing
or
--code--
import numpy as np
import tensorflow as tf
import matplotlib
import matplotlib.pyplot as plt
import datetime
import os
import importlib
import pickle
import time
from pathlib import Path
gpus = tf.config.experimental.list_physical_devices('GPU')
print(gpus)
"""Generate samples for training"""
my_dtype = 'float32'
tf.keras.backend.set_floatx(my_dtype)
num_smpl = 500
amp1 = 0.5
x = 0.5*np.random.rand(num_smpl, 1).astype(np.float32)
y = amp1 * np.heaviside(x, 0) - amp1*0.2*np.heaviside(x-0.15, 0) \
+ amp1*0.2*np.heaviside(x-0.17, 0)
mu, sigma = 0, 0.001 # mean and standard deviation
white_noise = np.random.normal(mu, sigma, num_smpl).reshape(-1, 1)
y = y + white_noise
fig = plt.figure()
ax = fig.add_subplot(111)
ax.scatter(x, y, s=5, edgecolors='k')
"""Create model using a funtional API"""
layer_input = tf.keras.layers.Input(shape=(1,), dtype=my_dtype)
layer_dense = tf.keras.layers.Dense(8, activation='tanh', dtype=my_dtype)(layer_input)
layer_dense = tf.keras.layers.Dense(8, activation='tanh', dtype=my_dtype)(layer_dense)
layer_output = tf.keras.layers.Dense(1, name='', dtype=my_dtype)(layer_dense)
model = tf.keras.Model(inputs=layer_input, outputs=layer_output, name='my_model')
model.summary()
epoch = 1
optimizer = tf.keras.optimizers.Adam(learning_rate=0.01, epsilon=1e-09)
loss_fn = tf.keras.losses.MeanSquaredError()
"""Training using a custom training loop"""
while epoch < 100:
with tf.GradientTape() as t1:
output = model(x, training=True)
mse = loss_fn(output, y)
grads = t1.gradient(mse, model.trainable_weights)
optimizer.apply_gradients(zip(grads, model.trainable_weights))
current_time = datetime.datetime.now().strftime("%Y%m%d-%H%M%S")
if epoch % 10 == 0:
print('Epoch: {}, Loss: {}'.format(epoch, mse))
epoch = epoch + 1
"""Normal variables can be saved using pickle"""
print('Cur. locaiton: ' + os.getcwd())
a = np.arange(0., 10.)
print(a)
with open('test.pkl', 'wb') as file:
pickle.dump(a, file)
"""***How to save optimizer?***"""
with open('my_opti.pkl', 'wb') as file:
pickle.dump(optimizer, file)
'

restoring weights of an already saved Tensorflow .pb model

I have seen many posts about restoring an already saved TF models here, but none could answer my question. Using TF 1.0.0
Specifically, I am interested in seeing the weights for inceptionv3 model which is publicly available in .pb file here. I managed to restore it back using a small chunk of Python code and can access the graphs high-level view in tensorboard:
from tensorflow.python.platform import gfile
INCEPTION_LOG_DIR = '/tmp/inception_v3_log'
if not os.path.exists(INCEPTION_LOG_DIR):
os.makedirs(INCEPTION_LOG_DIR)
with tf.Session() as sess:
model_filename = './model/tensorflow_inception_v3_stripped_optimized_quantized.pb'
with gfile.FastGFile(model_filename, 'rb') as f:
graph_def = tf.GraphDef()
graph_def.ParseFromString(f.read())
_= tf.import_graph_def(graph_def,name='')
writer = tf.train.SummaryWriter(INCEPTION_LOG_DIR, graph_def)
writer=tf.summary.FileWriter(INCEPTION_LOG_DIR, graph_def)
writer.close()
However, I failed to access any layers' weights.
tensors= tf.import_graph_def(graph_def,name='')
returns empty, even if I add the arbitrary return_elements=. Does it have any weights at all? If yes, what is the appropriate procedure here? Thanks.

use this code to print your tensor's value :
with tf.Session() as sess:
print sess.run('your_tensor_name')
you can use this code to retrieve tensor names:
op = sess.graph.get_operations()
for m in op :
print(m.values())

There is a difference between restoring weights and printing them. The former one denotes that one would like to import the weight values from already saved ckpt files for retraining or inference while the latter may be for inspection. Also .pb file encodes model parameters as tf.constant() ops. As a result, the model parameters would not appear in tf.trainable_variables(), hence you can't use .pb directly to restore the weights. From your question I take that you just want to 'see' the weights for inspection.
Let us first load the graph from .pb file.
import tensorflow as tf
from tensorflow.python.platform import gfile
GRAPH_PB_PATH = './model/tensorflow_inception_v3_stripped_optimized_quantized.pb' #path to your .pb file
with tf.Session(config=config) as sess:
print("load graph")
with gfile.FastGFile(GRAPH_PB_PATH,'rb') as f:
graph_def = tf.GraphDef()
graph_def.ParseFromString(f.read())
sess.graph.as_default()
tf.import_graph_def(graph_def, name='')
graph_nodes=[n for n in graph_def.node]
Now when you freeze a graph to .pb file your variables are converted to Const type and the weights which were trainabe variables would also be stored as Const in .pb file. graph_nodes contains all the nodes in graph. But we are interested in all the Const type nodes.
wts = [n for n in graph_nodes if n.op=='Const']
Each element of wts is of NodeDef type. It has several atributes such as name, op etc. The values can be extracted as follows -
from tensorflow.python.framework import tensor_util
for n in wts:
print "Name of the node - %s" % n.name
print "Value - "
print tensor_util.MakeNdarray(n.attr['value'].tensor)
Hope this solves your concern.

You can use this code to get the names of tensor.
[tensor.name for tensor in tf.get_default_graph().as_graph_def().node]

Just small utils to print .pb model weights:
import argparse
import tensorflow as tf
from tensorflow.python.framework import tensor_util
def print_pb_weights(pb_filepath):
graph_def = tf.GraphDef()
with tf.gfile.GFile(pb_filepath, "rb") as f:
graph_def.ParseFromString(f.read())
tf.import_graph_def(graph_def, name='')
for node in graph_def.node:
if node.op == 'Const':
print('-' * 60)
print('op:', node.op)
print('name:', node.name)
arr = tensor_util.MakeNdarray(node.attr['value'].tensor)
print('shape:', list(arr.shape))
print(arr)
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument('pb_filepath')
args = parser.parse_args()
print_pb_weights(args.pb_filepath)

principal component analysis in RNN

If I want to project sequence(features) A,B,and C to target sequence with tensorflow LSTM, how can I know the importance of each features affected the target? Does the principal component analysis help? If pca can helps, how to do?
The structure（columns） of data set such as below:
A sequence
B sequence
C sequence
Target sequence

What would the principal components of this sequence be? What you could do is take the PCA of A sequence, B sequence and C sequence and visualise that...
Here is a simple tutorial on visualising PCA with Tensorboard: http://www.pinchofintelligence.com/simple-introduction-to-tensorboard-embedding-visualisation/
%matplotlib inline
import matplotlib.pyplot as plt
import tensorflow as tf
import numpy as np
import os
from tensorflow.contrib.tensorboard.plugins import projector
from tensorflow.examples.tutorials.mnist import input_data
LOG_DIR = 'minimalsample'
NAME_TO_VISUALISE_VARIABLE = "mnistembedding"
TO_EMBED_COUNT = 500
path_for_mnist_sprites = os.path.join(LOG_DIR,'mnistdigits.png')
path_for_mnist_metadata = os.path.join(LOG_DIR,'metadata.tsv')
mnist = input_data.read_data_sets("MNIST_data/", one_hot=False)
batch_xs, batch_ys = mnist.train.next_batch(TO_EMBED_COUNT)
embedding_var = tf.Variable(batch_xs, name=NAME_TO_VISUALISE_VARIABLE)
summary_writer = tf.summary.FileWriter(LOG_DIR)
config = projector.ProjectorConfig()
embedding = config.embeddings.add()
embedding.tensor_name = embedding_var.name
# Specify where you find the metadata
embedding.metadata_path = path_for_mnist_metadata #'metadata.tsv'
# Specify where you find the sprite (we will create this later)
embedding.sprite.image_path = path_for_mnist_sprites #'mnistdigits.png'
embedding.sprite.single_image_dim.extend([28,28])
# Say that you want to visualise the embeddings
projector.visualize_embeddings(summary_writer, config)
sess = tf.InteractiveSession()
sess.run(tf.global_variables_initializer())
saver = tf.train.Saver()
saver.save(sess, os.path.join(LOG_DIR, "model.ckpt"), 1)
with open(path_for_mnist_metadata,'w') as f:
f.write("Index\tLabel\n")
for index,label in enumerate(batch_ys):
f.write("%d\t%d\n" % (index,label))
Hope this helps you think about PCA!

open tensorflow graph from file

I'm trying to use tensorflow for study and i don't undestand how to open and use saved early in file my graph with type tf.Graph. Something like this:
import tensorflow as tf
my_graph = tf.Graph()
with g.as_default():
x = tf.Variable(0)
b = tf.constant(-5)
k = tf.constant(2)
y = k*x + b
tf.train.write_graph(my_graph, '.', 'graph.pbtxt')
f = open('graph.pbtxt', "r")
# Do something with "f" to get my saved graph and use it below in
# tf.Session(graph=...) instead of dots
with tf.Session(graph=...) as sess:
tf.initialize_all_variables().run()
y1 = sess.run(y, feed_dict={x: 5})
y2 = sess.run(y, feed_dict={x: 10})
print(y1, y2)

You have to load file contents, parse it to GraphDef and then import.
It will be imported into current graph. You may want to wrap it with graph.as_default(): context manager.
import tensorflow as tf
from tensorflow.core.framework import graph_pb2 as gpb
from google.protobuf import text_format as pbtf
gdef = gpb.GraphDef()
with open('my-graph.pbtxt', 'r') as fh:
graph_str = fh.read()
pbtf.Parse(graph_str, gdef)
tf.import_graph_def(gdef)

One option: take a look at the Tensorflow MetaGraph save/restore support, documented here: https://www.tensorflow.org/versions/r0.11/how_tos/meta_graph/index.html

I solved this problem this way: first, i name needed calculation in my graph "output" and then save this model in code below...
import tensorflow as tf
x = tf.placeholder(dtype=tf.float64, shape=[], name="input")
a = tf.Variable(111, name="var1", dtype=tf.float64)
b = tf.Variable(-666, name="var2", dtype=tf.float64)
y = tf.add(x, a, name="output")
saver = tf.train.Saver()
with tf.Session() as sess:
tf.initialize_all_variables().run()
print(sess.run(y, feed_dict={x: 555}))
save_path = saver.save(sess, "model.ckpt", meta_graph_suffix='meta', write_meta_graph=True)
print("Model saved in file: %s" % save_path)
Second, I need to run certain operation in graph, which i know by name "output". So I just restore model in another code and run my restored calculation by taking necessary graph parts with names "input" and "output" :
import tensorflow as tf
# Restore graph to another graph (and make it default graph) and variables
graph = tf.Graph()
with graph.as_default():
saver = tf.train.import_meta_graph("model.ckpt.meta")
y = graph.get_tensor_by_name("output:0")
x = graph.get_tensor_by_name("input:0")
with tf.Session() as sess:
saver.restore(sess, "model.ckpt")
print(sess.run(y, feed_dict={x: 888}))
# Variable out:
for var in tf.all_variables():
print("%s %.2f" % (var.name, var.eval()))

How to load pre-trained tensorflow model named inception by Google?

I have downloaded a tensorflow checkpoint model named inception_resnet_v2_2016_08_30.ckpt.
Do I need to create a graph (with all the variables) that were used when this checkpoint was created?
How do I make use of this model?

First of you have get the network architecture in memory. You can get the network architecture from here
Once you have this program with you, use the following approach to use the model:
from inception_resnet_v2 import inception_resnet_v2, inception_resnet_v2_arg_scope
height = 299
width = 299
channels = 3
X = tf.placeholder(tf.float32, shape=[None, height, width, channels])
with slim.arg_scope(inception_resnet_v2_arg_scope()):
logits, end_points = inception_resnet_v2(X, num_classes=1001,is_training=False)
With this you have all the network in memory, Now you can initialize the network with checkpoint file(ckpt) by using tf.train.saver:
saver = tf.train.Saver()
sess = tf.Session()
saver.restore(sess, "/home/pramod/Downloads/inception_resnet_v2_2016_08_30.ckpt")
If you want to do bottle feature extraction, its simple like lets say you want to get features from last layer, then simply you have to declare predictions = end_points["Logits"] If you want to get it for other intermediate layer, you can get those names from the above program inception_resnet_v2.py
After that you can call: output = sess.run(predictions, feed_dict={X:batch_images})

Do I need to create a graph (with all the variables) that were used when this checkpoint was created?
No, you don't.
As for how to use checkpoint file (cpkt file)
1.This article (TensorFlow-Slim image classification library) tells you how to train your model from scratch
2.The following is an example code from google blog
import numpy as np
import os
import tensorflow as tf
import urllib2
from datasets import imagenet
from nets import inception
from preprocessing import inception_preprocessing
slim = tf.contrib.slim
batch_size = 3
image_size = inception.inception_v3.default_image_size
checkpoints_dir = '/root/code/model'
checkpoints_filename = 'inception_resnet_v2_2016_08_30.ckpt'
model_name = 'InceptionResnetV2'
sess = tf.InteractiveSession()
graph = tf.Graph()
graph.as_default()
def classify_from_url(url):
image_string = urllib2.urlopen(url).read()
image = tf.image.decode_jpeg(image_string, channels=3)
processed_image = inception_preprocessing.preprocess_image(image, image_size, image_size, is_training=False)
processed_images = tf.expand_dims(processed_image, 0)
# Create the model, use the default arg scope to configure the batch norm parameters.
with slim.arg_scope(inception.inception_resnet_v2_arg_scope()):
logits, _ = inception.inception_resnet_v2(processed_images, num_classes=1001, is_training=False)
probabilities = tf.nn.softmax(logits)
init_fn = slim.assign_from_checkpoint_fn(
os.path.join(checkpoints_dir, checkpoints_filename),
slim.get_model_variables(model_name))
init_fn(sess)
np_image, probabilities = sess.run([image, probabilities])
probabilities = probabilities[0, 0:]
sorted_inds = [i[0] for i in sorted(enumerate(-probabilities), key=lambda x:x[1])]
plt.figure()
plt.imshow(np_image.astype(np.uint8))
plt.axis('off')
plt.show()
names = imagenet.create_readable_names_for_imagenet_labels()
for i in range(5):
index = sorted_inds[i]
print('Probability %0.2f%% => [%s]' % (probabilities[index], names[index]))

Another way of loading a pre-trained Imagenet model is
ResNet50
import tensorflow as tf
from tensorflow.keras.applications.resnet50 import ResNet50
model = ResNet50()
model.summary()
InceptionV3
iport tensorflow as tf
from tensorflow.keras.applications.inception_v3 import InceptionV3
model = InceptionV3()
model.summary()
You can check a detailed explanation related to this here