I am using the tf.keras API to build my CNN model with the use of the tf.Dataset API to create a input pipeline for my model. The mnist dataset from the tf.keras.datasets is used for testing and prepared in the memory by executing the code:
(train_images,train_labels),(test_images,test_labels) = tf.keras.datasets.mnist.load_data()
and also some preprocessing to be compatible with my keras model:
Train_images = np.expand_dims(train_images,3).astype('float')/255.0
Test_images = np.expand_dims(test_images,3).astype('float')/255.0
Train_labels = tf.keras.utils.to_categorical(train_labels)
Test_labels = tf.keras.utils.to_categorical(test_labels)
Those data are stored in the memory as arrays and there are two option for creating a Dataset object. The first one is simply using tf.data.Dataset.from_tensor_slices:
image = tf.data.Dataset.from_tensor_slices((Train_images,Train_labels))
And input this resulting object to the model.fit():
model.fit(x=image,steps_per_epoch=1000)
OR input this dataset's iterator by:
iterator = image.make_one_shot_iterator()
model.fit(x=iterator,steps_per_epoch=1000)
Both of these two options just work fine since the dataset named image here is created using the data in the memory. However, according the the Importing Data here that we may like to avoid doing this because it copies data several times and takes up memory. So another option is creating such a dataset object based on tf.placeholder as well as the initialiable iterator:
X = tf.placeholder(tf.float32,shape = [60000,28,28,1])
Y = tf.placeholder(tf.float32,shape = [60000,10])
image2 = tf.data.Dataset.from_tensor_slices((X,Y))
iterator2 = image.make_initializable_iterator()
with tf.Session() as sess:
sess.run(iterator2.initializer,feed_dict={X:Train_images,Y:Train_labels}
sess.run(iterator2.get_next())
This kind of iterator works fine when using tf.Session() when fed with the data in the memory and avoids the multiple copies of the data. But I cannot find way to let it work with keras.model.fit() since you cannot really call the iterator.initializer or feed any data there. Is there a way to use this kind of iterator?
I don't think keras officially supports a case of passing initializable iterators, as you noted, there is no place to provide the placeholders and values mappings.
However a workaround is possible using keras callbacks:
import tensorflow as tf
import numpy as np
import pandas as pd
# Make sure only tensorflow.keras is imported, don't mix with keras
from tensorflow.keras import layers
import tensorflow.keras.backend as K
# example data
x_values = np.random.randn(200, 100).astype(np.float32)
y_labels = np.random.randint(low=0, high=9, size=200)
graph = tf.Graph()
with graph.as_default():
# make datasets from placeholders as in https://www.tensorflow.org/guide/datasets#reading_input_data
# X:
features_placeholder = tf.placeholder(tf.float32, x_values.shape, name='features')
dataset_x = tf.data.Dataset.from_tensor_slices({'x': features_placeholder})
# Y:
labels_placeholder = tf.placeholder(tf.float32, [None], name='labels')
dataset_y = tf.data.Dataset.from_tensor_slices({'y': labels_placeholder})
# compose datasets to make X-Y pairs for training
dataset0 = tf.data.Dataset.zip((dataset_x, dataset_y))
dataset0 = dataset0.batch(16).repeat()
# build model with keras
inputs = tf.keras.Input(name='x', shape=(x_values.shape[1],))
mlp1 = layers.Dense(16, name='mlp-1', activation='relu')
mlp1_out = mlp1(inputs)
output = layers.Dense(1, name='y', activation='linear')
output_out = output(mlp1_out)
model = tf.keras.Model(inputs=inputs, outputs=output_out)
# The compile step specifies the training configuration.
model.compile(optimizer=tf.train.RMSPropOptimizer(0.001), loss='mse', metrics=['mse'])
iterator = dataset0.make_initializable_iterator()
feed_dict = { labels_placeholder: y_labels, features_placeholder: x_values }
class InitIteratorCallback(tf.keras.callbacks.Callback):
"""
Ensures that placeholders in dataset are initialized before each epoch begins
"""
def on_epoch_begin(self, epoch, logs=None):
sess = K.get_session()
sess.run(iterator.initializer, feed_dict=feed_dict)
model.fit(iterator, callbacks=[InitIteratorCallback()],
epochs=10, steps_per_epoch=300)
Related
I am playing with the mnist_vae example and can't figure out how to properly save/load weights of the trained model.
enc_init_rng, dec_init_rng = random.split(random.PRNGKey(2))
_, init_encoder_params = encoder_init(enc_init_rng, (batch_size, 28 * 28))
_, init_decoder_params = decoder_init(dec_init_rng, (batch_size, 10))
init_params = init_encoder_params, init_decoder_params
opt_init, opt_update, get_params = optimizers.momentum(step_size, mass=0.9)
opt_state = opt_init(init_params)
after that, I train the model using opt_update and want to save it. However, I haven't found any function to save the optimizer state to the disk.
I tried to save parameters and initialize opt_state with them, but not all the information conserves, and the result opt_state_1 is not the original opt_state.
weights=get_params(opt_state)
jnp.save(file, weights)
weights = jnp.load(file,allow_pickle=True)
opt_state_1 = opt_init(init_params)
How do I properly save the model I trained?
import pickle
from jax.experimental import optimizers
trained_params = optimizers.unpack_optimizer_state(opt_state)
pickle.dump(trained_params, open(os.path.join(config["ckpt_path"], "best_ckpt.pkl"), "wb"))
best_params = pickle.load(open(os.path.join(config["ckpt_path"], "best_ckpt.pkl"), "rb"))
best_opt_state = optimizers.pack_optimizer_state(best_params)
I was able to train a model in TensorFlow with my own data. Input and Output of the model are images. I now tried to get the output of the predictions and save it to an png image file to see what's going on. Unfortunately I am getting an error when running the following function I created to test with predictions. My goal is to save the prediction that is also an image so I can open it with a normal image viewer.
Some more to the code. In my main I am creating an estimator
def predict_element(my_model, features):
eval_input_fn = tf.estimator.inputs.numpy_input_fn(
x=features,
num_epochs=1,
shuffle=False)
eval_results = my_model.predict(input_fn=eval_input_fn)
predictions = eval_results.next() #this returns a dict with my tensors
prediction_tensor = predictions["y"] #get the tensor from the dict
image_tensor = tf.reshape(prediction_tensor, [IMG_WIDTH, -1]) #reshape to a matrix due my returned tensor is a 1D flat one
decoded_image = tf.image.encode_png(image_tensor)
write_image = tf.write_file("output/my_output_image.png", decoded_image)
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
print(sess.run(write_image))
def get_input():
filename_dataset = tf.data.Dataset.list_files("features/*.png")
label_dataset = tf.data.Dataset.list_files("labels/*.png")
# Make a Dataset of image tensors by reading and decoding the files.
image_dataset = filename_dataset.map(lambda x: tf.cast(tf.image.decode_png(tf.read_file(x), channels=1),tf.float32))
l_dataset = label_dataset.map(lambda x: tf.cast(tf.image.decode_png(tf.read_file(x),channels=1),tf.float32))
image_reshape = image_dataset.map(lambda x: tf.reshape(x, [IM_WIDTH * IM_HEIGHT]))
label_reshape = l_dataset.map(lambda x: tf.reshape(x, [IM_WIDTH * IM_HEIGHT]))
iterator = image_reshape.make_one_shot_iterator()
iterator2 = label_reshape.make_one_shot_iterator()
next_img = iterator.get_next()
next_lbl = iterator2.get_next()
features = []
labels = []
# read all 10 images and labels and put it in the array
# so we can pass it to the estimator
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
for i in range(10):
t1, t2 = sess.run([next_img, next_lbl])
features.append(t1)
labels.append(t2)
return {"x": np.array(features)}, np.array(labels)
def main(unused_argv):
features, labels = get_input() # creating the features dict {"x": }
my_estimator = tf.estimator.Estimator(model_fn=my_cnn_model, model_dir="/tmp/my_model")
predict_element(my_estimator, features)
The error is
Graph is finalized and cannot be modified
With some easy print() statements I could see that retrieving the dict with
eval_results = my_model.predict(input_fn=eval_input_fn)
is probable the one which finalizes the graph.
I absolutely don't know what to do or where to look for a solution here. How could I save the output?
I tried this in my model_fn:
#the last layer of my network is dropout
predictions = {
"y": dropout
}
if mode == tf.estimator.ModeKeys.PREDICT:
reshape1 = tf.reshape(dropout, [-1,IM_WIDTH, IM_HEIGHT])
sliced = tf.slice(reshape1, [0,0,0], [1, IM_WIDTH, IM_HEIGHT])
encoded = tf.image.encode_png(tf.cast(sliced, dtype=tf.uint8))
outputfile = tf.write_file(params["output_path"], encoded)
return tf.estimator.EstimatorSpec(mode=mode, predictions=predictions)
My problem here is that I can't pass back the "outputfile" node so I can work with it.
Well your graph is finalized and cannot be modified. You can either add this tensorflow operations to your model (before running it) or simply write some python code which saves the images seperately (without using tensorflow). Maybe I'll find some old code of mine as an example.
You could also create a second graph, then you can use tensorflow without changing the existing model graph.
You have to distinguish between graph nodes and evaluated objects. tf.reshape doesn't take an array as input but a graph node.
https://www.tensorflow.org/programmers_guide/graphs
for everyone with the same problem here is my solution. I don't know if this is the proper way but it works.
In my predict function i created a second graph for the reshaping, slicing, encoding and saving like:
pred_dict = eval_results.next() #generator the predict function returns
preds = pred_dict["y"] #get the predictions from the dict
#create the second graph
g = tf.Graph()
with g.as_default():
inp = tf.Variable(preds)
reshape1 = tf.reshape(printnode, [IM_WIDTH, IM_HEIGHT, -1])
sliced = tf.slice(reshape1, [0,0,0], [ IM_WIDTH, IM_HEIGHT,1])
reshaped = tf.reshape(sliced, [IM_HEIGHT, IM_WIDTH, 1])
encoded = tf.image.encode_png(tf.image.convert_image_dtype(reshaped,tf.uint16))
outputfile = tf.write_file("/tmp/pred_output/prediction_img.png", encoded)
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
sess.run(outputfile)
I'm working on developing a CNN with the Cifar-10 dataset and to feed the data to the network, I am using the Dataset API to use feedable iterators with the handle placeholders: https://www.tensorflow.org/programmers_guide/datasets#creating_an_iterator. Personally I really like this method because it provides a clear and simple way to feed data to the network and switch between my testing and validation sets. However, when I save the graph at the end of training, the .meta file created is as large as the testing data I started with. I am using these operations to provide access later to the input placeholders and output operators:
tf.get_collection("validation_nodes")
tf.add_to_collection("validation_nodes", input_data)
tf.add_to_collection("validation_nodes", input_labels)
tf.add_to_collection("validation_nodes", predict)
And then use the following to save the graph:
Before training:
saver = tf.train.Saver()
After training:
save_path = saver.save(sess, "./my_model")
Is there a way to prevent TensorFlow from storing all the data in the graph? Thanks in advance!
You're creating a tf.constant for the dataset which is why it's added to the graph definition. The solution is to use an initializable iterator and define a placeholder. The first thing you do before you start running operations against the graph is to feed it the dataset. See the programmers guide under the "creating an iterator" section for an example.
https://www.tensorflow.org/programmers_guide/datasets#creating_an_iterator
I do exactly the same, so here is a copy/paste of the relevant parts of code that I use to achieve exactly your description (train/test sets of cifar10 using an initializable iterator):
def build_datasets(self):
""" Creates a train_iterator and test_iterator from the two datasets. """
self.imgs_4d_uint8_placeholder = tf.placeholder(tf.uint8, [None, 32, 32, 3], 'load_images_placeholder')
self.imgs_4d_float32_placeholder = tf.placeholder(tf.float32, [None, 32, 32, 3], 'load_images_float32_placeholder')
self.labels_1d_uint8_placeholder = tf.placeholder(tf.uint8, [None], 'load_labels_placeholder')
self.load_data_train = tf.data.Dataset.from_tensor_slices({
'data': self.imgs_4d_uint8_placeholder,
'labels': self.labels_1d_uint8_placeholder
})
self.load_data_test = tf.data.Dataset.from_tensor_slices({
'data': self.imgs_4d_uint8_placeholder,
'labels': self.labels_1d_uint8_placeholder
})
self.load_data_adversarial = tf.data.Dataset.from_tensor_slices({
'data': self.imgs_4d_float32_placeholder,
'labels': self.labels_1d_uint8_placeholder
})
# Train dataset pipeline
dataset_train = self.load_data_train
dataset_train = dataset_train.shuffle(buffer_size=50000)
dataset_train = dataset_train.repeat()
dataset_train = dataset_train.map(self._img_augmentation, num_parallel_calls=8)
dataset_train = dataset_train.map(self._img_preprocessing, num_parallel_calls=8)
dataset_train = dataset_train.batch(self.hyperparams['batch_size'])
dataset_train = dataset_train.prefetch(2)
self.iterator_train = dataset_train.make_initializable_iterator()
# Test dataset pipeline
dataset_test = self.load_data_test
dataset_test = dataset_test.map(self._img_preprocessing, num_parallel_calls=8)
dataset_test = dataset_test.batch(self.hyperparams['batch_size'])
self.iterator_test = dataset_test.make_initializable_iterator()
def init(self, sess):
self.cifar10 = Cifar10() # a class I wrote for loading cifar10
self.handle_train = sess.run(self.iterator_train.string_handle())
self.handle_test = sess.run(self.iterator_test.string_handle())
sess.run(self.iterator_train.initializer, feed_dict={self.handle: self.handle_train,
self.imgs_4d_uint8_placeholder: self.cifar10.train_data,
self.labels_1d_uint8_placeholder: self.cifar10.train_labels})
I like to perform image classification on our own large image libary (millions of labeled images) with tensorflow. I´m new to stackoverflow, python and tensorflow and worked myself through a few tutorials (mnist etc.) and got to the point, where i was able to prepare a TensorFlow datset from a dictionary including the absolute path to the images and the according labels. However, i´m stuck at the point using the dataset in a TensorFlow session. Here is my (example) code:
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import tensorflow as tf
import numpy as np
import time
import mymodule # I build my module to read the images and labels
from tensorflow.python.framework import ops
from tensorflow.python.framework import dtypes
from tensorflow.contrib.data import Iterator
beginTime = time.time()
batch_size = 100
learning_rate = 0.005
max_steps = 2
NUM_CLASSES = 25
def input_parser(img_path, label):
one_hot = tf.one_hot(label, NUM_CLASSES)
img_file = tf.read_file(img_path)
img_decoded = tf.image.decode_jpeg(img_file, channels = 3)
return img_decoded, one_hot
#Import Training data (returns the dicitonary with paths and labels)
train_dict = mymodule.getFileMap(labelList, imageList)
#Import Test data
test_dict = mymodule.getFileMap(labelList, imageList)
#Get train data
train_file_list, train_label_list = get_file_label_list(train_dict)
train_images_tensor = ops.convert_to_tensor(train_file_list, dtype=dtypes.string)
train_labels_tensor = ops.convert_to_tensor(train_label_list, dtype=dtypes.int64)
#Get test data
test_file_list, test_label_list = get_file_label_list(test_dict)
test_images_tensor = ops.convert_to_tensor(test_file_list, dtype=dtypes.string)
test_labels_tensor = ops.convert_to_tensor(test_label_list, dtype=dtypes.int64)
#Create TensorFlow Datset object
train_data = tf.data.Dataset.from_tensor_slices((train_images_tensor, train_labels_tensor))
test_data = tf.data.Dataset.from_tensor_slices((test_images_tensor, test_labels_tensor))
# Transform the datset so that it contains decoded images
# and one-hot vector labels
train_data = train_data.map(input_parser)
test_data = test_data.map(input_parser)
# Batching --> How to do it right?
#train_data = train_data.batch(batch_size = 100)
#test_data = train_data.batch(batch_size = 100)
#Define input placeholders
image_size = 990*990*3
images_placeholder = tf.placeholder(tf.float32, shape=[None, image_size])
labels_placeholder = tf.placeholder(tf.int64, shape=[None])
# Define variables (these afe the values we want to optimize)
weigths = tf.Variable(tf.zeros([image_size, NUM_CLASSES]))
biases = tf.Variable(tf.zeros([NUM_CLASSES]))
# Define the classifier´s result
logits = tf.matmul(images_placeholder, weigths) + biases
# Define the loss function
loss = tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(logits = logits, labels = labels_placeholder))
# Define the training operation
train_step = tf.train.GradientDescentOptimizer(learning_rate).minimize(loss)
# Operation comparing prediciton with true label
correct_prediciton = tf.equal(tf.argmax(logits, 1), labels_placeholder)
# Operation calculating the accuracy of our predicitons
accuracy = tf.reduce_mean(tf.cast(correct_prediciton, tf.float32))
#Create TensorFlow Iterator object
iterator = Iterator.from_structure(train_data.output_types,
train_data.output_shapes)
next_element = iterator.get_next()
#Create two initialization ops to switch between the datasets
train_init_op = iterator.make_initializer(train_data)
test_init_op = iterator.make_initializer(test_data)
with tf.Session() as sess:
#Initialize variables
sess.run(tf.global_variables_initializer())
sess.run(train_init_op)
for _ in range(10):
try:
elem = sess.run(next_element)
print(elem)
except tf.errors.OutOfRangeError:
print("End of training datset.")
break
Following this and this tutorial i could not solve the problem of how to use the (image and label) dataset in a tensorflow session for training. I was able to print out the datset by iterating through it, but wasn´t able to use it for learning.
I don´t understand how to access the images and labels seperately after they have been merged in the train_data = tf.data.Dataset.from_tensor_slices((train_images_tensor, train_labels_tensor)) operation, as requried by the 2nd tutorial. Also i don´t know how to implement batching correctly.
What i want to do in the session is basically this (from the 2nd tutorial):
# Generate input data batch
indices = np.random.choice(data_sets['images_train'].shape[0], batch_size)
images_batch = data_sets['images_train'][indices]
labels_batch = data_sets['labels_train'][indices]
# Periodically print out the model's current accuracy
if i % 100 == 0:
train_accuracy = sess.run(accuracy, feed_dict={
images_placeholder: images_batch, labels_placeholder: labels_batch})
print('Step {:5d}: training accuracy {:g}'.format(i, train_accuracy))
# Perform a single training step
sess.run(train_step, feed_dict={images_placeholder: images_batch,
labels_placeholder: labels_batch})
# After finishing the training, evaluate on the test set
test_accuracy = sess.run(accuracy, feed_dict={
images_placeholder: data_sets['images_test'],
labels_placeholder: data_sets['labels_test']})
print('Test accuracy {:g}'.format(test_accuracy))
endTime = time.time()
print('Total time: {:5.2f}s'.format(endTime - beginTime))
If anyone can tell me, how to access images and labels in the dataset sepearately and use it for training, i would be really thankful. Also a tip where and how to do the batching would be appreciated.
Thank you.
In your code, next_element is a tuple of two tensors, matching the structure of your datasets: i.e. it is a tuple whose first element is an image, and second element is a label. To access the individual tensors, you can do the following:
next_element = iterator.get_next()
next_image = next_element[0]
next_label = next_element[1]
# Or, in a single line:
next_image, next_label = iterator.get_next()
To batch a tf.data.Dataset, you can use the Dataset.batch() transformation. Your commented out code for this should simply work:
train_data = train_data.batch(batch_size = 100)
test_data = train_data.batch(batch_size = 100)
I have downloaded a tensorflow checkpoint model named inception_resnet_v2_2016_08_30.ckpt.
Do I need to create a graph (with all the variables) that were used when this checkpoint was created?
How do I make use of this model?
First of you have get the network architecture in memory. You can get the network architecture from here
Once you have this program with you, use the following approach to use the model:
from inception_resnet_v2 import inception_resnet_v2, inception_resnet_v2_arg_scope
height = 299
width = 299
channels = 3
X = tf.placeholder(tf.float32, shape=[None, height, width, channels])
with slim.arg_scope(inception_resnet_v2_arg_scope()):
logits, end_points = inception_resnet_v2(X, num_classes=1001,is_training=False)
With this you have all the network in memory, Now you can initialize the network with checkpoint file(ckpt) by using tf.train.saver:
saver = tf.train.Saver()
sess = tf.Session()
saver.restore(sess, "/home/pramod/Downloads/inception_resnet_v2_2016_08_30.ckpt")
If you want to do bottle feature extraction, its simple like lets say you want to get features from last layer, then simply you have to declare predictions = end_points["Logits"] If you want to get it for other intermediate layer, you can get those names from the above program inception_resnet_v2.py
After that you can call: output = sess.run(predictions, feed_dict={X:batch_images})
Do I need to create a graph (with all the variables) that were used when this checkpoint was created?
No, you don't.
As for how to use checkpoint file (cpkt file)
1.This article (TensorFlow-Slim image classification library) tells you how to train your model from scratch
2.The following is an example code from google blog
import numpy as np
import os
import tensorflow as tf
import urllib2
from datasets import imagenet
from nets import inception
from preprocessing import inception_preprocessing
slim = tf.contrib.slim
batch_size = 3
image_size = inception.inception_v3.default_image_size
checkpoints_dir = '/root/code/model'
checkpoints_filename = 'inception_resnet_v2_2016_08_30.ckpt'
model_name = 'InceptionResnetV2'
sess = tf.InteractiveSession()
graph = tf.Graph()
graph.as_default()
def classify_from_url(url):
image_string = urllib2.urlopen(url).read()
image = tf.image.decode_jpeg(image_string, channels=3)
processed_image = inception_preprocessing.preprocess_image(image, image_size, image_size, is_training=False)
processed_images = tf.expand_dims(processed_image, 0)
# Create the model, use the default arg scope to configure the batch norm parameters.
with slim.arg_scope(inception.inception_resnet_v2_arg_scope()):
logits, _ = inception.inception_resnet_v2(processed_images, num_classes=1001, is_training=False)
probabilities = tf.nn.softmax(logits)
init_fn = slim.assign_from_checkpoint_fn(
os.path.join(checkpoints_dir, checkpoints_filename),
slim.get_model_variables(model_name))
init_fn(sess)
np_image, probabilities = sess.run([image, probabilities])
probabilities = probabilities[0, 0:]
sorted_inds = [i[0] for i in sorted(enumerate(-probabilities), key=lambda x:x[1])]
plt.figure()
plt.imshow(np_image.astype(np.uint8))
plt.axis('off')
plt.show()
names = imagenet.create_readable_names_for_imagenet_labels()
for i in range(5):
index = sorted_inds[i]
print('Probability %0.2f%% => [%s]' % (probabilities[index], names[index]))
Another way of loading a pre-trained Imagenet model is
ResNet50
import tensorflow as tf
from tensorflow.keras.applications.resnet50 import ResNet50
model = ResNet50()
model.summary()
InceptionV3
iport tensorflow as tf
from tensorflow.keras.applications.inception_v3 import InceptionV3
model = InceptionV3()
model.summary()
You can check a detailed explanation related to this here