Related
I'd like to make TFRecords into TFRecordsDataset. And unpack the data while feeding it. This is my function for parsing.
def read_tfrecord(tfrecord, epochs, batch_size):
dataset = tf.data.TFRecordDataset(tfrecord)
def parse(record):
features = {
"x_img": tf.io.FixedLenFeature([], tf.string),
"x_img_shape": tf.io.FixedLenFeature([], tf.int64),
"x_spectrogram": tf.io.FixedLenFeature([], tf.string),
"x_spec_shape": tf.io.FixedLenFeature([], tf.int64),
"x_wave": tf.io.FixedLenFeature([], tf.string),
"x_wave_shape": tf.io.FixedLenFeature([], tf.int64),
"y":tf.io.FixedLenFeature([], tf.string),
"y_shape": tf.io.FixedLenFeature([], tf.int64)}
example = tf.io.parse_single_example(record, features)
x_image = tf.io.decode_image(example["x_img"])
x_spec = tf.io.decode_image(example["x_spectrogram"])
x_wave = tf.cast(example["x_wave"], tf.string)
y = tf.io.decode_image(example["y"])
return x_image, x_spec, x_wave, y
dataset = dataset.map(parse)
dataset = dataset.shuffle(buffer_size=5)
dataset = dataset.prefetch(buffer_size=batch_size) #
dataset = dataset.batch(batch_size, drop_remainder=True)
dataset = dataset.repeat(epochs)
return dataset
I made an iterator to do the task:
train_files = path + "train.tfrecords"
EPOCHS = 100
BATCH_SIZE = 5
train_dataset = read_tfrecord(tfrecord=train_files,
epochs=EPOCHS,
batch_size=BATCH_SIZE)
iterator = iter(train_dataset)
x_img, x_spec, x_wave, y = next(iterator)
But I got this error:
InvalidArgumentError: Key: x_img_shape. Can't parse serialized Example.
[[{{node ParseSingleExample/ParseExample/ParseExampleV2}}]]
The code that serializes the data:
with tf.io.TFRecordWriter(tfrcpath) as file_writer:
for video in sorted(video_list):
print(video)
x_img = frame_extraction(raw_loc, video, interval)
x_spec, x_wave = audio_extraction(raw_loc, video, interval)
y = ground_truth_extraction(raw_loc, video, ground_truth, interval)
record_bytes = tf.train.Example(
features= tf.train.Features(
feature={
"x_img": tf.train.Feature(float_list=tf.train.FloatList(value= x_img.flatten())),
"x_img_shape": tf.train.Feature(int64_list=tf.train.Int64List(value= x_img.shape)),
"x_spec": tf.train.Feature(float_list=tf.train.FloatList(value= x_spec.flatten())),
"x_spec_shape": tf.train.Feature(int64_list=tf.train.Int64List(value= x_spec.shape)),
"x_wave": tf.train.Feature(float_list=tf.train.FloatList(value= x_wave.flatten())),
"x_wave_shape": tf.train.Feature(int64_list=tf.train.Int64List(value= x_wave.shape)),
"y": tf.train.Feature(float_list=tf.train.FloatList(value= y.flatten())),
"y_shape": tf.train.Feature(float_list=tf.train.FloatList(value= y.shape)),
})).SerializeToString()
file_writer.write(record_bytes)
What I have tried:
Solution in link suggests adding the shape of the data. So I tried update it as "x_img_shape": tf.io.FixedLenFeature([3], tf.int64), but didn't solve my issue.
My questions are:
What is the error meaning?
Why is the error only shows at x_img_shape but not x_img? x_img_shape is only used to record the shape of the images
Since I did flatten the images when writing theam as TFRecords, do I need to reshape the training images back? If so, how do I do that?
I was trying to make a prediction from a loaded tensorflow model. Though I'm not sure if it's correct how I previously saved it, specifically I have doubts about code inside serving_input_fn() function (MAX_SEQ_LENGTH=128):
def serving_input_fn():
feature_spec = { "input_ids" : tf.FixedLenFeature([None,MAX_SEQ_LENGTH], tf.int64),
"input_mask" : tf.FixedLenFeature([None,MAX_SEQ_LENGTH], tf.int64),
"segment_ids" : tf.FixedLenFeature([None,MAX_SEQ_LENGTH], tf.int64),
"label_ids" : tf.FixedLenFeature([None], tf.int64) }
serialized_tf_example = tf.placeholder(dtype=tf.string,shape=[None],name='input_example_tensor')
receiver_tensors = {'example': serialized_tf_example}
features = tf.parse_example(serialized_tf_example, feature_spec)
return tf.estimator.export.ServingInputReceiver(features, receiver_tensors)
estimator.export_saved_model('gs://bucket/trained_model', serving_input_receiver_fn=serving_input_fn)
When I try to predict from loaded model:
from tensorflow.contrib import predictor
predict_fn = predictor.from_saved_model(LOAD_PATH)
input_features_test = convert_examples_to_features( test_examples,label_list, MAX_SEQ_LENGTH, tokenizer)
predictions = predict_fn({'example':input_features_test[0]})
it returns this error:
ValueError: Cannot feed value of shape () for Tensor
'input_example_tensor:0', which has shape '(?,)'
How should I change serving_input_fn() method?
If you want to reproduce it: github_repo (you should download variables from here and put it in trained_model/1608370941/ folder)
This is the tutorial I followed to fine tuned BERT model on google cloud TPU.
I saved the model using following serving_input_fn() function (found in this tutorial):
def serving_input_fn():
label_ids = tf.placeholder(tf.int32, [None], name='label_ids')
input_ids = tf.placeholder(tf.int32, [None, MAX_SEQ_LENGTH], name='input_ids')
input_mask = tf.placeholder(tf.int32, [None, MAX_SEQ_LENGTH], name='input_mask')
segment_ids = tf.placeholder(tf.int32, [None, MAX_SEQ_LENGTH], name='segment_ids')
input_fn = tf.estimator.export.build_raw_serving_input_receiver_fn({
'label_ids': label_ids,
'input_ids': input_ids,
'input_mask': input_mask,
'segment_ids': segment_ids,
})()
return input_fn
Then I loaded it using the code mentioned below:
from tensorflow.contrib import predictor
predict_fn = predictor.from_saved_model(LOAD_PATH_GCP)
I converted input string to BERT model input features with following method:
def convert_single_string_to_input_dict(example_string_prep, vocab_file_path, max_seq_length):
#Inizialize BERT tokenizer
tokenizer = tokenization.FullTokenizer(vocab_file_path, do_lower_case=True)
token_a = tokenizer.tokenize(example_string_prep)
tokens = []
segments_ids = []
segment_ids = []
tokens.append("[CLS]")
segment_ids.append(0)
for token in token_a:
tokens.append(token)
segment_ids.append(0)
tokens.append('[SEP]')
segment_ids.append(0)
input_ids = tokenizer.convert_tokens_to_ids(tokens)
input_mask = [1] * len(input_ids)
while len(input_ids) < max_seq_length:
input_ids.append(0)
input_mask.append(0)
segment_ids.append(0)
label_id = [0]
padding = [0] * max_seq_length
print(len(input_ids),len(input_mask),len(segment_ids),len(label_id))
return {"input_ids":[input_ids,padding], "input_mask":[input_mask,padding], "segment_ids":[segment_ids,padding], "label_ids":label_id}
Finally I made a prediction using loaded model:
MAX_SEQ_LENGTH = 128
VOCAB_FILE_PATH = 'path/to/vocab'
example_features = convert_single_string_to_input_dict(example_string, VOCAB_FILE_PATH, MAX_SEQ_LENGTH)
prediction = predict_fn(example_features)['probabilities'][0]
prediction_dict = {'POS': round(prediction[1],4), 'NEG': round(prediction[0],4)}
pprint(f"prediction: {prediction_dict}")
I am trying to save a fine tuned bert model. I have ran the code correctly - it works fine, and in the ipython console I am able to call getPrediction and have it result the result.
I have my weight files saved (highest being model.ckpt-333.data-00000-of-00001
I have no idea how I would go about saving the model to be reuseable.
I am using bert-tensorflow.
import json
import pandas as pd
import tensorflow as tf
import tensorflow_hub as hub
from datetime import datetime
from sklearn.model_selection import train_test_split
import os
print("tensorflow version : ", tf.__version__)
print("tensorflow_hub version : ", hub.__version__)
#Importing BERT modules
import bert
from bert import run_classifier
from bert import optimization
from bert import tokenization
#set output directory of the model
OUTPUT_DIR = 'model'
##markdown Whether or not to clear/delete the directory and create a new one
DO_DELETE = False ##param {type:"boolean"}
if DO_DELETE:
try:
tf.gfile.DeleteRecursively(OUTPUT_DIR)
except:
pass
tf.io.gfile.makedirs(OUTPUT_DIR)
print('***** Model output directory: {} *****'.format(OUTPUT_DIR))
### Load the data
data = pd.read_csv("data/bbc-text.csv")
data.columns = ['category', 'text']
print('*****Data Loaded: {} *****'.format(data.head()))
#check to see if any null values are present.
print('*****Empty Data: {} *****'.format(data[data.isnull().any(axis=1)]))
#encode category variable into numeric
data.category = pd.Categorical(data.category)
data['code'] = data.category.cat.codes
from sklearn.model_selection import train_test_split
train, test = train_test_split(data, test_size=0.2, random_state=200)
## 2 -- Data Visualisation
print(data.code.unique())
import matplotlib.pyplot as plt
train['code'].value_counts().plot(kind = 'bar')
DATA_COLUMN = 'text'
LABEL_COLUMN = 'code'
label_list = [0, 1, 2, 3, 4]
plt.show()
## 2 -- Data Preprocessing
train_InputExamples = train.apply(lambda x: bert.run_classifier.InputExample(guid=None,
text_a = x[DATA_COLUMN],
text_b = None,
label = x[LABEL_COLUMN]), axis = 1)
test_InputExamples = test.apply(lambda x: bert.run_classifier.InputExample(guid=None,
text_a = x[DATA_COLUMN],
text_b = None,
label = x[LABEL_COLUMN]), axis = 1)
# This is a path to an uncased (all lowercase) version of BERT
BERT_MODEL_HUB = "https://tfhub.dev/google/bert_uncased_L-12_H-768_A-12/1"
def create_tokenizer_from_hub_module():
"""Get the vocab file and casing info from the Hub module."""
with tf.Graph().as_default():
bert_module = hub.Module(BERT_MODEL_HUB)
tokenization_info = bert_module(signature="tokenization_info", as_dict=True)
with tf.compat.v1.Session() as sess:
vocab_file, do_lower_case = sess.run([tokenization_info["vocab_file"],
tokenization_info["do_lower_case"]])
return bert.tokenization.FullTokenizer(
vocab_file=vocab_file, do_lower_case=do_lower_case)
tokenizer = create_tokenizer_from_hub_module()
# We'll set sequences to be at most 128 tokens long.
MAX_SEQ_LENGTH = 128
# Convert our train and validation features to InputFeatures that BERT understands.
train_features = bert.run_classifier.convert_examples_to_features(train_InputExamples, label_list, MAX_SEQ_LENGTH, tokenizer)
test_features = bert.run_classifier.convert_examples_to_features(test_InputExamples, label_list, MAX_SEQ_LENGTH, tokenizer)
#Example on first observation in the training set
print("Example of train[0] as a training set")
print("Sentence : ", train_InputExamples.iloc[0].text_a)
print("-"*30)
print("Tokens : ", tokenizer.tokenize(train_InputExamples.iloc[0].text_a))
print("-"*30)
print("Input IDs : ", train_features[0].input_ids)
print("-"*30)
print("Input Masks : ", train_features[0].input_mask)
print("-"*30)
print("Segment IDs : ", train_features[0].segment_ids)
## 3. Creating a Multiclass Classifier
def create_model(is_predicting, input_ids, input_mask, segment_ids, labels,
num_labels):
bert_module = hub.Module(
BERT_MODEL_HUB,
trainable=True)
bert_inputs = dict(
input_ids=input_ids,
input_mask=input_mask,
segment_ids=segment_ids)
bert_outputs = bert_module(
inputs=bert_inputs,
signature="tokens",
as_dict=True)
# Use "pooled_output" for classification tasks on an entire sentence.
# Use "sequence_outputs" for token-level output.
output_layer = bert_outputs["pooled_output"]
hidden_size = output_layer.shape[-1].value
# Create our own layer to tune for politeness data.
output_weights = tf.compat.v1.get_variable(
"output_weights", [num_labels, hidden_size],
initializer=tf.truncated_normal_initializer(stddev=0.02))
output_bias = tf.compat.v1.get_variable(
"output_bias", [num_labels], initializer=tf.zeros_initializer())
with tf.compat.v1.variable_scope("loss"):
# Dropout helps prevent overfitting
output_layer = tf.nn.dropout(output_layer, keep_prob=0.9)
logits = tf.matmul(output_layer, output_weights, transpose_b=True)
logits = tf.nn.bias_add(logits, output_bias)
log_probs = tf.nn.log_softmax(logits, axis=-1)
# Convert labels into one-hot encoding
one_hot_labels = tf.one_hot(labels, depth=num_labels, dtype=tf.float32)
predicted_labels = tf.squeeze(tf.argmax(log_probs, axis=-1, output_type=tf.int32))
# If we're predicting, we want predicted labels and the probabiltiies.
if is_predicting:
return (predicted_labels, log_probs)
# If we're train/eval, compute loss between predicted and actual label
per_example_loss = -tf.reduce_sum(one_hot_labels * log_probs, axis=-1)
loss = tf.reduce_mean(per_example_loss)
return (loss, predicted_labels, log_probs)
#A function that adapts our model to work for training, evaluation, and prediction.
# model_fn_builder actually creates our model function
# using the passed parameters for num_labels, learning_rate, etc.
def model_fn_builder(num_labels, learning_rate, num_train_steps,
num_warmup_steps):
"""Returns `model_fn` closure for TPUEstimator."""
def model_fn(features, labels, mode, params): # pylint: disable=unused-argument
"""The `model_fn` for TPUEstimator."""
input_ids = features["input_ids"]
input_mask = features["input_mask"]
segment_ids = features["segment_ids"]
label_ids = features["label_ids"]
is_predicting = (mode == tf.estimator.ModeKeys.PREDICT)
# TRAIN and EVAL
if not is_predicting:
(loss, predicted_labels, log_probs) = create_model(
is_predicting, input_ids, input_mask, segment_ids, label_ids, num_labels)
train_op = bert.optimization.create_optimizer(
loss, learning_rate, num_train_steps, num_warmup_steps, use_tpu=False)
# Calculate evaluation metrics.
def metric_fn(label_ids, predicted_labels):
accuracy = tf.compat.v1.metrics.accuracy(label_ids, predicted_labels)
true_pos = tf.compat.v1.metrics.true_positives(
label_ids,
predicted_labels)
true_neg = tf.compat.v1.metrics.true_negatives(
label_ids,
predicted_labels)
false_pos = tf.compat.v1.metrics.false_positives(
label_ids,
predicted_labels)
false_neg = tf.compat.v1.metrics.false_negatives(
label_ids,
predicted_labels)
return {
"eval_accuracy": accuracy,
"true_positives": true_pos,
"true_negatives": true_neg,
"false_positives": false_pos,
"false_negatives": false_neg
}
eval_metrics = metric_fn(label_ids, predicted_labels)
if mode == tf.estimator.ModeKeys.TRAIN:
return tf.estimator.EstimatorSpec(mode=mode,
loss=loss,
train_op=train_op)
else:
return tf.estimator.EstimatorSpec(mode=mode,
loss=loss,
eval_metric_ops=eval_metrics)
else:
(predicted_labels, log_probs) = create_model(
is_predicting, input_ids, input_mask, segment_ids, label_ids, num_labels)
predictions = {
'probabilities': log_probs,
'labels': predicted_labels
}
return tf.estimator.EstimatorSpec(mode, predictions=predictions)
# Return the actual model function in the closure
return model_fn
# Compute train and warmup steps from batch size
# These hyperparameters are copied from this colab notebook (https://colab.sandbox.google.com/github/tensorflow/tpu/blob/master/tools/colab/bert_finetuning_with_cloud_tpus.ipynb)
BATCH_SIZE = 16
LEARNING_RATE = 2e-5
NUM_TRAIN_EPOCHS = 3.0
# Warmup is a period of time where the learning rate is small and gradually increases--usually helps training.
WARMUP_PROPORTION = 0.1
# Model configs
SAVE_CHECKPOINTS_STEPS = 300
SAVE_SUMMARY_STEPS = 100
# Compute train and warmup steps from batch size
num_train_steps = int(len(train_features) / BATCH_SIZE * NUM_TRAIN_EPOCHS)
num_warmup_steps = int(num_train_steps * WARMUP_PROPORTION)
# Specify output directory and number of checkpoint steps to save
run_config = tf.estimator.RunConfig(
model_dir=OUTPUT_DIR,
save_summary_steps=SAVE_SUMMARY_STEPS,
save_checkpoints_steps=SAVE_CHECKPOINTS_STEPS)
# Specify output directory and number of checkpoint steps to save
run_config = tf.estimator.RunConfig(
model_dir=OUTPUT_DIR,
save_summary_steps=SAVE_SUMMARY_STEPS,
save_checkpoints_steps=SAVE_CHECKPOINTS_STEPS)
#Initializing the model and the estimator
model_fn = model_fn_builder(
num_labels=len(label_list),
learning_rate=LEARNING_RATE,
num_train_steps=num_train_steps,
num_warmup_steps=num_warmup_steps)
estimator = tf.estimator.Estimator(
model_fn=model_fn,
config=run_config,
params={"batch_size": BATCH_SIZE})
# Create an input function for training. drop_remainder = True for using TPUs.
train_input_fn = bert.run_classifier.input_fn_builder(
features=train_features,
seq_length=MAX_SEQ_LENGTH,
is_training=True,
drop_remainder=False)
# Create an input function for validating. drop_remainder = True for using TPUs.
test_input_fn = run_classifier.input_fn_builder(
features=test_features,
seq_length=MAX_SEQ_LENGTH,
is_training=False,
drop_remainder=False)
# #Training the model
print(f'Beginning Training!')
current_time = datetime.now()
estimator.train(input_fn=train_input_fn, max_steps=num_train_steps)
print("Training took time ", datetime.now() - current_time)
#Evaluating the model with Validation set
accuracy = estimator.evaluate(input_fn=test_input_fn, steps=None)
# A method to get predictions
def getPrediction(in_sentences):
# A list to map the actual labels to the predictions
labels = ["business", "entertainment", "politics", "sports", "tech"]
# Transforming the test data into BERT accepted form
input_examples = [run_classifier.InputExample(guid="", text_a=x, text_b=None, label=0) for x in in_sentences]
# Creating input features for Test data
input_features = run_classifier.convert_examples_to_features(input_examples, label_list, MAX_SEQ_LENGTH, tokenizer)
# Predicting the classes
predict_input_fn = run_classifier.input_fn_builder(features=input_features, seq_length=MAX_SEQ_LENGTH,
is_training=False, drop_remainder=False)
predictions = estimator.predict(predict_input_fn)
return [(sentence, prediction['probabilities'], prediction['labels'], labels[prediction['labels']]) for
sentence, prediction in zip(in_sentences, predictions)]
pred_sentences = list(test['text'])
predictions = getPrediction(pred_sentences)
enc_labels = []
act_labels = []
for i in range(len(predictions)):
enc_labels.append(predictions[i][2])
act_labels.append(predictions[i][3])
pd.DataFrame(enc_labels, columns = ['category']).to_excel('data/submission_bert.xlsx', index = False)
## Random tester
#Classifying random sentences
tests = getPrediction(['Mr.Modi is the Indian Prime Minister',
'Gaming machines are powered by efficient micro processores and GPUs',
'That HBO TV series is really good',
'A trillion dollar economy '
])
As the question clearly says to save the model, here is how it works:
import torch
torch.save(model, 'path/to/model')
saved_model = torch.load('path/to/model')
I think you can just rename your model.ckpt-333.data-00000-of-00001 to bert_model.ckpt and then use it in the same way you would use a non-finetuned model. For example, run
python run_classifier.py \
--task_name=MRPC \
--do_predict=true \
--data_dir=$GLUE_DIR/MRPC \
--vocab_file=$BERT_BASE_DIR/vocab.txt \
--bert_config_file=$BERT_BASE_DIR/bert_config.json \
--init_checkpoint=$TRAINED_CLASSIFIER
with --init_checkpoint pointing to your model's dir, or run bert-as-service
bert-serving-start -model_dir $TRAINED_CLASSIFIER
with the right -model_dir.
You can use these method:
model = MyModel(num_classes).to(device)
optimizer = AdamW(model.parameters(), lr=2e-5, weight_decay=1e-2)
output_model = './models/nameOfYourModel.pth'
# save
def save(model, optimizer):
# save
torch.save({
'model_state_dict': model.state_dict(),
'optimizer_state_dict': optimizer.state_dict()
}, output_model)
save(model, optimizer)
# load
checkpoint = torch.load(output_model, map_location='cpu')
model.load_state_dict(checkpoint['model_state_dict'])
optimizer.load_state_dict(checkpoint['optimizer_state_dict'])
Source: https://github.com/huggingface/transformers/issues/7849#issuecomment-718726121
I have created a binary classifier with Tensorflow BERT language model. Here is the link. After the model is trained, it saves the model and produces the following files.
Prediction code.
from tensorflow.contrib import predictor
#MODEL_FILE = 'graph.pbtxt'
with tf.Session() as sess:
predict_fn = predictor.from_saved_model(f'/content/drive/My Drive/binary_class/bert/graph.pbtxt')
predictions = predict_fn(pred_sentences)
print(predictions)
Error
OSError: SavedModel file does not exist at: /content/drive/My Drive/binary_class/bert/graph.pbtxt/{saved_model.pbtxt|saved_model.pb}
After digging in this issue. I came across tf.train.Saver() class for saving the model. I changed the estimator train code to following to save the model. I referred this link. I guess tensorflow estimators can be saved with it.
init_op = tf.global_variables_initializer()
with tf.Session() as sess:
sess.run(init_op)
# Do some work with the model.
saver = tf.train.Saver()
estimator.train(input_fn=train_input_fn, max_steps=num_train_steps)
# Save the variables to disk.
save_path = saver.save(sess, f'/content/drive/My Drive/binary_class/bert/tmp/model.ckpt')
And here is the error.
Beginning Training!
---------------------------------------------------------------------------
ValueError Traceback (most recent call last)
<ipython-input-38-0c9f9b70d76b> in <module>()
9 sess.run(init_op)
10 # Do some work with the model.
---> 11 saver = tf.train.Saver()
12 estimator.train(input_fn=train_input_fn, max_steps=num_train_steps)
13 # Save the variables to disk.
2 frames
/usr/local/lib/python3.6/dist-packages/tensorflow/python/training/saver.py in _build(self, checkpoint_path, build_save, build_restore)
860 return
861 else:
--> 862 raise ValueError("No variables to save")
863 self._is_empty = False
864
ValueError: No variables to save
The variables, weights are created in create_model function. What should I change to save my trained model?
Updates:
Code to save the model. I am not sure about the feature_spec and feature tensor.
feature_spec = {'x': tf.VarLenFeature(tf.string)}
def serving_input_receiver_fn():
serialized_tf_example = tf.placeholder(dtype=tf.string,
shape=[1], # batch size
name='input_example_tensor')
receiver_tensors = {'examples': serialized_tf_example}
features = tf.parse_example(serialized_tf_example, feature_spec)
return tf.estimator.export.ServingInputReceiver(features, receiver_tensors)
# Export the estimator
export_path = f'/content/drive/My Drive/binary_class/bert/'
estimator.export_saved_model(
export_path,
serving_input_receiver_fn=serving_input_receiver_fn)
I got this error :-
---------------------------------------------------------------------------
KeyError Traceback (most recent call last)
<ipython-input-55-209298910d1e> in <module>()
16 estimator.export_saved_model(
17 export_path,
---> 18 serving_input_receiver_fn=serving_input_receiver_fn)
4 frames
/usr/local/lib/python3.6/dist-packages/tensorflow_estimator/python/estimator/estimator.py in export_saved_model(self, export_dir_base, serving_input_receiver_fn, assets_extra, as_text, checkpoint_path, experimental_mode)
730 as_text=as_text,
731 checkpoint_path=checkpoint_path,
--> 732 strip_default_attrs=True)
733
734 def experimental_export_all_saved_models(
/usr/local/lib/python3.6/dist-packages/tensorflow_estimator/python/estimator/estimator.py in _export_all_saved_models(self, export_dir_base, input_receiver_fn_map, assets_extra, as_text, checkpoint_path, strip_default_attrs)
854 builder, input_receiver_fn_map, checkpoint_path,
855 save_variables, mode=ModeKeys.PREDICT,
--> 856 strip_default_attrs=strip_default_attrs)
857 save_variables = False
858
/usr/local/lib/python3.6/dist-packages/tensorflow_estimator/python/estimator/estimator.py in _add_meta_graph_for_mode(self, builder, input_receiver_fn_map, checkpoint_path, save_variables, mode, export_tags, check_variables, strip_default_attrs)
927 labels=getattr(input_receiver, 'labels', None),
928 mode=mode,
--> 929 config=self.config)
930
931 export_outputs = export_lib.export_outputs_for_mode(
/usr/local/lib/python3.6/dist-packages/tensorflow_estimator/python/estimator/estimator.py in _call_model_fn(self, features, labels, mode, config)
1144
1145 logging.info('Calling model_fn.')
-> 1146 model_fn_results = self._model_fn(features=features, **kwargs)
1147 logging.info('Done calling model_fn.')
1148
<ipython-input-17-119a3167bf33> in model_fn(features, labels, mode, params)
5 """The `model_fn` for TPUEstimator."""
6
----> 7 input_ids = features["input_ids"]
8 input_mask = features["input_mask"]
9 segment_ids = features["segment_ids"]
KeyError: 'input_ids'
The create_model function present in notebook takes some arguments. These features are passed to the model.
By updating the serving_input_fn function to following, the serving function works as expected.
Updated Code
def serving_input_receiver_fn():
feature_spec = {
"input_ids" : tf.FixedLenFeature([MAX_SEQ_LENGTH], tf.int64),
"input_mask" : tf.FixedLenFeature([MAX_SEQ_LENGTH], tf.int64),
"segment_ids" : tf.FixedLenFeature([MAX_SEQ_LENGTH], tf.int64),
"label_ids" : tf.FixedLenFeature([], tf.int64)
}
serialized_tf_example = tf.placeholder(dtype=tf.string,
shape=[None],
name='input_example_tensor')
print(serialized_tf_example.shape)
receiver_tensors = {'example': serialized_tf_example}
features = tf.parse_example(serialized_tf_example, feature_spec)
return tf.estimator.export.ServingInputReceiver(features, receiver_tensors)
export_path = '/content/drive/My Drive/binary_class/bert/'
estimator._export_to_tpu = False # this is important
estimator.export_saved_model(export_dir_base=export_path,serving_input_receiver_fn=serving_input_receiver_fn)
Using the feature_spec dict in the serving_input_receiver_fn did not work for me. I used the serving_input_fn below from someone else asking the same question.
To use the loaded estimator:
def serving_input_fn():
label_ids = tf.placeholder(tf.int32, [None], name='label_ids')
input_ids = tf.placeholder(tf.int32, [None, MAX_SEQ_LEN], name='input_ids')
input_mask = tf.placeholder(tf.int32, [None, MAX_SEQ_LEN], name='input_mask')
segment_ids = tf.placeholder(tf.int32, [None, MAX_SEQ_LEN], name='segment_ids')
input_fn = tf.estimator.export.build_raw_serving_input_receiver_fn(
{
'label_ids': label_ids,
'input_ids': input_ids,
'input_mask': input_mask,
'segment_ids': segment_ids
}
)()
return input_fn
export_path = '../testing'
estimator._export_to_tpu = False # this is important
estimator.export_saved_model(export_dir_base=export_path,serving_input_receiver_fn=serving_input_fn)
from tensorflow.contrib import predictor
predict_fn = predictor.from_saved_model('../testing/1589420991')
def predict(sentences, predict_fn):
labels = [0, 1]
input_examples = [
run_classifier.InputExample(
guid="",
text_a = x,
text_b = None,
label = 0
) for x in sentences] # here, "" is just a dummy label
input_features = run_classifier.convert_examples_to_features(
input_examples, labels, MAX_SEQ_LEN, tokenizer
)
all_input_ids = []
all_input_mask = []
all_segment_ids = []
all_label_ids = []
for feature in input_features:
all_input_ids.append(feature.input_ids)
all_input_mask.append(feature.input_mask)
all_segment_ids.append(feature.segment_ids)
all_label_ids.append(feature.label_id)
pred_dict = {
'input_ids': all_input_ids,
'input_mask': all_input_mask,
'segment_ids': all_segment_ids,
'label_ids': all_label_ids
}
predictions = predict_fn(pred_dict)
return [
(sentence, prediction, label)
for sentence, prediction, label in zip(pred_sentences, predictions['probabilities'], predictions['labels'])
]
pred_sentences = [
"That movie was absolutely awful",
"The acting was a bit lacking",
"The film was creative and surprising",
"Absolutely fantastic!",
]
predictions = predict(pred_sentences, predict_fn)
print(predictions)
[('That movie was absolutely awful',
array([-0.26713806, -1.4505868 ], dtype=float32),
0),
('The acting was a bit lacking',
array([-0.23832974, -1.5508994 ], dtype=float32),
0),
('The film was creative and surprising',
array([-0.2784096, -1.4146391], dtype=float32),
0),
('Absolutely fantastic!',
array([-0.29031944, -1.3784236 ], dtype=float32),
0),
('The patient has diabetes',
array([-0.33836085, -1.2480571 ], dtype=float32),
0),
('The patient does not have diabetes',
array([-0.29378486, -1.3682064 ], dtype=float32),
0)]
How would I go about changing the MNIST tutorial to use TFRecords instead of the odd format the tutorial downloads from the web?
I used build_image_data.py from the inception model to create my TFRecords containing 200x200 RGB images and intend to train this on a 1080Ti, but I can't find any good examples on how to load TFRecords and feed them into a convolutional neural network.
I did a similar thing as you intend doing. I also took the same script to build image data. My code for reading the data and training it is
import tensorflow as tf
height = 28
width = 28
tfrecords_train_filename = 'train-00000-of-00001'
tfrecords_test_filename = 'test-00000-of-00001'
def read_and_decode(filename_queue):
reader = tf.TFRecordReader()
_, serialized_example = reader.read(filename_queue)
features = tf.parse_single_example(
serialized_example,
features={
'image/height': tf.FixedLenFeature([], tf.int64),
'image/width': tf.FixedLenFeature([], tf.int64),
'image/colorspace': tf.FixedLenFeature([], dtype=tf.string, default_value=''),
'image/channels': tf.FixedLenFeature([], tf.int64),
'image/class/label': tf.FixedLenFeature([], tf.int64),
'image/class/text': tf.FixedLenFeature([], dtype=tf.string, default_value=''),
'image/format': tf.FixedLenFeature([], dtype=tf.string, default_value=''),
'image/filename': tf.FixedLenFeature([], dtype=tf.string, default_value=''),
'image/encoded': tf.FixedLenFeature([], dtype=tf.string, default_value='')
})
image_buffer = features['image/encoded']
image_label = tf.cast(features['image/class/label'], tf.int32)
# Decode the jpeg
with tf.name_scope('decode_jpeg', [image_buffer], None):
# decode
image = tf.image.decode_jpeg(image_buffer, channels=3)
# and convert to single precision data type
image = tf.image.convert_image_dtype(image, dtype=tf.float32)
image = tf.image.rgb_to_grayscale(image)
image_shape = tf.stack([height, width, 1])
image = tf.reshape(image, image_shape)
return image, image_label
def inputs(filename, batch_size, num_epochs):
if not num_epochs: num_epochs = None
with tf.name_scope('input'):
filename_queue = tf.train.string_input_producer([filename], num_epochs=None)
image, label = read_and_decode(filename_queue)
# Shuffle the examples and collect them into batch_size batches.
images, sparse_labels = tf.train.shuffle_batch(
[image, label], batch_size=batch_size, num_threads=2,
capacity=1000 + 3 * batch_size,
min_after_dequeue=1000)
return images, sparse_labels
image, label = inputs(filename=tfrecords_train_filename, batch_size=200, num_epochs=None)
image = tf.reshape(image, [-1, 784])
label = tf.one_hot(label - 1, 10)
# Create the model
x = tf.placeholder(tf.float32, [None, 784])
W = tf.Variable(tf.zeros([784, 10]))
b = tf.Variable(tf.zeros([10]))
y = tf.matmul(x, W) + b
y_ = tf.placeholder(tf.float32, [None, 10])
cross_entropy = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y))
train_step = tf.train.GradientDescentOptimizer(0.5).minimize(cross_entropy)
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(coord=coord)
correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
for i in range(1000):
img, lbl = sess.run([image, label])
sess.run(train_step, feed_dict={x: img, y_: lbl})
img, lbl = sess.run([image, label])
print(sess.run(accuracy, feed_dict={x: img, y_: lbl}))
coord.request_stop()
coord.join(threads)
This is a super easy model for classifying mnist. However I think it is also an extensible answer for how to train with TFRecord files. It does not yet take into account the evaluation data, since this needs more coordination to be done.