We Keep Coding

Writing summary.scalar with Dataset API and Keras

I use tensorflow Keras API and try to add custom scalar to the tensorboard but nothing except the loss is displayed.
Here is the code for the model:
embedding_in = Embedding(
input_dim=vocab_size + 1 + 1,
output_dim=dim,
mask_zero=True,
)
embedding_out = Embedding(
input_dim=vocab_size + 1 + 1,
output_dim=dim,
mask_zero=True,
)
input_a = Input((None,))
input_b = Input((None,))
input_c = Input((None, None))
emb_target = embedding_in(input_a)
emb_context = embedding_out(input_b)
emb_negatives = embedding_out(input_c)
emb_gru = GRU(dim, return_sequences=True)(emb_target)
num_negatives = tf.shape(input_c)[-1]
def make_logits(tensors):
emb_gru, emb_context, emb_negatives = tensors
true_logits = tf.reduce_sum(tf.multiply(emb_gru, emb_context), axis=2)
true_logits = tf.expand_dims(true_logits, -1)
sampled_logits = tf.squeeze(
tf.matmul(emb_negatives, tf.expand_dims(emb_gru, axis=2),
transpose_b=True), axis=3)
true_logits = true_logits*0
sampled_logits = sampled_logits*0
logits = K.concatenate([true_logits, sampled_logits], axis=-1)
return logits
logits = Lambda(make_logits)([emb_gru, emb_context, emb_negatives])
mean = tf.reduce_mean(logits)
tf.summary.scalar('mean_logits', mean)
model = keras.models.Model(inputs=[input_a, input_b, input_c], outputs=[logits])
In particular, I want to see the evolution of mean_logits scalar after each batch.
I create and compile the model like this:
model = build_model(dim, vocab_size)
model.compile(loss='binary_crossentropy', optimizer='sgd')
callbacks = [
keras.callbacks.TensorBoard(logdir, histogram_freq=1)
]
I use tf Dataset API to the model:
iterator = dataset.make_initializable_iterator()
with tf.Session() as sess:
sess.run(iterator.initializer)
sess.run(tf.tables_initializer())
model.fit(iterator, steps_per_epoch=100,
callbacks=callbacks,
validation_data=iterator,
validation_steps=1
)
However, I don't get any mean_logits graph in the tensorboard and it's not in the graphs.
How can I track mean_logits scalar in tensorboard after each batch?
I use tf 1.12 and keras 2.1.

I have also faced the same issue. It seems that Keras TensorBoard callback not gonna write all existing summaries automatically, but only those registered as metrics (and appear in logs dict). Updating the logs object is a nice trick as it allows to use the values in other callbacks, see Early stopping and learning rate schedule based on custom metric in Keras. I can see several possibilities:
1. Using Lambda callback
Something like this:
eval_callback = LambdaCallback(
on_epoch_end=lambda epoch, logs: logs.update(
{'mean_logits': K.eval(mean)}
))
2. Custom TensorBoard callback
You can also subclass the callback and define your own logic. For instance, my workaround for learning rate monitoring:
class Tensorboard(Callback):
def __init__(self,
log_dir='./log',
write_graph=True):
self.write_graph = write_graph
self.log_dir = log_dir
def set_model(self, model):
self.model = model
self.sess = K.get_session()
if self.write_graph:
self.writer = tf.summary.FileWriter(self.log_dir, self.sess.graph)
else:
self.writer = tf.summary.FileWriter(self.log_dir)
def on_epoch_end(self, epoch, logs={}):
logs.update({'learning_rate': float(K.get_value(self.model.optimizer.lr))})
self._write_logs(logs, epoch)
def _write_logs(self, logs, index):
for name, value in logs.items():
if name in ['batch', 'size']:
continue
summary = tf.Summary()
summary_value = summary.value.add()
if isinstance(value, np.ndarray):
summary_value.simple_value = value.item()
else:
summary_value.simple_value = value
summary_value.tag = name
self.writer.add_summary(summary, index)
self.writer.flush()
def on_train_end(self, _):
self.writer.close()
Here, I just add 'learning_rate' to logs explicitly. But this way can be much more flexible and powerful.
3. Metrics trick
Here is another interesting workaround. What you need to do is to pass a custom metric function to model's compile() call which returns aggregated summary tensor. The idea is to make Keras pass your aggregated summary operation to every session.run call and return it's result as metric:
x_entropy_t = K.sum(p_t * K.log(K.epsilon() + p_t), axis=-1, keepdims=True)
full_policy_loss_t = -res_t + X_ENTROPY_BETA * x_entropy_t
tf.summary.scalar("loss_entropy", K.sum(x_entropy_t))
tf.summary.scalar("loss_policy", K.sum(-res_t))
tf.summary.scalar("loss_full", K.sum(full_policy_loss_t))
summary_writer = tf.summary.FileWriter("logs/" + args.name)
def summary(y_true, y_pred):
return tf.summary.merge_all()
value_policy_model.compile(optimizer=Adagrad(), loss=loss_dict, metrics=[summary])
l = value_policy_model.train_on_batch(x_batch, y_batch)
l_dict = dict(zip(value_policy_model.metrics_names, l))
summary_writer.add_summary(l_dict['value_summary'], global_step=iter_idx)
summary_writer.flush()

Tensorflow - how to import MNIST database

I want to train a model using MNIST database. I'm working on Tensorflow tutorial Tensorflow tutorial. A suggested way to import database is to use mnist = input_data.read_data_sets(FLAGS.data_dir, one_hot=True) but I need to use something like:
with open('my/directory/train-images-idx3-ubyte.gz', 'rb') as f:
train_images = extract_images(f)
with open('my/directory/train-labels-idx1-ubyte.gz', 'rb') as f:
train_labels = extract_images(f)
...
Thats brings question how to adjust code to work with my train_images, train_lables, test_images, test_lables:
def main(_):
# Import data
mnist = input_data.read_data_sets(FLAGS.data_dir, one_hot=True)
# Create the model
x = tf.placeholder(tf.float32, [None, 784])
# Define loss and optimizer
y_ = tf.placeholder(tf.float32, [None, 10])
# Build the graph for the deep net
y_conv, keep_prob = deepnn(x)
with tf.name_scope('loss'):
cross_entropy = tf.nn.softmax_cross_entropy_with_logits(labels=y_,
logits=y_conv)
cross_entropy = tf.reduce_mean(cross_entropy)
with tf.name_scope('adam_optimizer'):
train_step = tf.train.AdamOptimizer(1e-4).minimize(cross_entropy)
with tf.name_scope('accuracy'):
correct_prediction = tf.equal(tf.argmax(y_conv, 1), tf.argmax(y_, 1))
correct_prediction = tf.cast(correct_prediction, tf.float32)
accuracy = tf.reduce_mean(correct_prediction)
graph_location = tempfile.mkdtemp()
print('Saving graph to: %s' % graph_location)
train_writer = tf.summary.FileWriter(graph_location)
train_writer.add_graph(tf.get_default_graph())
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
for i in range(20000):
batch = mnist.train.next_batch(50)
if i % 100 == 0:
train_accuracy = accuracy.eval(feed_dict={
x: batch[0], y_: batch[1], keep_prob: 1.0})
print('step %d, training accuracy %g' % (i, train_accuracy))
train_step.run(feed_dict={x: batch[0], y_: batch[1], keep_prob: 0.5})
print('test accuracy %g' % accuracy.eval(feed_dict={
x: mnist.test.images, y_: mnist.test.labels, keep_prob: 1.0}))
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument('--data_dir', type=str,
default='/tmp/tensorflow/mnist/input_data',
help='Directory for storing input data')
FLAGS, unparsed = parser.parse_known_args()
tf.app.run(main=main, argv=[sys.argv[0]] + unparsed)

A recommended way to import the mnist dataset with TF2 is the following:
from tensorflow.keras.datasets import mnist
(X_train, Y_train), (X_test, Y_test) = mnist.load_data()

Running a basic distributed MNIST solver in TensorFlow

I'm trying to learn a model to predict MNIST classes in distributed TensorFlow. I've read the main distributed TensorFlow page, but I don't understand what I run to create a distributed TensorFlow model.
I'm just using a linear classifier for the moment, based on the code here.
How do I run this model? The link I got the code from says that this command should be run in the terminal:
python dist_minst_softmax.py
--ps_hosts=localhost:2222,localhost:2223
--worker_hosts=localhost:2224,localhost:2225
--job_name=worker --task_index=1
If I run this in the terminal, I get the following messages:
2018-04-23 11:02:35.034319: I tensorflow/core/distributed_runtime/master.cc:221] CreateSession still waiting for response from worker: /job:ps/replica:0/task:0
2018-04-23 11:02:35.034375: I tensorflow/core/distributed_runtime/master.cc:221] CreateSession still waiting for response from worker: /job:worker/replica:0/task:0
This message just repeats indefinitely. So how do I start the training process?
For reference, the model is defined as follows:
import argparse
import sys
from tensorflow.examples.tutorials.mnist import input_data
import tensorflow as tf
FLAGS = None
def main(_):
ps_hosts = FLAGS.ps_hosts.split(",")
worker_hosts = FLAGS.worker_hosts.split(",")
cluster = tf.train.ClusterSpec({"ps": ps_hosts, "worker": worker_hosts})
server = tf.train.Server(cluster,
job_name=FLAGS.job_name,
task_index=FLAGS.task_index)
if FLAGS.job_name == "ps":
server.join()
elif FLAGS.job_name == "worker":
with tf.device(tf.train.replica_device_setter(
worker_device="/job:worker/task:%d" % FLAGS.task_index,
cluster=cluster)):
global_step = tf.contrib.framework.get_or_create_global_step()
with tf.name_scope("input"):
mnist = input_data.read_data_sets("./input_data", one_hot=True)
x = tf.placeholder(tf.float32, [None, 784], name="x-input")
y_ = tf.placeholder(tf.float32, [None, 10], name="y-input")
tf.set_random_seed(1)
with tf.name_scope("weights"):
W = tf.Variable(tf.zeros([784, 10]))
b = tf.Variable(tf.zeros([10]))
with tf.name_scope("model"):
y = tf.matmul(x, W) + b
with tf.name_scope("cross_entropy"):
cross_entropy = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y))
with tf.name_scope("train"):
train_step = tf.train.GradientDescentOptimizer(0.5).minimize(cross_entropy)
with tf.name_scope("acc"):
init_op = tf.initialize_all_variables()
correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
sv = tf.train.Supervisor(is_chief=(FLAGS.task_index == 0),
global_step=global_step,
init_op=init_op)
with sv.prepare_or_wait_for_session(server.target) as sess:
for _ in range(100):
batch_xs, batch_ys = mnist.train.next_batch(100)
sess.run(train_step, feed_dict={x: batch_xs, y_: batch_ys})
print(sess.run(accuracy, feed_dict={x: mnist.test.images, y_: mnist.test.labels}))
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.register("type", "bool", lambda v: v.lower() == "true")
# Flags for defining the tf.train.ClusterSpec
parser.add_argument(
"--ps_hosts",
type=str,
default="",
help="Comma-separated list of hostname:port pairs"
)
parser.add_argument(
"--worker_hosts",
type=str,
default="",
help="Comma-separated list of hostname:port pairs"
)
parser.add_argument(
"--job_name",
type=str,
default="",
help="One of 'ps', 'worker'"
)
# Flags for defining the tf.train.Server
parser.add_argument(
"--task_index",
type=int,
default=0,
help="Index of task within the job"
)
FLAGS, unparsed = parser.parse_known_args()
print(FLAGS, unparsed)
tf.app.run(main=main, argv=[sys.argv[0]] + unparsed)

You should first initialise your ps_server, and then launch your worker. Example with one ps and one worker:
python dist_minst_softmax.py
--ps_hosts=localhost:2222
--worker_hosts=localhost:2223
--job_name=ps --task_index=0
python dist_minst_softmax.py
--ps_hosts=localhost:2222
--worker_hosts=localhost:2223
--job_name=worker --task_index=0
I couldn't run the example code you gave me, since my computer doesn't have BLAS configured, but at least it tried to perform some operations...

MonitoredTrainingSession save and restore model

I'm trying to extend the example https://www.tensorflow.org/deploy/distributed outlined here but I'm having trouble saving the model. I'm running this in docker container available at gcr.io/tensorflow/tensorflow:1.5.0-gpu-py3. I started two processes one for 'ps' and one for 'worker' and the ps process is simply this code:
import tensorflow as tf
def main(_):
cluster = tf.train.ClusterSpec({"ps":["localhost:2222"],"worker":["localhost:2223"]})
server = tf.train.Server(cluster, job_name="ps", task_index=0)
server.join()
if __name__ == "__main__":
tf.app.run()
The worker code is the following and is based on the mnist examples and the distributed article above:
import tensorflow as tf
from tensorflow.examples.tutorials.mnist import input_data
data_dir = "/data"
checkpoint_dir = "/tmp/train_logs"
def main(_):
cluster = tf.train.ClusterSpec({"ps":["localhost:2222"],"worker":["localhost:2223"]})
server = tf.train.Server(cluster, job_name="worker", task_index=0)
mnist = input_data.read_data_sets(data_dir, one_hot=True)
with tf.device(tf.train.replica_device_setter(worker_device="/job:worker/task:0", cluster=cluster)):
x = tf.placeholder(tf.float32, [None,784], name="x_input")
W = tf.Variable(tf.zeros([784,10]))
b = tf.Variable(tf.zeros([10]))
y = tf.placeholder(tf.float32, [None,10])
model = tf.matmul(x, W) + b
cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits_v2(labels=y, logits=model))
global_step = tf.train.get_or_create_global_step()
train_op = tf.train.GradientDescentOptimizer(0.5).minimize(cost, global_step=global_step)
prediction = tf.equal(tf.argmax(model,1), tf.argmax(y,1))
accuracy = tf.reduce_mean(tf.cast(prediction, tf.float32))
hooks = [tf.train.StopAtStepHook(last_step=101)]
with tf.train.MonitoredTrainingSession(master=server.target, is_chief=True, checkpoint_dir=checkpoint_dir, hooks=hooks) as sess:
while not sess.should_stop():
batch_xs, batch_ys = mnist.train.next_batch(1000)
sess.run(train_op, feed_dict={x: batch_xs, y: batch_ys})
latest_checkpoint = tf.train.latest_checkpoint(checkpoint_dir)
#saver = tf.train.Saver()
saver = tf.train.import_meta_graph(latest_checkpoint+".meta", clear_devices=True)
with tf.Session() as sess:
saver.restore(sess,latest_checkpoint) # "/tmp/train_logs/model.ckpt"
acc = sess.run(accuracy, feed_dict={x: mnist.test.images,y: mnist.test.labels});
print("Test accuracy = "+"{:5f}".format(acc))
if __name__ == "__main__":
tf.app.run()
The examples I've found all seem to end without showing how to use the model. The above code fails on the saver.restore() line with the following error:
InvalidArgumentError (see above for traceback): Cannot assign a device for operation 'save/RestoreV2_2':
Operation was explicitly assigned to /job:ps/task:0/device:CPU:0
but available devices are [ /job:localhost/replica:0/task:0/device:CPU:0, /job:localhost/replica:0/task:0/device:GPU:0 ].
Make sure the device specification refers to a valid device.
Also, as shown above I tried both saver = tf.train.Saver() and saver = tf.train.import_meta_graph(latest_checkpoint+".meta", clear_devices=True) with no success. Same error is shown in either case.
I don't really understand the with tf.device(...): statement. In one iteration I commented out this line (and unindented the statements below it) and the code ran without errors. But I think this is not correct and would like to understand the correct way for this to work.

train two seperate tensorflow models at once

I am new in tensorflow, i am trying to train to tensorflow models which are connected with a dot product output layer. The input are two 2048 float vectors.
When I run the script I always get these errors:
Traceback (most recent call last):
File "classifier.py", line 120, in
_, summary = sess.run([optimizer, merged], feed_dict={x1: batch_x1s, x2: batch_x2s})
File "/Users/Joachim/work/tensorflow/virtualenv/tensorflow/lib/python3.6/site-packages/tensorflow/python/client/session.py", line 789, in run
run_metadata_ptr)
File "/Users/Joachim/work/tensorflow/virtualenv/tensorflow/lib/python3.6/site-packages/tensorflow/python/client/session.py", line 968, in _run
np_val = np.asarray(subfeed_val, dtype=subfeed_dtype)
File "/Users/Joachim/work/tensorflow/virtualenv/tensorflow/lib/python3.6/site-packages/numpy/core/numeric.py", line 531, in asarray
return array(a, dtype, copy=False, order=order)
ValueError: setting an array element with a sequence.
Here is my code:
import tensorflow as tf
import sys
import math
import os
import numpy as np
import json
import argparse
from sklearn.model_selection import train_test_split
from tqdm import tqdm
from tensorflow.python.platform import gfile
from progress.bar import Bar
bottleneck_dir = 'bottlenecks'
### LOAD DATA FROM BOTTLENECKS
data_inputs = []
data_labels = []
data_expected_result=[]
bottleneck_list = []
file_glob = os.path.join(bottleneck_dir, '*.txt')
bottleneck_list.extend(gfile.Glob(file_glob))
for bottleneck_file in bottleneck_list:
bottleneck = open(bottleneck_file)
bottleneck_string = bottleneck.read()
bottleneck_values = [float(x) for x in bottleneck_string.split(',')]
imageName=bottleneck_file.split('.')[0]
helper=False
for i in range(len(data_labels)):
if imageName==data_labels[i]:
if 'search' in bottleneck_file:
data_inputs[i][0]=bottleneck_values
else:
data_inputs[i][1]=bottleneck_values
helper=true
if helper!=True:
if 'search' in bottleneck_file:
data_inputs.append([bottleneck_values,[]])
else:
data_inputs.append([[],bottleneck_values])
data_expected_result.append(1);
data_inputs_x1 = [i[0] for i in data_inputs]
data_inputs_x2 = [i[1] for i in data_inputs]
# Setting hyperparameters
learning_rate = 0.01
batch_size = 4
epochs = 1
log_batch_step = 50
n_features = np.size(data_inputs, 1)
tf.reset_default_graph()
graph = tf.get_default_graph()
inputVectorSize=2048
outputVectorSize=2048
x1 = tf.placeholder(tf.float32, [None, inputVectorSize], name='x1')#input layer
x2 = tf.placeholder(tf.float32, [None, inputVectorSize], name='x2')#input layer
dense1 = tf.layers.dense(inputs=x1, units=inputVectorSize, activation=tf.nn.relu)
logits1 = tf.layers.dense(inputs=dense1, units=outputVectorSize, activation=tf.nn.relu)
logits1_normalized=tf.nn.softmax(logits1)
dense2 = tf.layers.dense(inputs=x2, units=inputVectorSize, activation=tf.nn.relu)
logits2 = tf.layers.dense(inputs=dense2, units=outputVectorSize, activation=tf.nn.relu)
logits2_normalized=tf.nn.softmax(logits2)
output = tf.reduce_sum( tf.multiply( logits1_normalized, logits2_normalized), 1, keep_dims=True )
# Defining loss of network
loss = data_expected_result-output
tf.summary.scalar('loss', loss)
# Setting optimiser
optimizer = tf.train.AdamOptimizer(learning_rate).minimize(loss)
# Define accuracy
accuracy = loss
tf.summary.scalar('accuracy', accuracy)
# For saving checkpoint after training
saver = tf.train.Saver()
merged = tf.summary.merge_all()
# use in command line: tensorboard --logdir=path/to/log --> to view tensorboard
# Run tensorflow session
with tf.Session() as sess:
init = tf.global_variables_initializer()
sess.run(init)
train_writer = tf.summary.FileWriter('log', sess.graph)
tf.train.write_graph(sess.graph_def, '', 'savedgraph.pbtxt', as_text=False)
# Running the training in batches
batch_count = int(math.ceil(len(data_inputs)/batch_size))
for epoch_i in range(epochs):
batches_pbar = tqdm(range(batch_count), desc='Epoch {:>2}/{}'.format(epoch_i+1, epochs), unit='batches')
# The training cycle
for batch_i in batches_pbar:
# Get a batch of training features and labels
batch_start = batch_i*batch_size
batch_x1s = data_inputs_x1[batch_start:batch_start + batch_size]
batch_x2s = data_inputs_x2[batch_start:batch_start + batch_size]
# Run optimizer
_, summary = sess.run([optimizer, merged], feed_dict={x1: batch_x1s, x2: batch_x2s})
train_writer.add_summary(summary, batch_i)
# Check accuracy against validation data
val_accuracy, val_loss = sess.run([accuracy, loss], feed_dict={x1: data_inputs_x1[0:len(data_inputs-1)], x2: data_inputs_x2[0:len(data_inputs-1)]})
print("After epoch {}, Loss: {}, Accuracy: {}".format(epoch_i+1, val_loss, val_accuracy))
test_accuracy, test_loss = sess.run([accuracy, loss], feed_dict={x1: data_inputs_x1[0:len(data_inputs-1)], x2: data_inputs_x2[0:len(data_inputs-1)]})
print ("TEST LOSS: {}, TEST ACCURACY: {}".format(test_loss, test_accuracy))
g = tf.get_default_graph()
saver.save(sess, 'savedgraph')
Can anyone show my what to do to fix the problem?

You need to feed array not list. change the lines where used list as feed_dict input.
batch_x1s = np.asarray(data_inputs_x1[batch_start:batch_start + batch_size])
batch_x2s = np.asarray(data_inputs_x2[batch_start:batch_start + batch_size])
...
test_accuracy, test_loss = sess.run([accuracy, loss], feed_dict=
{x1:np.asarray(data_inputs_x1[0:len(data_inputs-1)]), x2:
np.asarray(data_inputs_x2[0:len(data_inputs-1)])})

I found the Problem, it was a problem with the input data.
import tensorflow as tf
import sys
import math
import os
import numpy as np
import json
import argparse
from sklearn.model_selection import train_test_split
from tqdm import tqdm
from tensorflow.python.platform import gfile
from progress.bar import Bar
bottleneck_dir = 'bottlenecks'
### LOAD DATA FROM BOTTLENECKS
data_inputs = []
data_labels = []
data_expected_result=[]
bottleneck_list = []
file_glob = os.path.join(bottleneck_dir, '*.txt')
bottleneck_list.extend(gfile.Glob(file_glob))
for bottleneck_file in bottleneck_list:
bottleneck = open(bottleneck_file)
bottleneck_string = bottleneck.read()
bottleneck_values = [float(x) for x in bottleneck_string.split(',')]
imageName=bottleneck_file.split('.')[0]
helper=False
for i in range(len(data_labels)):
if imageName==data_labels[i]:
if 'search' in bottleneck_file:
data_inputs[i][0]=np.asarray(bottleneck_values)
else:
data_inputs[i][1]=np.asarray(bottleneck_values)
helper=True
if helper!=True:
if 'search' in bottleneck_file:
data_inputs.append([bottleneck_values,[]])
else:
data_inputs.append([[],bottleneck_values])
data_expected_result.append(1);
data_labels.append(imageName);
data_inputs_x1 = [i[0] for i in data_inputs]
data_inputs_x2 = [i[1] for i in data_inputs]
for i in range(len(data_inputs_x2)):
print(len(data_inputs_x2[i]))
# Setting hyperparameters
learning_rate = 0.01
batch_size = 4
epochs = 1
log_batch_step = 50
n_features = np.size(data_inputs, 1)
tf.reset_default_graph()
graph = tf.get_default_graph()
inputVectorSize=2048
outputVectorSize=2048
x1 = tf.placeholder(tf.float32, [None, inputVectorSize], name='x1')#input layer
x2 = tf.placeholder(tf.float32, [None, inputVectorSize], name='x2')#input layer
dense1 = tf.layers.dense(inputs=x1, units=inputVectorSize, activation=tf.nn.relu)
logits1 = tf.layers.dense(inputs=dense1, units=outputVectorSize, activation=tf.nn.relu)
logits1_normalized=tf.nn.softmax(logits1)
dense2 = tf.layers.dense(inputs=x2, units=inputVectorSize, activation=tf.nn.relu)
logits2 = tf.layers.dense(inputs=dense2, units=outputVectorSize, activation=tf.nn.relu)
logits2_normalized=tf.nn.softmax(logits2)
output = tf.reduce_sum( tf.multiply( logits1_normalized, logits2_normalized), 1, keep_dims=True )
# Defining loss of network
loss = tf.reduce_sum(tf.subtract(1.0,output));
tf.summary.scalar('loss', loss)
# Setting optimiser
optimizer = tf.train.AdamOptimizer(learning_rate).minimize(loss)
# Define accuracy
accuracy = loss
tf.summary.scalar('accuracy', accuracy)
# For saving checkpoint after training
saver = tf.train.Saver()
merged = tf.summary.merge_all()
# use in command line: tensorboard --logdir=path/to/log --> to view tensorboard
# Run tensorflow session
with tf.Session() as sess:
init = tf.global_variables_initializer()
sess.run(init)
train_writer = tf.summary.FileWriter('log', sess.graph)
tf.train.write_graph(sess.graph_def, '', 'savedgraph.pbtxt', as_text=False)
# Running the training in batches
batch_count = int(math.ceil(len(data_inputs)/batch_size))
for epoch_i in range(epochs):
batches_pbar = tqdm(range(batch_count), desc='Epoch {:>2}/{}'.format(epoch_i+1, epochs), unit='batches')
# The training cycle
for batch_i in batches_pbar:
# Get a batch of training features and labels
batch_start = batch_i*batch_size
batch_x1s = np.asarray(data_inputs_x1[batch_start:batch_start + batch_size])
batch_x2s = np.asarray(data_inputs_x2[batch_start:batch_start + batch_size])
# Run optimizer
_, summary = sess.run([optimizer, merged], feed_dict={x1: batch_x1s, x2: batch_x2s})
train_writer.add_summary(summary, batch_i)
# Check accuracy against validation data
val_accuracy, val_loss = sess.run([accuracy, loss], feed_dict={x1: np.asarray(data_inputs_x1), x2: np.asarray(data_inputs_x2)})
print("After epoch {}, Loss: {}, Accuracy: {}".format(epoch_i+1, val_loss, val_accuracy))
test_accuracy, test_loss = sess.run([accuracy, loss], feed_dict={x1: np.asarray(data_inputs_x1), x2: np.asarray(data_inputs_x2)})
print ("TEST LOSS: {}, TEST ACCURACY: {}".format(test_loss, test_accuracy))
g = tf.get_default_graph()
saver.save(sess, 'savedgraph')