To build a test case code, I simply want to pass a tensor to a graph and have the model decide whether to multiply by 2 once or twice. The decision is based on evaluation of a boolean. Here is the code:
class what_to_do():
def __init__(self):
self.conditionally_determined_A = None
self.conditionally_determined_B = None
self.truth_values = tf.placeholder(shape=[None, 1],
dtype=tf.bool)
self.input = tf.placeholder(shape=[None, 1],
dtype=tf.float32)
# The question is, can this statement evaluate the conditional and then direct the
# implementation of the model's components.
_ = tf.cond(tf.constant(True),
lambda: self.real_conditional(),
lambda: self.fake_condition())
self.input_A = tf.Variable(initial_value=self.conditionally_determined_A,
trainable=False,
validate_shape=False)
self.const_A = tf.constant([2.])
self.operation_A1 = tf.multiply(self.input_A, self.const_A)
self.input_B = tf.Variable(initial_value=self.conditionally_determined_B,
trainable=False,
validate_shape=False)
self.const_B = tf.constant([2.])
self.operation_B1 = tf.multiply(self.input_B, self.const_B)
self.output = self.operation_B1
# These functions will serve as the condition coordinators for the model
def real_conditional(self):
print('in loop')
self.conditionally_determined_B = self.input
self.conditionally_determined_A = None
return 1
def fake_condition(self):
print('not in loop')
self.conditionally_determined_B = None
self.conditionally_determined_A = self.input
return 0
tf.reset_default_graph()
model = what_to_do()
data_set = np.array([[2,True], [2,True], [2,True], [2,False], [2,False], [2,False], [2,False]])
i, t = np.split(ary=data_set,
indices_or_sections=2,
axis=-1)
init = tf.global_variables_initializer()
with tf.Session() as sess:
sess.run(init)
output = sess.run(fetches=[model.output],
feed_dict={model.input_A:i, model.truth_values:t})
I've run into trouble trying to get tf.cond() to handle the tensor. It complains about ranks and the likes (perhaps another question).
What I have done to the code is to just define the condition as True or False, thus executing the proper function. If it is True, the graph should start at input_B and not worry about input_A at all.
Any help setting up tf.cond() to manipulate the graph dynamically?
Related
I made a module, that needs an extra loss term, e.g.
class MyModule:
def forward(self, x):
out = f(x)
extra_loss = loss_f(self.parameters(), x)
return out, extra_loss
I can't figure out how to make this module embeddable, for example, into a Sequential model: any regular module like Linear put after this one will fail because extra_loss causes the input to Linear to be a tuple, which Linear does not support.
So what I am looking for is extracting that extra loss after running the model forward
my_module = MyModule()
model = Sequential(
my_module,
Linear(my_module_outputs, 1)
)
output = model(x)
my_module_loss = ????
loss = mse(label, output) + my_module_loss
Does module composability support this scenario?
IMHO, hooks here is overreaction. Provided extra_loss is additive, we can use global variable like this:
class MyModule:
extra_loss =0
def forward(self, x):
out = f(x)
MyModule.extra_loss += loss_f(self.parameters(), x)
return out
output = model(x)
loss = mse(label, output) + MyModule.extra_loss
MyModule.extra_loss =0
You can register a hook in this case. A hook can be registered on a Tensor or a nn.Module. A hook is a function that is executed when the either forward or backward is called. In this case, we want to attach a forward hook without deattaching itself from the graph so that backward pass can happen.
import torch.nn as nn
act_out = {}
def get_hook(name):
def hook(m, input, output):
act_out[name] = output
return hook
class MyModule(torch.nn.Module):
def __init__(self, input, out, device=None):
super().__init__()
self.model = nn.Linear(input,out)
def forward(self,x):
return self.model(x), torch.sum(x) #our extra loss
class MyModule1(torch.nn.Module):
def __init__(self, input, out, device=None):
super().__init__()
self.model = nn.Linear(input,out)
def forward(self, pair):
x, loss = pair
return self.model(x)
model = nn.Sequential(
MyModule(5,10),
MyModule1(10,1)
)
for name, module in model.named_children():
print(name, module)
if name == '0':
module.register_forward_hook(get_hook(name))
x = torch.tensor([1,2,3,4,5]).float()
out = model(x)
print(act_out)
loss = myanotherloss(out)+act_out['0'][1] # this is the extra loss
# further processing
Note: I am using name == '0' because this is the only module where I want to attach the hook.
Note: Another notable point is nn.Sequential doesn't allow multiple inputs. In this case, it is simply considered as a tuple and then from that tuple we are using the loss and the input.
I want to use model.predict in batch generator, what a possible ways of achieve this?
Seems one option is to load model on init and on epoch end:
class DataGenerator(keras.utils.Sequence):
def __init__(self, model_name):
# Load model
# ...
def on_epoch_end(self):
# Load model
In my experience, predicting another model while training will bring errors.
You should probably simply append your training model after your generator model.
Suppose you have:
generator_model (the one you want to use inside the generator)
training_model (the one you want to train)
Then
generatorInput = Input(shapeOfTheGeneratorInput)
generatorOutput = generator_model(generatorInput)
trainingOutput = training_model(generatorOutput)
entireModel = Model(generatorInput,trainingOutput)
Make sure that the generator model has all layers untrainable before compiling:
genModel = entireModel.layers[1]
for l in genModel.layers:
l.trainable = False
entireModel.compile(optimizer=optimizer,loss=loss)
Now use the generator regularly.
Predicting inside the generator:
class DataGenerator(keras.utils.Sequence):
def __init__(self, model_name, modelInputs, batchSize):
self.genModel = load_model(model_name)
self.inputs = modelInputs
self.batchSize = batchSize
def __len__(self):
l,rem = divmod(len(self.inputs), self.batchSize)
return (l + (1 if rem > 0 else 0))
def __getitem__(self,i):
items = self.inputs[i*self.batchSize:(i+1)*self.batchSize]
items = doThingsWithItems(items)
predItems = self.genModel.predict_on_batch(items)
#the following is the only reason not to chain models
predItems = doMoreThingsWithItems(predItems)
#do something to get Y_train_items as well
return predItems, y_train_items
If you do find the error I mentioned, you can sacrifice the parallel generation capabilities and do some manual loops:
for e in range(epochs):
for i in range(batches):
x,y = generator[i]
model.train_on_batch(x,y)
As my question,I define a class.In init, I new some tf value.And init global,init local.Even in main script.
class DNN():
def __init__(self):
kernel_shape = [3,3]
self.c11w = tf.Variable(tf.truncated_normal(kernel_shape + [3, 64], stddev=0.1), name = 'W'))
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
sess.run(tf.local_variables_initializer())
Like this.And uninitiated error.So I call tf.report_uninitialized_variables(), just empty.
So,I want to know why and how, thanks much.
In TF, variables values live in session only. Once session is closed there are no values anymore, thus in your current code you create variables, initialise them, and then discard them, all in the constructor.
Typical pattern for integrating TF to OO code in python would be something among the lines of:
class MLobject():
def __init__(self):
self._graph = tf.Graph() # separate graph per instance
with self._graph.as_default():
variable_1 = ....
...
self._initialiser = tf.global_variables_initializer()
self._session = tf.Session(graph=self._graph) # store session in a field
self._graph.finalize() # For safety, this should not be modified anymore
def fit(self, X, y):
self._session.run(self._initialiser)
... # execute training using self._session
I'm using tensorflow to predict outputs of a neural network. I have a class where I have described the neural network and I have a main file where the predictions are being made, and based on the results, the weights are updated. However, the predictions seem to be really slow. Here is how my code looks like:
class NNPredictor():
def __init__(self):
self.input = tf.placeholder(...)
...
self.output = (...) #Neural network output
def predict_output(self, sess, input):
return sess.run(tf.squeeze(self.output), feed_dict = {self.input: input})
Here's how the main file looks like:
sess = tf.Session()
predictor = NNPredictor()
input = #some initial value
for i in range(iter):
output = predictor.predict_output(sess, input)
input = #some function of output
However, if I use the following function definition in the class:
def predict_output(self):
return self.output
And have the main file as follows:
sess = tf.Session()
predictor = NNPredictor()
input = #some initial value
output_op = predictor.predict_value()
for i in range(iter):
output = np.squeeze(sess.run(output_op, feed_dict = {predictor.input: input}))
input = #some function of output
The code runs almost 20-30x faster. I don't seem to understand how things are working here, and I'd like to know what the best practice would be.
This has to do with the underlying memory accesses masked by Python. Here's some sample code to illustrate this idea:
import time
runs = 10000000
class A:
def __init__(self):
self.val = 1
def get_val(self):
return self.val
# Using method to then call object attribute
obj = A()
start = time.time()
total = 0
for i in xrange(runs):
total += obj.get_val()
end = time.time()
print end - start
# Using object attribute directly
start = time.time()
total = 0
for i in xrange(runs):
total += obj.val
end = time.time()
print end - start
# Assign to local_var first
start = time.time()
total = 0
local_var = obj.get_val()
for i in xrange(runs):
total += local_var
end = time.time()
print end - start
On my computer, it runs in the following timing:
1.49576115608
0.656110048294
0.551875114441
Specific to your case, you're calling the object method in the first case but not doing it in the second case. If you're calling your code many times in this way, there would be significant performance differences.
I have defined two classes of models, x and y.
class x():
def __init__(self, x_inp1, x_inp2):
# do sth...
def step(self, session, encoder_inputs):
input_feed = {}
for l in range(encoder_size):
input_feed[self.encoder_inputs[l].name] = encoder_inputs[l]
...
output_feed = [x_output]
return session.run(x_output)
class y():
def __init__(self, y_inp1, y_inp2):
# do sth...
def step(self, encoder_inputs):
input_feed = {}
for l in range(encoder_size):
input_feed[self.encoder_inputs[l].name] = encoder_inputs[l]
...
They have quite similar functions. And then I define another class to group them up.
class gp():
def __init__(self, x_inp1, x_inp2, y_inp1, y_inp2):
with tf.variable_scope('x'):
self.x_model = x(x_inp1, x_inp2)
with tf.variable_scope('y'):
self.y_model = y(y_inp1, y_inp2)
def step(self, session, encoder_inputs):
x_output = self.x_model.step(session, encoder_inputs)
y_output = self.y_model.step(session, x_output)
...
Please notice that the y_model takes the output of x_model as input. And I run the gp() in the main function:
with tf.Session() as sess:
gp_m = gp(x_inp1, x_inp2, y_inp1, y_inp2)
gp_m.step(sess, x_inp1, x_inp2, y_inp1, y_inp2)
And after running x_output = self.x_model.step(encoder_inputs) and begin to do y_output = self.y_model.step(x_output), I got such an error:
InvalidArgumentError (see above for traceback): You must feed a value for placeholder tensor 'x/encoder0' with dtype int32
[[Node: x/encoder0 = Placeholder[dtype=DT_INT32, shape=[], _device="/job:localhost/replica:0/task:0/cpu:0"]()]]
Please notice this error points to the x_model even the step function of it has been finished. I wonder how can I use the output of x_model as the input of y_model without any error? Thanks in advance!
You should defer the calls to session.run to be outside the step functions. The problem here is that trying to run Y triggers X because they are connected in the graph.
Instead, it might be better to fully separate your graph build and graph run stages of your program, so you know what placeholders to provide when.