tensorflow Attempting to use uninitialized value W in a class - python

As my question,I define a class.In init, I new some tf value.And init global,init local.Even in main script.
class DNN():
def __init__(self):
kernel_shape = [3,3]
self.c11w = tf.Variable(tf.truncated_normal(kernel_shape + [3, 64], stddev=0.1), name = 'W'))
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
sess.run(tf.local_variables_initializer())
Like this.And uninitiated error.So I call tf.report_uninitialized_variables(), just empty.
So,I want to know why and how, thanks much.

In TF, variables values live in session only. Once session is closed there are no values anymore, thus in your current code you create variables, initialise them, and then discard them, all in the constructor.
Typical pattern for integrating TF to OO code in python would be something among the lines of:
class MLobject():
def __init__(self):
self._graph = tf.Graph() # separate graph per instance
with self._graph.as_default():
variable_1 = ....
...
self._initialiser = tf.global_variables_initializer()
self._session = tf.Session(graph=self._graph) # store session in a field
self._graph.finalize() # For safety, this should not be modified anymore
def fit(self, X, y):
self._session.run(self._initialiser)
... # execute training using self._session

Related

Can I define a method as an attribute?

The topic above is a bit ambiguous, the explaination is below:
class Trainer:
"""Object used to facilitate training."""
def __init__(
self,
# params: Namespace,
params,
model,
device=torch.device("cpu"),
optimizer=None,
scheduler=None,
wandb_run=None,
early_stopping: callbacks.EarlyStopping = None,
):
# Set params
self.params = params
self.model = model
self.device = device
# self.optimizer = optimizer
self.optimizer = self.get_optimizer()
self.scheduler = scheduler
self.wandb_run = wandb_run
self.early_stopping = early_stopping
# list to contain various train metrics
# TODO: how to add more metrics? wandb log too. Maybe save to model artifacts?
self.history = DefaultDict(list)
#staticmethod
def get_optimizer(
model: models.CustomNeuralNet,
optimizer_params: global_params.OptimizerParams(),
):
"""Get the optimizer for the model.
Args:
model (models.CustomNeuralNet): [description]
optimizer_params (global_params.OptimizerParams): [description]
Returns:
[type]: [description]
"""
return getattr(torch.optim, optimizer_params.optimizer_name)(
model.parameters(), **optimizer_params.optimizer_params
)
Notice that initially I passed in optimizer in the constructor, where I will be calling it outside this class. However, I now put get_optimizer inside the class itself (for consistency purpose, but unsure if it is ok). So, should I still define self.optimizer = self.get_optimizer() or just use self.get_optimizer() at the designated places in the class? The former encourages some readability for me.
Addendum: I now put the instance inside the .fit() method where I will call say 5 times to train the model 5 times. In this scenario, even though there won't be any obvious issue as we are using optimizer once per call, will it still be better to not define self.optimizer here?
def fit(
self,
train_loader: torch.utils.data.DataLoader,
valid_loader: torch.utils.data.DataLoader,
fold: int = None,
):
"""[summary]
Args:
train_loader (torch.utils.data.DataLoader): [description]
val_loader (torch.utils.data.DataLoader): [description]
fold (int, optional): [description]. Defaults to None.
Returns:
[type]: [description]
"""
self.optimizer = self.get_optimizer(
model=self.model, optimizer_params=OPTIMIZER_PARAMS
)
self.scheduler = self.get_scheduler(
optimizer=self.optimizer, scheduler_params=SCHEDULER_PARAMS
)
There is a difference between the two: calling your get_optimizer will instantiate a new torch.optim.<optimizer> every time. In contrast, setting self.optimizer and accessing it numerous times later will only create a single optimizer instance.

Creating a new object every time you call a function in python

I have a recommender system that I need to train, and I included the entire training procedure inside a function:
def train_model(data):
model = Recommender()
Recommender.train(data)
pred = Recommender.predict(data)
return pred
something like this. Now if I want to train this inside a loop, for different datasets, like:
preds_list = []
data_list = [dataset1, dataset2, dataset3...]
for data_subset in data_list:
preds = train_model(data_subset)
preds_list += [preds]
How can I make sure that every time I call the train_model function, a brand new instance of a recommender is created, not an old one, trained on the previous dataset?
You are already creating a new instance everytime you execute train_model. The thing you are not using the new instance.
You probably meant:
def train_model(data):
model = Recommender()
model.train(data)
pred = model.predict(data)
return pred
Use the instance you've instantiated, not the class
class Recommender:
def __init__(self):
self.id = self
def train(self, data):
return data
def predict(self, data):
return data + str(self.id)
def train_model(data):
model = Recommender()
model.train(data)
return model.predict(data)
data = 'a data '
x = {}
for i in range(3):
x[i] = train_model(data)
print(x[i])
# a data <__main__.Recommender object at 0x11cefcd10>
# a data <__main__.Recommender object at 0x11e0471d0>
# a data <__main__.Recommender object at 0x11a064d50>

Can tf.cond() be used to dynamically define the graph

To build a test case code, I simply want to pass a tensor to a graph and have the model decide whether to multiply by 2 once or twice. The decision is based on evaluation of a boolean. Here is the code:
class what_to_do():
def __init__(self):
self.conditionally_determined_A = None
self.conditionally_determined_B = None
self.truth_values = tf.placeholder(shape=[None, 1],
dtype=tf.bool)
self.input = tf.placeholder(shape=[None, 1],
dtype=tf.float32)
# The question is, can this statement evaluate the conditional and then direct the
# implementation of the model's components.
_ = tf.cond(tf.constant(True),
lambda: self.real_conditional(),
lambda: self.fake_condition())
self.input_A = tf.Variable(initial_value=self.conditionally_determined_A,
trainable=False,
validate_shape=False)
self.const_A = tf.constant([2.])
self.operation_A1 = tf.multiply(self.input_A, self.const_A)
self.input_B = tf.Variable(initial_value=self.conditionally_determined_B,
trainable=False,
validate_shape=False)
self.const_B = tf.constant([2.])
self.operation_B1 = tf.multiply(self.input_B, self.const_B)
self.output = self.operation_B1
# These functions will serve as the condition coordinators for the model
def real_conditional(self):
print('in loop')
self.conditionally_determined_B = self.input
self.conditionally_determined_A = None
return 1
def fake_condition(self):
print('not in loop')
self.conditionally_determined_B = None
self.conditionally_determined_A = self.input
return 0
tf.reset_default_graph()
model = what_to_do()
data_set = np.array([[2,True], [2,True], [2,True], [2,False], [2,False], [2,False], [2,False]])
i, t = np.split(ary=data_set,
indices_or_sections=2,
axis=-1)
init = tf.global_variables_initializer()
with tf.Session() as sess:
sess.run(init)
output = sess.run(fetches=[model.output],
feed_dict={model.input_A:i, model.truth_values:t})
I've run into trouble trying to get tf.cond() to handle the tensor. It complains about ranks and the likes (perhaps another question).
What I have done to the code is to just define the condition as True or False, thus executing the proper function. If it is True, the graph should start at input_B and not worry about input_A at all.
Any help setting up tf.cond() to manipulate the graph dynamically?

Handling tensorflow session in a class

I'm using tensorflow to predict outputs of a neural network. I have a class where I have described the neural network and I have a main file where the predictions are being made, and based on the results, the weights are updated. However, the predictions seem to be really slow. Here is how my code looks like:
class NNPredictor():
def __init__(self):
self.input = tf.placeholder(...)
...
self.output = (...) #Neural network output
def predict_output(self, sess, input):
return sess.run(tf.squeeze(self.output), feed_dict = {self.input: input})
Here's how the main file looks like:
sess = tf.Session()
predictor = NNPredictor()
input = #some initial value
for i in range(iter):
output = predictor.predict_output(sess, input)
input = #some function of output
However, if I use the following function definition in the class:
def predict_output(self):
return self.output
And have the main file as follows:
sess = tf.Session()
predictor = NNPredictor()
input = #some initial value
output_op = predictor.predict_value()
for i in range(iter):
output = np.squeeze(sess.run(output_op, feed_dict = {predictor.input: input}))
input = #some function of output
The code runs almost 20-30x faster. I don't seem to understand how things are working here, and I'd like to know what the best practice would be.
This has to do with the underlying memory accesses masked by Python. Here's some sample code to illustrate this idea:
import time
runs = 10000000
class A:
def __init__(self):
self.val = 1
def get_val(self):
return self.val
# Using method to then call object attribute
obj = A()
start = time.time()
total = 0
for i in xrange(runs):
total += obj.get_val()
end = time.time()
print end - start
# Using object attribute directly
start = time.time()
total = 0
for i in xrange(runs):
total += obj.val
end = time.time()
print end - start
# Assign to local_var first
start = time.time()
total = 0
local_var = obj.get_val()
for i in xrange(runs):
total += local_var
end = time.time()
print end - start
On my computer, it runs in the following timing:
1.49576115608
0.656110048294
0.551875114441
Specific to your case, you're calling the object method in the first case but not doing it in the second case. If you're calling your code many times in this way, there would be significant performance differences.

How to take the output of one model as the input of another one with Tensorflow r-1.0?

I have defined two classes of models, x and y.
class x():
def __init__(self, x_inp1, x_inp2):
# do sth...
def step(self, session, encoder_inputs):
input_feed = {}
for l in range(encoder_size):
input_feed[self.encoder_inputs[l].name] = encoder_inputs[l]
...
output_feed = [x_output]
return session.run(x_output)
class y():
def __init__(self, y_inp1, y_inp2):
# do sth...
def step(self, encoder_inputs):
input_feed = {}
for l in range(encoder_size):
input_feed[self.encoder_inputs[l].name] = encoder_inputs[l]
...
They have quite similar functions. And then I define another class to group them up.
class gp():
def __init__(self, x_inp1, x_inp2, y_inp1, y_inp2):
with tf.variable_scope('x'):
self.x_model = x(x_inp1, x_inp2)
with tf.variable_scope('y'):
self.y_model = y(y_inp1, y_inp2)
def step(self, session, encoder_inputs):
x_output = self.x_model.step(session, encoder_inputs)
y_output = self.y_model.step(session, x_output)
...
Please notice that the y_model takes the output of x_model as input. And I run the gp() in the main function:
with tf.Session() as sess:
gp_m = gp(x_inp1, x_inp2, y_inp1, y_inp2)
gp_m.step(sess, x_inp1, x_inp2, y_inp1, y_inp2)
And after running x_output = self.x_model.step(encoder_inputs) and begin to do y_output = self.y_model.step(x_output), I got such an error:
InvalidArgumentError (see above for traceback): You must feed a value for placeholder tensor 'x/encoder0' with dtype int32
[[Node: x/encoder0 = Placeholder[dtype=DT_INT32, shape=[], _device="/job:localhost/replica:0/task:0/cpu:0"]()]]
Please notice this error points to the x_model even the step function of it has been finished. I wonder how can I use the output of x_model as the input of y_model without any error? Thanks in advance!
You should defer the calls to session.run to be outside the step functions. The problem here is that trying to run Y triggers X because they are connected in the graph.
Instead, it might be better to fully separate your graph build and graph run stages of your program, so you know what placeholders to provide when.

Categories