I have the following Graph neural network model and I am not able to get the learnable parameters of the model to do optimization.
from torch.nn.parameter import Parameter
from torch.nn.modules.module import Module
class Graphconvlayer(nn.Module):
def __init__(self,adj,input_feature_neurons,output_neurons):
super(Graphconvlayer, self).__init__()
self.adj=adj
self.input_feature_neurons=input_feature_neurons
self.output_neurons=output_neurons
self.weights=Parameter(torch.normal(mean=0.0,std=torch.ones(input_feature_neurons,output_neurons)))
self.bias=Parameter(torch.normal(mean=0.0,std=torch.ones(output_neurons)))
def forward(self,inputfeaturedata):
output1= torch.mm(self.adj,inputfeaturedata)
print(output1.shape)
print(self.weights.shape)
print(self.bias.shape)
output2= torch.matmul(output1,self.weights)+ self.bias
return output2
class GCN(nn.Module):
def __init__(self,adj,input_feature_neurons,output_neurons,lr,dropoutvalue,hidden,data):
super(GCN, self).__init__()
self.adj=adj
self.input_feature_neurons=input_feature_neurons
self.output_neurons=output_neurons
self.lr=lr
self.dropoutvalue=dropoutvalue
self.hidden=hidden
self.data=data
self.gcn1 = Graphconvlayer(adj,input_feature_neurons,hidden)
self.gcn2 = Graphconvlayer(adj,hidden,output_neurons)
def forward(self,x):
x= F.relu(self.gcn1(x))
x= F.dropout(x,self.dropoutvalue)
x= self.gcn2(x)
print("opop")
return F.log_softmax(x,dim=1)
for n, p in a.named_parameters():
print(n, p.shape)
>>>
gcn1.weights torch.Size([1433, 2708])
gcn1.bias torch.Size([2708])
gcn2.weights torch.Size([2708, 7])
gcn2.bias torch.Size([7])
>>>
optimizer= optim.Adam(a.named_parameters()),lr=0.001)
>>>
NameError: name 'optim' is not defined
When I pass it as a dict(a.named_parameters()), I am able to print the values, but can not pass it to the optimization function. Can anyone guide me through this?
Related
I made a module, that needs an extra loss term, e.g.
class MyModule:
def forward(self, x):
out = f(x)
extra_loss = loss_f(self.parameters(), x)
return out, extra_loss
I can't figure out how to make this module embeddable, for example, into a Sequential model: any regular module like Linear put after this one will fail because extra_loss causes the input to Linear to be a tuple, which Linear does not support.
So what I am looking for is extracting that extra loss after running the model forward
my_module = MyModule()
model = Sequential(
my_module,
Linear(my_module_outputs, 1)
)
output = model(x)
my_module_loss = ????
loss = mse(label, output) + my_module_loss
Does module composability support this scenario?
IMHO, hooks here is overreaction. Provided extra_loss is additive, we can use global variable like this:
class MyModule:
extra_loss =0
def forward(self, x):
out = f(x)
MyModule.extra_loss += loss_f(self.parameters(), x)
return out
output = model(x)
loss = mse(label, output) + MyModule.extra_loss
MyModule.extra_loss =0
You can register a hook in this case. A hook can be registered on a Tensor or a nn.Module. A hook is a function that is executed when the either forward or backward is called. In this case, we want to attach a forward hook without deattaching itself from the graph so that backward pass can happen.
import torch.nn as nn
act_out = {}
def get_hook(name):
def hook(m, input, output):
act_out[name] = output
return hook
class MyModule(torch.nn.Module):
def __init__(self, input, out, device=None):
super().__init__()
self.model = nn.Linear(input,out)
def forward(self,x):
return self.model(x), torch.sum(x) #our extra loss
class MyModule1(torch.nn.Module):
def __init__(self, input, out, device=None):
super().__init__()
self.model = nn.Linear(input,out)
def forward(self, pair):
x, loss = pair
return self.model(x)
model = nn.Sequential(
MyModule(5,10),
MyModule1(10,1)
)
for name, module in model.named_children():
print(name, module)
if name == '0':
module.register_forward_hook(get_hook(name))
x = torch.tensor([1,2,3,4,5]).float()
out = model(x)
print(act_out)
loss = myanotherloss(out)+act_out['0'][1] # this is the extra loss
# further processing
Note: I am using name == '0' because this is the only module where I want to attach the hook.
Note: Another notable point is nn.Sequential doesn't allow multiple inputs. In this case, it is simply considered as a tuple and then from that tuple we are using the loss and the input.
I'm experimenting with my model's architecture and I would like to have several predefined blocks of layers that I could mix at will. I thought that creating a different class for each of this block structure would make it easier, and I figured that subclassing the Model class in tf.keras was the way to go. So I have done the following (Toy example, yet long. Sorry!).
class PoolingBlock(Model):
def __init__(self, filters, stride, name):
super(PoolingBlock, self).__init__(name=name)
self.bn = BatchNormalization()
self.conv1 = Conv1D(filters=filters, kernel_size=1, padding='same')
self.mp1 = MaxPooling1D(stride, padding='same')
def call(self, input_tensor, training=False, mask=None):
x = self.bn(input_tensor)
x = tf.nn.relu(x)
x = self.conv1(x)
x = self.mp1(x)
return x
class ModelA(Model):
def __init__(self, n_dense, filters, stride, name):
super(ModelA, self).__init__(name=name)
self.d1 = Dense(n_dense, "DenseLayer1")
self.pb1 = PoolingBlock(filters, stride, name="PoolingBlock_1")
self.d2 = Dense(n_dense, "DenseLayer2")
def call(self, inputs, training=False, mask=None):
x = inputs
x = self.d1(x)
x = self.pb1(x)
x = self.d2(x)
return x
model = ModelA(100, 10, 2, 'ModelA')
model.build(input_shape=x.shape)
Then I continue with model.compile(...) and model.fit(...) as usual. But when training, I receive this warning:
WARNING:tensorflow:Entity < bound method PoolingBlock.call of
< model.PoolingBlock object at 0x7fe09ca04208 > > could not be
transformed and will be executed as-is. Please report this to the
AutgoGraph team. When filing the bug, set the verbosity to 10 (on
Linux, export AUTOGRAPH_VERBOSITY=10) and attach the full output.
Cause: converting < bound method PoolingBlock.call of
< model.PoolingBlock object at 0x7fe09ca04208 > >: AttributeError:
module 'gast' has no attribute 'Num'
I don't understand what that means. I am wondering if my training is going as I have planned, if this way of subclassing is correct and solid, if I can suppress this warning somehow.
Kindly try to downgrade the version of gast
pip install gast==0.2.2
And then re-train the network
I am trying to write the derivative function given in pseudo code in my mwe below. It is supposed to calculate the numerical derivative of the cost of the prediction of my neural network with respect to a parameter of one of its layers.
My problem is I don't know how to pass and access an instance of NeuralNetwork and an instance of Layer from within the function (or method?) at the same time.
Looking into e.g. Passing a class to another class (Python) did not provide an answer to me.
import copy
class NeuralNetwork:
def __init__(self):
self.first_layer = Layer()
self.second_layer = Layer()
def cost(self):
# not the actual cost but not of interest
return self.first_layer.a + self.second_layer.a
class Layer:
def __init__(self):
self.a = 1
''' pseudocode
def derivative(NeuralNetwork, Layer):
stepsize = 0.01
cost_unchanged = NeuralNetwork.cost()
NN_deviated = copy.deepcopy(NeuralNetwork)
NN_deviated.Layer.a += stepsize
cost_deviated = NN_deviated.cost()
return (cost_deviated - cost_unchanged)/stepsize
'''
NN = NeuralNetwork()
''' pseudocode
derivative_first_layer = derivative(NN, first_layer)
derivative_second_layer = derivative(NN, second_layer)
'''
I'm reading the book "Make Your Own Neural Network" by Tariq Rashid.
This is my code:
import numpy
class neuralNetwork:
def _init_(self,inputnodes,hiddennodes,outputnodes,learningrate):
self.inodes=inputnodes
self.hnodes=hiddennodes
self.onodes=outputnodes
self.lr=learningrate
pass
def train():
pass
def query():
pass
self.wih=(numpy.random.rand(self.hnodes,self.inodes)-0.5)
self.who=(numpy.random.rand(self.onodes,self.hnodes)-0.5)
It produces this error:
NameError: name 'self' is not defined
What am I doing wrong?
You need to first install the package with the command :
pip3 install numpy
on your shell (I assume you use Python 3).
After you need to write on the top your code :
import numpy
EDIT : With a quick search on Google, I found this :
class neuralNetwork:
# initialise the neural network:
def __init__(self, inputnodes, hiddennodes, outputnodes, learningrate):
#set number of nodes in each input, hidden, output layer:
self.inodes = inputnodes #why can't we immediately use the inputnodes?
self.hnodes = hiddennodes
self.onodes = outputnodes
#Setting the weights:
self.wih = np.random.normal(0.0, pow(self.hnodes, -0.5),(self.hnodes,self.inodes))
self.who = np.random.normal(0.0, pow(self.onodes, -0.5),(self.onodes, self.hnodes))
#learning rate:
self.lr = learningrate
#activation function:
self.activation_function = lambda x: scipy.special.expit(x)
pass
You should check Python tutorials first and be careful with indentation.
I have defined two classes of models, x and y.
class x():
def __init__(self, x_inp1, x_inp2):
# do sth...
def step(self, session, encoder_inputs):
input_feed = {}
for l in range(encoder_size):
input_feed[self.encoder_inputs[l].name] = encoder_inputs[l]
...
output_feed = [x_output]
return session.run(x_output)
class y():
def __init__(self, y_inp1, y_inp2):
# do sth...
def step(self, encoder_inputs):
input_feed = {}
for l in range(encoder_size):
input_feed[self.encoder_inputs[l].name] = encoder_inputs[l]
...
They have quite similar functions. And then I define another class to group them up.
class gp():
def __init__(self, x_inp1, x_inp2, y_inp1, y_inp2):
with tf.variable_scope('x'):
self.x_model = x(x_inp1, x_inp2)
with tf.variable_scope('y'):
self.y_model = y(y_inp1, y_inp2)
def step(self, session, encoder_inputs):
x_output = self.x_model.step(session, encoder_inputs)
y_output = self.y_model.step(session, x_output)
...
Please notice that the y_model takes the output of x_model as input. And I run the gp() in the main function:
with tf.Session() as sess:
gp_m = gp(x_inp1, x_inp2, y_inp1, y_inp2)
gp_m.step(sess, x_inp1, x_inp2, y_inp1, y_inp2)
And after running x_output = self.x_model.step(encoder_inputs) and begin to do y_output = self.y_model.step(x_output), I got such an error:
InvalidArgumentError (see above for traceback): You must feed a value for placeholder tensor 'x/encoder0' with dtype int32
[[Node: x/encoder0 = Placeholder[dtype=DT_INT32, shape=[], _device="/job:localhost/replica:0/task:0/cpu:0"]()]]
Please notice this error points to the x_model even the step function of it has been finished. I wonder how can I use the output of x_model as the input of y_model without any error? Thanks in advance!
You should defer the calls to session.run to be outside the step functions. The problem here is that trying to run Y triggers X because they are connected in the graph.
Instead, it might be better to fully separate your graph build and graph run stages of your program, so you know what placeholders to provide when.