Related
I have a pytorch text multiclass classifier model based on a XLMR based architecture. Due to IP reasons I can't share the architecture code. I have tried putting as much detail as I can. Please point out if more information is needed. But it outputs 28 classes from 'class0' to 'class27' with probability scores that add to 1.
I am trying to use shap package to explain the results. I have wrapped my model into huggingface's custom pipeline object and I get the following output for 1 input text:
pipe = CustomPipeline(model = model, tokenizer = base_tokenizer)
output = pipe(list_of_inputs) # list_of_inputs = ['this is test input']
Output:
[[{"label": "class0","score": 0.01500235591083765},{"label": "class1","score": 0.001698049483820796},{"label": "class2","score": 0.0019644589629024267},{"label": "class3","score": 0.0004418794414959848},{"label": "class4","score": 5.9095666074426845e-05},{"label": "class5","score": 0.0007908751722425222},{"label": "class6","score": 0.002379569923505187},{"label": "class7","score": 0.0035733324475586414},{"label": "class8","score": 0.0014360857894644141},{"label": "class9","score": 0.0007365105557255447},{"label": "class10","score": 0.0014471099711954594},{"label": "class11","score": 0.0011013210751116276},{"label": "class12","score": 0.0010048456024378538},{"label": "class13","score": 0.000885132874827832},{"label": "class14","score": 0.0022015925496816635},{"label": "class15","score": 0.0013197452062740922},{"label": "class16","score": 0.0037292027845978737},{"label": "class17","score": 0.004212632775306702},{"label": "class18","score": 0.9481304287910461},{"label": "class19","score": 0.001469381619244814},{"label": "class20","score": 0.0009713817853480577},{"label": "class21","score": 0.0018773127812892199},{"label": "class22","score": 0.0009251375449821353},{"label": "class23","score": 0.0007248060428537428},{"label": "class24","score": 0.00031718137324787676},{"label": "class25","score": 0.0011144360760226846},{"label": "class26","score": 0.0002294857840752229},{"label": "class27","score": 0.00025681318948045373}]]
The output is in same format as the notebook specified by shap package.
Now, when I try to use shap Explainer:
pipe = UDTMPipeline(model = model, tokenizer = base_tokenizer)
explainer = shap.Explainer(pipe)
shap_values = explainer(list_of_inputs)
shap.plots.text(shap_values)
Error:
File "C:\pipeline.py", line 104, in udtm_xai
shap_values = explainer(query_data)
File "C:\Users\Miniconda3\envs\dtlr_udtm\lib\site-packages\shap\explainers\_partition.py", line 136, in __call__
return super().__call__(
File "C:\Users\Miniconda3\envs\dtlr_udtm\lib\site-packages\shap\explainers\_explainer.py", line 266, in __call__
row_result = self.explain_row(
File "C:\Users\Miniconda3\envs\dtlr_udtm\lib\site-packages\shap\explainers\_partition.py", line 161, in explain_row
self._curr_base_value = fm(m00.reshape(1, -1), zero_index=0)[0] # the zero index param tells the masked model what the baseline is
File "C:\Users\Miniconda3\envs\dtlr_udtm\lib\site-packages\shap\utils\_masked_model.py", line 67, in __call__
return self._full_masking_call(masks, batch_size=batch_size)
File "C:\Users\Miniconda3\envs\dtlr_udtm\lib\site-packages\shap\utils\_masked_model.py", line 144, in _full_masking_call
outputs = self.model(*joined_masked_inputs)
File "C:\Users\Miniconda3\envs\dtlr_udtm\lib\site-packages\shap\models\_transformers_pipeline.py", line 35, in __call__
output[i, self.label2id[obj["label"]]] = sp.special.logit(obj["score"]) if self.rescale_to_logits else obj["score"]
KeyError: 'class0'
The code is unable to find 'class0'. In the postprocess function of pipeline class, I read a file containing label mappings, obtain the softmax scores from _forward function of pipeline class and create a dictionary in the final format to send as output:
class CustomPipeline(Pipeline): # Ignore no indentation below, formatting issue in stackoverflow
def _sanitize_parameters(self, **kwargs):
self.mapping_json = json.loads(open("mapping_file.json", "r", encoding = "utf-8").read().strip())
return {}, {}, {}
def preprocess(self, inputs):
inputs_df = pd.DataFrame([inputs], columns = ["Query"])
inference_dataloader = getInferenceDataloader(inputs_df, self.tokenizer, batch_size = 16)
return inference_dataloader
def softmax_with_temp(self, input, t):
ex = torch.exp(input/t)
sum = torch.sum(ex,0)
return ex / sum
def _forward(self, model_inputs):
if torch.cuda.is_available():
device = torch.device('cuda')
else:
device = torch.device('cpu')
final_pred_labels = []
final_scores = []
batch_count = 0
self.model.eval()
for batch in model_inputs:
batch_count += 1
b_input_ids = batch[0].to(device)
b_input_mask = batch[1].to(device)
b_input_task = torch.full((b_input_ids.shape[0],), -1, dtype=torch.int32).to(device)
with torch.no_grad():
result = self.model((b_input_ids, b_input_mask, b_input_task))
logits = result
logits = logits.detach().cpu().numpy()
pred_softmax_t = self.softmax_with_temp(torch.from_numpy(logits[0]), 2).numpy()
return pred_softmax_t
def postprocess(self, model_outputs):
output_list = [{"label":"class0", "score":float(model_outputs[0])}]
index = 1
for label in self.mapping_json.keys(): #self.mapping_json contains label names that has been read from a file
output_list.append({"label":label, "score":float(model_outputs[index])}) #model_outputs is a list of 28 floating scores
index += 1
return output_list
Am I missing some to define any label based variable which is why I am getting 'class0' key error?
I am a beginner to machine learning and trying to train a model on counting the amount of numbers below 0.5 in a 1D Vector with the length of 10. The input vectors contain number between 0 and 1. I generate the input data and the labels in my script instead of having them in a seperate file, because the data is so simple.
This is the Code:
import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
class MyNet(nn.Module):
def __init__(self):
super(MyNet, self).__init__()
self.lin1 = nn.Linear(10,10)
self.lin2 = nn.Linear(10,1)
def forward(self,x):
x = self.lin1(x)
x = F.relu(x)
x = self.lin2(x)
return x
net = MyNet()
net.to(device)
def train():
criterion = nn.MSELoss()
optimizer = optim.SGD(net.parameters(), lr=0.1)
for epochs in range(100):
target = 0
data = torch.rand(10)
for entry in data:
if entry < 0.5:
target += 1
# print(target)
# print(data)
data = data.to(device)
out = net(data)
# print(out)
target = torch.Tensor(target)
target = target.to(device)
loss = criterion(out, target)
print(loss)
net.zero_grad()
loss.backward()
optimizer.step()
def test():
acc_error = 0
for i in range(100):
test_data = torch.rand(10)
test_data.to(device)
test_target = 0
for entry in test_data:
if entry < 0.5:
test_target += 1
out = net(test_data)
error = test_target - out
if error < 0:
error *= -1
acc_error += error
overall_error = acc_error / 100
print(overall_error)
train()
test()
This is the error:
Traceback (most recent call last):
File "test1.py", line 70, in <module>
test()
File "test1.py", line 59, in test
out = net(test_data)
File "/vol/fob-vol7/mi18/radtklau/SP/sem_project/lib64/python3.6/site-packages/torch/nn/modules/module.py", line 889, in _call_impl
result = self.forward(*input, **kwargs)
File "test1.py", line 15, in forward
x = self.lin1(x)
File "/vol/fob-vol7/mi18/radtklau/SP/sem_project/lib64/python3.6/site-packages/torch/nn/modules/module.py", line 889, in _call_impl
result = self.forward(*input, **kwargs)
File "/vol/fob-vol7/mi18/radtklau/SP/sem_project/lib64/python3.6/site-packages/torch/nn/modules/linear.py", line 94, in forward
return F.linear(input, self.weight, self.bias)
File "/vol/fob-vol7/mi18/radtklau/SP/sem_project/lib64/python3.6/site-packages/torch/nn/functional.py", line 1753, in linear
return torch._C._nn.linear(input, weight, bias)
RuntimeError: Tensor for 'out' is on CPU, Tensor for argument #1 'self' is on CPU, but expected them to be on GPU (while checking arguments for addmm)
The other posts regarding the topic have not solved my problem. Maybe somebody can help. Thanks!
Notice how your error message traces back to test, while train works fine.
You've transfered your data correctly in train:
data = data.to(device)
But not in test:
test_data.to(device)
Instead it should be reassigned to test_data, since torch.Tensor.to makes a copy:
test_data = test_data.to(device)
I try to repeat some code on kaggle. I use tensorflow 2.3.1, Below is my code:
input_shape = (size, size, 3)
in_lay = tf.keras.layers.Input(shape = input_shape)
in_lay = tf.keras.layers.Input(shape = input_shape)
base_pretrained_model = tf.keras.applications.VGG16(input_shape = input_shape,
include_top = False, weights = 'imagenet')
base_pretrained_model.trainable = False
pt_depth = base_pretrained_model.get_output_shape_at(0)[-1]
pt_features = base_pretrained_model(in_lay)
bn_features = tf.keras.layers.BatchNormalization()(pt_features)
……
I get the error at the line " pt_depth = base_pretrained_model.get_output_shape_at(0)[-1]". The error is:
pt_depth = base_pretrained_model.get_output_shape_at(0)[-1]
File "/home/xxxx/anaconda3/envs/py36/lib/python3.6/site-packages/tensorflow/python/keras/engine/base_layer.py", line 2030, in get_output_shape_at
'output shape')
File "/home/xxxx/anaconda3/envs/py36/lib/python3.6/site-packages/tensorflow/python/keras/engine/base_layer.py", line 2603, in _get_node_attribute_at_index
'and thus has no defined ' + attr_name + '.')
RuntimeError: The layer has never been called and thus has no defined output shape.
What's the cause?
Thanks
Try this method instead:
pt_depth = model.layers[0].compute_output_shape(input_shape)
I tried the fix by Andrey and failed at another step in my code. So it doesn't work for me.
I got the solution from this page: https://github.com/tensorflow/tensorflow/issues/44857. We need to call the model with a input before getting the outshape, like following:
in_lay = Input(t_x.shape[1:])
base_pretrained_model = VGG16(input_shape = t_x.shape[1:], include_top = False, weights = 'imagenet')
base_pretrained_model.trainable = False
pt_features = base_pretrained_model(in_lay)
pt_depth = base_pretrained_model.get_output_shape_at(0)[-1]
And then my program can proceed.
Environment: Win10, Python 3.6.8, Tensorflow-gpu 1.13.1
I have a model based on tf.estimator.DNNRegressor which I've trained locally. Now I'm having an issue getting the predict_input_fn function to work with real-time data. I have it working by reading a CSV file with a single entry but would like to pass in a numpy array or some other form of data from memory and have the model make a prediction.
Here's a code snippet:
tf.enable_eager_execution()
model = tf.estimator.DNNRegressor(
hidden_units=[128,128],
feature_columns=feature_columns,
model_dir=model_dir)
def predict_input_fn():
test_array = np.array([-0.057,-0.0569]) # shortened, normally 56 floats
defaults = [tf.float32] * 56
filenames = ['./prediction.csv']
dataset_old = tf.data.experimental.CsvDataset( # this works
filenames=filenames,
record_defaults=defaults)
dataset = tf.data.Dataset.from_tensor_slices(test_array) # this doesn't :(
dataset = dataset.batch(1)
features = tf.data.experimental.get_single_element(dataset)
return dict(zip(feature_names, features)), None
predictor = model.predict(input_fn=predict_input_fn)
print(next(predictor))
I'm getting the following error even though eager execution is enabled:
TypeError: Tensor objects are only iterable when eager execution is enabled. To iterate over this tensor use tf.map_fn.
The data format I'm feeding in for training is 56 x float32 variables as features and 1 x float32 variable as a label. As mentioned it works perfectly when using the tf.data.experimental.CsvDataset() function but for the life of me I can't figure out how to feed in a basic array.
I've tried putting the array in a container array, using the standard make_one_shot_iterator() function on the dataset, using .constant() to try and build a tensor before passing it to tensor_slices. Have gone through the docs and every example I can find but still can't get it working.
Struggling a bit, any help would be greatly appreciated.
Full code below:
import tensorflow as tf
# import tensorflowjs as tfjs
import json
import numpy as np
tf.enable_eager_execution()
model_dir = './dnn3/'
feature_columns = []
feature_names = []
feature_spec = {}
for i in range(56):
feature_columns.append(tf.feature_column.numeric_column(key="jkey"+str(i)))
feature_names.append("jkey"+str(i))
feature_spec["jkey"+str(i)] = tf.FixedLenFeature([1],tf.float32)
def serving_input_receiver_fn():
serialized_tf_example = tf.placeholder(dtype=tf.string,shape=[None],name='input_tensors')
receiver_tensors = {'inputs': serialized_tf_example}
features = tf.parse_example(serialized_tf_example, feature_spec)
return tf.estimator.export.ServingInputReceiver(features, receiver_tensors)
def train_input_fn():
defaults = [tf.float32] * 57
filenames = ['./training.csv'] # can add to this array
dataset = tf.data.experimental.CsvDataset(
filenames=filenames,
record_defaults=defaults)
dataset = dataset.shuffle(1000000)
dataset = dataset.repeat(1000) # epochs
dataset = dataset.batch(128)
iter = dataset.make_one_shot_iterator()
next = iter.get_next()
features = next[:-1]
label = next[-1]
return dict(zip(feature_names, features)), label
def eval_input_fn():
defaults = [tf.float32] * 57
filenames = ['./evaluation.csv']
dataset = tf.data.experimental.CsvDataset(
filenames=filenames,
record_defaults=defaults)
dataset = dataset.shuffle(10000)
dataset = dataset.repeat(100)
dataset = dataset.batch(100)
iter = dataset.make_one_shot_iterator()
next = iter.get_next()
features = next[:-1]
label = next[-1]
return dict(zip(feature_names, features)), label
def predict_input_fn():
test_array = [[-0.057,-0.0569,-0.0569,-0.0759,-0.0568,-0.0379,-0.0379,-0.0567,-0.0946,0.0378,0.0566,0.0755,0,0,0,0,-0.0189,0,0,0,0,0.0379,0.0378,0.0568,0,0.0378,0,0.0758,0.0379,0.0379,0.057,0.0758,0.0379,0.0381,-0.152,0.2089,-0.0952,0.0568,-0.2485,.4667,-0.0803,-0.0775,-0.0832,-0.054,-0.0989,0.0063,0.0037,-0.0342,0.0007,0.0281,0.0187,-0.0065,0.0423,0.078,-0.0285,-0.0093],[-0.057,-0.0569,-0.0569,-0.0759,-0.0568,-0.0379,-0.0379,-0.0567,-0.0946,0.0378,0.0566,0.0755,0,0,0,0,-0.0189,0,0,0,0,0.0379,0.0378,0.0568,0,0.0378,0,0.0758,0.0379,0.0379,0.057,0.0758,0.0379,0.0381,-0.152,0.2089,-0.0952,0.0568,-0.2485,.4667,-0.0803,-0.0775,-0.0832,-0.054,-0.0989,0.0063,0.0037,-0.0342,0.0007,0.0281,0.0187,-0.0065,0.0423,0.078,-0.0285,-0.0093]]
defaults = [tf.float32] * 56
filenames = ['./prediction.csv']
dataset2 = tf.data.experimental.CsvDataset(
filenames=filenames,
record_defaults=defaults)
test_tensor = tf.constant(test_array)
#test_dict = dict(zip(feature_names, test_array))
dataset = tf.data.Dataset.from_tensor_slices(test_tensor)
dataset = dataset.batch(1)
features = tf.data.experimental.get_single_element(dataset)
return dict(zip(feature_names, features)), None
model = tf.estimator.DNNRegressor(
hidden_units=[128,128],
feature_columns=feature_columns,
model_dir=model_dir)
def train():
for i in range(5):
model.train(input_fn=train_input_fn, steps=10000)
eval_result = model.evaluate(input_fn=eval_input_fn)
print(eval_result)
def train_eval():
train_spec = tf.estimator.TrainSpec(train_input_fn,max_steps=20000)
eval_spec = tf.estimator.EvalSpec(eval_input_fn)
tf.estimator.train_and_evaluate(model,train_spec,eval_spec)
def save():
save_dir = model.export_savedmodel(export_dir_base=model_dir+'saved', serving_input_receiver_fn=serving_input_receiver_fn)
print('Saved to ',save_dir)
def predict():
predictor = model.predict(input_fn=predict_input_fn)
print(next(predictor))
predict()
# tfjs.converters.save_keras_model(model, './tfjs/')
And full error response:
WARNING:tensorflow:From C:\Users\jimba\AppData\Local\Programs\Python\Python36\lib\site-packages\tensorflow\python\framework\op_def_library.py:263: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.
Instructions for updating:
Colocations handled automatically by placer.
Traceback (most recent call last):
File "train.py", line 93, in <module>
predict()
File "train.py", line 91, in predict
print(next(predictor))
File "C:\Users\jimba\AppData\Local\Programs\Python\Python36\lib\site-packages\tensorflow_estimator\python\estimator\estimator.py", line 609, in predict
input_fn, model_fn_lib.ModeKeys.PREDICT)
File "C:\Users\jimba\AppData\Local\Programs\Python\Python36\lib\site-packages\tensorflow_estimator\python\estimator\estimator.py", line 967, in _get_features_from_input_fn
result = self._call_input_fn(input_fn, mode)
File "C:\Users\jimba\AppData\Local\Programs\Python\Python36\lib\site-packages\tensorflow_estimator\python\estimator\estimator.py", line 1079, in _call_input_fn
return input_fn(**kwargs)
File "train.py", line 67, in predict_input_fn
return dict(zip(feature_names, features)), None
File "C:\Users\jimba\AppData\Local\Programs\Python\Python36\lib\site-packages\tensorflow\python\framework\ops.py", line 442, in __iter__
"Tensor objects are only iterable when eager execution is "
TypeError: Tensor objects are only iterable when eager execution is enabled. To iterate over this tensor use tf.map_fn
I wrote a python class, but when using it, it can not find any of the functions inside. I tested it on windows and MAC, it does not work both. Here is the some of the code of the class I defined:(I here is not the full code as it it too many codes and it's not allowed to copy here)
import numpy as np
import tensorflow as tf
from tensorflow.examples.tutorials.mnist import input_data
import matplotlib.pyplot as plt
class VanillaInfoGAN(object):
def __init__(self,learning_rate,batch_size,num_epoches,z_dim,c_cat_dim,c_cont_dim,model_num,print_every=30,show_every=50,
store_path = './ckpt/',training_picture_path='./pictures/'):
self.learning_rate = learning_rate
self.batch_size = batch_size
self.num_epoches = num_epoches
self.z_dim = z_dim
self.c_cat_dim = c_cat_dim
self.c_cont_dim = c_cont_dim
self.print_every = print_every
self.show_every = show_every
self.data =input_data.read_data_sets('MNIST',one_hot=True)
self.data_dim = 784
self.store_path = store_path
self.model_num = model_num
self.training_picture_path = training_picture_path
self.real_data_placeholder = tf.placeholder(tf.float32,[None,self.data_dim])
self.z_placeholder = tf.placeholder(tf.float32,[None,self.z_dim])
self.c_cat_placeholder = tf.placeholder(tf.float32,[None,self.c_cat_dim])
self.c_cont_placeholder = tf.placeholder(tf.float32,[None,self.c_cont_dim])
self.c = tf.concat([self.c_cat_placeholder,self.c_cont_placeholder],axis=1)
self.z_c = tf.concat([self.z_placeholder,self.c_cat_placeholder,self.c_cont_placeholder],axis=1)
self.g_out = self.generator()
d_out_real = self.discriminator(self.real_data_placeholder)
d_out_fake = self.discriminator(self.g_out,reuse=True)
q_out = self.q_net(self.g_out)
self.g_loss,self.d_loss,self.q_loss = self.build_loss(self.g_out,d_out_real,d_out_fake,q_out)
self.g_opt,self.d_opt,self.q_opt = self.optimizer(self.g_loss,self.d_loss,self.q_loss)
self.saver= tf.train.Saver()
print('Model graph has built')
def generator(self,reuse=False):
with tf.variable_scope('generator',reuse=reuse):
layer = tf.layers.dense(self.z_c,128,activation = tf.nn.relu)
layer = tf.layers.dense(layer,self.data_dim,activation = tf.nn.sigmoid)
return layer
def discriminator(self,d_input,reuse=False):
with tf.variable_scope('discriminator',reuse=reuse):
layer = tf.layers.dense(d_input,128,activation = tf.nn.relu)
layer = tf.layers.dense(layer,1,activation = tf.nn.sigmoid)
return layer
def q_net(self,g_out,reuse=False):
with tf.variable_scope('Q',reuse=reuse):
layer = tf.layers.dense(g_out,128,activation = tf.nn.relu)
layer = tf.layers.dense(layer,self.c_cat_dim+self.c_cont_dim,activation = None)
layer_cat = tf.nn.softmax(layer[:,:self.c_cat_dim])
layer_cont =tf.nn.sogmoid(layer[:,self.c_cat_dim:])
q_out = tf.concat([layer_cat,layer_cont],axis=1)
return q_out
Here is the running warpper I used to create the object:
from VanillaInfoGAN import VanillaInfoGAN
gan = VanillaInfoGAN(learning_rate = learning_rate,batch_size=batch_size,num_epoches=num_epoches,z_dim=z_dim,c_cat_dim=c_cat_dim,c_cont_dim=c_cont_dim,
model_num=model_num,print_every=print_every,show_every=show_every)
I got error:
File "<ipython-input-2-5252a711a145>", line 1, in <module>
runfile('/Users/shiyanpei/Documents/embeddings/infogan/test/run.py', wdir='/Users/shiyanpei/Documents/embeddings/infogan/test')
File "/Users/shiyanpei/Applications/anaconda3/lib/python3.6/site-packages/spyder/utils/site/sitecustomize.py", line 692, in runfile
execfile(filename, namespace)
File "/Users/shiyanpei/Applications/anaconda3/lib/python3.6/site-packages/spyder/utils/site/sitecustomize.py", line 101, in execfile
exec(compile(f.read(), filename, 'exec'), namespace)
File "/Users/shiyanpei/Documents/embeddings/infogan/test/run.py", line 28, in <module>
model_num=model_num,print_every=print_every,show_every=show_every)
File "/Users/shiyanpei/Documents/embeddings/infogan/test/VanillaInfoGAN.py", line 40, in __init__
self.g_out = self.generator()
AttributeError: 'VanillaInfoGAN' object has no attribute 'generator'
I have wrote many other classes but I have not seen this error yet
Can Anybody help?
Thanks!!
Your indentation is wrong:
class VanillaInfoGAN(object):
def __init__(self,learning_rate,batch_size,num_epoches,z_dim,c_cat_dim,c_cont_dim,model_num,print_every=30,show_every=50,
store_path = './ckpt/',training_picture_path='./pictures/'):
# a lot of lines skipped
self.g_out = self.generator()
# a lot of lines skipped
def generator(self,reuse=False):
with tf.variable_scope('generator',reuse=reuse):
Python code is executed from top to bottom. def generator(self,reuse=False): is not even executed when you call self.generator. Make generator a property of the class instead by outdenting it:
class VanillaInfoGAN(object):
def __init__(self,learning_rate,batch_size,num_epoches,z_dim,c_cat_dim,c_cont_dim,model_num,print_every=30,show_every=50,
store_path = './ckpt/',training_picture_path='./pictures/'):
# a lot of line skipped
self.g_out = self.generator()
# a lot of line skipped
def generator(self,reuse=False):
with tf.variable_scope('generator',reuse=reuse):
While you're at it, also strongly consider to split the code into multiple classes and methods; its excessive complexity makes it very easy to commit mistakes.