I am new to Tensorflow. I am trying to write a function in python using Tensorflow that operates on a sparse matrix input. Normally I would define a tensorflow placeholder, but apparently there is no placeholder for sparse matrices.
What is the proper way to define a function that operates on sparse data in tensorflow and pass values into it?
Specifically, I am trying to rewrite the fundamental example of a multilayer perceptron, found here https://github.com/aymericdamien/TensorFlow-Examples/blob/master/examples/3_NeuralNetworks/multilayer_perceptron.py, to accept sparse input instead of dense.
As a dummy example, how would you write a function that looks something like this?
import tensorflow as tf
x = tf.placeholder("sparse")
y = tf.placeholder("float", [None, n_classes])
# Create model
def sparse_multiply(x, y):
outlayer = tf.sparse_tensor_dense_matmul(x, y)
return out_layer
pred = multiply(x, y)
# Launch the graph
with tf.Session() as sess:
result = sess.run(pred, feed_dict={x: x_input, y: y_input})
Someone at the link https://github.com/tensorflow/tensorflow/issues/342 recommended, as a workaround, passing in the elements needed to construct the sparse matrix and then creating the sparse matrix on the fly within the function. That seems a little hacky, and I get errors when I try to construct it that way.
Any help, especially answers with code, would be greatly appreciated!
I think I figured it out. The suggestion I linked to actually did work, I just needed to correct all the inputs to have consistent types. Here is the dummy example I listed in the question, coded correctly:
import tensorflow as tf
import sklearn.feature_extraction
import numpy as np
def convert_csr_to_sparse_tensor_inputs(X):
coo = X.tocoo()
indices = np.mat([coo.row, coo.col]).transpose()
return indices, coo.data, coo.shape
X = ____ #Some sparse 2 x 8 csr matrix
y_input = np.asarray([1, 1, 1, 1, 1, 1, 1, 1])
y_input.shape = (8,1)
x_indices, x_values, x_shape = convert_csr_to_sparse_tensor_inputs(X)
# tf Graph input
y = tf.placeholder(tf.float64)
values = tf.placeholder(tf.float64)
indices = tf.placeholder(tf.int64)
shape = tf.placeholder(tf.int64)
# Create model
def multiply(values, indices, shape, y):
x_tensor = tf.SparseTensor(indices, values, shape)
out_layer = tf.sparse_tensor_dense_matmul(x_tensor, y)
return out_layer
pred = multiply(values, indices, shape, y)
# Launch the graph
with tf.Session() as sess:
result = sess.run(pred, feed_dict={values: x_values, indices: x_indices, shape: x_shape, y: y_input})
Related
I want to create a custom keras layer that compute the "predict" of a Self Organizing Map.
Here is my implementation of the layer
`
class SOMLayer(keras.layers.Layer):
def __init__(self, X_train, y_train, w, n=50, input_shape=265):
super(SOMLayer, self).__init__()
self.inputs = None
self.X_train = X_train
self.y_train = y_train
self._input_len = input_shape
# weights of pre-trained SOM, two versions: _weights is a tensor, _weights_nc is a numpy array
self._weights = tf.convert_to_tensor(w, dtype=tf.float32)
self._weights_nc = w
self._activation_map = zeros((n, n))
# activation distance calc., lblmap version works with numpy arrays, the other one with tf tensors
self._activation_distance = self._manhattan_distance
self._activation_distance_lblmap = self._manhattan_distance_lblmap
self.winmap = self.labels_map(X_train, y_train)
self.default_class = sum(list(self.winmap.values())).most_common()[0][0]
#tf.function
def call(self, inputs):
ret_arr = np.array(np.full(6, 0.01), ndmin=2)
win_position = self.winner(inputs)
if win_position.ref() in self.winmap:
ret_arr[0][self.winmap[win_position.ref()].most_common()[0][0]] = 0.95
else:
ret_arr[0][self.default_class] = 0.95
return ret_arr
def _activate(self, x):
"""Updates matrix activation_map, in this matrix
the element i,j is the response of the neuron i,j to x."""
self._activation_map = self._activation_distance(x, self._weights)
def _activate_lblmap(self, x):
"""Updates matrix activation_map, in this matrix
the element i,j is the response of the neuron i,j to x."""
self._activation_map = self._activation_distance_lblmap(x, self._weights_nc)
def _manhattan_distance(self, x, w):
return tf.linalg.norm(tf.subtract(x, w), ord=1, axis=-1)
def _manhattan_distance_lblmap(self, x, w):
return linalg.norm(subtract(x, w), ord=1, axis=-1)
def _check_input_len(self, data):
"""Checks that the data in input is of the correct shape."""
data_len = len(data[0])
if self._input_len != data_len:
msg = 'Received %d features, expected %d.' % (data_len,
self._input_len)
raise ValueError(msg)
#tf.function
def winner(self, x):
"""Computes the coordinates of the winning neuron for the sample x."""
self._activate(x)
return tf.unravel_index(tf.argmin(self._activation_map, output_type=tf.int32),
tf.shape(self._activation_map))
def winner_lblmap(self, x):
"""Computes the coordinates of the winning neuron for the sample x."""
self._activate_lblmap(x)
return unravel_index(self._activation_map.argmin(),
self._activation_map.shape)
def labels_map(self, data, labels):
"""Returns a dictionary wm where wm[(i,j)] is a dictionary
that contains the number of samples from a given label
that have been mapped in position i,j.
Parameters
----------
data : np.array or list
Data matrix.
label : np.array or list
Labels for each sample in data.
"""
self._check_input_len(data)
if not len(data) == len(labels):
raise ValueError('data and labels must have the same length.')
winmap = defaultdict(list)
for x, l in zip(data, labels):
winmap[self.winner_lblmap(x)].append(l)
for position in winmap:
winmap[position] = Counter(winmap[position])
return winmap
`
There are two implementation of the same method for methods like "activate", "manhattan_distance" and "winner".
The implementation with "_lblmap" in the name works with numpy arrays (is the impl. of the minisom python library) and the other impl. uses tensorflow tensors because the input of the layer when the model execute is a tensor and i was not able to convert that tensor in a numpy array.
The problem is in the "winner" method, the winner method that works with numpy array return this:
self.winner_lblmap numpy array implementation
(0, 2)
that is the position (like x,y coordinates) of the Best Matching Unit that will determine the class of the input, and its called inside "labels_map" method.
The winner method called inside "call" method works with tensorflow tensors because it will manage the input passed when you use model.predict(...) and this input is a tensor.
The output of this implementation is:
self.winner tensorflow tensors implementation
Tensor("UnravelIndex:0", shape=(2, 50), dtype=int32)
And looking the shape it's easy to see that is not the correct shape of the desired output, and also i'm not able to directly access the data inside this tensor (or previuos calculated ones).
I tried to force "eager evaluation" in different ways to try to visualize the data but without success.
I tried also to tune the "tf.linalg.norm" parameters but the shape of the result is (0,).
Is there a way to access the data inside the tensor and obtain tha same result of the "winner_lblmap" method that works with numpy arrays?
I'm using tensorflow 1.14.0. I would like to know how I can type cast list into tensor. I get this error when trying to use tf.convert_to_tensor(). Appreciate any help
Failed to convert object of type to Tensor. Contents: [None]. Consider casting elements to a supported type.
Here is my code
def testtf4():
x = tf.placeholder(tf.float32, shape=[None])
y = tf.placeholder(tf.float32, shape=[None])
op = tf.placeholder(tf.float32, shape=[None,3])
print("\nshape of x,y", x.shape, y.shape)
arr = np.genfromtxt("C:\\Data\\Training_and_codes\\ML\\TF Samples\\Data.csv", delimiter=",");
gradmulx_op = tf.gradients(op[:,0],x)
gradmuly_op = tf.gradients(op[:,0],y)
tgradmulx_op = tf.convert_to_tensor(gradmulx_op)
tgradmuly_op = tf.convert_to_tensor(gradmuly_op)
print("\nshape of gradmul tensors", tgradmulx_op.shape, tgradmuly_op.shape)
with tf.Session() as sess:
print("started session......\n")
input_feed={}
input_feed[x]=arr[:,0]
input_feed[y]=arr[:,1]
input_feed[op]=arr[:,2:4]
[gradx, grady] = sess.run([tgradmulx_op, tgradmuly_op],input_feed)
print("x gradient",gradx)
print("y gradient",grady)
Your problem does not have to do with tf.convert_to_tensor, but with the fact that your are trying to compute some gradients that do not exist. You have these two placeholders:
x = tf.placeholder(tf.float32, shape=[None])
op = tf.placeholder(tf.float32, shape=[None, 3])
And then you try to get the following gradients:
gradmulx_op = tf.gradients(op[:, 0], x)
gradmuly_op = tf.gradients(op[:, 0], y)
For these gradients to exist (that is, not be None), the value of op[:, 0] would have to be the result of one or more differentiable operations using x and y. For example, if op were defined as:
op = tf.stack([2 * x + 3 * y, x - 1, 2 * y + 2], axis=1)
Then it would work, because op[:, 0] would be computed from x and y (and possibly other values), so there is a gradient between the tensors. Or, put it a different way, changing x or y changes the value of op[:, 0]. TensorFlow keeps track of the operations used to compute each value and uses that information to automatically compute the gradients.
But op is not calculated from x and y, in fact it is not calculated from anything, since it is a placeholder, it is just a given value. A change in x or y does not entail a change in op. So there is no gradients between those tensors. I am not sure what you are trying to achieve with your code, but you probably need to rethink what exactly is the result that you want to compute.
I need to write a custom Op in python, which will generate an output based on a model and another op that will update the model. In the following sample code, I have a very simple model of just a scaler, w (but in reality it will be a nxm matrix). I figured out how to "read" the model as demonstrated in the custom_model_read_op function (in reality much more complicated). However, how can I create something similar that will update w in some custom complicated way (using custom_model_update_op)? I assume this is possible given the fact that Optimizer ops like SGD are able to do this. Thanks in advance!
import tensorflow as tf
import numpy
# Create a model
w = tf.Variable(numpy.random.randn(), name="weight")
X = tf.placeholder(tf.int32, shape=(), name="X")
def custom_model_read_op(i, w):
y = i*float(w)
return y
y = tf.py_func(custom_model_read_op, [X, w], [tf.float64], name="read_func")
def custom_model_update_op(i, w):
==> # How to update w (the model stored in a Variable above) based on the value of i and some crazy logic?
return 0
crazy_update = tf.py_func(custom_model_update_op, [X, w], [tf.int64], name="update_func")
with tf.Session() as sess:
tf.global_variables_initializer().run()
for i in range(10):
y_out, __ = sess.run([y, crazy_update], feed_dict={X: i})
print("y=", "{:.4f}".format(y_out[0]))
Well, I'm not sure this is the best way, but it accomplishes when I need. I don't have a py_func where the update on w occurs, but I do update it in the read_op, passing it back as a return value, and finally using the assign function to modify it outside of the custom op. If any Tensorflow experts can confirm that this is a good legitimate way to do that, I'd appreciate it.
import tensorflow as tf
import numpy
# Create a model
w = tf.Variable(numpy.random.randn(), name="weight")
X = tf.placeholder(tf.int32, shape=(), name="X")
def custom_model_read_op(i, w):
y = i*float(w)
w = custom_model_update(w)
return y, w
y = tf.py_func(custom_model_read_op, [X, w], [tf.float64, tf.float64], name="read_func")
def custom_model_update(w):
# update w (the model stored in a Variable above) based on the vaue of i and some crazy logic
return w + 1
with tf.Session() as sess:
tf.global_variables_initializer().run()
for i in range(10):
y_out, w_modified = sess.run(y, feed_dict={X: i})
print("y=", "{:.4f}".format(y_out))
assign_op = w.assign(w_modified)
sess.run(assign_op)
print("w=", "{:.4f}".format(sess.run(w)))
I am trying to process a tensor of variable size, in a python way that would be something like:
# X is of shape [m, n]
for x in X:
process(x)
I have tried to use tf.scan, the thing is that I want to process every sub-tensor, so I have tried to use a nested scan, but I was enable to do it, because tf.scan work with the accumulator, if not found it will take the first entry of the elems as initializer, which I don't want to do.
As an example, suppose I want to add one to every element of my tensor (this is just an example), and I want to process it element by element. If I run the code bellow, I will only have one added to a sub-tensor, because scan consider the first tensor as initializer, along with the first element of every sub-tensor.
import numpy as np
import tensorflow as tf
batch_x = np.random.randint(0, 10, size=(5, 10))
x = tf.placeholder(tf.float32, shape=[None, 10])
def inner_loop(x_in):
return tf.scan(lambda _, x_: x_ + 1, x_in)
outer_loop = tf.scan(lambda _, input_: inner_loop(input_), x, back_prop=True)
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
rs = sess.run(outer_loop, feed_dict={x: batch_x})
Any suggestions ?
To loop over a tensor you could try tf.unstack
Unpacks the given dimension of a rank-R tensor into rank-(R-1) tensors.
So adding 1 to each tensor would look something like:
import tensorflow as tf
x = tf.placeholder(tf.float32, shape=(None, 10))
x_unpacked = tf.unstack(x) # defaults to axis 0, returns a list of tensors
processed = [] # this will be the list of processed tensors
for t in x_unpacked:
# do whatever
result_tensor = t + 1
processed.append(result_tensor)
output = tf.concat(processed, 0)
with tf.Session() as sess:
print(sess.run([output], feed_dict={x: np.zeros((5, 10))}))
Obviously you can further unpack each tensor from the list to process it, down to single elements. To avoid lots of nested unpacking though, you could maybe try flattening x with tf.reshape(x, [-1]) first, and then loop over it like
flattened_unpacked = tf.unstack(tf.reshape(x, [-1])
for elem in flattened_unpacked:
process(elem)
In this case elem is a scalar.
Most of tensorflow built-in functions could be applied elementwise. So you could just pass a tensor into a function. Like:
outer_loop = inner_loop(x)
However, if you have some function that could not be applied this way (it's really tempting to see that function), you could use map_fn.
Say, your function simply adds 1 to every element of a tensor (or whatever):
inputs = tf.placeholder...
def my_elementwise_func(x):
return x + 1
def recursive_map(inputs):
if tf.shape(inputs).ndims > 0:
return tf.map_fn(recursive_map, inputs)
else:
return my_elementwise_func(inputs)
result = recursive_map(inputs)
I am new to tensorflow and neural networks, and I am trying to create a model that just multiples two float values together.
I wasn't sure how many neurons I would want, but I picked 10 neurons and tried to see where I could go from that. I figured that would probably introduce enough complexity in order to semi-accurately learn that operation.
Anyways, here is my code:
import tensorflow as tf
import numpy as np
# Teach how to multiply
def generate_data(how_many):
data = np.random.rand(how_many, 2)
answers = data[:, 0] * data[:, 1]
return data, answers
sess = tf.InteractiveSession()
# Input data
input_data = tf.placeholder(tf.float32, shape=[None, 2])
correct_answers = tf.placeholder(tf.float32, shape=[None])
# Use 10 neurons--just one layer for now, but it'll be fully connected
weights_1 = tf.Variable(tf.truncated_normal([2, 10], stddev=.1))
bias_1 = tf.Variable(.1)
# Output of this will be a [None, 10]
hidden_output = tf.nn.relu(tf.matmul(input_data, weights_1) + bias_1)
# Weights
weights_2 = tf.Variable(tf.truncated_normal([10, 1], stddev=.1))
bias_2 = tf.Variable(.1)
# Softmax them together--this will be [None, 1]
calculated_output = tf.nn.softmax(tf.matmul(hidden_output, weights_2) + bias_2)
cross_entropy = tf.reduce_mean(correct_answers * tf.log(calculated_output))
optimizer = tf.train.GradientDescentOptimizer(.5).minimize(cross_entropy)
sess.run(tf.initialize_all_variables())
for i in range(1000):
x, y = generate_data(100)
sess.run(optimizer, feed_dict={input_data: x, correct_answers: y})
error = tf.reduce_sum(tf.abs(calculated_output - correct_answers))
x, y = generate_data(100)
print("Total Error: ", error.eval(feed_dict={input_data: x, correct_answers: y}))
It seems that the error is always around 7522.1, which very very bad for just 100 data points, so I assume it is not learning.
My questions: Is my machine learning? If so, what can I do to make it more accurate? If not, how can I make it learn?
There are a few major issues with the code. Aaron has already identified some of them, but there's another important one: calculated_output and correct_answers are not the same shape, so you're creating a 2D matrix when you subtract them. (The shape of calculated_output is (100, 1) and the shape of correct_answers is (100).) So you need to adjust the shape (for example, by using tf.squeeze on calculated_output).
This problem also doesn't really require any non-linearities, so you could get by with no activations and only one layer. The following code gets a total error of about 6 (~0.06 error on average for each test point). Hope that helps!
import tensorflow as tf
import numpy as np
# Teach how to multiply
def generate_data(how_many):
data = np.random.rand(how_many, 2)
answers = data[:, 0] * data[:, 1]
return data, answers
sess = tf.InteractiveSession()
input_data = tf.placeholder(tf.float32, shape=[None, 2])
correct_answers = tf.placeholder(tf.float32, shape=[None])
weights_1 = tf.Variable(tf.truncated_normal([2, 1], stddev=.1))
bias_1 = tf.Variable(.0)
output_layer = tf.matmul(input_data, weights_1) + bias_1
mean_squared = tf.reduce_mean(tf.square(correct_answers - tf.squeeze(output_layer)))
optimizer = tf.train.GradientDescentOptimizer(.1).minimize(mean_squared)
sess.run(tf.initialize_all_variables())
for i in range(1000):
x, y = generate_data(100)
sess.run(optimizer, feed_dict={input_data: x, correct_answers: y})
error = tf.reduce_sum(tf.abs(tf.squeeze(output_layer) - correct_answers))
x, y = generate_data(100)
print("Total Error: ", error.eval(feed_dict={input_data: x, correct_answers: y}))
The way you are using softmax is weird. Softmax is normally used when you want to have a probability distribution over a set of classes. In your code it looks like you have a one dimensional output. The softmax is not helping you there.
The cross entropy loss function is appropriate in classification problems but you are doing regression. You should try using a mean squared error loss function instead.