I currently trained a logistic model for a decision boundary that looks like this:
using the following code that I got online:
x_min, x_max = xbatch[:, 0].min() - .5, xbatch[:, 0].max() + .5
y_min, y_max = xbatch[:, 1].min() - .5, xbatch[:, 1].max() + .5
h = 0.05
# Generate a grid of points with distance h between them
xx, yy = np.meshgrid(np.arange(x_min, x_max, h), np.arange(y_min, y_max, h))
X = np.vstack( ( xx.reshape(1, np.product(xx.shape)), yy.reshape(1, np.product(yy.shape)) ) ).T
# Predict the function value for the whole grid
z1 = np.dot(X, w1_pred)+b1_pred
h1 = 1 / (1 + np.exp(-z1))
z2 = np.dot(h1, w2_pred)+b2_pred
y_hat = 1 / (1 + np.exp(-z2))
pred = np.round(y_hat)
Z = pred.reshape(xx.shape)
# Plot the contour and training examples
plt.contourf(xx, yy, Z)
plt.scatter(xbatch[:, 0], xbatch[:, 1], c=ybatch, s=40, edgecolors="grey", alpha=0.9)
My question is this:
is there a way to plot the decision line without meshgrid or contour?
I would like to just plot the wave sigmoid function on the graph. without the colours or contours so it looks like this:
Use contour with level=[0.5] for sigmoid should work.
A Synthetic training set:
train_X = np.random.multivariate_normal([2.2, 2.2], [[0.1,0],[0,0.1]], 150)
train_Y = np.zeros(150)
train_X = np.concatenate((train_X, np.random.multivariate_normal([1.4, 1.3], [[0.05,0],[0,0.3]], 50)), axis=0)
train_Y = np.concatenate((train_Y, np.ones(50)))
train_X = np.concatenate((train_X, np.random.multivariate_normal([1.3, 2.9], [[0.05,0],[0,0.05]], 50)), axis=0)
train_Y = np.concatenate((train_Y, np.ones(50)))
train_X = np.concatenate((train_X, np.random.multivariate_normal([2.5, 0.95], [[0.1,0],[0,0.1]], 50)), axis=0)
train_Y = np.concatenate((train_Y, np.ones(50)))
An example model:
x = tf.placeholder(tf.float32, [None, 2])
y = tf.placeholder(tf.float32, [None,1])
#Input to hidden units
w_i_h = tf.Variable(tf.truncated_normal([2, 2],mean=0, stddev=0.1))
b_i_h = tf.Variable(tf.zeros([2]))
hidden = tf.sigmoid(tf.matmul(x, w_i_h) + b_i_h)
#hidden to output
w_h_o = tf.Variable(tf.truncated_normal([2, 1],mean=0, stddev=0.1))
b_h_o = tf.Variable(tf.zeros([1]))
logits = tf.sigmoid(tf.matmul(hidden, w_h_o) + b_h_o)
cost = tf.reduce_mean(tf.square(logits-y))
optimizer = tf.train.GradientDescentOptimizer(0.5).minimize(cost)
correct_prediction = tf.equal(tf.sign(logits-0.5), tf.sign(y-0.5))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
#Initialize all variables
init = tf.global_variables_initializer()
#Launch the graph
with tf.Session() as sess:
sess.run(init)
for epoch in range(3000):
_, c = sess.run([optimizer, cost], feed_dict={x:train_X, y:np.reshape(train_Y, (train_Y.shape[0],1))})
if epoch%1000 == 0:
print('Epoch: %d' %(epoch+1), 'cost = {:0.4f}'.format(c), end='\r')
acc = sess.run([accuracy] , feed_dict={x:train_X, y:np.reshape(train_Y, (train_Y.shape[0],1))})
print('\n Accuracy:', acc)
xx, yy = np.mgrid[0:3.5:0.1, 0:3.5:0.1]
grid = np.c_[xx.ravel(), yy.ravel()]
pred_1 = sess.run([logits], feed_dict={x:grid})
The output:
Z = np.array(pred_1).reshape(xx.shape)
plt.contour(xx, yy, Z, levels=[0.5], cmap='gray')
plt.scatter(train_X[:,0], train_X[:,1], s=20, c=train_Y, cmap='jet', vmin=0, vmax=1)
plt.show()
:
Related
I'm new to Tensorflow and working on an assignment but running into a problem.
I know the code is a hodgepodge of both Tensorflow v1 and v2 but it is what was given to me in my assignment. I had to fill some missing sections to get it to work up to this point.
import numpy as np
import tensorflow as tf
import matplotlib.pyplot as plt
tf.compat.v1.disable_eager_execution()
#1) Generate the synthetic data using the following Python code snippet.
# Generate synthetic data
N = 100
# Zeros form a Gaussian centered at (-1, -1)
x_zeros = np.random.multivariate_normal(mean=np.array((-1, -1)), cov=.1*np.eye(2), size=(N//2,))
y_zeros = np.zeros((N//2,))
# Ones form a Gaussian centered at (1, 1)
x_ones = np.random.multivariate_normal(mean=np.array((1, 1)), cov=.1*np.eye(2), size=(N//2,))
y_ones = np.ones((N//2,))
x_np = np.vstack([x_zeros, x_ones])
y_np = np.concatenate([y_zeros, y_ones])
# Plot x_zeros and x_ones on the same graph
plt.scatter(x_zeros[:,0], x_zeros[:,1], label='class 0')
plt.scatter(x_ones[:,0], x_ones[:,1], label='class 1')
plt.legend()
plt.show()
#3) Generate a TensorFlow graph.
with tf.name_scope("placeholders"):
x = tf.constant(x_np, dtype=tf.float32)
y = tf.constant(y_np, dtype=tf.float32)
with tf.name_scope("weights"):
W = tf.Variable(tf.random.normal((2, 1)))
b = tf.Variable(tf.random.normal((1,)))
with tf.name_scope("prediction"):
y_logit = tf.squeeze(tf.matmul(x, W) + b)
# the sigmoid gives the class probability of 1
y_one_prob = tf.sigmoid(y_logit)
# Rounding P(y=1) will give the correct prediction.
y_pred = tf.round(y_one_prob)
with tf.name_scope("loss"):
# Compute the cross-entropy term for each datapoint
entropy = tf.nn.sigmoid_cross_entropy_with_logits(logits=y_logit, labels=y)
# Sum all contributions
l = tf.reduce_sum(entropy)
with tf.name_scope("optim"):
train_op = tf.compat.v1.train.AdamOptimizer(.01).minimize(l)
with tf.name_scope("summaries"):
tf.compat.v1.summary.scalar("loss", l)
merged = tf.compat.v1.summary.merge_all()
train_writer = tf.compat.v1.summary.FileWriter('logistic-train', tf.compat.v1.get_default_graph())
#4) Train the model, get the weights, and make predictions.
with tf.compat.v1.Session() as sess:
sess.run(tf.compat.v1.global_variables_initializer())
# Train model
for i in range(100):
_, summary, loss = sess.run([train_op, merged, l], {x: x_np, y: y_np})
train_writer.add_summary(summary, i)
# Get weights and bias
w_final, b_final = sess.run([W, b])
# Make predictions
y_pred_np = sess.run(y_pred, {x: x_np})
train_writer.close()
#5) Plot the predicted outputs on top of the data.
x_min, x_max = x_np[:, 0].min() - .5, x_np[:, 0].max() + .5
y_min, y_max = x_np[:, 1].min() - .5, x_np[:, 1].max() + .5
xx, yy = np.meshgrid(np.linspace(x_min, x_max, 100),
np.linspace(y_min, y_max, 100))
X_grid = np.c_[xx.ravel(), yy.ravel()]
with tf.compat.v1.Session() as sess:
y_grid = sess.run(y_one_prob, {x: X_grid})
plt.contourf(xx, yy, y_grid.reshape(xx.shape), cmap=plt.cm.Paired)
plt.scatter(x_np[y_np == 0, 0], x_np[y_np == 0, 1], c="red")
plt.scatter(x_np[y_np == 1, 0], x_np[y_np == 1, 1], c="blue")
plt.show()
Warning Raised:
---------------------------------------------------------------------------
ValueError Traceback (most recent call last)
<ipython-input-2-d4ec6e7c74ca> in <module>
112
113 with tf.compat.v1.Session() as sess:
--> 114 y_grid = sess.run(y_one_prob, {x: X_grid})
115
116 plt.contourf(xx, yy, y_grid.reshape(xx.shape), cmap=plt.cm.Paired)
1 frames
/usr/local/lib/python3.8/dist-packages/tensorflow/python/client/session.py in _run(self, handle, fetches, feed_dict, options, run_metadata)
1163 if (not is_tensor_handle_feed and
1164 not subfeed_t.get_shape().is_compatible_with(np_val.shape)):
-> 1165 raise ValueError(
1166 f'Cannot feed value of shape {str(np_val.shape)} for Tensor '
1167 f'{subfeed_t.name}, which has shape '
ValueError: Cannot feed value of shape (10000, 2) for Tensor placeholders_1/Const:0, which has shape (100, 2)
The bug is coming from this line here:
y_grid = sess.run(y_one_prob, {x: X_grid})
I tried to find a relevant answer on google and got some similar ones, but I can't get any of them to work.
Anyone help? Thanks a lot!
I'm plotting contours to visually represent my neural network classification with a given set of data. I've noticed these black dots appear as the model is training through all its epochs. What do they represent?
Code is as follows (skipped explaining how neural network was constructed):
# Plotting Function
def plot_model(sess, model, xy, labels, feature_lambda, title=''):
from pandas import DataFrame
xx, yy = np.meshgrid(np.linspace(-1.2,1.2,400), np.linspace(-1.2,1.2,400))
prediction = sess.run(model, feed_dict={X: np.array([feature_lambda(xxval, yyval) for xxval, yyval in zip(xx.flatten(), yy.flatten())])})
Z = prediction.reshape(xx.shape)
df = DataFrame(dict(x=xy[:,0], y=xy[:,1], label=labels.flatten()))
markers = {0:'bs', 1:'r^'}
_, ax = plt.subplots(figsize=(5, 5))
cs = ax.contourf(xx, yy, Z, 20, cmap='coolwarm', alpha=.9)
ax.clabel(cs, colors='gray')
cs = ax.contour(xx, yy, Z, cmap='gray', levels=[0, 0.5, 1.0], linestyles='--', linewidths=2)
ax.clabel(cs, colors='k')
predictions = sess.run(model, feed_dict={X: xy})
for k, xy0 in df[['x', 'y']].iterrows():
x0, y0 = xy0.values
plt.plot(x0, y0, markers[labels[k][0]], mec='k')
ax.set_xlim([-1.1, 1.1])
ax.set_ylim([-1.1, 1.1])
plt.grid(linestyle='--', alpha=0.5)
plt.title(title)
plt.show()
# Generate data
xy, labels = make_blobs(n_samples=200, center_box=(-1, 1), centers=6, cluster_std=0.1, random_state=20)
labels = labels % 2
labels = labels.reshape(-1, 1)
ftr_fn = lambda x, y: [x, y]
features = np.array([ftr_fn(xval, yval) for xval, yval in xy])
# Stochastic Method
batch_size = 25
init = tf.global_variables_initializer()
sess = tf.Session()
sess.run(init)
for epoch in range(MaxEpoch):
if epoch % 5 == 0:
curr_loss = sess.run(loss, feed_dict={X: features, y: labels})
plot_model(sess, yhat, xy, labels, ftr_fn, 'Epoch {} \n (loss={:.3f})'.format(epoch, curr_loss))
for X_batch, y_batch in generate_batches(batch_size, features_shuffled, labels_shuffled):
sess.run(train, feed_dict={X: X_batch, y: y_batch})
loss_final = sess.run(loss, feed_dict={X: features, y: labels})
plot_model(sess, yhat, xy, labels, ftr_fn, 'Epoch 25 \n (loss={:.3f})'.format(loss_final))
Epoch 0
Epoch 10 - Bottom Right Corner
Epoch 25 - Bottom
I have generated a balanced dataset of 4000 examples, 2000 for the negative class and 2000 for the positive one. Then, I've build a neural net with one single hidden layer and 3 neurons with a ReLU activation function and an output layer with a sigmoid. The cost function is a standard cross-entropy function and I chose Adam as optimizer. Using minibatches of 15 examples, after 1000 epochs of running the final accuracy 96.37%, so I am assuming that the model is doing well on the test set. But when I want to display the decision boundary, that's what I get:
I cannot figure out if the problem is a code error or the model just needs mode training. Script I'm using for this:
# implement a neural network that finds a decision boundary under a
constraint on the second hidden layer with tensorflow
import numpy as np
from sklearn.utils import shuffle
from sklearn.preprocessing import normalize
from sklearn.model_selection import train_test_split
import tensorflow as tf
from tf_utils import random_mini_batches
import matplotlib.pyplot as plt
def generate_dataset():
np.random.seed(2)
# positive class samples
d1_x = np.random.normal(5, 10, 1000)
d1_y = np.random.normal(5, 2, 1000)
d2_x = np.random.normal(40, 20, 1000)
d2_y = np.random.normal(2, 1, 1000)
# negative class samples
d3_x = np.random.normal(60, 5, 2000)
d3_y = np.random.normal(10, 1, 2000)
plt.scatter(d1_x, d1_y, color='b')
plt.scatter(d2_x, d2_y, color='b')
plt.scatter(d3_x, d3_y, color='r')
Y = np.zeros((4000, 1))
d_x = np.concatenate([d1_x, d2_x, d3_x])
d_y = np.concatenate([d1_y, d2_y, d3_y])
d_x = d_x.reshape(d_x.shape[0], 1)
d_y = d_y.reshape(d_y.shape[0], 1)
X = np.concatenate([d_x, d_y], axis=1)
Y[2000:] = 1
return X, Y
# define a tensorflow model 5-3-1 with two hideen layers and the output
being scalar
costs = []
print_cost = True
learning_rate = .0009
minibatch_size = 15
num_epochs = 1000
XX, YY = generate_dataset()
XX, YY = shuffle(XX, YY)
X_norm = normalize(XX)
X_train, X_test, y_train, y_test = train_test_split(X_norm, YY,
test_size=0.2, random_state=42)
X_train = np.transpose(X_train)
y_train = np.transpose(y_train)
X_test = np.transpose(X_test)
y_test = np.transpose(y_test)
# define train and test sets
m = XX.shape[1] # input dimension
n = YY.shape[1] # output dimension
X = tf.placeholder(tf.float32, shape = [m, None], name = 'X')
y = tf.placeholder(tf.float32, shape = [n, None], name = 'y')
# model parameters
n1 = 3 # output dimension of the first hidden layer
#n2 = 4 # output dimension of the second hidden layer
#n3 = 2
W1 = tf.get_variable("W1", [n1, m],
initializer=tf.contrib.layers.xavier_initializer(seed=1))
b1 = tf.get_variable("b1", [n1 ,1], initializer=tf.zeros_initializer)
#W2 = tf.get_variable("W2", [n2, n1],
initializer=tf.contrib.layers.xavier_initializer(seed=1))
#b2 = tf.get_variable("b2", [n2, 1], initializer=tf.zeros_initializer)
#W3 = tf.get_variable("W3", [n3, n2],
initializer=tf.contrib.layers.xavier_initializer(seed=1))
#b3 = tf.get_variable("b3", [n3, 1], initializer=tf.zeros_initializer)
W4 = tf.get_variable("W4", [n, n1],
initializer=tf.contrib.layers.xavier_initializer(seed=1))
b4 = tf.get_variable("b4", [n, 1], initializer=tf.zeros_initializer)
# forward propagation
z1 = tf.add(tf.matmul(W1, X), b1)
a1 = tf.nn.relu(z1)
#z2 = tf.add(tf.matmul(W2, a1), b2)
#a2 = tf.nn.relu(z2)
#z3 = tf.add(tf.matmul(W3, a2), b3)
#a3 = tf.nn.relu(z3)
z4 = tf.add(tf.matmul(W4, a1), b4)
pred = tf.nn.sigmoid(z4)
# cost function
cost = tf.reduce_mean(tf.losses.log_loss(labels=y, predictions=pred)) #
logit is the probability estimate given by the model --> this is what is used inside the formula, not the net input z
# ADAM optimizer
optimizer =
tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(cost)
# metrics
correct_prediction = tf.less_equal(tf.abs(pred - y), 0.5)
accuracy = tf.reduce_mean(tf.cast(correct_prediction, "float"))
init = tf.global_variables_initializer()
with tf.Session() as sess:
seed = 1
sess.run(init)
for epoch in range(num_epochs):
epoch_cost = 0
seed += 1
num_minibatches = int(X_train.shape[0] / minibatch_size)
minibatches = random_mini_batches(X_train, y_train, minibatch_size, seed)
for minibatch in minibatches:
(minibatch_X, minibatch_Y) = minibatch
_, minibatch_cost = sess.run([optimizer, cost], feed_dict={X:minibatch_X, y:minibatch_Y})
epoch_cost += minibatch_cost / minibatch_size
# Print the cost every epoch
if print_cost == True and epoch % 100 == 0:
print("Cost after epoch %i: %f" % (epoch, epoch_cost))
if print_cost == True and epoch % minibatch_size == 0:
costs.append(epoch_cost)
#plt.plot(costs)
#plt.show()
cp, val_accuracy = sess.run([correct_prediction, accuracy], feed_dict={X: X_test, y: y_test})
# plot the cost
# plt.plot(np.squeeze(costs))
# plt.ylabel('cost'), feed_dict={X: X_test, y: y_test})
# plt.xlabel('iterations (per fives)')
# plt.title("Learning rate =" + str(learning_rate))
# plt.show()
cmap = plt.get_cmap('Paired')
# Define region of interest by data limits
xmin, xmax = min(XX[:, 0]) - 1, max(XX[:, 0]) + 1
ymin, ymax = min(XX[:, 1]) - 1, max(XX[:, 1]) + 1
steps = 100
x_span = np.linspace(xmin, xmax, steps)
y_span = np.linspace(ymin, ymax, steps)
xx, yy = np.meshgrid(x_span, y_span)
A = np.concatenate([[xx.ravel()], [yy.ravel()]], axis=0)
A = normalize(A, axis=0)
# Make predictions across region of interest
predictions = sess.run(pred, feed_dict={X: A})
# Plot decision boundary in region of interest
z = predictions.reshape(xx.shape)
plt.contourf(xx, yy, z, cmap=cmap, alpha=.5)
plt.show()
# Get predicted labels on training data and plot
#train_labels = model.predict(X)
#ax.scatter(X[:, 0], X[:, 1], c=y, cmap=cmap, lw=0)
I am doing a nonlinear gradient descent, the main part of code as follows:
m = len(train_x)
Y = train_y
W = np.array([0] * 10, dtype='float')
# create new features from x
X = np.c_[
np.ones(m),
train_x,
train_x ** 0.5,
train_x ** 2,
train_x ** 3,
train_x ** 4,
train_x ** 5,
train_x ** 6,
train_x ** 7,
train_x ** 8,
]
# normalize
for wi in range(len(W)):
X[:, wi] /= X[:, wi].max()
num_iters = 200
lr = 0.1
for i in np.arange(1, 1+num_iters):
diff = X # W - Y
for wi in range(len(W)):
# update using derivative
k = diff if wi == 0 else diff * X[:, wi]
W[wi] -= lr * np.sum(k) / m
if i % 20 == 0:
cost = np.square(diff).sum() / 2 / m
print(f'iteration={i} cost={cost}')
y = X # W
plt.scatter(train_x, y, color='red', label='predication')
plt.scatter(train_x, train_y, label='train data')
plt.scatter(test_x, test_y, label='test data')
plt.legend()
plt.show()
I get the following output:
May I know what am I doing wrong here?
How can I get output fit to the curve?
update
I tried to randomize the initial weights but get some very similar results:
W = np.array([np.random.rand() for _ in range(10)])
I'm trying to build a model in tensor flow with models like f_i(x) = m_ix + b_i such that:
f(x) = [f_1(x), f_2(x)]^T [x, x] + b
this is just an exercise. My difficulty is in understanding how to concatenate two tensors:
# Model 1
f1 = tf.add(tf.mul(X, W), b)
# Model 2
f2 = tf.add(tf.mul(X, W2), b2)
# Concatenate 1 & 2
fi = tf.concat(0, [f1, f2])
# Final model
pred = tf.add(tf.mul(fi, W3), b3)
Unfortunately this doesn't seem to work.
Here's the full example:
'''
A linear regression learning algorithm example using TensorFlow library.
Author: Aymeric Damien (original author) # I am altering it though
Project: https://github.com/aymericdamien/TensorFlow-Examples/
'''
from __future__ import print_function
import tensorflow as tf
import numpy
import matplotlib.pyplot as plt
rng = numpy.random
# Parameters
learning_rate = 0.01
training_epochs = 1000
display_step = 50
# Training Data
train_X = numpy.asarray(
[3.3, 4.4, 5.5, 6.71, 6.93, 4.168, 9.779, 6.182, 7.59, 2.167,
7.042, 10.791, 5.313, 7.997, 5.654, 9.27, 3.1])
train_Y = numpy.asarray(
[1.7, 2.76, 2.09, 3.19, 1.694, 1.573, 3.366, 2.596, 2.53, 1.221,
2.827, 3.465, 1.65, 2.904, 2.42, 2.94, 1.3])
n_samples = train_X.shape[0]
# tf Graph Input
X = tf.placeholder("float")
Y = tf.placeholder("float")
# Set model weights
W = tf.Variable(rng.randn(), name="weight")
b = tf.Variable(rng.randn(), name="bias")
W2 = tf.Variable(rng.randn(), name="weight2")
b2 = tf.Variable(rng.randn(), name="bias2")
W3 = tf.Variable([rng.randn(), rng.randn()], name="weight3")
b3 = tf.Variable(rng.randn(), name="bias3")
# Model 1
f1 = tf.add(tf.mul(X, W), b)
# Model 2
f2 = tf.add(tf.mul(X, W2), b2)
# Concatenate 1 & 2
fi = tf.concat(0, [f1, f2])
# Final model
pred = tf.add(tf.mul(fi, W3), b3)
# Mean squared error
cost = tf.reduce_sum(tf.pow(pred - Y, 2)) / (2 * n_samples)
# Gradient descent
optimizer = tf.train.GradientDescentOptimizer(learning_rate).minimize(cost)
# Initializing the variables
init = tf.initialize_all_variables()
# Launch the graph
with tf.Session() as sess:
sess.run(init)
# Fit all training data
for epoch in range(training_epochs):
for (x, y) in zip(train_X, train_Y):
sess.run(optimizer, feed_dict={X: x, Y: y})
# Display logs per epoch step
if (epoch + 1) % display_step == 0:
c = sess.run(cost, feed_dict={X: train_X, Y: train_Y})
print("Epoch:", '%04d' % (epoch + 1), "cost=", "{:.9f}".format(c), \
"W=", sess.run(W), "b=", sess.run(b))
print("Optimization Finished!")
training_cost = sess.run(cost, feed_dict={X: train_X, Y: train_Y})
print("Training cost=", training_cost, "W=", sess.run(W), "b=", sess.run(b),
'\n')
# Graphic display
plt.plot(train_X, train_Y, 'ro', label='Original data')
plt.plot(train_X, sess.run(W) * train_X + sess.run(b), label='Fitted line')
plt.legend()
plt.show()
# Testing example, as requested (Issue #2)
test_X = numpy.asarray([6.83, 4.668, 8.9, 7.91, 5.7, 8.7, 3.1, 2.1])
test_Y = numpy.asarray([1.84, 2.273, 3.2, 2.831, 2.92, 3.24, 1.35, 1.03])
print("Testing... (Mean square loss Comparison)")
testing_cost = sess.run(
tf.reduce_sum(tf.pow(pred - Y, 2)) / (2 * test_X.shape[0]),
feed_dict={X: test_X, Y: test_Y}) # same function as cost above
print("Testing cost=", testing_cost)
print("Absolute mean square loss difference:", abs(
training_cost - testing_cost))
plt.plot(test_X, test_Y, 'bo', label='Testing data')
plt.plot(train_X, sess.run(W) * train_X + sess.run(b), label='Fitted line')
plt.legend()
plt.show()
One way to achieve a similar result without the headache of tf.concat is
pred = tf.add(tf.add(f1, f2), b3)