In the tensorflow MNIST tutorial the mnist.train.next_batch(100) function comes very handy. I am now trying to implement a simple classification myself. I have my training data in a numpy array. How could I implement a similar function for my own data to give me the next batch?
sess = tf.InteractiveSession()
tf.global_variables_initializer().run()
Xtr, Ytr = loadData()
for it in range(1000):
batch_x = Xtr.next_batch(100)
batch_y = Ytr.next_batch(100)
The link you posted says: "we get a "batch" of one hundred random data points from our training set". In my example I use a global function (not a method like in your example) so there will be a difference in syntax.
In my function you'll need to pass the number of samples wanted and the data array.
Here is the correct code, which ensures samples have correct labels:
import numpy as np
def next_batch(num, data, labels):
'''
Return a total of `num` random samples and labels.
'''
idx = np.arange(0 , len(data))
np.random.shuffle(idx)
idx = idx[:num]
data_shuffle = [data[ i] for i in idx]
labels_shuffle = [labels[ i] for i in idx]
return np.asarray(data_shuffle), np.asarray(labels_shuffle)
Xtr, Ytr = np.arange(0, 10), np.arange(0, 100).reshape(10, 10)
print(Xtr)
print(Ytr)
Xtr, Ytr = next_batch(5, Xtr, Ytr)
print('\n5 random samples')
print(Xtr)
print(Ytr)
And a demo run:
[0 1 2 3 4 5 6 7 8 9]
[[ 0 1 2 3 4 5 6 7 8 9]
[10 11 12 13 14 15 16 17 18 19]
[20 21 22 23 24 25 26 27 28 29]
[30 31 32 33 34 35 36 37 38 39]
[40 41 42 43 44 45 46 47 48 49]
[50 51 52 53 54 55 56 57 58 59]
[60 61 62 63 64 65 66 67 68 69]
[70 71 72 73 74 75 76 77 78 79]
[80 81 82 83 84 85 86 87 88 89]
[90 91 92 93 94 95 96 97 98 99]]
5 random samples
[9 1 5 6 7]
[[90 91 92 93 94 95 96 97 98 99]
[10 11 12 13 14 15 16 17 18 19]
[50 51 52 53 54 55 56 57 58 59]
[60 61 62 63 64 65 66 67 68 69]
[70 71 72 73 74 75 76 77 78 79]]
In order to shuffle and sampling each mini-batch, the state whether a sample has been selected inside the current epoch should also be considered. Here is an implementation which use the data in the above answer.
import numpy as np
class Dataset:
def __init__(self,data):
self._index_in_epoch = 0
self._epochs_completed = 0
self._data = data
self._num_examples = data.shape[0]
pass
#property
def data(self):
return self._data
def next_batch(self,batch_size,shuffle = True):
start = self._index_in_epoch
if start == 0 and self._epochs_completed == 0:
idx = np.arange(0, self._num_examples) # get all possible indexes
np.random.shuffle(idx) # shuffle indexe
self._data = self.data[idx] # get list of `num` random samples
# go to the next batch
if start + batch_size > self._num_examples:
self._epochs_completed += 1
rest_num_examples = self._num_examples - start
data_rest_part = self.data[start:self._num_examples]
idx0 = np.arange(0, self._num_examples) # get all possible indexes
np.random.shuffle(idx0) # shuffle indexes
self._data = self.data[idx0] # get list of `num` random samples
start = 0
self._index_in_epoch = batch_size - rest_num_examples #avoid the case where the #sample != integar times of batch_size
end = self._index_in_epoch
data_new_part = self._data[start:end]
return np.concatenate((data_rest_part, data_new_part), axis=0)
else:
self._index_in_epoch += batch_size
end = self._index_in_epoch
return self._data[start:end]
dataset = Dataset(np.arange(0, 10))
for i in range(10):
print(dataset.next_batch(5))
the output is:
[2 8 6 3 4]
[1 5 9 0 7]
[1 7 3 0 8]
[2 6 5 9 4]
[1 0 4 8 3]
[7 6 2 9 5]
[9 5 4 6 2]
[0 1 8 7 3]
[9 7 8 1 6]
[3 5 2 4 0]
the first and second (3rd and 4th,...) mini-batch correspond to one whole epoch..
I use Anaconda and Jupyter.
In Jupyter if you run ?mnist you get:
File: c:\programdata\anaconda3\lib\site-packages\tensorflow\contrib\learn\python\learn\datasets\base.py
Docstring: Datasets(train, validation, test)
In folder datesets you shall find mnist.py which contains all methods including next_batch.
The answer which is marked up above I tried the algorithm by that algorithm I am not getting results so I searched on kaggle and I saw really amazing algorithm which worked really well. Best result try this. In below algorithm **Global variable takes the input you declared above in which you read your data set.**
epochs_completed = 0
index_in_epoch = 0
num_examples = X_train.shape[0]
# for splitting out batches of data
def next_batch(batch_size):
global X_train
global y_train
global index_in_epoch
global epochs_completed
start = index_in_epoch
index_in_epoch += batch_size
# when all trainig data have been already used, it is reorder randomly
if index_in_epoch > num_examples:
# finished epoch
epochs_completed += 1
# shuffle the data
perm = np.arange(num_examples)
np.random.shuffle(perm)
X_train = X_train[perm]
y_train = y_train[perm]
# start next epoch
start = 0
index_in_epoch = batch_size
assert batch_size <= num_examples
end = index_in_epoch
return X_train[start:end], y_train[start:end]
If you would not like to get shape mismatch error in your tensorflow session run
then use the below function instead of the function provided in the first solution above (https://stackoverflow.com/a/40995666/7748451) -
def next_batch(num, data, labels):
'''
Return a total of `num` random samples and labels.
'''
idx = np.arange(0 , len(data))
np.random.shuffle(idx)
idx = idx[:num]
data_shuffle = data[idx]
labels_shuffle = labels[idx]
labels_shuffle = np.asarray(labels_shuffle.values.reshape(len(labels_shuffle), 1))
return data_shuffle, labels_shuffle
Yet another implementation:
from typing import Tuple
import numpy as np
class BatchMaker(object):
def __init__(self, feat: np.array, lab: np.array) -> None:
if len(feat) != len(lab):
raise ValueError("Expected feat and lab to have the same number of samples")
self.feat = feat
self.lab = lab
self.indexes = np.arange(len(feat))
np.random.shuffle(self.indexes)
self.pos = 0
# "BatchMaker, BatchMaker, make me a batch..."
def next_batch(self, batch_size: int) -> Tuple[np.array, np.array]:
if self.pos + batch_size > len(self.feat):
np.random.shuffle(self.indexes)
self.pos = 0
batch_indexes = self.indexes[self.pos: self.pos + batch_size]
self.pos += batch_size
return self.feat[batch_indexes], self.lab[batch_indexes]
Related
I have a dataset X where each data point (each row) is in a particular order.
To totally shuffle the X, I use something like this:
shufX = torch.randperm(len(X))
X=X[shufX]
Say I just want to mildly shuffle (maybe shift positions of a few data points) without totally shuffling. I would like to have a parameter p, such that when p=0, it does not shuffle , and when p=1, it totally shuffles like the code about. This way, I can adjust the amount of shuffling to be mild or more extensive.
I attempted this but realized it could result in duplicate data points, which is not what I want.
p = 0.1
mask = torch.bernoulli(p*torch.ones(len(X))).bool()
shufX = torch.randperm(len(X))
X1=X[shufX]
C = torch.where(mask1, X, X1)
Create a shuffle function which only swaps a limited number of items.
import numpy as np
from random import randrange, seed
def shuffle( arr_in, weight = 1.0 ):
count = len( arr_in )
n = int( count * weight ) # Set the number of iterations
for ix in range( n ):
ix0 = randrange( count )
ix1 = randrange( count )
arr_in[ ix0 ], arr_in[ ix1 ] = arr_in[ ix1 ], arr_in[ ix0 ]
# Swap the items from the two chosen indices
seed ( 1234 )
arr = np.arange(50)
shuffle( arr, 0.25 )
print( arr )
# [ 7 15 42 3 4 44 28 0 8 29 10 11 12 13 14 22 16 17 18 19 20 21
# 1 23 24 25 26 27 49 9 41 31 32 33 34 35 36 5 38 30 40 39 2 43
# 37 45 46 47 48 6]
Even with a weight of 1.0 some of the items ( on average ) won't be moved. You can play with the parameters to the function to get the behaviour you need.
I am trying to define my own custom layer in keras. In the call function, where the logic of the class lies, I am dealing with the tensor object.
After finding the maximum value from a shredded slice of a tensor object, I want to assign it to a different tensor, but I am getting ERROR
"Sliced assignment is only supported for variables"
I have tried Sess.eval() in the call function of a class which does not solve the problem
mid_arr = x[i:spliti,j:splitj] #shredded slice
num = tf.reduce_max(mid_arr) #max vlaue from shred slice
res_arr = res_arr.assign( tf.where (res_arr[m][n],num, res_arr) ) #assign it
Specifying the solution here (Answer Section) even though it is present in Comments Section (thanks to jdehesa), for the benefit of the community.
Complete 2.x compatible code (work around) to perform sliced assignment of a Tensor is shown below:
import tensorflow as tf
def replace_slice(input_, replacement, begin, size=None):
inp_shape = tf.shape(input_)
if size is None:
size = tf.shape(replacement)
else:
replacement = tf.broadcast_to(replacement, size)
padding = tf.stack([begin, inp_shape - (begin + size)], axis=1)
replacement_pad = tf.pad(replacement, padding)
mask = tf.pad(tf.ones_like(replacement, dtype=tf.bool), padding)
return tf.where(mask, replacement_pad, input_)
def replace_slice_in(tensor):
return _SliceReplacer(tensor)
class _SliceReplacer:
def __init__(self, tensor):
self._tensor = tensor
def __getitem__(self, slices):
return _SliceReplacer._Inner(self._tensor, slices)
def with_value(self, replacement): # Just for convenience in case you skip the indexing
return _SliceReplacer._Inner(self._tensor, (...,)).with_value(replacement)
class _Inner:
def __init__(self, tensor, slices):
self._tensor = tensor
self._slices = slices
def with_value(self, replacement):
begin, size = _make_slices_begin_size(self._tensor, self._slices)
return replace_slice(self._tensor, replacement, begin, size)
# This computes begin and size values for a set of slices
def _make_slices_begin_size(input_, slices):
if not isinstance(slices, (tuple, list)):
slices = (slices,)
inp_rank = tf.rank(input_)
inp_shape = tf.shape(input_)
# Did we see a ellipsis already?
before_ellipsis = True
# Sliced dimensions
dim_idx = []
# Slice start points
begins = []
# Slice sizes
sizes = []
for i, s in enumerate(slices):
if s is Ellipsis:
if not before_ellipsis:
raise ValueError('Cannot use more than one ellipsis in slice spec.')
before_ellipsis = False
continue
if isinstance(s, slice):
start = s.start
stop = s.stop
if s.step is not None:
raise ValueError('Step value not supported.')
else: # Assumed to be a single integer value
start = s
stop = s + 1
# Dimension this slice refers to
i_dim = i if before_ellipsis else inp_rank - (len(slices) - i)
dim_size = inp_shape[i_dim]
# Default slice values
start = start if start is not None else 0
stop = stop if stop is not None else dim_size
# Fix negative indices
start = tf.cond(tf.convert_to_tensor(start >= 0), lambda: start, lambda: start + dim_size)
stop = tf.cond(tf.convert_to_tensor(stop >= 0), lambda: stop, lambda: stop + dim_size)
dim_idx.append([i_dim])
begins.append(start)
sizes.append(stop - start)
# For empty slice specs like [...]
if not dim_idx:
return tf.zeros_like(inp_shape), inp_shape
# Make full begin and size array (including omitted dimensions)
begin_full = tf.scatter_nd(dim_idx, begins, [inp_rank])
size_mask = tf.scatter_nd(dim_idx, tf.ones_like(sizes, dtype=tf.bool), [inp_rank])
size_full = tf.where(size_mask,
tf.scatter_nd(dim_idx, sizes, [inp_rank]),
inp_shape)
return begin_full, size_full
#with tf.Graph().as_default():
x = tf.reshape(tf.range(60), (4, 3, 5))
x2 = replace_slice_in(x)[:2, ..., -3:].with_value([100, 200, 300])
print('Tensor before Changing is \n', x)
print('\n')
print('Tensor after Changing is \n', x2)
Output of the above code is shown below:
Tensor before Changing is
tf.Tensor(
[[[ 0 1 2 3 4]
[ 5 6 7 8 9]
[10 11 12 13 14]]
[[15 16 17 18 19]
[20 21 22 23 24]
[25 26 27 28 29]]
[[30 31 32 33 34]
[35 36 37 38 39]
[40 41 42 43 44]]
[[45 46 47 48 49]
[50 51 52 53 54]
[55 56 57 58 59]]], shape=(4, 3, 5), dtype=int32)
Tensor after Changing is
tf.Tensor(
[[[ 0 1 100 200 300]
[ 5 6 100 200 300]
[ 10 11 100 200 300]]
[[ 15 16 100 200 300]
[ 20 21 100 200 300]
[ 25 26 100 200 300]]
[[ 30 31 32 33 34]
[ 35 36 37 38 39]
[ 40 41 42 43 44]]
[[ 45 46 47 48 49]
[ 50 51 52 53 54]
[ 55 56 57 58 59]]], shape=(4, 3, 5), dtype=int32)
1 i#:coding:utf-8
2 #0导入模块，生成模拟数据集
3 import tensorflow as tf
4 import numpy as np
5 BATCH_SIZE = 8
6 seed = 23455
7
8 #给予seed产生随机数
9 rng = np.random.RandomState(seed)
10 #随机数返回32行2列矩阵 表示32组 体积和重量 作为输入数据集
11 X = rng.rand(32,3)
12
13 Y = [[int(x0+x1<1)] for (x0,x1) in X]
14 print "X:\n",X
15 print "Y:\n",Y
16
17 #1定义神经网络的输入，参数和输出，定义向前传播过程
18 x = tf.placeholder(tf.float32, shape=(None, 2))
19 y_= tf.placeholder(tf.float32, shape=(None, 1))
20
21 w1= tf.variable(tf.random([2,3], stddev=1, seed=1))
22 w2= tf.variable(tf.random([3,1], stddev=1, seed=1))
23
24 a =tf.matmul(x,w1)
25 y =tf.matmul(a,w2)
26
27 #定义损失函数集反向传播方法
28 loss = tf.reduce_mean(tf.square(y-y_))
29 #train_step = tf.train.MomentumOptimizer(0.001,0.9).minimize(loss)
30 #train_step = tf.train.AdamOptimizer(0.001).minimize(loss)
31
32 #3生成会话，训练steps轮
33 with tf.Session() as sess:
34 init_op = tf.global_variables_initializer()
35 sess.run(init_op)
36 # 输出目前未经训练的参数取值。
37 print "w1:\n", sess.run(w1)
38 print "w2:\n", sess.run(w2)
39 print "\n"
40
41 #train the model
42 STEPS=3000
43 for i in range(STEPS):
44 start =(i*BATCH_SIZE) % 32
45 end = start + BATCH_SIZE
46 sess.run(train_step, feed_dict={x: X[start:end], y_: Y[start:end]})
47 if i % 500 == 0:
48 total_loss = sess.run(loss, feed_dict={x: X, y_: Y})
49 print("After %d training steps(s), loss on all data in %g" % (i, total_loss))
50
51 #output the trained value of variables
52 print "\n"
53 print "w1:\n", sess.run(w1)
54 print "w2:\n", sess.run(w2)
File "tf3_6.py", line 13, in
Y = [[int(x0+x1<1)] for (x0,x1) in X]
ValueError: too many values to unpack.
The code i don't think it is wrong but i still noticed the value error so i hope you guys to help me cope this question thanks a lot
The shape of X is (32, 3), but in your list comprehension, you are only trying to unpack 2 values:
Y = [[int(x0+x1<1)] for (x0,x1) in X]
Either change the shape of your array of rands:
X = rng.rand(32,2)
Or throw away the third rand in your list comp:
Y = [[int(x0+x1<1)] for (x0,x1, _) in X]
I am confused about the follow code:
import tensorflow as tf
import numpy as np
from tensorflow.python.framework import ops
from tensorflow.python.ops import array_ops
from tensorflow.python.ops import control_flow_ops
from tensorflow.python.ops import math_ops
from tensorflow.python.framework import dtypes
'''
Randomly crop a tensor, then return the crop position
'''
def random_crop(value, size, seed=None, name=None):
with ops.name_scope(name, "random_crop", [value, size]) as name:
value = ops.convert_to_tensor(value, name="value")
size = ops.convert_to_tensor(size, dtype=dtypes.int32, name="size")
shape = array_ops.shape(value)
check = control_flow_ops.Assert(
math_ops.reduce_all(shape >= size),
["Need value.shape >= size, got ", shape, size],
summarize=1000)
shape = control_flow_ops.with_dependencies([check], shape)
limit = shape - size + 1
begin = tf.random_uniform(
array_ops.shape(shape),
dtype=size.dtype,
maxval=size.dtype.max,
seed=seed) % limit
return tf.slice(value, begin=begin, size=size, name=name), begin
sess = tf.InteractiveSession()
size = [10]
a = tf.constant(np.arange(0, 100, 1))
print (a.eval())
a_crop, begin = random_crop(a, size = size, seed = 0)
print ("offset: {}".format(begin.eval()))
print ("a_crop: {}".format(a_crop.eval()))
a_slice = tf.slice(a, begin=begin, size=size)
print ("a_slice: {}".format(a_slice.eval()))
assert (tf.reduce_all(tf.equal(a_crop, a_slice)).eval() == True)
sess.close()
outputs:
[ 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23
24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47
48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71
72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95
96 97 98 99]
offset: [46]
a_crop: [89 90 91 92 93 94 95 96 97 98]
a_slice: [27 28 29 30 31 32 33 34 35 36]
There are two tf.slice options:
(1). called in function random_crop, such as tf.slice(value, begin=begin, size=size, name=name)
(2). called as a_slice = tf.slice(a, begin=begin, size=size)
The parameters (values, begin and size) of those two slice operations are the same.
However, why the printed values a_crop and a_slice are different and tf.reduce_all(tf.equal(a_crop, a_slice)).eval() is True?
Thanks
EDIT1
Thanks #xdurch0, I understand the first question now.
Tensorflow random_uniform seems like a random generator.
import tensorflow as tf
import numpy as np
sess = tf.InteractiveSession()
size = [10]
np_begin = np.random.randint(0, 50, size=1)
tf_begin = tf.random_uniform(shape = [1], minval=0, maxval=50, dtype=tf.int32, seed = 0)
a = tf.constant(np.arange(0, 100, 1))
a_slice = tf.slice(a, np_begin, size = size)
print ("a_slice: {}".format(a_slice.eval()))
a_slice = tf.slice(a, np_begin, size = size)
print ("a_slice: {}".format(a_slice.eval()))
a_slice = tf.slice(a, tf_begin, size = size)
print ("a_slice: {}".format(a_slice.eval()))
a_slice = tf.slice(a, tf_begin, size = size)
print ("a_slice: {}".format(a_slice.eval()))
sess.close()
output
a_slice: [42 43 44 45 46 47 48 49 50 51]
a_slice: [42 43 44 45 46 47 48 49 50 51]
a_slice: [41 42 43 44 45 46 47 48 49 50]
a_slice: [29 30 31 32 33 34 35 36 37 38]
The confusing thing here is that tf.random_uniform (like every random operation in TensorFlow) produces a new, different value on each evaluation call (each call to .eval() or, in general, each call to tf.Session.run). So if you evaluate a_crop you get one thing, if you then evaluate a_slice you get a different thing, but if you evaluate tf.reduce_all(tf.equal(a_crop, a_slice)) you get True, because all is being computed in a single evaluation step, so only one random value is produced and it determines the value of both a_crop and a_slice. Another example is this, if you run tf.stack([a_crop, a_slice]).eval() you will get a tensor with to equal rows; again, only one random value was produced. More generally, if you call tf.Session.run with multiple tensors to evaluate, all the computations in that call will use the same random values.
As a side note, if you actually need a random value in a computation that you want to maintain for a later computation, the easiest thing would be to just retrieve if with tf.Session.run, along with any other needed computation, to feed it back later through feed_dict; or you could have a tf.Variable and store the random value there. A more advanced possibility would be to use partial_run, an experimental API that allows you to evaluate part of the computation graph and continue evaluating it later, while maintaining the same state (i.e. the same random values, among other things).
So i'm trying to create a greedy algorithm for a knapsack problem. The txt file below is the knap20.txt file. The first line gives the number of items, in this case 20. The last line gives the capacity of the knapsack, in this case 524. The remaining lines give the index, value and weight of each item.
My function is to ideally return the solution in a list and the value of the weights
From what I can tell by my results, my program is working correctly. Is it working as you would expect, and how can i improve it?
txt file
20
1 91 29
2 60 65
3 61 71
4 9 60
5 79 45
6 46 71
7 19 22
8 57 97
9 8 6
10 84 91
11 20 57
12 72 60
13 32 49
14 31 89
15 28 2
16 81 30
17 55 90
18 43 25
19 100 82
20 27 19
524
python file
import os
import matplotlib.pyplot as plt
def get_optimal_value(capacity, weights, values):
value = 0.
numItems = len(values)
valuePerWeight = sorted([[values[i] / weights[i], weights[i]] for i in range(numItems)], reverse=True)
while capacity > 0 and numItems > 0:
maxi = 0
idx = None
for i in range(numItems):
if valuePerWeight[i][1] > 0 and maxi < valuePerWeight[i][0]:
maxi = valuePerWeight[i][0]
idx = i
if idx is None:
return 0.
if valuePerWeight[idx][1] <= capacity:
value += valuePerWeight[idx][0]*valuePerWeight[idx][1]
capacity -= valuePerWeight[idx][1]
else:
if valuePerWeight[idx][1] > 0:
value += (capacity / valuePerWeight[idx][1]) * valuePerWeight[idx][1] * valuePerWeight[idx][0]
return values, value
valuePerWeight.pop(idx)
numItems -= 1
return value
def read_kfile(fname):
print('file started')
with open(fname) as kfile:
print('fname found', fname)
lines = kfile.readlines() # reads the whole file
n = int(lines[0])
c = int(lines[n+1])
vs = []
ws = []
lines = lines[1:n+1] # Removes the first and last line
for l in lines:
numbers = l.split() # Converts the string into a list
vs.append(int(numbers[1])) # Appends value, need to convert to int
ws.append(int(numbers[2])) # Appends weigth, need to convert to int
return n, c, vs, ws
dir_path = os.path.dirname(os.path.realpath(__file__)) # Get the directory where the file is located
os.chdir(dir_path) # Change the working directory so we can read the file
knapfile = 'knap20.txt'
nitems, capacity, values, weights = read_kfile(knapfile)
val1,val2 = get_optimal_value(capacity, weights, values)
print ('values',val1)
print('value',val2)
result
values [91, 60, 61, 9, 79, 46, 19, 57, 8, 84, 20, 72, 32, 31, 28, 81, 55, 43, 100, 27]
value 733.2394366197183