Related
I am trying to properly represent multiple normal priors with a function call in the joint distribution and run mcmc.sample_chain with NUTS kernel. I used the excellent blog post here for a sketch with the following MCMC implementation.
def trace_fn(_, pkr):
return (
pkr.inner_results.inner_results.target_log_prob,
pkr.inner_results.inner_results.leapfrogs_taken,
pkr.inner_results.inner_results.has_divergence,
pkr.inner_results.inner_results.energy,
pkr.inner_results.inner_results.log_accept_ratio
)
def run_nuts(
target_log_prob_fn,
inits,
trace_fn=trace_fn,
bijectors_list=None,
num_steps=5000,
num_burnin=500,
n_chains=n_chains):
step_size = np.random.rand(n_chains, 1)*.5 + 1.
if not isinstance(inits, list):
inits = [inits]
if bijectors_list is None:
bijectors_list = [tfb.Identity()]*len(inits)
kernel = tfp.mcmc.DualAveragingStepSizeAdaptation(
tfp.mcmc.TransformedTransitionKernel(
inner_kernel=tfp.mcmc.NoUTurnSampler(
target_log_prob_fn,
step_size=[step_size]*len(inits)
),
bijector=bijectors_list
),
target_accept_prob=.8,
num_adaptation_steps=int(0.8*num_burnin),
step_size_setter_fn=lambda pkr, new_step_size: pkr._replace(
inner_results=pkr.inner_results._replace(step_size=new_step_size)
),
step_size_getter_fn=lambda pkr: pkr.inner_results.step_size,
log_accept_prob_getter_fn=lambda pkr: pkr.inner_results.log_accept_ratio,
)
res = tfp.mcmc.sample_chain(
num_results=num_steps,
num_burnin_steps=num_burnin,
current_state=inits,
kernel=kernel,
trace_fn=trace_fn
)
return res
I can get the MCMC working when individually specifying the priors but not when declaring them as a batch.
This works
dtype=tf.float32
root = tfd.JointDistributionCoroutine.Root
def basic_logistic(data_df):
def _generator():
a = yield root(tfd.Sample(tfd.Normal(0,10),1, name='a'))
b = yield root(tfd.Sample(tfd.Normal(0,10),1, name='b'))
c = yield root(tfd.Sample(tfd.Normal(0,10),1, name='c'))
l = a+tf.cast(data_df['x1'],dtype)*b + tf.cast(data_df['x2'],dtype)*c
print(l)
y = yield tfd.Independent(
tfd.Bernoulli(
logits = l,
name = 'success'
),
reinterpreted_batch_ndims=1
)
return tfd.JointDistributionCoroutine(_generator)
arm_0_test = basic_logistic(arm_0_test_df)
arm_0_log_prob = lambda *args: arm_0_test.log_prob(args + (tf.cast(arm_0_test_df['y'],dtype),))
n_chains=3
arm0_res = run_nuts(arm_0_log_prob, [tf.ones((n_chains,1)), tf.ones((n_chains,1)), tf.ones((n_chains,1))])
This does not
dtype=tf.float32
root = tfd.JointDistributionCoroutine.Root
def basic_logistic_multiple(X_df):
X_df_copy = X_df.copy()
n_features = X_df_copy.shape[1] + 1 # have to include intercept term
prior_means = [0 for i in range(n_features)] # list of prior means
print(prior_means)
prior_sds = [10 for i in range(n_features)] # list of prior sds
X_df_copy.insert(0, 'intercept', np.ones(X_df_copy.shape[0])) # Add column of 1s for intercept
X = tf.convert_to_tensor(X_df_copy, dtype=dtype)
def _generator():
beta = yield root(tfd.Sample(
tfd.Normal(prior_means, prior_sds, name='beta')
))
print(beta)
l = tf.tensordot(X, beta, axes=1)
# l = tf.reshape(l, (l.shape[0], ))
print(l)
y = yield tfd.Independent(
tfd.Bernoulli(
logits = l,
name = 'success'
),
reinterpreted_batch_ndims=1
)
return tfd.JointDistributionCoroutine(_generator)
arm_0_test = basic_logistic_multiple(arm_0_test_df)
arm_0_log_prob = lambda *args: arm_0_test.log_prob(args + (tf.cast(arm_0_test_df['y'],dtype),))
n_chains=3
init_beta, _ = arm_0_test.sample(n_chains)
init_beta = tf.zeros_like(init_beta)
arm0_res = run_nuts(arm_0_log_prob, [init_beta,])
I get the following error
ValueError: Dimensions must be equal, but are 3 and 1000000 for '{{node mcmc_sample_chain/dual_averaging_step_size_adaptation___init__/_bootstrap_results/transformed_kernel_bootstrap_results/NoUTurnSampler/.bootstrap_results/process_args/maybe_call_fn_and_grads/value_and_gradients/value_and_gradient/JointDistributionCoroutine_CONSTRUCTED_AT_top_level/log_prob/add_1}} = AddV2[T=DT_FLOAT](mcmc_sample_chain/dual_averaging_step_size_adaptation___init__/_bootstrap_results/transformed_kernel_bootstrap_results/NoUTurnSampler/.bootstrap_results/process_args/maybe_call_fn_and_grads/value_and_gradients/value_and_gradient/JointDistributionCoroutine_CONSTRUCTED_AT_top_level/log_prob/add, mcmc_sample_chain/dual_averaging_step_size_adaptation___init__/_bootstrap_results/transformed_kernel_bootstrap_results/NoUTurnSampler/.bootstrap_results/process_args/maybe_call_fn_and_grads/value_and_gradients/value_and_gradient/JointDistributionCoroutine_CONSTRUCTED_AT_top_level/log_prob/Independentsuccess/log_prob/Sum)' with input shapes: [3,3], [1000000].
I can sample from both Jointdistributions fine so I believe it is something clashing in the sample_chain function. Possibly my initial state declaration?
Thanks for any help!
I have some code that is mapping data to a matrix... Most of this is all setup so that my problem can be easily reproduced but the only part I need to speed up is what's after the comment # the part I want to speed up
import numpy as np
# config
matrix_height = 100
matrix_width = 200
# fake data
x_data = np.array(range(10000))
y_data = [{i:i for i in range(100)} for t in range(len(x_data))]
# fake mapping
x_to_index = {x: np.random.randint(matrix_width) for x in x_data }
y_to_index = {}
for y_dict in y_data:
for y_key, y_val in y_dict.items():
y_start = np.random.randint(matrix_height-2)
y_to_index[y_key] = (y_start, y_start+2 )
# data that must be stored
total_matrix = np.zeros([matrix_height, matrix_width]).astype(int)
n_matrix = np.zeros([matrix_height, matrix_width]).astype(int)
latest_value = np.zeros([matrix_height, matrix_width]).astype(int)
# the part I want to speed up
for x, y_dict in zip(x_data, y_data):
x_index = x_to_index[x]
for y_key, y_data in y_dict.items():
y_slice = slice(*y_to_index[y_key])
total_matrix[ y_slice, x_index ] += y_data
latest_value[ y_slice, x_index ] = y_data
n_matrix[ y_slice, x_index ] += 1
Again, I'm only concerned with the part below the comment # the part I want to speed up.
I'm not sure how to speed this up but it seems like it should be possible to use a mapping function that can do this all in parallel...
I'm looking for a marked improvement in that last section. Any ideas?
Tailored to number of cores.
For total_matrix, addition is commutative.
For latest_value, apply the split cols in reverse order.
import numpy as np
import time
import multiprocessing as mp
def func(cols, zz, tm, nm, lv, setOrder):
for c in cols:
for t in zz:
tm[slice(*t[0]), c] += t[1]
lv[slice(*t[0]), c] = t[1]
nm[slice(*t[0]), c] += 1
return [tm, nm, lv, setOrder]
if __name__ == '__main__':
mp.freeze_support()
matrix_height = 100
matrix_width = 200
total_range = 10000
x_data = np.array(range(total_range))
y_data = [{i:i for i in range(matrix_height)} for t in range(len(x_data))]
x_to_index = {x: np.random.randint(matrix_width) for x in x_data}
# potential general purpose cols list
#cols = np.random.randint(0, total_range, (1, total_range))[0]
cols = [np.int(x_to_index[k]) for k in x_to_index]
y_to_index = {}
for y_dict in y_data:
for y_key, y_val in y_dict.items():
y_start = np.random.randint(matrix_height-2)
y_to_index[y_key] = (y_start, y_start+2)
# potential general purpose rows list
#rows = [(np.random.randint(matrix_height), np.random.randint(matrix_height)) for x in range(matrix_height)]
rows = [y_to_index[k] for k in y_to_index]
# potential general purpose y data
#y_dat = np.random.randint(0, matrix_height, (1, matrix_height))[0]
y_dat = [i for i in range(matrix_height)]
o_total_matrix = np.zeros([matrix_height, matrix_width]).astype(int)
o_n_matrix = np.zeros([matrix_height, matrix_width]).astype(int)
o_latest_value = np.zeros([matrix_height, matrix_width]).astype(int)
startTime = time.time()
for x, y_dict in zip(x_data, y_data):
x_index = x_to_index[x]
for y_key, y_data in y_dict.items():
y_slice = slice(*y_to_index[y_key])
o_total_matrix[ y_slice, x_index ] += y_data
o_latest_value[ y_slice, x_index ] = y_data
o_n_matrix[ y_slice, x_index ] += 1
print('original time was {0:5.2f} sec'.format(time.time() - startTime))
procs = mp.cpu_count()
i_tm = [np.zeros([matrix_height, matrix_width]).astype(int)] * procs
i_nm = [np.zeros([matrix_height, matrix_width]).astype(int)] * procs
i_lv = [np.zeros([matrix_height, matrix_width]).astype(int)] * procs
zz = list(zip(rows, y_dat))
procs_split = np.array_split(cols, procs)
itup = []
for x in range(procs):
itup.append(((list(procs_split[x])), zz, i_tm[x], i_nm[x], i_lv[x], x))
startTime = time.time()
with mp.Pool(processes=procs) as pool:
ret = pool.starmap(func, itup)
i_total_matrix = ret[0][0]
i_n_matrix = ret[0][1]
for x in range(1, procs):
i_total_matrix = np.add(i_total_matrix, ret[x][0])
i_n_matrix = np.add(i_n_matrix, ret[x][1])
colOrder = [0] * procs
for x in range(procs):
colOrder[x] = (procs-1) - ret[x][3]
i_latest_value = ret[colOrder[0]][2]
for x in range(1, procs):
np.putmask(i_latest_value, i_latest_value == 0, ret[x][2])
print('improved time was {0:5.2f} sec'.format(time.time() - startTime))
comparison = i_total_matrix == o_total_matrix
if not comparison.all():
print('ERROR TOTAL MATRIX')
comparison = i_n_matrix == o_n_matrix
if not comparison.all():
print('ERROR N MATRIX')
comparison = i_latest_value == o_latest_value
if not comparison.all():
print('ERROR LATEST VALUE')
After a trial run, the results showed approx. twice as fast:
original time was 7.12 sec
improved time was 2.29 sec
I am trying to solve a linear programming problem using IBM's CPLEX Python API. It involves two sets of equality constraints. The code below works fine when we use either one of the two sets of constraints, but fails to find a solution when both sets of constraints are used.
The constraints are:
First constraint: Wx' = c', where W = [[20,0,0],[0,20,30]], x = [a,b,c], c=[20,30]
Second constraint: Vx' = e', where V = [[1,1,0],[0,0,1]], x = [a,b,c], c=[1,1]
Objective function: minimize a + c
One solution which meets both sets of constrains is a=1, b=0, c=1.
There is an error in the way I am introducing the two sets of constrains in Cplex Python. My code is below. To check that the code works with either set of constraints by itself comment out on of the sets of constraints.
import cplex
from cplex.exceptions import CplexError
import sys
def populatebynonzero(prob):
my_obj = [1.0, 0.0, 1.0]
my_ub = [1.0] * len(my_obj)
my_lb = [0.0] * len(my_obj)
my_colnames = ["a", "b", "c"]
prob.objective.set_sense(prob.objective.sense.minimize)
prob.variables.add(obj = my_obj, ub = my_ub, lb = my_lb ,names = my_colnames)
# first set of equality constraints: Wx' = c', where W = [[20,0,0],[0,20,30]], x = [a,b,c], c=[20,30]
my_rhs = [20.0, 30.0]
my_rownames = ["c1", "c2"]
my_sense = "E" * len(my_rownames)
rows = [0,1,1]
cols = [0,1,2]
vals = [20.0,20.0,30.0]
prob.linear_constraints.add(rhs = my_rhs, senses = my_sense,names = my_rownames)
prob.linear_constraints.set_coefficients(zip(rows, cols, vals))
# second set of equality constraints: Vx' = e', where V = [[1,1,0],[0,0,1]], x = [a,b,c], c=[1,1]
my_rhs = [1.0, 1.0]
my_rownames = ["e1", "e2"]
my_sense = "E" * len(my_rownames)
rows = [0,0,1]
cols = [0,1,2]
vals = [1.0,1.0,1.0]
prob.linear_constraints.add(rhs = my_rhs, senses = my_sense,names = my_rownames)
prob.linear_constraints.set_coefficients(zip(rows, cols, vals))
def lpex1():
try:
my_prob = cplex.Cplex()
handle = populatebynonzero(my_prob)
my_prob.solve()
except CplexError, exc:
print exc
return
numrows = my_prob.linear_constraints.get_num()
numcols = my_prob.variables.get_num()
print
# solution.get_status() returns an integer code
print "Solution status = " , my_prob.solution.get_status(), ":",
# the following line prints the corresponding string
print my_prob.solution.status[my_prob.solution.get_status()]
print "Solution value = ", my_prob.solution.get_objective_value()
slack = my_prob.solution.get_linear_slacks()
pi = my_prob.solution.get_dual_values()
x = my_prob.solution.get_values()
dj = my_prob.solution.get_reduced_costs()
for i in range(numrows):
print "Row %d: Slack = %10f Pi = %10f" % (i, slack[i], pi[i])
for j in range(numcols):
print "Column %d: Value = %10f Reduced cost = %10f" % (j, x[j], dj[j])
my_prob.write("lpex1.lp")
print x, "SOLUTIONS"
lpex1()
It works if the two sets of constraints are combined into one matrix in the following way, though it would be good to find a solution which does not have to do combining
import cplex
from cplex.exceptions import CplexError
import sys
def populatebynonzero(prob):
my_obj = [1.0, 0.0, 1.0]
my_ub = [1.0] * len(my_obj)
my_lb = [0.0] * len(my_obj)
my_colnames = ["a", "b", "c"]
prob.objective.set_sense(prob.objective.sense.minimize)
prob.variables.add(obj = my_obj, ub = my_ub, lb = my_lb ,names = my_colnames)
#combined constraints
my_rhs = [20.0, 30.0, 1.0, 1.0]
my_rownames = ["c1", "c2", "e1", "e2"]
my_sense = "E" * len(my_rownames)
rows = [0,1,1,2,2,3]
cols = [0,1,2,0,1,2]
vals = [20.0,20.0,30.0,1,1,1]
prob.linear_constraints.add(rhs = my_rhs, senses = my_sense,names = my_rownames)
prob.linear_constraints.set_coefficients(zip(rows, cols, vals))
"""
# first set of equality constraints: Wx' = c', where W = [[20,0,0],[0,20,30]], x = [a,b,c], c=[20,30]
my_rhs = [20.0, 30.0]
my_rownames = ["c1", "c2"]
my_sense = "E" * len(my_rownames)
rows = [0,1,1]
cols = [0,1,2]
vals = [20.0,20.0,30.0]
prob.linear_constraints.add(rhs = my_rhs, senses = my_sense,names = my_rownames)
prob.linear_constraints.set_coefficients(zip(rows, cols, vals))
# second set of equality constraints: Vx' = e', where V = [[1,1,0],[0,0,1]], x = [a,b,c], c=[1,1]
my_rhs = [1.0, 1.0]
my_rownames = ["e1", "e2"]
my_sense = "E" * len(my_rownames)
rows = [0,0,1]
cols = [0,1,2]
vals = [1.0,1.0,1.0]
prob.linear_constraints.add(rhs = my_rhs, senses = my_sense,names = my_rownames)
prob.linear_constraints.set_coefficients(zip(rows, cols, vals))
"""
def lpex1():
try:
my_prob = cplex.Cplex()
handle = populatebynonzero(my_prob)
my_prob.solve()
except CplexError, exc:
print exc
return
numrows = my_prob.linear_constraints.get_num()
numcols = my_prob.variables.get_num()
print
# solution.get_status() returns an integer code
print "Solution status = " , my_prob.solution.get_status(), ":",
# the following line prints the corresponding string
print my_prob.solution.status[my_prob.solution.get_status()]
print "Solution value = ", my_prob.solution.get_objective_value()
slack = my_prob.solution.get_linear_slacks()
pi = my_prob.solution.get_dual_values()
x = my_prob.solution.get_values()
dj = my_prob.solution.get_reduced_costs()
for i in range(numrows):
print "Row %d: Slack = %10f Pi = %10f" % (i, slack[i], pi[i])
for j in range(numcols):
print "Column %d: Value = %10f Reduced cost = %10f" % (j, x[j], dj[j])
my_prob.write("lpex1.lp")
print x, "SOLUTIONS"
lpex1()
When you are attempting to create the second set of constraints, you are using the wrong indices for the rows:
# second set of equality constraints: Vx' = e', where V = [[1,1,0],[0,0,1]], x = [a,b,c], c=[1,1]
my_rhs = [1.0, 1.0]
my_rownames = ["e1", "e2"]
my_sense = "E" * len(my_rownames)
rows = [0,0,1] # <- HERE
cols = [0,1,2]
vals = [1.0,1.0,1.0]
That is, you're using the row indices 0 and 1, which were created for the first set of constraints. Instead, you should do something like the following:
# second set of equality constraints: Vx' = e', where V = [[1,1,0],[0,0,1]], x = [a,b,c], c=[1,1]
my_rhs = [1.0, 1.0]
my_rownames = ["e1", "e2"]
my_sense = "E" * len(my_rownames)
cols = [0,1,2]
vals = [1.0,1.0,1.0]
rowindices = list(prob.linear_constraints.add(rhs = my_rhs, senses = my_sense,names = my_rownames))
assert len(rowindices) == 2
rows = [rowindices[0], rowindices[0], rowindices[1]]
prob.linear_constraints.set_coefficients(zip(rowindices, cols, vals))
Above, we get the new row indices from the call to prob.linear_constriants.add. With this change, the script works fine.
In addition, it's a good idea to write out the problem in LP format to make sure that it looks correct, like so:
prob.write("model.lp")
This program works for the value of the variable "n" set to 4, as it's the case in the following code:
from __future__ import division
from numpy import zeros
import numpy as np
import matplotlib.pyplot as plt
from numpy.linalg import linalg
import math
def getA(kappa):
matrix = zeros((n, n), float)
for i in range(n):
for j in range(n):
matrix[i][j] = 2*math.cos((2*math.pi/n)*(abs(j-i))*kappa)
return matrix
def getF(csi, a):
csiInv = linalg.inv(csi)
valueF = np.dot(csiInv, a)
valueF = np.dot(valueF, csiInv)
valueF = np.dot(valueF, a)
traceF = valueF.trace()
return 0.5 * traceF
def getG(csi, f, a):
csiInv = linalg.inv(csi)
valueG = np.dot(csiInv, a)
valueG = np.dot(valueG, csiInv)
valueG = valueG / (2 * f)
return valueG
def getE(g, k):
#m = 10 ^ -6
#kinv = linalg.inv(k + np.eye(k.shape[1])*m)
kinv = linalg.inv(k)
#kinv = linalg.pinv(k)
ktrans = k.transpose()
#ktransinv = linalg.pinv(ktrans)
#ktransinv = linalg.inv(ktrans + np.eye(ktrans.shape[1])*m)
ktransinv = linalg.inv(ktrans)
e = np.dot(ktransinv,g)
e = np.dot(e, kinv)
return e
def getW(k, a, e):
ktrans = k.transpose()
w = np.dot(k, a)
w = np.dot(w, ktrans)
w = np.dot(w, e)
valuew = w.trace()
return valuew
def getV(csi, e, e2, k):
v = np.dot(csi, k)
v = np.dot(v, e)
v = np.dot(v, k)
v = np.dot(v, csi)
v = np.dot(v, k)
v = np.dot(v, e2)
v = np.dot(v, k)
traceV = v.trace()
return traceV
handle_2 = open("test.txt", "w")
n = 4
power_spectrum_k = np.zeros(n, float)
for i in range(n):
power = math.exp(-(2*math.pi*i/n)*(2*math.pi*i/n))
power_spectrum_k[i] = power
# ora posso chiamare l'anti-trasformata
inverse_transform = np.fft.ifft(power_spectrum_k)
print 'inverse_transform:', inverse_transform
CSI = zeros((n, n))
for i in range(n):
for j in range(n):
CSI[i][j] = inverse_transform[abs(i-j)]
betaArray = zeros(n, float)
WabArray = zeros((6, n), float)
correlation = zeros((6, 6), float)
list = [1, 2, 3, 4, 5, 6]
K = zeros((n, n), float)
for i in range(n):
for j in range(i+1):
i_shifted = i + 2
j_shifted = j + 1
print "###############################"
print i_shifted
print j
component1 = ((3.0*70.0*70.0*0.3)/(2.0*300000.0*300000.0))
component2 = ((j_shifted*(i_shifted-j_shifted))/(i_shifted))
component3 = (1.0+(70.0/300000.0)*j_shifted)
print component1
print component2
print component3
K[i][j] = component1*component2*component3
#print 'DetK:', np.linalg.det(K)
print 'K:\n', K
counter = 0
for alpha in list:
counter2 = 0
Aa = getA(alpha)
Faa = getF(CSI, Aa)
Ga = getG(CSI, Faa, Aa)
Ea = getE(Ga, K)
#print 'Ea:', Ea
V_alphaalpha = getV(CSI, Ea, Ea, K)
for beta in xrange(n):
Ab = getA(beta + 1)
#print "Calling getW with K=", K, "\n Ab=", Ab, "\nEa=", Ea
W_ab = getW(K, Ab, Ea)
#print "\nGot W_ab=", W_ab
betaArray[beta] = beta + 1
WabArray[counter][beta] = W_ab
output_string = " {0} {1} \n".format(str(beta + 1), str(W_ab))
handle_2.write(output_string)
Fbb = getF(CSI, Ab)
Gb = getG(CSI, Fbb, Ab)
Eb = getE(Gb, K)
#print "Beta array"
#print betaArray
#print "Wab array"
#print WabArray
for gamma in list:
Ac = getA(gamma)
Fcc = getF(CSI, Ac)
Gc = getG(CSI, Fcc, Ac)
Ec = getE(Gc, K)
V_alphagamma = getV(CSI, Ea, Ec, K)
V_gammagamma = getV(CSI, Ec, Ec, K)
C_alphagamma = V_alphagamma/(math.sqrt(V_alphaalpha * V_gammagamma))
correlation[counter][counter2] = C_alphagamma
counter2 = counter2 + 1
counter = counter + 1
print 'correlation:\n', correlation
WabArray_all = []
betaArray_all = []
for b in range(0, len(WabArray), 1):
for n in betaArray:
betaArray_all.append(n)
for n in WabArray[b]:
WabArray_all.append(n)
Now, as soon as I get n = 5 and any other value bigger than 4, I receive the error:
line 148, in <module>
C_alphagamma = V_alphagamma/(math.sqrt(V_alphaalpha * V_gammagamma))
ValueError: math domain error
which I interpret as a math error due to the fact that I performing the square root of a negative value. Nevertheless, I cannot understand where exactly is the error, in the sense that I cannot understand why changing from 4 to, say, 5 makes the difference. Has anyone any idea of what is going wrong?
math.sqrt is unable to calculate the negative square root
>>> math.sqrt(-1)
Traceback (most recent call last):
File "<stdin>", line 1, in <module>
ValueError: math domain error
If this is valid complex math, import sqrt from cmath instead:
>>> cmath.sqrt(-1)
1j
It is hard to say where the problem is without understanding the algorythm. You will have to do more investigations. IMHO, you should ty to dump the values to better understand the problem but only when thing goes wrong to avoid too much output. Something like :
try:
C_alphagamma = V_alphagamma/(math.sqrt(V_alphaalpha * V_gammagamma))
except ValueError as e:
print alpha, Aa, Faa, Ga, Ea, V_alphaalpha
print gamma, Ac, Fcc, Gc, Ec, V_alphagamma, V_gammagamma
raise e
This is only an idea and other values might be more pertinent
I've set up numpy.seterr as follows:
np.seterr(invalid='raise', over ='raise', under='raise')
And I'm getting the following error:
c = beta[j,i] + oneminusbeta[j,i]
FloatingPointError: overflow encountered in double_scalars
I've checked what beta[j,i] and oneminusbeta[j,i] are at the point of crash, and these are their values:
beta: -131.340389182
oneminusbeta: 0.0
Please note, this line of addition (beta[j,i] + oneminusbeta[j,i]) has run for thousands of lines in a loop (that performs image classification) before crashing here at this point.
How can I deal with this? Is it necessary to change the type of the numpy arrays?
This is how I've initialized them:
beta = np.empty([m,n])
oneminusbeta = np.empty([m,n])
Is it possible to cast the individual values before adding them up? Rather than changing the entire array declarations? Or is this even a serious issue? Would it be safe to simply turn off the numpy.seterr configuration and let the calculations go ahead without raising the error?
Edit
Someone suggested below, and I suspected as well, that the values being added shouldn't cause an overflow. Then how can I find out where the overflow is really happening?
This is my code:
epthreshold = 709
enthreshold = -708
f.write("weights["+str(i)+", " + str(j)+"] = math.exp(beta: " +str(beta[j,i])+ " + oneminusbeta: " + str(oneminusbeta[j,i])+")\n" )
c = beta[j,i] + oneminusbeta[j,i]
weights[i,j] = math.exp(np.clip(c, enthreshold, epthreshold))
And when I check my log file, this is the line I get:
weights[5550, 13] = math.exp(beta: -131.340389182 + oneminusbeta: 0.0)
Edit 2
Here's the rest of my code, where variables n,m and H have already been initialized to integer values:
import numba
import numpy as np
import statsmodels.api as sm
weights = np.empty([n,m])
for curr_n in range(n):
for curr_m in range(m):
weights[curr_n,curr_m] = 1.0/(n)
beta = np.empty([m,n])
oneminusbeta = np.empty([m,n])
for curr_class in range(m):
for curr_sample in range(n):
beta[curr_class,curr_sample] = 1./m
epthreshold = 709 # positive exponential threshold
enthreshold = -708
for h in range(H):
print "Boosting round %d ... " % h
z = np.empty([n,m])
for j in range(m): # computing working responses and weights, Step 2(a)(i)
for i in range(no_samples):
i_class = y[i] #get the correct class for the current sample
if h == 0:
z[i,j] = (int(j==i_class) - beta[j,i])/((beta[j,i])*(1. - beta[j,i]))
weights[i,j] = beta[j,i]*(1. - beta[j,i])
else:
if j == i_class:
z[i,j] = math.exp(np.clip(-beta[j,i],enthreshold, epthreshold))
else:
z[i,j] = -math.exp(np.clip(oneminusbeta[j,i], enthreshold, epthreshold))
f.write("weights["+str(i)+", " + str(j)+"] = math.exp(beta: " +str(beta[j,i])+ " + oneminusbeta: " + str(oneminusbeta[j,i])+")\n" )
c = beta[j,i] + oneminusbeta[j,i]
weights[i,j] = math.exp(np.clip(c, enthreshold, epthreshold))
g_h = np.zeros([1,1])
j = 0
# Calculating regression coefficients per class
# building the parameters per j class
for y1_w in zip(z.T, weights.T):
y1, w = y1_w
temp_g = sm.WLS(y1, X, w).fit() # Step 2(a)(ii)
if np.allclose(g_h,0):
g_h = temp_g.params
else:
g_h = np.c_[g_h, temp_g.params]
j = j + 1
if np.allclose(g,0):
g = g_h
else:
g = g + g_h # Step(2)(a)(iii)
# now set g(x), function coefficients according to new formula, step (2)(b)
sum_g = g.sum(axis=1)
for j in range(m):
diff = (g[:,j] - ((1./m) * sum_g))
g[:,j] = ((m-1.)/m) * diff
g_per_round[h,:,j] = g[:,j]
#Now computing beta, Step 2(c)...."
Q = 0.
e = 0.
for j in range(m):
# Calculating beta and oneminusbeta for class j
aj = 0.0
for i in range(no_samples):
i_class = y[i]
X1 = X[i].reshape(1, no_features)
g1 = g[:,j].reshape(no_features, 1)
gc = g[:,i_class].reshape(no_features, 1)
dot = 1. + float(np.dot(X1, g1)) - float(np.dot(X1,gc))
aj = dot
sum_e = 0.
a_q = []
a_q.append(0.)
for j2 in range(m): # calculating sum of e's except for all j except where j=i_class
if j2 != i_class: # g based on j2, not necessarily g1?
g2 = g[:,j2].reshape(no_features, 1)
dot1 = 1. + float(np.dot(X1, g2)) - float(np.dot(X1,gc))
e2 = math.exp(np.clip(dot1,enthreshold, epthreshold))
sum_e = sum_e + e2
a_q.append(dot1)
if (int(j==i_class) == 1):
a_q_arr = np.array(a_q)
alpha = np.array(a_q_arr[1:])
Q = mylogsumexp(f,a_q_arr, 1, 0)
sumalpha = mylogsumexp(f,alpha, 1, 0)
beta[j,i] = -Q
oneminusbeta[j,i] = sumalpha - Q
else:
alpha = a_q
alpha = np.array(alpha[1:])
a_q_arr = np.array(a_q)
Q = mylogsumexp(f,a_q_arr, 0, aj)
sumalpha = log(math.exp(np.clip(Q, enthreshold, epthreshold)) - math.exp(np.clip(aj, enthreshold, epthreshold)))
beta[j,i] = aj - Q
oneminusbeta[j,i] = sumalpha - Q
and the function mylogsumexp is:
def mylogsumexp(f, a, is_class, maxaj, axis=None, b=None):
np.seterr(over="raise", under="raise", invalid="raise")
threshold = -sys.float_info.max
maxthreshold = sys.float_info.max
epthreshold = 709 # positive exponential threshold
enthreshold = -708
a = asarray(a)
if axis is None:
a = a.ravel()
else:
a = rollaxis(a, axis)
if is_class == 1:
a_max = a.max(axis=0)
else:
a_max = maxaj
#bnone = " none "
if b is not None:
a_max = maxaj
b = asarray(b)
if axis is None:
b = b.ravel()
else:
b = rollaxis(b, axis)
a = np.clip(a - a_max, enthreshold, epthreshold)
midout = np.sum(np.exp(a), axis=0)
midout = 1.0 + np.clip(midout - math.exp(a_max), threshold, maxthreshold)
out = np.log(midout)
else:
a = np.clip(a - a_max, enthreshold, epthreshold)
out = np.log(np.sum(np.exp(a)))
out += a_max
if out == float("inf"):
out = maxthreshold
if out == float("-inf"):
out = threshold
return out