We Keep Coding

how to pack my numpy variables and arrays when calling curve_fit?

This is my standalone code to reproduce the problem:
import numpy as np
from scipy.optimize import curve_fit
def find_vector_of_minor_axis_from_chunk(data):
n = 20 # number of points
time = np.linspace(0, 2 * np.pi, n)
guess_center_point = data.mean(1)
guess_center_point = guess_center_point[np.newaxis, :].transpose()
guess_a_phase = 0
guess_b_phase = 0
guess_a = 1
guess_b = 1
guess_a_axis_vector = np.array([[1], [0], [0]])
guess_b_axis_vector = np.array([[0], [1], [0]])
p0 = np.array([guess_center_point,
guess_a, guess_a_axis_vector, guess_a_phase,
guess_b, guess_b_axis_vector, guess_b_phase])
def ellipse_func(t, center_point, a, a_axis_vector, a_phase, b, b_axis_vector, b_phase):
return center_point + a * a_axis_vector * np.sin(t * a_phase) + b * b_axis_vector * np.sin(t + b_phase)
popt, pcov = curve_fit(ellipse_func, time, data, p0=p0)
center_point, a, a_axis_vector, a_phase, b, b_axis_vector, b_phase = popt
print(str(a_axis_vector, b_axis_vector))
shorter_vector = a_axis_vector
if np.abs(a_axis_vector) > np.aps(b_axis_vector):
shorter_vector = b_axis_vector
return shorter_vector
def main():
data = np.array([[-4.62767933, -4.6275775, -4.62735346, -4.62719652, -4.62711625, -4.62717975,
-4.62723845, -4.62722407, -4.62713901, -4.62708749, -4.62703238, -4.62689101,
-4.62687185, -4.62694013, -4.62701082, -4.62700483, -4.62697488, -4.62686825,
-4.62675683, -4.62675204],
[-1.58625998, -1.58625039, -1.58619648, -1.58617611, -1.58620606, -1.5861833,
-1.5861821, -1.58619169, -1.58615814, -1.58616893, -1.58613179, -1.58615934,
-1.58611262, -1.58610782, -1.58613179, -1.58614017, -1.58613059, -1.58612699,
-1.58607428, -1.58610183],
[-0.96714786, -0.96713827, -0.96715984, -0.96715145, -0.96716703, -0.96712869,
-0.96716104, -0.96713228, -0.96719698, -0.9671838, -0.96717062, -0.96717062,
-0.96715744, -0.96707717, -0.96709275, -0.96706519, -0.96715026, -0.96711791,
-0.96713588, -0.96714786]])
print(str(find_vector_of_minor_axis_from_chunk(data)))
if __name__ == '__main__':
main()
That gives me this traceback:
Traceback (most recent call last):
File "C:/Users/X/PycharmProjects/lissajous-achse/ellipse_fit.py", line 52, in <module>
main()
File "C:/Users/X/PycharmProjects/lissajous-achse/ellipse_fit.py", line 49, in main
print(str(find_vector_of_minor_axis_from_chunk(data)))
File "C:/Users/X/PycharmProjects/lissajous-achse/ellipse_fit.py", line 25, in find_vector_of_minor_axis_from_chunk
popt, pcov = curve_fit(ellipse_func, time, data, p0=p0)
File "C:\Users\X\PycharmProjects\lissajous-achse\venv\lib\site-packages\scipy\optimize\minpack.py", line 763, in curve_fit
res = leastsq(func, p0, Dfun=jac, full_output=1, **kwargs)
File "C:\Users\X\PycharmProjects\lissajous-achse\venv\lib\site-packages\scipy\optimize\minpack.py", line 392, in leastsq
raise TypeError('Improper input: N=%s must not exceed M=%s' % (n, m))
TypeError: Improper input: N=7 must not exceed M=3
Process finished with exit code 1
My code is an adaption of the second answer here. The problem causing the error message is solved by simple packing of variables here.
Why does the problem not surface in the mentioned second answer? And how can I pack my variables, which consist of several 3d vectors and individual scalars, to solve this problem? How do i pass in my t, which is a constant and should not be optimized?

Apparently python is quite smart regarding the length of the fields of the arguments, depending on the the initial guesses. So i could just pass in ONE variable, and split it up inside the function like so:
import numpy as np
from scipy.optimize import minimize
def find_vector_of_minor_axis_from_chunk(data):
n = 20 # number of points
guess_center_point = data.mean(1)
guess_center_point = guess_center_point[np.newaxis, :].transpose()
guess_a_phase = 0.0
guess_b_phase = 0.0
guess_a = 1.0
guess_b = 1.0
guess_a_axis_vector = np.array([[1.0], [0.0], [0.0]])
guess_b_axis_vector = np.array([[0.0], [1.0], [0.0]])
p0 = np.array([guess_center_point,
guess_a, guess_a_axis_vector, guess_a_phase,
guess_b, guess_b_axis_vector, guess_b_phase])
def ellipse_func(x, data):
center_point = x[0]
a = x[1]
a_axis_vector = x[2]
a_phase = x[3]
b = x[4]
b_axis_vector = x[5]
b_phase = x[6]
t = np.linspace(0, 2 * np.pi, n)
error = center_point + a * a_axis_vector * np.sin(t * a_phase) + b * b_axis_vector * np.sin(t + b_phase) - data
error_sum = np.sum(error**2)
print(str(error_sum))
return error_sum
popt, pcov = minimize(ellipse_func, p0, args=(data))
center_point, a, a_axis_vector, a_phase, b, b_axis_vector, b_phase = popt
print(str(a_axis_vector, b_axis_vector))
shorter_vector = a_axis_vector
if np.abs(a_axis_vector) > np.aps(b_axis_vector):
shorter_vector = b_axis_vector
return shorter_vector
def main():
data = np.array([[-4.62767933, -4.6275775, -4.62735346, -4.62719652, -4.62711625, -4.62717975,
-4.62723845, -4.62722407, -4.62713901, -4.62708749, -4.62703238, -4.62689101,
-4.62687185, -4.62694013, -4.62701082, -4.62700483, -4.62697488, -4.62686825,
-4.62675683, -4.62675204],
[-1.58625998, -1.58625039, -1.58619648, -1.58617611, -1.58620606, -1.5861833,
-1.5861821, -1.58619169, -1.58615814, -1.58616893, -1.58613179, -1.58615934,
-1.58611262, -1.58610782, -1.58613179, -1.58614017, -1.58613059, -1.58612699,
-1.58607428, -1.58610183],
[-0.96714786, -0.96713827, -0.96715984, -0.96715145, -0.96716703, -0.96712869,
-0.96716104, -0.96713228, -0.96719698, -0.9671838, -0.96717062, -0.96717062,
-0.96715744, -0.96707717, -0.96709275, -0.96706519, -0.96715026, -0.96711791,
-0.96713588, -0.96714786]])
print(str(find_vector_of_minor_axis_from_chunk(data)))
if __name__ == '__main__':
main()
Also i fixed some floating point vs integer errors in the vector for the initial values.
However now I get a different error:
Traceback (most recent call last):
File "C:/Users/X/PycharmProjects/lissajous-achse/ellipse_fit.py", line 61, in <module>
main()
File "C:/Users/X/PycharmProjects/lissajous-achse/ellipse_fit.py", line 58, in main
print(str(find_vector_of_minor_axis_from_chunk(data)))
File "C:/Users/X/PycharmProjects/lissajous-achse/ellipse_fit.py", line 34, in find_vector_of_minor_axis_from_chunk
popt, pcov = minimize(ellipse_func, p0, args=(data))
File "C:\Users\X\PycharmProjects\lissajous-achse\venv\lib\site-packages\scipy\optimize\_minimize.py", line 604, in minimize
return _minimize_bfgs(fun, x0, args, jac, callback, **options)
File "C:\Users\X\PycharmProjects\lissajous-achse\venv\lib\site-packages\scipy\optimize\optimize.py", line 1063, in _minimize_bfgs
if isinf(rhok): # this is patch for numpy
ValueError: The truth value of an array with more than one element is ambiguous. Use a.any() or a.all()
I guess that
The truth value of an array with more than one element is ambiguous.
Use a.any() or a.all()
is some internal error, stemming from the internal decision matrix how to proceed. I don't know how I caused it and how to fix it. When i figure out how it is done properly, I will come back and edit this answer.

Python "ValueError: setting an array element with a sequence." I don't see which part is not working

The goal with the following code is to plug the function "ddW" into odeint to find W for a given X (later on). I'm using the print function to make sure functions run).
import matplotlib.pyplot as plt
import numpy as np
from scipy.integrate import odeint
import pint
u = pint.UnitRegistry()
R = 8.31446261815324 * u.J/(u.mol*u.K)
Rgas = R.to (u.atm*u.L/(u.mol*u.K))
P0 = 10 * u.atm
T = 400 * u.K
v0 = 2 * u.L/u.min
α = 0.02 * u.kg**-1
k = 1.4 * u.L**2/(u.mol*u.kg*u.min)
FA0 = 0.5*P0*v0/(Rgas*T) #Assuming I.G.
def ddW(param,w): #param: [X,P]
X = param[0]
P = u.Quantity(param[1]).magnitude * u.atm
W = u.Quantity(w).magnitude * u.kg
d = np.zeros(2)
d[0] = k*FA0/(v0**2*P0) * P*W*(1-X) #dX/dW
d[1] = -α/2 *P0**2 * (1-X)/P #dP/dW
return d
param0 = [0,P0]
Wrange = np.linspace(0,100) *u.kg
#PBR = odeint(ddW,param0,Wrange)
#plt.plot(Wrange,PBR)
print(ddW(param0,0))
Traceback (most recent call last):
File "<ipython-input-1-83022bc3b5da>", line 1, in <module>
runfile(REDACTED, wdir=REDACTED)
File "C:\Users\Spencer\Anaconda3\lib\site-packages\spyder\utils\site\sitecustomize.py", line 705, in runfile
execfile(filename, namespace)
File "C:\Users\Spencer\Anaconda3\lib\site-packages\spyder\utils\site\sitecustomize.py", line 102, in execfile
exec(compile(f.read(), filename, 'exec'), namespace)
File "REDACTED", line 43, in <module>
print(ddW(param0,0))
File "REDACTED", line 33, in ddW
d[1] = -α/2 *P0**2 * (1-X)/P #dP/dW
ValueError: setting an array element with a sequence.
I previously had the same error message for what is now line 32 (defining d[0]) until I defined W in line 29. What is bothering me is that none of the inputs for d[1] (as far as I can tell) have size >1, so it should fit.

Use the following to create an array. The array which you are creating is of type float and the values which you are getting in d[0] and d[1] are not float.
d = np.array(np.zeros(2), dtype=np.object)

Assuming your fix for line 32 (defining d[0]) was to replace w with W where you took the magnitude of w, you need to do the same for the quantity you are trying to put into d[1].
The right side of = for line 32 is a dimensionless Quantity object, whereas the right side of = for line 33 is a Quantity object with dimension standard_atmosphere / kilogram which can't be put into array d as is because it needs to be magnitude only so it can be converted to type float.
Try this if you want to keep your array d as type float:
val = -a/2 *P0**2 * (1-X)/P #dP/dW
d[1] = val.magnitude

scipy.optimize.shgo ValueError: The truth value of an array with more than one element is ambiguous. Use a.any() or a.all()

I am trying to fit a function y(x,T,p) to get the coefficients a,b,c,d,e,f. The data for y,x,T,p are known. With a global optimizer I want to find a good starting point. shgo seems to be the only one that accept constraints.
import numpy as np
import matplotlib.pyplot as plt
from scipy.optimize import shgo
# test data
x = np.array([0.1,0.2,0.3,1])
T = np.array([300,300,300,300])
p = np.array([67.2,67.2,67.2,67.2])
y = np.array([30,50,55,67.2])
# function
def func(pars,x,T,p):
a,b,c,d,e,f = pars
return x*p+x*(1-x)*(a+b*T+c*T**2+d*x+e*x*T+f*x*T**2)*p
# residual
def resid(pars):
return ((func(pars,x,T,p) - y) ** 2).sum()
# constraint: derivation is positive in every data point
def der(pars):
a,b,c,d,e,f = pars
return -p*((3*f*T**2+3*e*T+3*d)*x**2+((2*c-2*f)*T**2+(2*b-2*e)*T-2*d+2*a)*x-c*T**2-b*T-a-1)
con1 = ({'type':'ineq', 'fun':der})
# minimizer shgo
bounds = [(-1,1),(-1,1),(-1,1),(-1,1),(-1,1),(-1,1)]
res = shgo(resid, bounds, constraints=con1)
print("a = %f , b = %f, c = %f, d = %f, e = %f, f = %f" % (res[0], res[1], res[2], res[3], res[4], res[5]))
# plotting
x0 = np.linspace(0, 1, 100)
fig, ax = plt.subplots()
fig.dpi = 80
ax.plot(x,y,'ro',label='data')
for i,txt in enumerate(T):
ax.annotate(txt,(x[i],y[i]))
ax.plot(x0, func(res.x, x0, 300,67.2), '-', label='fit1')
plt.xlabel('x')
plt.ylabel('y')
plt.legend()
plt.show()
With this I am getting ValueError: The truth value of an array with more than one element is ambiguous. Use a.any() or a.all()
I do not know what that Error means and other threads with the same Error does not realy help me to understand. When I use a local minimizer (scipy.optimize.minimize with the method cobyla) the error does not appear.
Can someone help me to understand my problem or even help to fix it?
Thanks
EDIT:
Traceback (most recent call last):
File "C:\Users\...\Python\Python36\site-packages\scipy\optimize\_shgo_lib\triangulation.py", line 759, in __getitem__
return self.cache[x]
KeyError: (0, 0, 0, 0, 0, 0)
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "C:/Users/.../test.py", line 70, in <module>
res = shgo(resid, bounds, constraints=con1)
File "C:\Users\...\Python\Python36\site-packages\scipy\optimize\_shgo.py", line 423, in shgo
shc.construct_complex()
File "C:\Users\...\Python\Python36\site-packages\scipy\optimize\_shgo.py", line 726, in construct_complex
self.iterate()
File "C:\Users\...\Python\Python36\site-packages\scipy\optimize\_shgo.py", line 869, in iterate
self.iterate_complex()
File "C:\Users\...\Python\Python36\site-packages\scipy\optimize\_shgo.py", line 890, in iterate_hypercube
self.g_args)
File "C:\Users\...\Python\Python36\site-packages\scipy\optimize\_shgo_lib\triangulation.py", line 121, in __init__
self.n_cube(dim, symmetry=symmetry)
File "C:\Users\...\Python\Python36\site-packages\scipy\optimize\_shgo_lib\triangulation.py", line 172, in n_cube
self.C0.add_vertex(self.V[origintuple])
File "C:\Users\...\Python\Python36\site-packages\scipy\optimize\_shgo_lib\triangulation.py", line 767, in __getitem__
index=self.index)
File "C:\Users\...\Python\Python36\site-packages\scipy\optimize\_shgo_lib\triangulation.py", line 681, in __init__
if g(self.x_a, *args) < 0.0:
ValueError: The truth value of an array with more than one element is ambiguous. Use a.any() or a.all()

The problem is, that der returns an array instead of a scalar value. Changing
con1 = ({'type':'ineq', 'fun':der})
to
con_list = [{'type':'ineq', 'fun': lambda x: der(x)[i_out]} for i_out in range(T.shape[0])]
removes the error. This transforms each output of der into its own inequality constraint.

Also, since your constraints are all written so that der(x)>=0, one can simply keep the definition of oyur constraint with vector output and then fetch the minimum of the outputs, i.e., take a scalar value constraint x -> \min (der(x)).

Creating a column of random numbers from a distribution with mean and std from other columns using python

I would like to be able to basically pass a column into the np.random.normal() function.
I have the following;
def calc_z(w,S,a1,a2,yt1,yt2):
mu = w * S
print 'Mu' , mu
sigma = mt.sqrt(0.5)
z = np.array(np.random.normal(mu,sigma))
u = [a1,a2,z]
yt = [yt1,yt2,1]
thetaset = np.random.rand(len(u))
m = [i for i in range(len(u))]
max_iter = 30
#Calculate E-step
for i in range(max_iter):
print 'Iteration:', i
print 'z:', z
print 'thetaset', thetaset
devLz = eq6(var,w,S,z,yt,u,thetaset,m)
dev2Lz2 = eq9(var,thetaset,u)
#Calculate M-Step
z = z - (devLz / dev2Lz2)
w = lambdaw * z
for i in range(len(thetaset)):
devLTheta = eq7(yt,u,thetaset,lambdatheta)
dev2LTheta2 = eq10(thetaset,u,lambdatheta)
thetaset = thetaset - (devLTheta / dev2LTheta2)
return float(z)
calc_z_udf = udf(calc_z,FloatType())
data.show()
data = data.withColumn('z', calc_z(data['w'],data['Org_Depth_Diff_S'],data['proximity_rank_a1'],data['cotravel_count_a2'],data['cotravel_yt1'],data['proximity_yt2']))
But when I pass S in, the np.random.normal function doesn't like being passed a column and gives me the following error;
Traceback (most recent call last):
File "/home/taylorr2/PySparkLatent3.py", line 125, in <module>
data = data.withColumn('z', calc_z(data['w'],data['Org_Depth_Diff_S'],data['proximity_rank_a1'],data['cotravel_count_a2'],data['cotravel_yt1'],data['proximity_yt2']))
File "/home/taylorr2/PySparkLatent3.py", line 90, in calc_z
z = np.array(np.random.normal(mu,sigma))
File "mtrand.pyx", line 1282, in mtrand.RandomState.normal (numpy/random/mtrand/mtrand.c:6920)
ValueError: setting an array element with a sequence.
I am trying to think of a way to get this function to accept this value or do this a different way.
Thanks!

scipy.optimize.leastsq error not an array of floats

This is my code
import os
import sys
import numpy as np
import scipy
from scipy.optimize import leastsq
def peval (inp_mat,p):
m0,m1,m2,m3,m4,m5,m6,m7 = p
out_mat = np.array(np.zeros(inp_mat.shape,dtype=np.float32))
mid = inp_mat.shape[0]/2
for xy in range(0,inp_mat.shape[0]):
if (xy<(inp_mat.shape[0]/2)):
out_mat[xy] = ( ( (inp_mat[xy+mid]*m0)+(inp_mat[xy]*m1)+ m2 ) /( (inp_mat[xy+mid]*m6)+(inp_mat[xy]*m7)+1 ) )
else:
out_mat[xy] = ( ( (inp_mat[xy]*m3)+(inp_mat[xy-mid]*m4)+ m5 ) /( (inp_mat[xy]*m6)+(inp_mat[xy-mid]*m7)+1 ) )
return np.array(out_mat)
def residuals(p, out_mat, inp_mat):
m0,m1,m2,m3,m4,m5,m6,m7 = p
err=np.array(np.zeros(inp_mat.shape,dtype=np.float32))
if (out_mat.shape == inp_mat.shape):
for xy in range(0,inp_mat.shape[0]):
err[xy] = err[xy]+ (out_mat[xy] -inp_mat[xy])
return np.array(err)
f = open('/media/anilil/Data/Datasets/repo/txt_op/vid.txt','r')
x = np.loadtxt(f,dtype=np.int16,comments='#',delimiter='\t')
nof = x.shape[0]/72 # Find the number of frames
x1 = x.reshape(-1,60,40)
x1_1= x1[0,:,:].flatten()
x1_2= x1[1,:,:].flatten()
x= []
y= []
for xy in range(1,50,1):
y.append(x1[xy,:,:].flatten())
x.append(x1[xy-1,:,:].flatten())
x=np.array(x,dtype=np.float32)
y=np.array(y,dtype=np.float32)
length = x1_1.shape#initail guess
p0 = np.array([1,1,1,1,1,1,1,1],dtype=np.float32)
abc=leastsq(residuals, p0,args=(y,x))
print ('Size of first matrix is '+str(x1_1.shape))
print ('Size of first matrix is '+str(x1_2.shape))
print ("Done with program")
I have tried adding np.array in most places with no use.
Could someone please help me ?
Another question here is do I give the output of the residuals() as a single value by adding all errorsnp.sum(err,axis=1). or leave it the way it is ?
When I return np.sum(err,axis=1) in the function residuals(). There is no change in the initial guess. It just remains the same.
I.E error is for each item in the input output mapping. or a combined error overall ?
Example data.
Output
ValueError: object too deep for desired array
Traceback (most recent call last):
File "/media/anilil/Data/charm/mv_clean/.idea/nose_reduction_mpeg.py", line 49, in <module>
abc=leastsq(residuals, p0,args=(y,x))
File "/usr/lib/python2.7/dist-packages/scipy/optimize/minpack.py", line 378, in leastsq
gtol, maxfev, epsfcn, factor, diag)
minpack.error: Result from function call is not a proper array of floats.

leastsq requires a 1D array to be returned from your residuals function.
Currently you calculate the residuals for the whole image and return that as a 2D array.
The simple fix would be to flatten the array of residuals (turning your 2D array into a 1D one).
So instead of returning
return np.array(err)
Do this instead
return err.flatten()
Note that err is already a numpy array so doesn't need to be cast before the return (I guess that slipped in when you were trying to debug it!)