Related
Let us assume that we have a closed form analytical function y = a * x + b. We have also experimental data to which this function should be fitted. Thus, error minimization by fitting right a and b, might be done. Attached Python 3.8 code do this easily.
import time
import numpy as np
import matplotlib.pyplot as plt
from scipy.optimize import minimize
class Model:
def __init__(self, x_exp, y_exp, params_init):
self.x_exp = x_exp
self.y_exp = y_exp
self.params_init = params_init
self.possible_b = [-1, 0, 1, 2, 3, 4]
self.method = "brute"
self.tolerance = 1 * 10**(-10)
self.res = None
def _fun_linear(self, x, a, b):
y = a * x + b
return y
def _model(self, params):
a = params[0]
b = params[1]
y_fit = self._fun_linear(self.x_exp, a, b)
err = np.linalg.norm(y_fit - self.y_exp)
return err
def fit(self):
time_beg = time.time()
self.res = minimize(fun=self._model,
x0=self.params_init)
time_end = time.time()
print(time_end - time_beg)
print(self.res.x)
print(self.res)
def f(x, a, b):
return a * x + b
x_exp = np.linspace(0, 10, 1000)
y_exp = f(x_exp, a=1, b=2) + np.random.normal(0, 0.5, 1000)
model = Model(x_exp, y_exp, [0, 0])
model.fit()
y_fit = f(x_exp, a=model.res.x[0], b=model.res.x[1])
fig, ax = plt.subplots()
ax.plot(x_exp, y_exp)
ax.plot(x_exp, y_fit)
fig.show()
Now, I would like do chose b value from my own list defined in the self.possible_b instead to approximate it by the used minimization method. Its some-kind of fixed values constrain to b.
Is it possible? How such kind of algorithms are called if they exist? Any libraries are able to do this?
I am trying to learn how to optimize data fits in higher dimensions using python / numpy / scipy. Having successfully used scipy to fit a line of the form y = f(x), I tried extending the logic to fit an ellipse of the form z = f(x, y); both are shown below. I'm hoping this approach can be generalized to fit shapes in higher dimensions (ie, sphere).
import numpy as np
from scipy.optimize import minimize
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D
## LINE EXAMPLE
npoints = 30
x = np.linspace(-5, 5, npoints)
m, b = 3, -4 # slope, y-intercept
initial_parameter_guess = (2.5, -1)
y = m * x + b # exact line
noise = np.random.uniform(-1, 1, size=x.size)
yn = y + noise # line with noise
def get_residuals(prms, x, y):
""" """
return (y - (prms[0] * x + prms[1]))**2
def f_error(prms, x, y):
""" """
resid = get_residuals(prms, x, y)
return np.sum(resid)
result = minimize(f_error, x0=initial_parameter_guess, args=(x, yn))
# print(result)
yf = result.x[0] * x + result.x[1]
fig, ax = plt.subplots()
ax.scatter(x, yn, color='b', marker='.')
ax.plot(x, yf, color='r', alpha=0.5)
ax.grid(color='k', alpha=0.3, linestyle=':')
plt.show()
plt.close(fig)
Applying this logic to the case of an ellipse,
## ELLIPSE EXAMPLE
npoints = 75
theta = np.random.uniform(0, 2*np.pi, size=npoints)
a, b = (3, 5)
initial_parameter_guess = (2.5, 6)
xnoise = np.random.uniform(-1, 1, size=theta.size)
ynoise = np.random.uniform(-1, 1, size=theta.size)
x = a**2 * np.cos(theta)
xn = x + xnoise
y = b**2 * np.sin(theta)
yn = y + ynoise
def get_residuals(prms, x, y):
""" """
return 1 - ((x/prms[0])**2 + (y/prms[1])**2)
def f_error(prms, x, y):
""" """
resid = get_residuals(prms, x, y)
return np.sum(resid)
result = minimize(f_error, x0=initial_parameter_guess, args=(xn, yn))
# print(result)
The minimize algorithm via scipy does not find the optimal parameters; the following output is shown with print(result):
fun: -4243.611573066522
hess_inv: array([[41.44400248, 39.68101343],
[39.68101343, 37.99343048]])
jac: array([-1496.81719971, 2166.68896484])
message: 'Desired error not necessarily achieved due to precision loss.'
nfev: 174
nit: 1
njev: 42
status: 2
success: False
x: array([-1.51640333, 2.93879038])
I have seen another solution to this problem, which use the matrix formulation of least squares, but the example is a bit difficult for me to follow. Almost all approaches I've seen are based on the approach in the posted link. I'd prefer using minimize unless the linear algebra approach was better for a reason that I am currently unaware of.
Anyways, can my approach above be adapted/tweaked in a way that will work and can be generalized for higher dimensions?
Two issues with the code . Instead of minimizing the sum of residuals , minimize the sum of squares of residuals. Secondly , elliptic equation should be defined as x=a*np.cos(theta) and y=b*np.sin(theta)
npoints = 75
theta = np.random.uniform(0, 2*np.pi, size=npoints)
a, b = (3, 5)
initial_parameter_guess = (2.5, 6)
xnoise = np.random.uniform(-1, 1, size=theta.size)
ynoise = np.random.uniform(-1, 1, size=theta.size)
x = a * np.cos(theta)
xn = x + xnoise
y = b * np.sin(theta)
yn = y + ynoise
def get_residuals(prms, x, y):
""" """
return 1 - ((x/prms[0])**2 + (y/prms[1])**2)
def f_error(prms, x, y):
""" """
resid = get_residuals(prms, x, y)
return np.sum(np.square(resid))
result = minimize(f_error, x0=initial_parameter_guess,args=(xn, yn))
result
fun: 5.85099318913613
hess_inv: array([[ 0.07025572, -0.02902779],
[-0.02902779, 0.12040811]])
jac: array([-5.96046448e-08, 1.19209290e-07])
message: 'Optimization terminated successfully.'
nfev: 48
nit: 10
njev: 12
status: 0
success: True
x: array([3.35248219, 5.13728323])
I have checked python non linear ODE with 2 variables , which is not my case. Maybe my case is not called as nonlinear ODE, correct me please.
The question isFrenet Frame actually, in which there are 3 vectors T(s), N(s) and B(s); the parameter s>=0. And there are 2 scalar with known math formula expression t(s) and k(s). I have the initial value T(0), N(0) and B(0).
diff(T(s), s) = k(s)*N(s)
diff(N(s), s) = -k(s)*T(s) + t(s)*B(s)
diff(B(s), s) = -t(s)*N(s)
Then how can I get T(s), N(s) and B(s) numerically or symbolically?
I have checked scipy.integrate.ode but I don't know how to pass k(s)*N(s) into its first parameter at all
def model (z, tspan):
T = z[0]
N = z[1]
B = z[2]
dTds = k(s) * N # how to express function k(s)?
dNds = -k(s) * T + t(s) * B
dBds = -t(s)* N
return [dTds, dNds, dBds]
z = scipy.integrate.ode(model, [T0, N0, B0]
Here is a code using solve_ivp interface from Scipy (instead of odeint) to obtain a numerical solution:
import numpy as np
from scipy.integrate import solve_ivp
from scipy.integrate import cumtrapz
import matplotlib.pylab as plt
# Define the parameters as regular Python function:
def k(s):
return 1
def t(s):
return 0
# The equations: dz/dt = model(s, z):
def model(s, z):
T = z[:3] # z is a (9, ) shaped array, the concatenation of T, N and B
N = z[3:6]
B = z[6:]
dTds = k(s) * N
dNds = -k(s) * T + t(s) * B
dBds = -t(s)* N
return np.hstack([dTds, dNds, dBds])
T0, N0, B0 = [1, 0, 0], [0, 1, 0], [0, 0, 1]
z0 = np.hstack([T0, N0, B0])
s_span = (0, 6) # start and final "time"
t_eval = np.linspace(*s_span, 100) # define the number of point wanted in-between,
# It is not necessary as the solver automatically
# define the number of points.
# It is used here to obtain a relatively correct
# integration of the coordinates, see the graph
# Solve:
sol = solve_ivp(model, s_span, z0, t_eval=t_eval, method='RK45')
print(sol.message)
# >> The solver successfully reached the end of the integration interval.
# Unpack the solution:
T, N, B = np.split(sol.y, 3) # another way to unpack the z array
s = sol.t
# Bonus: integration of the normal vector in order to get the coordinates
# to plot the curve (there is certainly better way to do this)
coords = cumtrapz(T, x=s)
plt.plot(coords[0, :], coords[1, :]);
plt.axis('equal'); plt.xlabel('x'); plt.xlabel('y');
T, N and B are vectors. Therefore, there are 9 equations to solve: z is a (9,) array.
For constant curvature and no torsion, the result is a circle:
thanks for your example. And I thought it again, found that since there is formula for dZ where Z is matrix(T, N, B), we can calculate Z[i] = Z[i-1] + dZ[i-1]*deltaS according to the concept of derivative. Then I code and find this idea can solve the circle example. So
is Z[i] = Z[i-1] + dZ[i-1]*deltaS suitable for other ODE? will it fail in some situation, or does scipy.integrate.solve_ivp/scipy.integrate.ode supply advantage over the direct usage of Z[i] = Z[i-1] + dZ[i-1]*deltaS?
in my code, I have to normalize Z[i] because ||Z[i]|| is not always 1. Why does it happen? A float numerical calculation error?
my answer to my question, at least it works for the circle
import numpy as np
from scipy.integrate import cumtrapz
import matplotlib.pylab as plt
# Define the parameters as regular Python function:
def k(s):
return 1
def t(s):
return 0
def dZ(s, Z):
return np.array(
[k(s) * Z[1], -k(s) * Z[0] + t(s) * Z[2], -t(s)* Z[1]]
)
T0, N0, B0 = np.array([1, 0, 0]), np.array([0, 1, 0]), np.array([0, 0, 1])
deltaS = 0.1 # step to calculate dZ/ds
num = int(2*np.pi*1/deltaS) + 1 # how many points on the curve we have to calculate
T = np.zeros([num, ], dtype=object)
N = np.zeros([num, ], dtype=object)
B = np.zeros([num, ], dtype=object)
T[0] = T0
N[0] = N0
B[0] = B0
for i in range(num-1):
temp_dZ = dZ(i*deltaS, np.array([T[i], N[i], B[i]]))
T[i+1] = T[i] + temp_dZ[0]*deltaS
T[i+1] = T[i+1]/np.linalg.norm(T[i+1]) # have to do this
N[i+1] = N[i] + temp_dZ[1]*deltaS
N[i+1] = N[i+1]/np.linalg.norm(N[i+1])
B[i+1] = B[i] + temp_dZ[2]*deltaS
B[i+1] = B[i+1]/np.linalg.norm(B[i+1])
coords = cumtrapz(
[
[i[0] for i in T], [i[1] for i in T], [i[2] for i in T]
]
, x=np.arange(num)*deltaS
)
plt.figure()
plt.plot(coords[0, :], coords[1, :]);
plt.axis('equal'); plt.xlabel('x'); plt.xlabel('y');
plt.show()
I found that the equation I listed in the first post does not work for my curve. So I read Gray A., Abbena E., Salamon S-Modern Differential Geometry of Curves and Surfaces with Mathematica. 2006 and found that for arbitrary curve, Frenet equation should be written as
diff(T(s), s) = ||r'||* k(s)*N(s)
diff(N(s), s) = ||r'||*(-k(s)*T(s) + t(s)*B(s))
diff(B(s), s) = ||r'||* -t(s)*N(s)
where ||r'||(or ||r'(s)||) is diff([x(s), y(s), z(s)], s).norm()
now the problem has changed to be some different from that in the first post, because there is no r'(s) function or discrete data array. So I think this is suitable for a new reply other than comment.
I met 2 questions while trying to solve the new equation:
how can we program with r'(s) if scipy's solve_ivp is used?
I try to modify my gaussian solution, but the result is totally wrong.
thanks again
import numpy as np
from scipy.integrate import cumtrapz
import matplotlib.pylab as plt
# Define the parameters as regular Python function:
def k(s):
return 1
def t(s):
return 0
def dZ(s, Z, r_norm):
return np.array([
r_norm * k(s) * Z[1],
r_norm*(-k(s) * Z[0] + t(s) * Z[2]),
r_norm*(-t(s)* Z[1])
])
T0, N0, B0 = np.array([1, 0, 0]), np.array([0, 1, 0]), np.array([0, 0, 1])
deltaS = 0.1 # step to calculate dZ/ds
num = int(2*np.pi*1/deltaS) + 1 # how many points on the curve we have to calculate
T = np.zeros([num, ], dtype=object)
N = np.zeros([num, ], dtype=object)
B = np.zeros([num, ], dtype=object)
R0 = N0
T[0] = T0
N[0] = N0
B[0] = B0
for i in range(num-1):
r_norm = np.linalg.norm(R0)
temp_dZ = dZ(i*deltaS, np.array([T[i], N[i], B[i]]), r_norm)
T[i+1] = T[i] + temp_dZ[0]*deltaS
T[i+1] = T[i+1]/np.linalg.norm(T[i+1])
N[i+1] = N[i] + temp_dZ[1]*deltaS
N[i+1] = N[i+1]/np.linalg.norm(N[i+1])
B[i+1] = B[i] + temp_dZ[2]*deltaS
B[i+1] = B[i+1]/np.linalg.norm(B[i+1])
R0 = R0 + T[i]*deltaS
coords = cumtrapz(
[
[i[0] for i in T], [i[1] for i in T], [i[2] for i in T]
]
, x=np.arange(num)*deltaS
)
plt.figure()
plt.plot(coords[0, :], coords[1, :]);
plt.axis('equal'); plt.xlabel('x'); plt.xlabel('y');
plt.show()
I implemented gradient descent for linear regression in Python for a data read from csv file. below is the code:
import numpy as np
import scipy.optimize as op
from sklearn import preprocessing
import matplotlib.pyplot as plt
from matplotlib import style
def CostFunc(theta,x,y):
m,n = x.shape;
theta = theta.reshape((n,1));
#y = y.reshape((m,1));
#x=x.reshape((m,1))
J = 0.5*sum(np.square((X.dot(theta)-y)))[0]/m;
return J;
def gradientDescent(X, y, theta, alpha, num_iters):
m,n = X.shape;
initial_iter = num_iters
J = np.zeros((num_iters,1));
while num_iters > 0:
theta = theta - (alpha*np.sum(((X.dot(theta)-y)*X),axis=0)/m).reshape((n,1))
J[initial_iter-num_iters][0]=CostFunc(theta,X,y)
num_iters=num_iters-1
return theta,J;
data = np.loadtxt(open("ex1data2.txt","rb"),delimiter=",",skiprows=1)
nr,nc = data.shape
X=data[:,0:nc - 1]
X=preprocessing.scale(X)
X=np.insert(X,0,1,axis=1)
y= data[:,[nc - 1]]
m , n = X.shape;
#initial_theta = np.zeros(len(n));
initial_theta = np.zeros((n,1));
theta,J=gradientDescent(X,y,initial_theta,0.01,400)
J=J.reshape(400)
count=0
grp=np.zeros((400,2));
for value in J:
grp[count][0] = count+1
grp[count][1] = value
count = count+1
plt.plot(grp)
plt.legend(loc=4)
plt.xlabel('iter')
plt.ylabel('cost')
plt.show()
print(theta)
optimal_theta = Result.x;
I ran 400 iterations and got a decreasing plot of cost as followsiteration vs cost plot
Then I used one of the custom algorithms on the same data with the following code
import numpy as np
import scipy.optimize as op
from sklearn import preprocessing
def Sigmoid(z):
return 1/(1 + np.exp(-z));
def Gradient(theta,x,y):
m , n = x.shape
theta = theta.reshape((n,1));
#y = y.reshape((m,1))
#x=x.reshape((m,1))
#sigmoid_x_theta = Sigmoid(x.dot(theta));
grad = (np.sum(((X.dot(theta)-y)*X),axis=0)/m);
return grad.flatten();
def CostFunc(theta,x,y):
m,n = x.shape;
theta = theta.reshape((n,1));
#y = y.reshape((m,1));
#x=x.reshape((m,1))
J = 0.5*sum(np.square((X.dot(theta)-y)))[0]/m;
return J;
data = np.loadtxt(open("ex1data2.txt","rb"),delimiter=",",skiprows=1)
nr,nc = data.shape
X=data[:,0:nc - 1]
X=preprocessing.scale(X)
X=np.insert(X,0,1,axis=1)
y= data[:,[nc - 1]]
m , n = X.shape;
#initial_theta = np.zeros(len(n));
initial_theta = np.zeros((n,1));
Result = op.minimize(fun = CostFunc,
x0 = initial_theta,
args = (X, y),
method = 'TNC',
jac = Gradient);
print("====================")
print(Result)
#optimal_theta = Result.x;
The theta values of the first case were
[[ 333032.07465084] [ 100130.7408761 ] [ 3699.66611303]]
While from the second I get the following output
fun: 2066502781.7118049
jac: array([ -6.45345806e-11, 7.84261236e-10, -2.42370452e-10])
message: 'Local minimum reached (|pg| ~= 0)'
nfev: 27
nit: 13
status: 0
success: True
x: array([ 339119.45652174, 110248.92165868, -6226.22670554])
I am assuming x: array([ 339119.45652174, 110248.92165868, -6226.22670554]) to be the theta value.
Why am I getting the values different for 2 cases?
I am currently working with astronomical data among which I have comet images. I would like to remove the background sky gradient in these images due to the time of capture (twilight). The first program I developed to do so took user selected points from Matplotlib's "ginput" (x,y) pulled the data for each coordinate (z) and then gridded the data in a new array with SciPy's "griddata."
Since the background is assumed to vary only slightly, I would like to fit a 3d low order polynomial to this set of (x,y,z) points. However, the "griddata" does not allow for an input order:
griddata(points,values, (dimension_x,dimension_y), method='nearest/linear/cubic')
Any ideas on another function that may be used or a method for developing a leas-squares fit that will allow me to control the order?
Griddata uses a spline fitting. A 3rd order spline is not the same thing as a 3rd order polynomial (instead, it's a different 3rd order polynomial at every point).
If you just want to fit a 2D, 3rd order polynomial to your data, then do something like the following to estimate the 16 coefficients using all of your data points.
import itertools
import numpy as np
import matplotlib.pyplot as plt
def main():
# Generate Data...
numdata = 100
x = np.random.random(numdata)
y = np.random.random(numdata)
z = x**2 + y**2 + 3*x**3 + y + np.random.random(numdata)
# Fit a 3rd order, 2d polynomial
m = polyfit2d(x,y,z)
# Evaluate it on a grid...
nx, ny = 20, 20
xx, yy = np.meshgrid(np.linspace(x.min(), x.max(), nx),
np.linspace(y.min(), y.max(), ny))
zz = polyval2d(xx, yy, m)
# Plot
plt.imshow(zz, extent=(x.min(), y.max(), x.max(), y.min()))
plt.scatter(x, y, c=z)
plt.show()
def polyfit2d(x, y, z, order=3):
ncols = (order + 1)**2
G = np.zeros((x.size, ncols))
ij = itertools.product(range(order+1), range(order+1))
for k, (i,j) in enumerate(ij):
G[:,k] = x**i * y**j
m, _, _, _ = np.linalg.lstsq(G, z)
return m
def polyval2d(x, y, m):
order = int(np.sqrt(len(m))) - 1
ij = itertools.product(range(order+1), range(order+1))
z = np.zeros_like(x)
for a, (i,j) in zip(m, ij):
z += a * x**i * y**j
return z
main()
The following implementation of polyfit2d uses the available numpy methods numpy.polynomial.polynomial.polyvander2d and numpy.polynomial.polynomial.polyval2d
#!/usr/bin/env python3
import unittest
def polyfit2d(x, y, f, deg):
from numpy.polynomial import polynomial
import numpy as np
x = np.asarray(x)
y = np.asarray(y)
f = np.asarray(f)
deg = np.asarray(deg)
vander = polynomial.polyvander2d(x, y, deg)
vander = vander.reshape((-1,vander.shape[-1]))
f = f.reshape((vander.shape[0],))
c = np.linalg.lstsq(vander, f)[0]
return c.reshape(deg+1)
class MyTest(unittest.TestCase):
def setUp(self):
return self
def test_1(self):
self._test_fit(
[-1,2,3],
[ 4,5,6],
[[1,2,3],[4,5,6],[7,8,9]],
[2,2])
def test_2(self):
self._test_fit(
[-1,2],
[ 4,5],
[[1,2],[4,5]],
[1,1])
def test_3(self):
self._test_fit(
[-1,2,3],
[ 4,5],
[[1,2],[4,5],[7,8]],
[2,1])
def test_4(self):
self._test_fit(
[-1,2,3],
[ 4,5],
[[1,2],[4,5],[0,0]],
[2,1])
def test_5(self):
self._test_fit(
[-1,2,3],
[ 4,5],
[[1,2],[4,5],[0,0]],
[1,1])
def _test_fit(self, x, y, c, deg):
from numpy.polynomial import polynomial
import numpy as np
X = np.array(np.meshgrid(x,y))
f = polynomial.polyval2d(X[0], X[1], c)
c1 = polyfit2d(X[0], X[1], f, deg)
np.testing.assert_allclose(c1,
np.asarray(c)[:deg[0]+1,:deg[1]+1],
atol=1e-12)
unittest.main()
According to the principle of Least squares, and imitate Kington's style,
while move argument m to argument m_1 and argument m_2.
import numpy as np
import matplotlib.pyplot as plt
import itertools
# w = (Phi^T Phi)^{-1} Phi^T t
# where Phi_{k, j + i (m_2 + 1)} = x_k^i y_k^j,
# t_k = z_k,
# i = 0, 1, ..., m_1,
# j = 0, 1, ..., m_2,
# k = 0, 1, ..., n - 1
def polyfit2d(x, y, z, m_1, m_2):
# Generate Phi by setting Phi as x^i y^j
nrows = x.size
ncols = (m_1 + 1) * (m_2 + 1)
Phi = np.zeros((nrows, ncols))
ij = itertools.product(range(m_1 + 1), range(m_2 + 1))
for h, (i, j) in enumerate(ij):
Phi[:, h] = x ** i * y ** j
# Generate t by setting t as Z
t = z
# Generate w by solving (Phi^T Phi) w = Phi^T t
w = np.linalg.solve(Phi.T.dot(Phi), (Phi.T.dot(t)))
return w
# t' = Phi' w
# where Phi'_{k, j + i (m_2 + 1)} = x'_k^i y'_k^j
# t'_k = z'_k,
# i = 0, 1, ..., m_1,
# j = 0, 1, ..., m_2,
# k = 0, 1, ..., n' - 1
def polyval2d(x_, y_, w, m_1, m_2):
# Generate Phi' by setting Phi' as x'^i y'^j
nrows = x_.size
ncols = (m_1 + 1) * (m_2 + 1)
Phi_ = np.zeros((nrows, ncols))
ij = itertools.product(range(m_1 + 1), range(m_2 + 1))
for h, (i, j) in enumerate(ij):
Phi_[:, h] = x_ ** i * y_ ** j
# Generate t' by setting t' as Phi' w
t_ = Phi_.dot(w)
# Generate z_ by setting z_ as t_
z_ = t_
return z_
if __name__ == "__main__":
# Generate x, y, z
n = 100
x = np.random.random(n)
y = np.random.random(n)
z = x ** 2 + y ** 2 + 3 * x ** 3 + y + np.random.random(n)
# Generate w
w = polyfit2d(x, y, z, m_1=3, m_2=2)
# Generate x', y', z'
n_ = 1000
x_, y_ = np.meshgrid(np.linspace(x.min(), x.max(), n_),
np.linspace(y.min(), y.max(), n_))
z_ = np.zeros((n_, n_))
for i in range(n_):
z_[i, :] = polyval2d(x_[i, :], y_[i, :], w, m_1=3, m_2=2)
# Plot
plt.imshow(z_, extent=(x_.min(), y_.max(), x_.max(), y_.min()))
plt.scatter(x, y, c=z)
plt.show()
If anyone is looking for fitting a polynomial of a specific order (rather than polynomials where the highest power is equal to order, you can make this adjustment to the accepted answer's polyfit and polyval:
instead of:
ij = itertools.product(range(order+1), range(order+1))
which, for order=2 gives [(0, 0), (0, 1), (0, 2), (1, 0), (1, 1), (1, 2), (2, 0), (2, 1), (2, 2)] (aka up to a 4th degree polynomial), you can use
def xy_powers(order):
powers = itertools.product(range(order + 1), range(order + 1))
return [tup for tup in powers if sum(tup) <= order]
This returns [(0, 0), (0, 1), (0, 2), (1, 0), (1, 1), (2, 0)] for order=2