I'm able to use numpy.polynomial to fit terms to 1D polynomials like f(x) = 1 + x + x^2. How can I fit multidimensional polynomials, like f(x,y) = 1 + x + x^2 + y + yx + y x^2 + y^2 + y^2 x + y^2 x^2? It looks like numpy doesn't support multidimensional polynomials at all: is that the case? In my real application, I have 5 dimensions of input and I am interested in hermite polynomials. It looks like the polynomials in scipy.special are also only available for one dimension of inputs.
# One dimension of data can be fit
x = np.random.random(100)
y = np.sin(x)
params = np.polynomial.polynomial.polyfit(x, y, 6)
np.polynomial.polynomial.polyval([0, .2, .5, 1.5], params)
array([ -5.01799432e-08, 1.98669317e-01, 4.79425535e-01,
9.97606096e-01])
# When I try two dimensions, it fails.
x = np.random.random((100, 2))
y = np.sin(5 * x[:,0]) + .4 * np.sin(x[:,1])
params = np.polynomial.polynomial.polyvander2d(x, y, [6, 6])
---------------------------------------------------------------------------
ValueError Traceback (most recent call last)
<ipython-input-13-5409f9a3e632> in <module>()
----> 1 params = np.polynomial.polynomial.polyvander2d(x, y, [6, 6])
/usr/local/lib/python2.7/site-packages/numpy/polynomial/polynomial.pyc in polyvander2d(x, y, deg)
1201 raise ValueError("degrees must be non-negative integers")
1202 degx, degy = ideg
-> 1203 x, y = np.array((x, y), copy=0) + 0.0
1204
1205 vx = polyvander(x, degx)
ValueError: could not broadcast input array from shape (100,2) into shape (100)
I got annoyed that there is no simple function for a 2d polynomial fit of any number of degrees so I made my own. Like the other answers it uses numpy lstsq to find the best coefficients.
import numpy as np
from scipy.linalg import lstsq
from scipy.special import binom
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D
def _get_coeff_idx(coeff):
idx = np.indices(coeff.shape)
idx = idx.T.swapaxes(0, 1).reshape((-1, 2))
return idx
def _scale(x, y):
# Normalize x and y to avoid huge numbers
# Mean 0, Variation 1
offset_x, offset_y = np.mean(x), np.mean(y)
norm_x, norm_y = np.std(x), np.std(y)
x = (x - offset_x) / norm_x
y = (y - offset_y) / norm_y
return x, y, (norm_x, norm_y), (offset_x, offset_y)
def _unscale(x, y, norm, offset):
x = x * norm[0] + offset[0]
y = y * norm[1] + offset[1]
return x, y
def polyvander2d(x, y, degree):
A = np.polynomial.polynomial.polyvander2d(x, y, degree)
return A
def polyscale2d(coeff, scale_x, scale_y, copy=True):
if copy:
coeff = np.copy(coeff)
idx = _get_coeff_idx(coeff)
for k, (i, j) in enumerate(idx):
coeff[i, j] /= scale_x ** i * scale_y ** j
return coeff
def polyshift2d(coeff, offset_x, offset_y, copy=True):
if copy:
coeff = np.copy(coeff)
idx = _get_coeff_idx(coeff)
# Copy coeff because it changes during the loop
coeff2 = np.copy(coeff)
for k, m in idx:
not_the_same = ~((idx[:, 0] == k) & (idx[:, 1] == m))
above = (idx[:, 0] >= k) & (idx[:, 1] >= m) & not_the_same
for i, j in idx[above]:
b = binom(i, k) * binom(j, m)
sign = (-1) ** ((i - k) + (j - m))
offset = offset_x ** (i - k) * offset_y ** (j - m)
coeff[k, m] += sign * b * coeff2[i, j] * offset
return coeff
def plot2d(x, y, z, coeff):
# regular grid covering the domain of the data
if x.size > 500:
choice = np.random.choice(x.size, size=500, replace=False)
else:
choice = slice(None, None, None)
x, y, z = x[choice], y[choice], z[choice]
X, Y = np.meshgrid(
np.linspace(np.min(x), np.max(x), 20), np.linspace(np.min(y), np.max(y), 20)
)
Z = np.polynomial.polynomial.polyval2d(X, Y, coeff)
fig = plt.figure()
ax = fig.gca(projection="3d")
ax.plot_surface(X, Y, Z, rstride=1, cstride=1, alpha=0.2)
ax.scatter(x, y, z, c="r", s=50)
plt.xlabel("X")
plt.ylabel("Y")
ax.set_zlabel("Z")
plt.show()
def polyfit2d(x, y, z, degree=1, max_degree=None, scale=True, plot=False):
"""A simple 2D polynomial fit to data x, y, z
The polynomial can be evaluated with numpy.polynomial.polynomial.polyval2d
Parameters
----------
x : array[n]
x coordinates
y : array[n]
y coordinates
z : array[n]
data values
degree : {int, 2-tuple}, optional
degree of the polynomial fit in x and y direction (default: 1)
max_degree : {int, None}, optional
if given the maximum combined degree of the coefficients is limited to this value
scale : bool, optional
Wether to scale the input arrays x and y to mean 0 and variance 1, to avoid numerical overflows.
Especially useful at higher degrees. (default: True)
plot : bool, optional
wether to plot the fitted surface and data (slow) (default: False)
Returns
-------
coeff : array[degree+1, degree+1]
the polynomial coefficients in numpy 2d format, i.e. coeff[i, j] for x**i * y**j
"""
# Flatten input
x = np.asarray(x).ravel()
y = np.asarray(y).ravel()
z = np.asarray(z).ravel()
# Remove masked values
mask = ~(np.ma.getmask(z) | np.ma.getmask(x) | np.ma.getmask(y))
x, y, z = x[mask].ravel(), y[mask].ravel(), z[mask].ravel()
# Scale coordinates to smaller values to avoid numerical problems at larger degrees
if scale:
x, y, norm, offset = _scale(x, y)
if np.isscalar(degree):
degree = (int(degree), int(degree))
degree = [int(degree[0]), int(degree[1])]
coeff = np.zeros((degree[0] + 1, degree[1] + 1))
idx = _get_coeff_idx(coeff)
# Calculate elements 1, x, y, x*y, x**2, y**2, ...
A = polyvander2d(x, y, degree)
# We only want the combinations with maximum order COMBINED power
if max_degree is not None:
mask = idx[:, 0] + idx[:, 1] <= int(max_degree)
idx = idx[mask]
A = A[:, mask]
# Do the actual least squares fit
C, *_ = lstsq(A, z)
# Reorder coefficients into numpy compatible 2d array
for k, (i, j) in enumerate(idx):
coeff[i, j] = C[k]
# Reverse the scaling
if scale:
coeff = polyscale2d(coeff, *norm, copy=False)
coeff = polyshift2d(coeff, *offset, copy=False)
if plot:
if scale:
x, y = _unscale(x, y, norm, offset)
plot2d(x, y, z, coeff)
return coeff
if __name__ == "__main__":
n = 100
x, y = np.meshgrid(np.arange(n), np.arange(n))
z = x ** 2 + y ** 2
c = polyfit2d(x, y, z, degree=2, plot=True)
print(c)
It doesn't look like polyfit supports fitting multivariate polynomials, but you can do it by hand, with linalg.lstsq. The steps are as follows:
Gather the degrees of monomials x**i * y**j you wish to use in the model. Think carefully about it: your current model already has 9 parameters, if you are going to push to 5 variables then with the current approach you'll end up with 3**5 = 243 parameters, a sure road to overfitting. Maybe limit to the monomials of __total_ degree at most 2 or three...
Plug the x-points into each monomial; this gives a 1D array. Stack all such arrays as columns of a matrix.
Solve a linear system with aforementioned matrix and with the right-hand side being the target values (I call them z because y is confusing when you also use x, y for two variables).
Here it is:
import numpy as np
x = np.random.random((100, 2))
z = np.sin(5 * x[:,0]) + .4 * np.sin(x[:,1])
degrees = [(i, j) for i in range(3) for j in range(3)] # list of monomials x**i * y**j to use
matrix = np.stack([np.prod(x**d, axis=1) for d in degrees], axis=-1) # stack monomials like columns
coeff = np.linalg.lstsq(matrix, z)[0] # lstsq returns some additional info we ignore
print("Coefficients", coeff) # in the same order as the monomials listed in "degrees"
fit = np.dot(matrix, coeff)
print("Fitted values", fit)
print("Original values", y)
I believe you have misunderstood what polyvander2d does and how it should be used. polyvander2d() returns the pseudo-Vandermonde matrix of degrees deg and sample points (x, y).
Here, y is not the value(s) of the polynomial at point(s) x but rather it is the y-coordinate of the point(s) and x is the x-coordinate. Roughly speaking, the returned array is a set of combinations of (x**i) * (y**j) and x and y are essentially 2D "mesh-grids". Therefore, both x and y must have identical shapes.
Your x and y, however, arrays have different shapes:
>>> x.shape
(100, 2)
>>> y.shape
(100,)
I do not believe numpy has a 5D-polyvander of the form polyvander5D(x, y, z, v, w, deg). Notice, all the variables here are coordinates and not the values of the polynomial p=p(x,y,z,v,w). You, however, seem to be using y (in the 2D case) as f.
It appears that numpy does not have 2D or higher equivalents for the polyfit() function. If your intention is to find the coefficients of the best-fitting polynomial in higher-dimensions, I would suggest that you generalize the approach described here: Equivalent of `polyfit` for a 2D polynomial in Python
The option isn't there because nobody wants to do that. Combine the polynomials linearly (f(x,y) = 1 + x + y + x^2 + y^2) and solve the system of equations yourself.
Related
I'm trying to write a function to evaluate the probability mass function for the bivariate poisson distribution.
This is easy when all of the parameters (x, y, theta1, theta2, theta0) are scalars, but tricky to scale up without loops to allow these parameters to be vectors. I need it to scale such that, for:
theta0 being a scalar - the "correlation parameter" in the equation
theta1 and theta2 having length l
x, y both having length n
the output array would have shape (l, n, n). For example, a slice [j, :, :] from the output array would look like:
The first part (the constant, before the summation) I think i've figured out:
import numpy as np
from scipy.special import factorial
def constant(theta1, theta2, theta0, x, y):
exponential_part = np.exp(-(theta1 + theta2 + theta0)).reshape(-1, 1, 1)
x = np.tile(x, (len(x), 1)).transpose()
y = np.tile(y, (len(y), 1))
double_factorial = (np.power(np.array(theta1).reshape(-1, 1, 1), x)/factorial(x)) * \
(np.power(np.array(theta2).reshape(-1, 1, 1), y)/factorial(y))
return exponential_part * double_factorial
But I'm struggling with the summation part. How can I vectorize a summation where the limits depend on variable arrays?
I think I have this figured out, based on the approach that #w-m suggests: calculate every possible summation term which could appear, based on the maximum x or y value which appears, and use a mask to get rid of the ones you don't want. Assuming you have your x and y terms go from 0 to N, in consecutive order, this is calculating up to three times more terms than are actually required, but this is offset by getting to use vectorization.
Reference implementation
I wrote this by first writing a pure-Python reference implementation, which just implements your problem using loops. With 4 nested loops, it's not exactly fast, but it's handy to have while testing the numpy version.
import numpy as np
from scipy.special import factorial, comb
import operator as op
from functools import reduce
def choose(n, r):
# https://stackoverflow.com/a/4941932/530160
r = min(r, n-r)
numer = reduce(op.mul, range(n, n-r, -1), 1)
denom = reduce(op.mul, range(1, r+1), 1)
return numer // denom # or / in Python 2
def reference_impl_constant(s_theta1, s_theta2, s_theta0, s_x, s_y):
# Cast to float to prevent overflow
s_theta1 = float(s_theta1)
s_theta2 = float(s_theta2)
s_theta0 = float(s_theta0)
s_x = float(s_x)
s_y = float(s_y)
term1 = np.exp(-(s_theta1 + s_theta2 + s_theta0))
term2 = (s_theta1 ** s_x / factorial(s_x))
term3 = (s_theta2 ** s_y / factorial(s_y))
assert term1 >= 0
assert term2 >= 0
assert term3 >= 0
return term1 * term2 * term3
def reference_impl_constant_loop(theta1, theta2, theta0, x, y):
theta_len = theta1.shape[0]
xy_len = x.shape[0]
constant_array = np.zeros((theta_len, xy_len, xy_len))
for i in range(theta_len):
for j in range(xy_len):
for k in range(xy_len):
s_theta1 = theta1[i]
s_theta2 = theta2[i]
s_theta0 = theta0
s_x = x[j]
s_y = y[k]
constant_term = reference_impl_constant(s_theta1, s_theta2, s_theta0, s_x, s_y)
assert constant_term >= 0
constant_array[i, j, k] = constant_term
return constant_array
def reference_impl_summation(s_theta1, s_theta2, s_theta0, s_x, s_y):
sum_ = 0
for i in range(min(s_x, s_y) + 1):
sum_ += choose(s_x, i) * choose(s_y, i) * factorial(i) * ((s_theta0/s_theta1/s_theta2) ** i)
assert sum_ >= 0
return sum_
def reference_impl_summation_loop(theta1, theta2, theta0, x, y):
theta_len = theta1.shape[0]
xy_len = x.shape[0]
summation_array = np.zeros((theta_len, xy_len, xy_len))
for i in range(theta_len):
for j in range(xy_len):
for k in range(xy_len):
s_theta1 = theta1[i]
s_theta2 = theta2[i]
s_theta0 = theta0
s_x = x[j]
s_y = y[k]
summation_term = reference_impl_summation(s_theta1, s_theta2, s_theta0, s_x, s_y)
assert summation_term >= 0
summation_array[i, j, k] = summation_term
return summation_array
def reference_impl(theta1, theta2, theta0, x, y):
# all array inputs must be 1D
assert len(theta1.shape) == 1
assert len(theta2.shape) == 1
assert len(x.shape) == 1
assert len(y.shape) == 1
# theta vectors must have same length
theta_len = theta1.shape[0]
assert theta2.shape[0] == theta_len
# x and y must have same length
xy_len = x.shape[0]
assert y.shape[0] == xy_len
# theta0 is scalar
assert isinstance(theta0, (int, float))
constant_array = np.zeros((theta_len, xy_len, xy_len))
output = np.zeros((theta_len, xy_len, xy_len))
constant_array = reference_impl_constant_loop(theta1, theta2, theta0, x, y)
summation_array = reference_impl_summation_loop(theta1, theta2, theta0, x, y)
output = constant_array * summation_array
return output
Numpy implementation
I split the implementation of this across two functions.
The fast_constant() function calculates everything to the left of the summation symbol. The fast_summation() function calculates everything inside the summation symbol.
import numpy as np
from scipy.special import factorial, comb
def fast_summation(theta1, theta2, theta0, x, y):
x = np.tile(x, (len(x), 1)).transpose()
y = np.tile(y, (len(y), 1))
sum_limit = np.minimum(x, y)
max_sum_limit = np.max(sum_limit)
i = np.arange(max_sum_limit + 1).reshape(-1, 1, 1)
summation_mask = (i <= sum_limit)
theta_ratio = (theta0 / (theta1 * theta2)).reshape(-1, 1, 1, 1)
theta_to_power = np.power(theta_ratio, i)
terms = comb(x, i) * comb(y, i) * factorial(i) * theta_to_power
# mask out terms which aren't part of sum
terms *= summation_mask
# axis 0 is theta
# axis 1 is i
# axis 2 & 3 are x and y
# so sum across axis 1
terms = terms.sum(axis=1)
return terms
def fast_constant(theta1, theta2, theta0, x, y):
theta1 = theta1.astype('float64')
theta2 = theta2.astype('float64')
exponential_part = np.exp(-(theta1 + theta2 + theta0)).reshape(-1, 1, 1)
# x and y must be 1D
assert len(x.shape) == 1
assert len(y.shape) == 1
# x and y must have same shape
assert x.shape == y.shape
x_len, y_len = x.shape[0], y.shape[0]
x = x.reshape((x_len, 1))
y = y.reshape((1, y_len))
double_factorial = (np.power(np.array(theta1).reshape(-1, 1, 1), x)/factorial(x)) * \
(np.power(np.array(theta2).reshape(-1, 1, 1), y)/factorial(y))
return exponential_part * double_factorial
def fast_impl(theta1, theta2, theta0, x, y):
return fast_summation(theta1, theta2, theta0, x, y) * fast_constant(theta1, theta2, theta0, x, y)
Benchmarking
Assuming that X and Y range from 0 to 20, and that theta is centered somewhere inside that range, I get the result that the numpy version is roughly 280 times faster than the pure python reference.
Numerical stability
I'm unsure how numerically stable this is. For example, when I center theta at 100, I get a floating-point overflow. Typically, when computing an expression which has lots of choose and factorial expressions inside it, you'll use some mathematical equivalent which results in smaller intermediate sums. In this case I have so little understanding of the math that I don't know how you'd do that.
I'm interested in plotting a real-valued function f(x,y,z)=a, where (x,y,z) is a 3D point on the sphere and a is a real number. I calculate the Cartesian coordinates of the points of the sphere as follows, but I have no clue on how to visualize the value of f on each of those points.
import plotly.graph_objects as go
import numpy as np
fig = go.Figure(layout=go.Layout(title=go.layout.Title(text=title), hovermode=False))
# Create mesh grid for spherical coordinates
phi, theta = np.mgrid[0.0:np.pi:100j, 0.0:2.0 * np.pi:100j]
# Get Cartesian mesh grid
x = np.sin(phi) * np.cos(theta)
y = np.sin(phi) * np.sin(theta)
z = np.cos(phi)
# Plot sphere surface
self.fig.add_surface(x=x, y=y, z=z, opacity=0.35)
fig.show()
I would imagine/expect/like a visualization like this
Additionally, I also have the gradient of f calculated in closed-form (i.e., for each (x,y,z) I calculate the 3D-dimensional gradient of f). Is there a way of plotting this vector field, similarly to what is shown in the figure above?
Here's an answer that's far from perfect, but hopefully that's enough for you to build on.
For the sphere itself, I don't know of any "shortcut" to do something like that in plotly, so my approach is simply to manually create a sphere mesh. Generating the vertices is simple, for example like you did - the slightly more tricky part is figuring out the vertex indices for the triangles (which depends on the vertex generation scheme). There are various algorithms to do that smoothly (i.e. generating a sphere with no "tip"), I hacked something crude just for the demonstration. Then we can use the Mesh3d object to display the sphere along with the intensities and your choice of colormap:
N = 100 # Sphere resolution (both rings and segments, can be separated to different constants)
theta, z = np.meshgrid(np.linspace(-np.pi, np.pi, N), np.linspace(-1, 1, N))
r = np.sqrt(1 - z ** 2)
x = r * np.cos(theta)
y = r * np.sin(theta)
x = x.ravel()
y = y.ravel()
z = z.ravel()
# Triangle indices
indices = np.arange(N * (N - 1) - 1)
i1 = np.concatenate([indices, (indices // N + 1) * N + (indices + 1) % N])
i2 = np.concatenate([indices + N, indices // N * N + (indices + 1) % N])
i3 = np.concatenate([(indices // N + 1) * N + (indices + 1) % N, indices])
# Point intensity function
def f(x, y, z):
return (np.cos(x * 2) + np.sin(y ** 2) + np.sin(z) + 3) / 6
fig = go.Figure(data=[
go.Mesh3d(
x=x,
y=y,
z=z,
colorbar_title='f(x, y, z)',
colorscale=[[0, 'gold'],
[0.5, 'mediumturquoise'],
[1, 'magenta']],
intensity = f(x, y, z),
i = i1,
j = i2,
k = i3,
name='y',
showscale=True
)
])
fig.show()
This yields the following interactive plot:
To add the vector field you can use the Cone plot; this requires some tinkering because when I simply draw the cones at the same x, y, z position as the sphere, some of the cones are partially or fully occluded by the sphere. So I generate another sphere, with a slightly larger radius, and place the cones there. I also played with some lighting parameters to make it black like in your example. The full code looks like this:
N = 100 # Sphere resolution (both rings and segments, can be separated to different constants)
theta, z = np.meshgrid(np.linspace(-np.pi, np.pi, N), np.linspace(-1, 1, N))
r = np.sqrt(1 - z ** 2)
x = r * np.cos(theta)
y = r * np.sin(theta)
x = x.ravel()
y = y.ravel()
z = z.ravel()
# Triangle indices
indices = np.arange(N * (N - 1) - 1)
i1 = np.concatenate([indices, (indices // N + 1) * N + (indices + 1) % N])
i2 = np.concatenate([indices + N, indices // N * N + (indices + 1) % N])
i3 = np.concatenate([(indices // N + 1) * N + (indices + 1) % N, indices])
# Point intensity function
def f(x, y, z):
return (np.cos(x * 2) + np.sin(y ** 2) + np.sin(z) + 3) / 6
# Vector field function
def grad_f(x, y, z):
return np.stack([np.cos(3 * y + 5 * x),
np.sin(z * y),
np.cos(4 * x - 3 * y + z * 7)], axis=1)
# Second sphere for placing cones
N2 = 50 # Smaller resolution (again rings and segments combined)
R2 = 1.05 # Slightly larger radius
theta2, z2 = np.meshgrid(np.linspace(-np.pi, np.pi, N2), np.linspace(-R2, R2, N2))
r2 = np.sqrt(R2 ** 2 - z2 ** 2)
x2 = r2 * np.cos(theta2)
y2 = r2 * np.sin(theta2)
x2 = x2.ravel()
y2 = y2.ravel()
z2 = z2.ravel()
uvw = grad_f(x2, y2, z2)
fig = go.Figure(data=[
go.Mesh3d(
x=x,
y=y,
z=z,
colorbar_title='f(x, y, z)',
colorscale=[[0, 'gold'],
[0.5, 'mediumturquoise'],
[1, 'magenta']],
intensity = f(x, y, z),
i = i1,
j = i2,
k = i3,
name='y',
showscale=True
),
go.Cone(
x=x2, y=y2, z=z2, u=uvw[:, 0], v=uvw[:, 1], w=uvw[:, 2], sizemode='absolute', sizeref=2, anchor='tail',
lighting_ambient=0, lighting_diffuse=0, opacity=.2
)
])
fig.show()
And yields this plot:
Hope this helps. There are a lot of tweaks to the display, and certainly better ways to construct a sphere mesh (e.g. see this article), so there should be a lot of freedom there (albeit at the cost of some work).
Good luck!
I have a nonuniformly sampled data that I am trying to apply a Gaussian filter to. I am using python's numpy library to solve this. The data is of XY type, here is how it looks like:
[[ -0.96 390.63523024]
[ -1.085 390.68523024]
[ -1.21 390.44023023]
...
[-76.695 390.86023024]
[-77.105 392.51023024]
[-77.155 392.10023024]]
And here is a link to the whole *.npz file.
Here is my approach:
I start with defining a Gaussian function
Then I start scanning the data with a while loop along the X axis
Within each step of the loop:
I select a portion of data that is within two cutoff lengths
shift the X axis of the selected data portion to make it symmetrical around 0
calculate my Gaussian function at every point, multiply with corresponding Y values, sum and divide by number of elements
Move to next point
Here is how code looks like:
import numpy as np
import matplotlib.pyplot as plt
xy = np.load('1D_data.npz')['arr_0']
def g_func(xx, w=1.0):
a = 0.47 * w
return (1 / a) * np.exp((xx / a) ** 2 * (-np.pi))
x, y, x_, y_ = xy[:, 0], xy[:, 1], [], []
counter, xi, ww = 0, x[0], 1.0
while xi > np.amin(x):
curr_x = x[(x < xi) & (x >= xi - 2 * ww)]
g, ysel = [], []
for i, els in enumerate(curr_x):
xil = els - curr_x[0] + abs(curr_x[0] - curr_x[-1]) / 2
g.append(g_func(xil, ww))
ysel.append(y[counter + i])
y_.append(np.sum(np.multiply(g, ysel)) / len(g))
x_.append(xi)
counter += 1
xi = x[counter]
plt.plot(x, y, '-k')
plt.plot(x_, y_, '-r')
plt.show()
The output doesn't look right though. (See the fig below) Even if discarding the edges, the convolution is very noisy and the values do not seem to correspond to the data. What am I possibly doing wrong?
You made one mistake in your code:
Before multiplying g with y_sel, y_sel is not centered.
The reason why y_sel should be centered is because we want to add the relative differences weighted by the Gaussian to the entry at the center. If you multiply g with y_sel directly, not just the values of the neighboring entries within the window, but also the value of the center entry will be weighted by the Gaussian. This will definitely change the function values dramatically.
Below is my solution using numpy
def g_func(xx, w=1.0):
mean = np.mean(xx)
a = 0.47 * w
return (1 / a) * np.exp(((xx-mean) / a) ** 2 * (-np.pi))
def get_convolution(array,half_window_size):
array = np.concatenate((np.repeat(array[0],half_window_size),
array,
np.repeat(array[-1],half_window_size)))
window_inds = [list(range(ind-half_window_size,ind+half_window_size+1)) \
for ind in range(half_window_size,len(array)-half_window_size)]
return np.take(array,window_inds)
xy = np.load('1D_data.npz')['arr_0']
x, y = xy[:, 0], xy[:, 1]
half_window_size = 4
x_conv = np.apply_along_axis(g_func,axis=1,arr=get_convolution(x,half_window_size=half_window_size))
y_conv = get_convolution(y,half_window_size=half_window_size)
y_mean = np.mean(y_conv,axis=1)
y_centered = y_conv - y_mean[:,None]
smoothed = np.sum(x_conv*y_centered,axis=1) / (half_window_size*2) + y_mean
fig,ax = plt.subplots(figsize=(10,6))
ax.plot(x, y, '-k')
ax.plot(x, smoothed, '-r')
running the code, the output is
UPDATE
In order to unify w with half_window_size, here is one possibility, the idea is to let the standard deviation of the Gaussian to be 2*half_window_size
def g_func(xx):
std = len(xx)
mean = np.mean(xx)
return 1 / (std*np.sqrt(2*np.pi)) * np.exp(-1/2*((xx-mean)/std)**2)
def get_convolution(array,half_window_size):
array = np.concatenate((np.repeat(array[0],half_window_size),
array,
np.repeat(array[-1],half_window_size)))
window_inds = [list(range(ind-half_window_size,ind+half_window_size+1)) \
for ind in range(half_window_size,len(array)-half_window_size)]
return np.take(array,window_inds)
xy = np.load('1D_data.npz')['arr_0']
x, y = xy[:, 0], xy[:, 1]
half_window_size = 4
x_conv = np.apply_along_axis(g_func,axis=1,arr=get_convolution(x,half_window_size=half_window_size))
y_conv = get_convolution(y,half_window_size=half_window_size)
y_mean = np.mean(y_conv,axis=1)
y_centered = y_conv - y_mean[:,None]
smoothed = np.sum(x_conv*y_centered,axis=1) / (half_window_size*2) + y_mean
fig,ax = plt.subplots(figsize=(10,6))
ax.plot(x, y, '-k')
ax.plot(x, smoothed, '-r')
I have a radially symmetric function evaluated on a 3D Cartesian grid. How can I numerically calculate the radial derivative of the function?
For a simple example (spherical Gaussian), calculate derivatives df/dx, df/dy and df/dz:
# Parameters
start = 0
end = 5
n = 20
# Variables
x = np.linspace(start, end, num=n)
y = np.linspace(start, end, num=n)
z = np.linspace(start, end, num=n)
dx = (end - start) / n
dy = (end - start) / n
dz = (end - start) / n
x_grid, y_grid, z_grid = np.meshgrid(x, y, z)
eval_xyz = np.exp(-(x_grid ** 2 + y_grid ** 2 + z_grid ** 2))
# Allocate
df_dx = np.zeros((n, n, n))
df_dy = np.zeros((n, n, n))
df_dz = np.zeros((n, n, n))
# Calculate Cartesian gradient numerically
for x in range(eval_xyz.shape[0] - 1):
for y in range(eval_xyz.shape[1] - 1):
for z in range(eval_xyz.shape[2] - 1):
df_dx[x, y, z] = (eval_xyz[x + 1, y, z] - eval_xyz[x, y, z]) / dx
df_dy[x, y, z] = (eval_xyz[x, y + 1, z] - eval_xyz[x, y, z]) / dy
df_dz[x, y, z] = (eval_xyz[x, y, z + 1] - eval_xyz[x, y, z]) / dz
Is it then possible to easily calculate the radial derivative df/dr from the Cartesian derivatives?
The trick is to express the radial derivatives as sum of Cartesian derivatives, taking into account theta and phi at each point which can be expressed in Cartesian coordiantes as:
The code therefore becomes:
theta_val = theta(i * dx, j * dy, k * dz)
phi_val = phi(i * dx, j * dy)
df_dr[i, j, k] = df_dx[i, j, k] * np.sin(theta_val) * np.cos(phi_val) \
+ df_dy[i, j, k] * np.sin(theta_val) * np.sin(phi_val) \
+ df_dz[i, j, k] * np.cos(theta_val)
Where theta and phi are calculated carefully to deal with divide by zero
def theta(x, y, z):
if x == 0 and y == 0 and z == 0:
return 0
elif z == 0:
return np.pi / 2
elif x == 0 and y == 0:
return 0
else:
return np.arctan(np.sqrt(x ** 2 + y ** 2) / z)
def phi(x, y):
if x == 0 and y == 0:
return 0
elif x == 0:
return np.pi / 2
elif y == 0:
return 0
else:
return math.atan2(y, x)
Your own answer is a step in the right direction, but there are some issues both in the answer and in the code generating the Cartesian derivatives.
These lines have a problem:
x = np.linspace(start, end, num=n)
dx = (end - start) / n
The step size is actually (end-start)/(n-1).
Here:
x_grid, y_grid, z_grid = np.meshgrid(x, y, z)
df_dx[x, y, z] = (eval_xyz[x + 1, y, z] - eval_xyz[x, y, z]) / dx
you fell in the trap of meshgrid's default setting: meshgrid(np.arange(n1), np.arange(n2)) will return arrays in the shape (n2, n1) unless you add the parameter indexing='ij'. Because you have size n in all dimensions, you will not get indexing errors to alert you, but you might be spending a lot of time trying to debug why the numbers make no sense.
When you manipulate multidimensional arrays, it's a good idea to set the sizes in different directions to slightly different values, so that you can easily check that the array shapes are what you want them to be.
Also, you should generally evaluate the derivative as (f[i+1]-f[i-1])/(2*dx), which is correct up to the second order in x.
for x in range(eval_xyz.shape[0] - 1):
for y in range(eval_xyz.shape[1] - 1):
for z in range(eval_xyz.shape[2] - 1):
When working with numpy, you should always try to vectorize operations rather than writing out for loops that potentially need to iterate over thousands of elements.
Here is code that calculates the Cartesian derivative and then the radial derivative.
import numpy as np
def get_cartesian_gradient(f, xyzsteps):
"""For f shape (nx, ny, nz), return gradient as (3, nx, ny, nz) shape.
xyzsteps is a (3,) array.
Note: edge points of the gradient array are set to NaN.
(Exercise for the reader to implement those).
"""
fshape = f.shape
grad = np.full((3,) + fshape, np.nan, dtype=np.float64)
sl, sm, sr = slice(0, -2), slice(1, -1), slice(2, None)
# Note: multiplying is faster than dividing.
grad[0, sm, sm, sm] = (f[sr, sm, sm] - f[sl, sm, sm]) * (0.5/xyzsteps[0])
grad[1, sm, sm, sm] = (f[sm, sr, sm] - f[sm, sl, sm]) * (0.5/xyzsteps[1])
grad[2, sm, sm, sm] = (f[sm, sm, sr] - f[sm, sm, sl]) * (0.5/xyzsteps[2])
return grad
def get_dfdr_from_cartesian(grad, x1s, y1s, z1s):
"""Return df/dr array from gradient(f).
grad.shape must be (3, nx, ny, nz)
return shape (nx, ny, nz).
"""
_, nx, ny, nz = grad.shape
# we need sin(theta), cos(theta), sin(phi), and cos(phi)
# rxy: shape (nx, ny, 1)
rxy = np.sqrt(x1s.reshape(-1, 1, 1)**2 + y1s.reshape(1, -1, 1)**2)
# r: shape (nx, ny, nz)
r = np.sqrt(rxy**2 + z1s.reshape(1, 1, -1)**2)
# change zeros to NaN
r = np.where(r==0, np.nan, r)
rxy = np.where(rxy==0, np.nan, rxy)
cos_theta = z1s.reshape(1, 1, -1) / r
sin_theta = rxy / r
cos_phi = x1s.reshape(-1, 1, 1) / rxy
sin_phi = y1s.reshape(1, -1, 1) / rxy
# and the derivative
dfdr = (grad[0]*cos_phi + grad[1]*sin_phi)*sin_theta + grad[2]*cos_theta
return dfdr
x1s = np.linspace(-1, 1, 19)
y1s = np.linspace(-1, 1, 21)
z1s = np.linspace(-1, 1, 23)
xs, ys, zs = np.meshgrid(x1s, y1s, z1s, indexing='ij')
xyzsteps = [x1s[1]-x1s[0], y1s[1]-y1s[0], z1s[1]-z1s[0]]
def func(x, y, z):
return x**2 + y**2 + z**2
def dfdr_analytical(x, y, z):
r = np.sqrt(x**2 + y**2 + z**2)
return 2*r
# grad has shape (3, nx, ny, nz)
grad = get_cartesian_gradient(func(xs, ys, zs), xyzsteps)
dfdr = get_dfdr_from_cartesian(grad, x1s, y1s, z1s)
# test
diff = dfdr - dfdr_analytical(xs, ys, zs)
assert np.nanmax(np.abs(diff)) < 1e-14
Note that I've chosen to return NaN values for points on the z-axis, because df/dr is not defined there unless f(x,y,z) is rotationally symmetric around the z-axis and has df/dr=0 in all directions. This is something that is not guaranteed for an arbitrary dataset.
The reason for replacing zeros in the denominators by np.nan using np.where is because dividing by zero will give warning messages, whereas dividing by nan won't.
I've trying to simulate a 2D Sérsic profile and then testing an extraction routine on it. However, when I do a test by extracting all the points lying along an ellipse supposedly aligned with an image, I get a periodic function. It is meant to be a straight line since all points along the ellipse should have equal intensity, although there will be a small amount of deviation due to rounding errors in the rough coordinate estimation (get_I()).
from __future__ import division
import numpy as np
import matplotlib.pyplot as plt
from scipy.interpolate import NearestNDInterpolator
def rotate(x, y, angle):
x1 = x*np.cos(angle) + y*np.sin(angle)
y1 = y*np.cos(angle) - x*np.sin(angle)
return x1, y1
def sersic_1d(R, mu0, h, n, zp=0):
exponent = (R / h) ** (1 / n)
I0 = np.exp((zp - mu0) / 2.5)
return I0 * np.exp(-1.* exponent)
def sersic_2d(x, y, e, i, mu0, h, n, zp=0):
xp, yp = rotate(x, y, i)
alpha = np.arctan2(yp, xp * (1-e))
a = xp / np.cos(alpha)
b = a * (1 - e)
# R2 = (a*a) + ((1 - (e*e)) * yp*yp)
return sersic_1d(a, mu0, h, n, zp)
def ellipse(x0, y0, a, e, i, theta):
b = a * (1 - e)
x = a * np.cos(theta)
y = b * np.sin(theta)
x, y = rotate(x, y, i)
return x + x0, y + y0
def get_I(x, y, Z):
return Z[np.round(x).astype(int), np.round(y).astype(int)]
if __name__ == '__main__':
n = np.linspace(-100,100,1000)
nx, ny = np.meshgrid(n, n)
Z = sersic_2d(nx, ny, 0.5, 0., 0, 50, 1, 25)
theta = np.linspace(0, 2*np.pi, 1000.)
a = 100.
e = 0.5
i = np.pi / 4.
x, y = ellipse(0, 0, a, e, i, theta)
I = get_I(x, y, Z)
plt.plot(I)
# plt.imshow(Z)
plt.show()
However, What I actually get is a massive periodic function. I've checked the alignment and it's correct and the float-> int rounding errors can't account for this kind of shift?
Any ideas?
There are two things that strike me as odd, one of which for sure is not what you wanted, the other I'm not sure about because astronomy is not my field of expertise.
The first is in your function get_I:
def get_I(x, y, Z):
return Z[np.round(x).astype(int), np.round(y).astype(int)]
When you call that function, x an y outline an ellipse, with its center at the origin (0,0). That means x and y both become negative at some point. The indexing you perfom in that function will then take values from the array's last elements, because Z[0,0] is in fact the top left corner of the image (which you plotted, but commented), while Z[-1, -1] is the bottom right corner. What you want is to take the values of Z that are on the ellipse contour, but both have to have the same center. To do that, you would first make sure you use an uneven amount of samples for n (which ultimately defines the shape of Z) and second, you would add an indexing offset:
def get_I(x, y, Z):
offset = Z.shape[0]//2
return Z[np.round(y).astype(int) + offset, np.round(x).astype(int) + offset]
...
n = np.linspace(-100,100,1001) # changed from 1000 to 1001 to ensure a point of origin is present and that the image exhibits point symmetry
Also notice that I changed the order of y and x in get_I: that's because you first index along the rows (for which we usually take the y-coordinate) and only then along the columns (which map to the x-coordinate in most conventions).
The second item that struck me as unusual is that your ellipse has its axes at an angle of pi/4 with respect to the horizontal axis, whereas your sersic (which maps to the 2D array of Z) does not have a tilt at all.
Changing all that, I end up with this code:
from __future__ import division
import numpy as np
import matplotlib.pyplot as plt
def rotate(x, y, angle):
x1 = x*np.cos(angle) + y*np.sin(angle)
y1 = y*np.cos(angle) - x*np.sin(angle)
return x1, y1
def sersic_1d(R, mu0, h, n, zp=0):
exponent = (R / h) ** (1 / n)
I0 = np.exp((zp - mu0) / 2.5)
return I0 * np.exp(-1.* exponent)
def sersic_2d(x, y, e, ang, mu0, h, n, zp=0):
xp, yp = rotate(x, y, ang)
alpha = np.arctan2(yp, xp * (1-e))
a = xp / np.cos(alpha)
b = a * (1 - e)
return sersic_1d(a, mu0, h, n, zp)
def ellipse(x0, y0, a, e, i, theta):
b = a * (1 - e) # half of a
x = a * np.cos(theta)
y = b * np.sin(theta)
x, y = rotate(x, y, i) # rotated by 45deg
return x + x0, y + y0
def get_I(x, y, Z):
offset = Z.shape[0]//2
return Z[np.round(y).astype(int) + offset, np.round(x).astype(int) + offset]
#return Z[np.round(y).astype(int), np.round(x).astype(int)]
if __name__ == '__main__':
n = np.linspace(-100,100,1001) # changed
nx, ny = np.meshgrid(n, n)
ang = 0;#np.pi / 4.
Z = sersic_2d(nx, ny, 0.5, ang=0, mu0=0, h=50, n=1, zp=25)
f, ax = plt.subplots(1,2)
dn = n[1]-n[0]
ax[0].imshow(Z, cmap='gray', aspect='equal', extent=[-100-dn/2, 100+dn/2, -100-dn/2, 100+dn/2])
theta = np.linspace(0, 2*np.pi, 1000.)
a = 20. # decreased long axis of ellipse to see the intensity-map closer to the "center of the galaxy"
e = 0.5
x, y = ellipse(0,0, a, e, ang, theta)
I = get_I(x, y, Z)
ax[0].plot(x,y) # easier to see where you want the intensities
ax[1].plot(I)
plt.show()
and this image:
The intensity variations look like quantisation noise to me, with the exception of the peaks, which are due to the asymptote in sersic_1d.