NumPy indexing with varying position - python

I have an array input_data of shape (A, B, C), and an array ind of shape (B,). I want to loop through the B axis and take the sum of elements C[B[i]] and C[B[i]+1]. The desired output is of shape (A, B). I have the following code which works, but I feel is inefficient due to index-based looping through the B axis. Is there a more efficient method?
import numpy as np
input_data = np.random.rand(2, 6, 10)
ind = [ 2, 3, 5, 6, 5, 4 ]
out = np.zeros( ( input_data.shape[0], input_data.shape[1] ) )
for i in range( len(ind) ):
d = input_data[:, i, ind[i]:ind[i]+2]
out[:, i] = np.sum(d, axis = 1)
Edited based on Divakar's answer:
import timeit
import numpy as np
N = 1000
input_data = np.random.rand(10, N, 5000)
ind = ( 4999 * np.random.rand(N) ).astype(np.int)
def test_1(): # Old loop-based method
out = np.zeros( ( input_data.shape[0], input_data.shape[1] ) )
for i in range( len(ind) ):
d = input_data[:, i, ind[i]:ind[i]+2]
out[:, i] = np.sum(d, axis = 1)
return out
def test_2():
extent = 2 # Comes from 2 in "ind[i]:ind[i]+2"
m,n,r = input_data.shape
idx = (np.arange(n)*r + ind)[:,None] + np.arange(extent)
out1 = input_data.reshape(m,-1)[:,idx].reshape(m,n,-1).sum(2)
return out1
print timeit.timeit(stmt = test_1, number = 1000)
print timeit.timeit(stmt = test_2, number = 1000)
print np.all( test_1() == test_2(), keepdims = True )
>> 7.70429363482
>> 0.392034666757
>> [[ True]]

Here's a vectorized approach using linear indexing with some help from broadcasting. We merge the last two axes of the input array, calculate the linear indices corresponding to the last two axes, perform slicing and reshape back to a 3D shape. Finally, we do summation along the last axis to get the desired output. The implementation would look something like this -
extent = 2 # Comes from 2 in "ind[i]:ind[i]+2"
m,n,r = input_data.shape
idx = (np.arange(n)*r + ind)[:,None] + np.arange(extent)
out1 = input_data.reshape(m,-1)[:,idx].reshape(m,n,-1).sum(2)
If the extent is always going to be 2 as stated in the question - "... sum of elements C[B[i]] and C[B[i]+1]", then you could simply do -
m,n,r = input_data.shape
ind_arr = np.array(ind)
axis1_r = np.arange(n)
out2 = input_data[:,axis1_r,ind_arr] + input_data[:,axis1_r,ind_arr+1]

You could also use integer array indexing combined with basic slicing:
import numpy as np
m,n,r = 2, 6, 10
input_data = np.arange(2*6*10).reshape(m, n, r)
ind = np.array([ 2, 3, 5, 6, 5, 4 ])
out = np.zeros( ( input_data.shape[0], input_data.shape[1] ) )
for i in range( len(ind) ):
d = input_data[:, i, ind[i]:ind[i]+2]
out[:, i] = np.sum(d, axis = 1)
out2 = input_data[:, np.arange(n)[:,None], np.add.outer(ind,range(2))].sum(axis=-1)
print(out2)
# array([[ 5, 27, 51, 73, 91, 109],
# [125, 147, 171, 193, 211, 229]])
assert np.allclose(out, out2)

Related

get minimum value across array of indices

I have an n-by-3 index array (think of triangles indexing points) and a list of float values associated with the triangles. I now want to get for each index ("point") the minimum value, i.e., check all rows which contain the index, say, 0, and get the minimum value from vals across the respective rows:
import numpy
a = numpy.array([
[0, 1, 2],
[2, 3, 0],
[1, 4, 2],
[2, 5, 3],
])
vals = numpy.array([0.1, 0.5, 0.3, 0.6])
out = [
numpy.min(vals[numpy.any(a == i, axis=1)])
for i in range(6)
]
# out = numpy.array([0.1, 0.1, 0.1, 0.5, 0.3, 0.6])
This solution is inefficient because it does a full array comparison for every i.
This problem is quite similar to numpy's ufuncs, but numpy.min.at doesn't exist.
Any hints?
Approach #1
One approach based on array-assignment to setup a 2D array filled up NaNs, using those a values as column indices (so assumes those to be integers), then mapping vals into it and looking for nan-skipped min values for the final output -
nr,nc = len(a),a.max()+1
m = np.full((nr,nc),np.nan)
m[np.arange(nr)[:,None],a] = vals[:,None]
out = np.nanmin(m,axis=0)
Approach #2
Another one again based on array-assignment, but uses masking and np.minimum.reduceat in favor of dealing with NaNs -
nr,nc = len(a),a.max()+1
m = np.zeros((nc,nr),dtype=bool)
m[a.T,np.arange(nr)] = 1
c = m.sum(1)
shift_idx = np.r_[0,c[:-1].cumsum()]
out = np.minimum.reduceat(np.broadcast_to(vals,m.shape)[m],shift_idx)
Approach #3
Another based on argsort (assuming you have all integers from 0 to a.max() in a) -
sidx = a.ravel().argsort()
c = np.bincount(a.ravel())
out = np.minimum.reduceat(vals[sidx//a.shape[1]],np.r_[0,c[:-1].cumsum()])
Approach #4
For memory efficiency and hence perf. and also to complete the set -
from numba import njit
#njit
def numba1(a, vals, out):
m,n = a.shape
for j in range(m):
for i in range(n):
e = a[j,i]
if vals[j] < out[e]:
out[e] = vals[j]
return out
def func1(a, vals, outlen=None): # feed in output length as outlen if known
if outlen is not None:
N = outlen
else:
N = a.max()+1
out = np.full(N,np.inf)
return numba1(a, vals, out)
You may switch to pd.GroupBy or itertools.groupby if your for loop goes way beyond 6.
For instance,
r = n.ravel()
pd.Series(np.arange(len(r))//3).groupby(r).apply(lambda s: vals[s].min())
This solution would be faster for long loops, and probably slower for small loops (< 50)
Here is one based on this Q&A:
If you have pythran, compile
file <stb_pthr.py>
import numpy as np
#pythran export sort_to_bins(int[:], int)
def sort_to_bins(idx, mx):
if mx==-1:
mx = idx.max() + 1
cnts = np.zeros(mx + 2, int)
for i in range(idx.size):
cnts[idx[i]+2] += 1
for i in range(2, cnts.size):
cnts[i] += cnts[i-1]
res = np.empty_like(idx)
for i in range(idx.size):
res[cnts[idx[i]+1]] = i
cnts[idx[i]+1] += 1
return res, cnts[:-1]
Otherwise the script will fall back to a sparse matrix based approach which is only slightly slower:
import numpy as np
try:
from stb_pthr import sort_to_bins
HAVE_PYTHRAN = True
except:
HAVE_PYTHRAN = False
from scipy.sparse import csr_matrix
def sort_to_bins_sparse(idx, mx):
if mx==-1:
mx = idx.max() + 1
aux = csr_matrix((np.ones_like(idx),idx,np.arange(idx.size+1)),
(idx.size,mx)).tocsc()
return aux.indices, aux.indptr
if not HAVE_PYTHRAN:
sort_to_bins = sort_to_bins_sparse
def f_op():
mx = a.max() + 1
return np.fromiter((np.min(vals[np.any(a == i, axis=1)])
for i in range(mx)),vals.dtype,mx)
def f_pp():
idx, bb = sort_to_bins(a.reshape(-1),-1)
res = np.minimum.reduceat(vals[idx//3], bb[:-1])
res[bb[:-1]==bb[1:]] = np.inf
return res
def f_div_3():
sidx = a.ravel().argsort()
c = np.bincount(a.ravel())
bb = np.r_[0,c.cumsum()]
res = np.minimum.reduceat(vals[sidx//a.shape[1]],bb[:-1])
res[bb[:-1]==bb[1:]] = np.inf
return res
a = np.array([
[0, 1, 2],
[2, 3, 0],
[1, 4, 2],
[2, 5, 3],
])
vals = np.array([0.1, 0.5, 0.3, 0.6])
assert np.all(f_op()==f_pp())
from timeit import timeit
a = np.random.randint(0,1000,(10000,3))
vals = np.random.random(10000)
assert len(np.unique(a))==1000
assert np.all(f_op()==f_pp())
print("1000/1000 labels, 10000 rows")
print("op ", timeit(f_op, number=10)*100, 'ms')
print("pp ", timeit(f_pp, number=100)*10, 'ms')
print("div", timeit(f_div_3, number=100)*10, 'ms')
a = 1 + 2 * np.random.randint(0,5000,(1000000,3))
vals = np.random.random(1000000)
nl = len(np.unique(a))
assert np.all(f_div_3()==f_pp())
print(f"{nl}/{a.max()+1} labels, 1000000 rows")
print("pp ", timeit(f_pp, number=10)*100, 'ms')
print("div", timeit(f_div_3, number=10)*100, 'ms')
a = 1 + 2 * np.random.randint(0,100000,(1000000,3))
vals = np.random.random(1000000)
nl = len(np.unique(a))
assert np.all(f_div_3()==f_pp())
print(f"{nl}/{a.max()+1} labels, 1000000 rows")
print("pp ", timeit(f_pp, number=10)*100, 'ms')
print("div", timeit(f_div_3, number=10)*100, 'ms')
Sample run (timings include #Divakar approach 3 for reference):
1000/1000 labels, 10000 rows
op 145.1122640981339 ms
pp 0.7944229000713676 ms
div 2.2905819199513644 ms
5000/10000 labels, 1000000 rows
pp 113.86540920939296 ms
div 417.2476712032221 ms
100000/200000 labels, 1000000 rows
pp 158.23634970001876 ms
div 486.13436080049723 ms
UPDATE: #Divakar's latest (approach 4) is hard to beat, being essentially a C implementation. Nothing wrong with that except that jitting is not an option but a requirement here (the unjitted code is no fun to run). If one accepts that, the same can, of course, be done with pythran:
pythran -O3 labeled_min.py
file <labeled_min.py>
import numpy as np
#pythran export labeled_min(int[:,:], float[:])
def labeled_min(A, vals):
mn = np.empty(A.max()+1)
mn[:] = np.inf
M,N = A.shape
for i in range(M):
v = vals[i]
for j in range(N):
c = A[i,j]
if v < mn[c]:
mn[c] = v
return mn
Both give another massive speedup:
from labeled_min import labeled_min
func1() # do not measure jitting time
print("nmb ", timeit(func1, number=100)*10, 'ms')
print("pthr", timeit(lambda:labeled_min(a,vals), number=100)*10, 'ms')
Sample run:
nmb 8.41792532010004 ms
pthr 8.104007659712806 ms
pythran comes out a few percent faster but this is only because I moved vals lookup out of the inner loop; without that they are all but equal.
For comparison, the previously best with and without non python helpers on the same problem:
pp 114.04887529788539 ms
pp (py only) 147.0821460010484 ms
Apparently, numpy.minimum.at exists:
import numpy
a = numpy.array([
[0, 1, 2],
[2, 3, 0],
[1, 4, 2],
[2, 5, 3],
])
vals = numpy.array([0.1, 0.5, 0.3, 0.6])
out = numpy.full(6, numpy.inf)
numpy.minimum.at(out, a.reshape(-1), numpy.repeat(vals, 3))

How to broadcast or vectorize a linear interpolation of a 2D array that uses scipy.ndimage map_coordinates?

I have recently hit a roadblock when it comes to performance. I know how to manually loop and do the interpolation from the origin cell to all the other cells by brute-forcing/looping each row and column in 2d array.
however when I process a 2D array of a shape say (3000, 3000), the linear spacing and the interpolation come to a standstill and severely hurt performance.
I am looking for a way I can optimize this loop, I am aware of vectorization and broadcasting just not sure how I can apply it in this situation.
I will explain it with code and figures
import numpy as np
from scipy.ndimage import map_coordinates
m = np.array([
[10,10,10,10,10,10],
[9,9,9,10,9,9],
[9,8,9,10,8,9],
[9,7,8,0,8,9],
[8,7,7,8,8,9],
[5,6,7,7,6,7]])
origin_row = 3
origin_col = 3
m_max = np.zeros(m.shape)
m_dist = np.zeros(m.shape)
rows, cols = m.shape
for col in range(cols):
for row in range(rows):
# Get spacing linear interpolation
x_plot = np.linspace(col, origin_col, 5)
y_plot = np.linspace(row, origin_row, 5)
# grab the interpolated line
interpolated_line = map_coordinates(m,
np.vstack((y_plot,
x_plot)),
order=1, mode='nearest')
m_max[row][col] = max(interpolated_line)
m_dist[row][col] = np.argmax(interpolated_line)
print(m)
print(m_max)
print(m_dist)
As you can see this is very brute force, and I have managed to broadcast all the code around this part but stuck on this part.
here is an illustration of what I am trying to achieve, I will go through the first iteration
1.) the input array
2.) the first loop from 0,0 to origin (3,3)
3.) this will return [10 9 9 8 0] and the max will be 10 and the index will be 0
5.) here is the output for the sample array I used
Here is an update of the performance based on the accepted answer.
To speed up the code, you could first create the x_plot and y_plot outside of the loops instead of creating them several times each one:
#this would be outside of the loops
num = 5
lin_col = np.array([np.linspace(i, origin_col, num) for i in range(cols)])
lin_row = np.array([np.linspace(i, origin_row, num) for i in range(rows)])
then you could access them in each loop by x_plot = lin_col[col] and y_plot = lin_row[row]
Second, you can avoid both loops by using map_coordinates on more than just one v_stack for each couple (row, col). To do so, you can create all the combinaisons of x_plot and y_plot by using np.tile and np.ravel such as:
arr_vs = np.vstack(( np.tile( lin_row, cols).ravel(),
np.tile( lin_col.ravel(), rows)))
Note that ravel is not used at the same place each time to get all the combinaisons. Now you can use map_coordinates with this arr_vs and reshape the result with the number of rows, cols and num to get each interpolated_line in the last axis of a 3D-array:
arr_map = map_coordinates(m, arr_vs, order=1, mode='nearest').reshape(rows,cols,num)
Finally, you can use np.max and np.argmax on the last axis of arr_map to get the results m_max and m_dist. So all the code would be:
import numpy as np
from scipy.ndimage import map_coordinates
m = np.array([
[10,10,10,10,10,10],
[9,9,9,10,9,9],
[9,8,9,10,8,9],
[9,7,8,0,8,9],
[8,7,7,8,8,9],
[5,6,7,7,6,7]])
origin_row = 3
origin_col = 3
rows, cols = m.shape
num = 5
lin_col = np.array([np.linspace(i, origin_col, num) for i in range(cols)])
lin_row = np.array([np.linspace(i, origin_row, num) for i in range(rows)])
arr_vs = np.vstack(( np.tile( lin_row, cols).ravel(),
np.tile( lin_col.ravel(), rows)))
arr_map = map_coordinates(m, arr_vs, order=1, mode='nearest').reshape(rows,cols,num)
m_max = np.max( arr_map, axis=-1)
m_dist = np.argmax( arr_map, axis=-1)
print (m_max)
print (m_dist)
and you get like expected:
#m_max
array([[10, 10, 10, 10, 10, 10],
[ 9, 9, 10, 10, 9, 9],
[ 9, 9, 9, 10, 8, 9],
[ 9, 8, 8, 0, 8, 9],
[ 8, 8, 7, 8, 8, 9],
[ 7, 7, 8, 8, 8, 8]])
#m_dist
array([[0, 0, 0, 0, 0, 0],
[0, 0, 2, 0, 0, 0],
[0, 2, 0, 0, 0, 0],
[0, 1, 0, 0, 0, 0],
[0, 2, 0, 0, 0, 0],
[1, 1, 2, 1, 2, 1]])
EDIT: lin_col and lin_row are related, so you can do faster:
if cols >= rows:
arr = np.arange(cols)[:,None]
lin_col = arr + (origin_col-arr)/(num-1.)*np.arange(num)
lin_row = lin_col[:rows] + np.linspace(0, origin_row - origin_col, num)[None,:]
else:
arr = np.arange(rows)[:,None]
lin_row = arr + (origin_row-arr)/(num-1.)*np.arange(num)
lin_col = lin_row[:cols] + np.linspace(0, origin_col - origin_row, num)[None,:]
Here is a sort-of-vectorized approach. It is not very optimized and there may be one or two index-off-by-one errors, but it may give you ideas.
Two examples a monochrome 384x512 test pattern and a "real" 3-channel 768x1024 image. Both are uint8.
This takes half a minute on my machine.
For larger images one would require more RAM than I have (8GB). Or one would have to break it down into smaller chunks.
And the code
import numpy as np
def rays(img, ctr):
M, N, *d = img.shape
aidx = 2*(slice(None),) + (img.ndim-2)*(None,)
m, n = ctr
out = np.empty_like(img)
offsI = np.empty(img.shape, np.uint16)
offsJ = np.empty(img.shape, np.uint16)
img4, out4, I4, J4 = ((x[m:, n:], x[m:, n::-1], x[m::-1, n:], x[m::-1, n::-1]) for x in (img, out, offsI, offsJ))
for i, o, y, x in zip(img4, out4, I4, J4):
for _ in range(2):
M, N, *d = i.shape
widths = np.arange(1, M+1, dtype=np.uint16).clip(None, N)
I = np.arange(M, dtype=np.uint16).repeat(widths)
J = np.ones_like(I)
J[0] = 0
J[widths[:-1].cumsum()] -= widths[:-1]
J = J.cumsum(dtype=np.uint16)
ii = np.arange(1, 2*M-1, dtype=np.uint16) // 2
II = ii.clip(None, I[:, None])
jj = np.arange(2*M-2, dtype=np.uint32) // 2 * 2 + 1
jj[0] = 0
JJ = ((1 + jj) * J[:, None] // (2*(I+1))[:, None]).astype(np.uint16).clip(None, J[:, None])
idx = i[II, JJ].argmax(axis=1)
II, JJ = (np.take_along_axis(ZZ[aidx] , idx[:, None], 1)[:, 0] for ZZ in (II, JJ))
y[I, J], x[I, J] = II, JJ
SH = II, JJ, *np.ogrid[tuple(map(slice, img.shape))][2:]
o[I, J] = i[SH]
i, o = i.swapaxes(0, 1), o.swapaxes(0, 1)
y, x = x.swapaxes(0, 1), y.swapaxes(0, 1)
return out, offsI, offsJ
from scipy.misc import face
f = face()
fr, *fidx = rays(f, (200, 400))
s = np.uint8((np.arange(384)[:, None] % 41 < 2)&(np.arange(512) % 41 < 2))
s = 255*s + 128*s[::-1, ::-1] + 64*s[::-1] + 32*s[:, ::-1]
sr, *sidx = rays(s, (200, 400))
import Image
Image.fromarray(f).show()
Image.fromarray(fr).show()
Image.fromarray(s).show()
Image.fromarray(sr).show()

Numpy reshape from (m,w,l) to (w,m,l) dimension

I'm working with financial time series data and a bit confused with numpy reshape function. My goal is to calculate log-returns for adj_close parameter.
inputs = np.array([df_historical_data[key][-W:], axis = 1).values for key in stock_list])
inputs.shape //(8, 820, 5)
prices = inputs[:, :, 0]
prices.shape //(8, 820)
prices[:,0]
array([ 4.17000004e+02, 4.68800000e+00, 8.47889000e-03,
3.18835850e+00, 3.58412583e+00, 8.35364850e-01,
5.54610005e-04, 3.33600003e-05]) //close prices of 8 stock for 0 day
However for my program, I need the shape of my inputs be (820, 8, 5) so I decided to reshape my numpy array
inputs = np.array([df_historical_data[key][-W:], axis = 1).values for key in stock_list]).reshape(820, 8, 5)
inputs.shape //(820, 8, 5)
prices = inputs[:, :, 0]
prices.shape //(820, 8)
prices[0]
array([ 417.00000354, 436.5100001 , 441.00000442, 440. ,
416.10000178, 409.45245 , 422.999999 , 432.48000001])
// close price of 1 stock for 8 days
// but should be the same as in the example above
Seems that I didn't reshaped my array properly.
Anyway I can't understand why such strange behaviour occurs.
What you need is transpose not reshape.
Let's assume we have an array as follows:
import numpy as np
m, w, l = 2, 3, 4
array1 = np.array([[['m%d w%d l%d' % (mi, wi, li) for li in range(l)] for wi in range(w)] for mi in range(m)])
print(array1.shape)
print(array1)
Reshape is probably not what you want, but here is how can you do it:
array2 = array1.reshape(w, m, l)
print(array2.shape)
print(array2)
Here is how transpose is done:
# originally
# 0, 1, 2
# m, w, l
# -------
# transposed
array3 = array1.transpose(1, 0, 2)
# w, m, l
print(array3.shape)
print(array3)

Find repeat values in numpy nd array

My data samples are each a numpy array of shape e.g. (100, 100, 9), and I have 10 of these concatenated into a single array foo of shape (10, 100, 100, 9). Over the 10 data samples, I'd like to find the indices of repeat values. So for example, if foo[0, 42, 42, 3] = 0.72 and foo[0, 42, 42, 7] = 0.72, I'd like an output that reflects this. What is an efficient way of doing so?
I'm thinking a boolean output array of shape (100, 100, 9), but is there a better approach than looping to compare each data sample (quadratic runtime for the number of data samples (10))?
In the below snippet, dups is the desired result: a boolean array that shows which indices are duplicates. There's also a delta threshold, so any difference in values <= this threshold is a duplicate.
delta = 0.
dups = np.zeros(foo.shape[1:], dtype=bool)
for i in xrange(foo.shape[0]):
for j in xrange(foo.shape[0]):
if i==j: continue
dups += abs(foo[i] - foo[j]) <= delta
Here is a solution using argsort on each sample. Not pretty, not fast but does the job.
import numpy as np
from timeit import timeit
def dupl(a, axis=0, make_dict=True):
a = np.moveaxis(a, axis, -1)
i = np.argsort(a, axis=-1, kind='mergesort')
ai = a[tuple(np.ogrid[tuple(map(slice, a.shape))][:-1]) + (i,)]
same = np.zeros(a.shape[:-1] + (a.shape[-1]+1,), bool)
same[..., 1:-1] = np.diff(ai, axis=-1) == 0
uniqs = np.where((same[..., 1:] & ~same[..., :-1]).ravel())[0]
same = (same[...,1:]|same[...,:-1]).ravel()
reps = np.split(i.ravel()[same], np.cumsum(same)[uniqs[1:]-1])
grps = np.searchsorted(uniqs, np.arange(0, same.size, a.shape[-1]))
keys = ai.ravel()[uniqs]
if make_dict:
result = np.empty(a.shape[:-1], object)
result.ravel()[:] = [dict(zip(*p)) for p in np.split(
np.array([keys, reps], object), grps[1:], axis=-1)]
return result
else:
return keys, reps, grps
a = np.random.randint(0,10,(10,100,100,9))
axis = 0
result = dupl(a, axis)
print('shape, axis, time (sec) for 10 trials:',
a.shape, axis, timeit(lambda: dupl(a, axis=axis), number=10))
print('same without creating dict:',
a.shape, axis, timeit(lambda: dupl(a, axis=axis, make_dict=False),
number=10))
#check
print("checking result")
am = np.moveaxis(a, axis, -1)
for af, df in zip(am.reshape(-1, am.shape[-1]), result.ravel()):
assert len(set(af)) + sum(map(len, df.values())) == len(df) + am.shape[-1]
for k, v in df.items():
assert np.all(np.where(af == k)[0] == v)
print("no errors")
prints:
shape, axis, time (sec) for 10 trials: (10, 100, 100, 9) 0 5.328339613042772
same without creating dict: (10, 100, 100, 9) 0 2.568383438978344
checking result
no errors

adding numpy arrays of differing shapes

I'd like to add two numpy arrays of different shapes, but without broadcasting, rather the "missing" values are treated as zeros. Probably easiest with an example like
[1, 2, 3] + [2] -> [3, 2, 3]
or
[1, 2, 3] + [[2], [1]] -> [[3, 2, 3], [1, 0, 0]]
I do not know the shapes in advance.
I'm messing around with the output of np.shape for each, trying to find the smallest shape which holds both of them, embedding each in a zero-ed array of that shape and then adding them. But it seems rather a lot of work, is there an easier way?
Thanks in advance!
edit: by "a lot of work" I meant "a lot of work for me" rather than for the machine, I seek elegance rather than efficiency: my effort getting the smallest shape holding them both is
def pad(a, b) :
sa, sb = map(np.shape, [a, b])
N = np.max([len(sa),len(sb)])
sap, sbp = map(lambda x : x + (1,)*(N-len(x)), [sa, sb])
sp = np.amax( np.array([ tuple(sap), tuple(sbp) ]), 1)
not pretty :-/
I'm messing around with the output of np.shape for each, trying to find the smallest shape which holds both of them, embedding each in a zero-ed array of that shape and then adding them. But it seems rather a lot of work, is there an easier way?
Getting the np.shape is trivial, finding the smallest shape that holds both is very easy, and of course adding is trivial, so the only "a lot of work" part is the "embedding each in a zero-ed array of that shape".
And yes, you can eliminate that, by just calling the resize method (or the resize function, if you want to make copies instead of changing them in-place). As the docs explain:
Enlarging an array: … missing entries are filled with zeros
For example, if you know the dimensionality statically:
>>> a1 = np.array([[1, 2, 3], [4, 5, 6]])
>>> a2 = np.array([[2], [2]])
>>> shape = [max(a.shape[axis] for a in (a1, a2)) for axis in range(2)]
>>> a1.resize(shape)
>>> a2.resize(shape)
>>> print(a1 + a2)
array([[3, 4, 3],
[4, 5, 6]])
This is the best I could come up with:
import numpy as np
def magic_add(*args):
n = max(a.ndim for a in args)
args = [a.reshape((n - a.ndim)*(1,) + a.shape) for a in args]
shape = np.max([a.shape for a in args], 0)
result = np.zeros(shape)
for a in args:
idx = tuple(slice(i) for i in a.shape)
result[idx] += a
return result
You can clean up the for loop a little if you know how many dimensions you expect on result, something like:
for a in args:
i, j = a.shape
result[:i, :j] += a
You may try my solution - for dimension 1 arrays you have to expand your arrays to
dimension 2 (as shown in the example below), before passing it to the function.
import numpy as np
import timeit
matrix1 = np.array([[0,10],
[1,20],
[2,30]])
matrix2 = np.array([[0,10],
[1,20],
[2,30],
[3,40]])
matrix3 = np.arange(0,0,dtype=int) # empty numpy-array
matrix3.shape = (0,2) # reshape to 0 rows
matrix4 = np.array([[0,10,100,1000],
[1,20,200,2000]])
matrix5 = np.arange(0,4000,1)
matrix5 = np.reshape(matrix5,(4,1000))
matrix6 = np.arange(0.0,4000,0.5)
matrix6 = np.reshape(matrix6,(20,400))
matrix1 = np.array([1,2,3])
matrix1 = np.expand_dims(matrix1, axis=0)
matrix2 = np.array([2,1])
matrix2 = np.expand_dims(matrix2, axis=0)
def add_2d_matrices(m1, m2, pos=(0,0), filler=None):
"""
Add two 2d matrices of different sizes or shapes,
offset by xy coordinates, whereat x is "from left to right" (=axis:1)
and y is "from top to bottom" (=axis:0)
Parameterse:
- m1: first matrix
- m2: second matrix
- pos: tuple (x,y) containing coordinates for m2 offset,
- filler: gaps are filled with the value of filler (or zeros)
Returns:
- 2d array (float):
containing filler-values, m1-values, m2-values
or the sum of m1,m2 (at overlapping areas)
Author:
Reinhard Daemon, Austria
"""
# determine shape of final array:
_m1 = np.copy(m1)
_m2 = np.copy(m2)
x,y = pos
y1,x1 = _m1.shape
y2,x2 = _m2.shape
xmax = max(x1, x2+x)
ymax = max(y1, y2+y)
# fill-up _m1 array with zeros:
y1,x1 = _m1.shape
diff = xmax - x1
_z = np.zeros((y1,diff))
_m1 = np.hstack((_m1,_z))
y1,x1 = _m1.shape
diff = ymax - y1
_z = np.zeros((diff,x1))
_m1 = np.vstack((_m1,_z))
# shift _m2 array by 'pos' and fill-up with zeros:
y2,x2 = _m2.shape
_z = np.zeros((y2,x))
_m2 = np.hstack((_z,_m2))
y2,x2 = _m2.shape
diff = xmax - x2
_z = np.zeros((y2,diff))
_m2 = np.hstack((_m2,_z))
y2,x2 = _m2.shape
_z = np.zeros((y,x2))
_m2 = np.vstack((_z,_m2))
y2,x2 = _m2.shape
diff = ymax - y2
_z = np.zeros((diff,x2))
_m2 = np.vstack((_m2,_z))
# add the 2 arrays:
_m3 = _m1 + _m2
# find and fill the "unused" positions within the summed array:
if filler not in (None,0,0.0):
y1,x1 = m1.shape
y2,x2 = m2.shape
x1min = 0
x1max = x1-1
y1min = 0
y1max = y1-1
x2min = x
x2max = x + x2-1
y2min = y
y2max = y + y2-1
for xx in range(xmax):
for yy in range(ymax):
if x1min <= xx <= x1max and y1min <= yy <= y1max:
continue
if x2min <= xx <= x2max and y2min <= yy <= y2max:
continue
_m3[yy,xx] = filler
return(_m3)
t1 = timeit.Timer("add_2d_matrices(matrix5, matrix6, pos=(1,1), filler=111.111)", \
"from __main__ import add_2d_matrices,matrix5,matrix6")
print("ran:",t1.timeit(number=10), "milliseconds")
print("\n\n")
my_res = add_2d_matrices(matrix1, matrix2, pos=(1,1), filler=99.99)
print(my_res)

Categories