i'm working on portefolio optimization so i want to minimize a function which computes the variance of its net result subject to its mean is equal or larger than a certain expected result exp :
minimize Var (Net Result) subject to E(Net result) >=exp
i defined my objective function as :
R=[]
length_to_split=[26,56,46,38,48,38,45,55,59,47]
from itertools import accumulate
def objective(z):
for i in range(0, len(dataframe)):
R.append((1- max(0, (dataframe.iloc[i,1]- z) /dataframe.iloc[i,1]) )*(dataframe.iloc[i,2]-dataframe.iloc[i,1]))
R_ann=[R[x - y: x] for x, y in zip(accumulate(length_to_split), length_to_split)] #each sublist
#contains results of one year
R_avg=sum([sum(r) for r in R_ann ] )/len([sum(r) for r in R_ann ]) #[sum(r) for r in R_ann ] will
#return 10 results corresponding to net results for each year from 2010 to 2019.
var = sum((Rj-R_avg)**2 for Rj in [sum(r) for r in R_ann ]) / len([sum(r) for r in R_ann ])
return var
dataframe is a data which contains 4 columns where the first one represents costs and the second one premiums and then i defined my constraint function as :
K=[]
from statistics import mean
def constraint(z):
exp=1000000
for i in range(0, len(dataframe)):
if dataframe.iloc[i,1]>z:
K.append((1- ((dataframe.iloc[i,1]-z) /dataframe.iloc[i,1])) * (dataframe.iloc[i,2]-dataframe.iloc[1,1]))
else:
K.append((dataframe.iloc[i,2]-dataframe.iloc[i,1]))
K_ann=[K[v - w: v] for v, w in zip(accumulate(length_to_split), length_to_split)]
return mean([sum(j) for j in K_ann]) -exp
When i tried to minimize the objective function as,
from scipy.optimize import minimize
con = {'type': 'ineq', 'fun': constraint}
guess=500000
minimize(objective,guess,constraints=con)
it gave me the error below :
---------------------------------------------------------------------------
TypeError Traceback (most recent call last)
<ipython-input-404-ff80aabebc42> in <module>
2 con = {'type': 'ineq', 'fun': constraint}
3 guess=500000
----> 4 minimize(objective,guess,constraints=con)
5 # initial guesses
~\Anaconda3\lib\site-packages\scipy\optimize\_minimize.py in minimize(fun, x0, args, method, jac, hess, hessp, bounds, constraints, tol, callback, options)
607 elif meth == 'slsqp':
608 return _minimize_slsqp(fun, x0, args, jac, bounds,
--> 609 constraints, callback=callback, **options)
610 elif meth == 'trust-constr':
611 return _minimize_trustregion_constr(fun, x0, args, jac, hess, hessp,
~\Anaconda3\lib\site-packages\scipy\optimize\slsqp.py in _minimize_slsqp(func, x0, args, jac, bounds, constraints, maxiter, ftol, iprint, disp, eps, callback, **unknown_options)
313 for c in cons['eq']]))
314 mieq = sum(map(len, [atleast_1d(c['fun'](x, *c['args']))
--> 315 for c in cons['ineq']]))
316 # m = The total number of constraints
317 m = meq + mieq
~\Anaconda3\lib\site-packages\scipy\optimize\slsqp.py in <listcomp>(.0)
313 for c in cons['eq']]))
314 mieq = sum(map(len, [atleast_1d(c['fun'](x, *c['args']))
--> 315 for c in cons['ineq']]))
316 # m = The total number of constraints
317 m = meq + mieq
<ipython-input-401-fe7f26be36f9> in constraint(z)
10 K.append((dataframe.iloc[i,2]-dataframe.iloc[i,1]))
11 K_ann=[K[v - w: v] for v, w in zip(accumulate(length_to_split), length_to_split)]
---> 12 return mean([sum(j) for j in K_ann]) -exp
13 #contrainte(900000)
14
~\Anaconda3\lib\statistics.py in mean(data)
309 if n < 1:
310 raise StatisticsError('mean requires at least one data point')
--> 311 T, total, count = _sum(data)
312 assert count == n
313 return _convert(total/n, T)
~\Anaconda3\lib\statistics.py in _sum(data, start)
145 for typ, values in groupby(data, type):
146 T = _coerce(T, typ) # or raise TypeError
--> 147 for n,d in map(_exact_ratio, values):
148 count += 1
149 partials[d] = partials_get(d, 0) + n
~\Anaconda3\lib\statistics.py in _exact_ratio(x)
227 return (x, None)
228 msg = "can't convert type '{}' to numerator/denominator"
--> 229 raise TypeError(msg.format(type(x).__name__))
230
231
TypeError: can't convert type 'ndarray' to numerator/denominator
Could someone help me solve this problem please? i could not get the correct result i think the problem is with my constraint function even when i fix this type of error.
Related
I am trying to calibrate a model using pykalman and the scipy optimiser. For some reasons scipy seem to think that my input is a masked array, but it is not. I have added the code below:
k = 0.00000000000001 #small sarting value
T = np.array([1, 2, 3, 4, 5] , dtype=int) #maturities
delta = 1e-9
x = k
transition_covariance=delta / (1 - delta) * np.eye(1)
def obj(x):
k = x[0]
Q = (1 - k) *np.eye(1)
Z = np.exp(-k*T)
Z =Z[:, None] #reshape to 2 dim
transition_covariance=delta / (1 - delta) * np.eye(1)
observation_covariance=0.001*np.eye(len(T))
kf = KalmanFilter(transition_matrices=np.eye(1)*(1-k),
observation_matrices= Z,
transition_covariance=transition_covariance,
observation_covariance=observation_covariance)
lk = kf.loglikelihood(X) # X is my observed data, 1265 rows × 5 columns
return -lk
x = np.array([k])
k =scipy.optimize.minimize(obj, x, method= 'TNC')
Running the scipy optimiser I get the error:
---------------------------------------------------------------------------
ValueError Traceback (most recent call last)
<ipython-input-16-8d87c9f2ced0> in <module>
1 # method that works: TNC, Nelder-Mead
----> 2 k =scipy.optimize.minimize(obj, x, method= 'TNC')
~\Anaconda3\lib\site-packages\scipy\optimize\_minimize.py in minimize(fun, x0, args, method, jac, hess, hessp, bounds, constraints, tol, callback, options)
618 callback=callback, **options)
619 elif meth == 'tnc':
--> 620 return _minimize_tnc(fun, x0, args, jac, bounds, callback=callback,
621 **options)
622 elif meth == 'cobyla':
~\Anaconda3\lib\site-packages\scipy\optimize\tnc.py in _minimize_tnc(fun, x0, args, jac, bounds, eps, scale, offset, mesg_num, maxCGit, maxiter, eta, stepmx, accuracy, minfev, ftol, xtol, gtol, rescale, disp, callback, finite_diff_rel_step, maxfun, **unknown_options)
373 messages = MSG_NONE
374
--> 375 sf = _prepare_scalar_function(fun, x0, jac=jac, args=args, epsilon=eps,
376 finite_diff_rel_step=finite_diff_rel_step,
377 bounds=new_bounds)
~\Anaconda3\lib\site-packages\scipy\optimize\optimize.py in _prepare_scalar_function(fun, x0, jac, args, bounds, epsilon, finite_diff_rel_step, hess)
259 # ScalarFunction caches. Reuse of fun(x) during grad
260 # calculation reduces overall function evaluations.
--> 261 sf = ScalarFunction(fun, x0, args, grad, hess,
262 finite_diff_rel_step, bounds, epsilon=epsilon)
263
~\Anaconda3\lib\site-packages\scipy\optimize\_differentiable_functions.py in __init__(self, fun, x0, args, grad, hess, finite_diff_rel_step, finite_diff_bounds, epsilon)
74
75 self._update_fun_impl = update_fun
---> 76 self._update_fun()
77
78 # Gradient evaluation
~\Anaconda3\lib\site-packages\scipy\optimize\_differentiable_functions.py in _update_fun(self)
164 def _update_fun(self):
165 if not self.f_updated:
--> 166 self._update_fun_impl()
167 self.f_updated = True
168
~\Anaconda3\lib\site-packages\scipy\optimize\_differentiable_functions.py in update_fun()
71
72 def update_fun():
---> 73 self.f = fun_wrapped(self.x)
74
75 self._update_fun_impl = update_fun
~\Anaconda3\lib\site-packages\scipy\optimize\_differentiable_functions.py in fun_wrapped(x)
68 def fun_wrapped(x):
69 self.nfev += 1
---> 70 return fun(x, *args)
71
72 def update_fun():
<ipython-input-13-48731c614f15> in obj(x)
19 observation_covariance=observation_covariance)
20
---> 21 lk = kf.loglikelihood(X) # X is my observed data, 1016 rows × 17 columns
22 return -lk
~\Anaconda3\lib\site-packages\pykalman\standard.py in loglikelihood(self, X)
1470
1471 # get likelihoods for each time step
-> 1472 loglikelihoods = _loglikelihoods(
1473 observation_matrices, observation_offsets, observation_covariance,
1474 predicted_state_means, predicted_state_covariances, Z
~\Anaconda3\lib\site-packages\pykalman\standard.py in _loglikelihoods(observation_matrices, observation_offsets, observation_covariance, predicted_state_means, predicted_state_covariances, observations)
165 + observation_covariance
166 )
--> 167 loglikelihoods[t] = log_multivariate_normal_density(
168 observation[np.newaxis, :],
169 predicted_observation_mean[np.newaxis, :],
~\Anaconda3\lib\site-packages\pykalman\utils.py in log_multivariate_normal_density(X, means, covars, min_covar)
71 lower=True)
72 cv_log_det = 2 * np.sum(np.log(np.diagonal(cv_chol)))
---> 73 cv_sol = solve_triangular(cv_chol, (X - mu).T, lower=True).T
74 log_prob[:, c] = - .5 * (np.sum(cv_sol ** 2, axis=1) + \
75 n_dim * np.log(2 * np.pi) + cv_log_det)
~\Anaconda3\lib\site-packages\scipy\linalg\basic.py in solve_triangular(a, b, trans, lower, unit_diagonal, overwrite_b, debug, check_finite)
332
333 a1 = _asarray_validated(a, check_finite=check_finite)
--> 334 b1 = _asarray_validated(b, check_finite=check_finite)
335 if len(a1.shape) != 2 or a1.shape[0] != a1.shape[1]:
336 raise ValueError('expected square matrix')
~\Anaconda3\lib\site-packages\scipy\_lib\_util.py in _asarray_validated(a, check_finite, sparse_ok, objects_ok, mask_ok, as_inexact)
259 if not mask_ok:
260 if np.ma.isMaskedArray(a):
--> 261 raise ValueError('masked arrays are not supported')
262 toarray = np.asarray_chkfinite if check_finite else np.asarray
263 a = toarray(a)
ValueError: masked arrays are not supported
Maybe something to do with an update in the scipy library? It worked with a different setup, so that's all I can think. My versions are:
Python 3.8.5
Scipy 1.5.2
Pykalman 0.9.5
Thanks!
I found the solution, which involves a small change in the utils.py file in the pykalman library (line 73):
try:
cv_sol = solve_triangular(cv_chol, (X - mu).T, lower=True).T
except ValueError:
cv_sol = np.linalg.solve(cv_chol, (X - mu).T).T
Source:
https://github.com/pykalman/pykalman/issues/83
i have a function like this
cos(x) - sin(y) + 0.2 = 0
i want to draw this function in plt.plot and find root (x,y) when function = 0
and this is my code
from scipy import optimize
import numpy as np
def func_2(x):
return np.cos(x[0])-np.sin(x[1])+0.2
result = optimize.root(func_2, [1])
and i got error
IndexError Traceback (most recent call last)
<ipython-input-366-6d13d246d69c> in <module>
14
15 result = optimize.root(func_1, [1])
---> 16 result1 = optimize.root(func_2, [1])
~\anaconda3\lib\site-packages\scipy\optimize\_root.py in root(fun, x0, args, method, jac, tol, callback, options)
185
186 if meth == 'hybr':
--> 187 sol = _root_hybr(fun, x0, args=args, jac=jac, **options)
188 elif meth == 'lm':
189 sol = _root_leastsq(fun, x0, args=args, jac=jac, **options)
~\anaconda3\lib\site-packages\scipy\optimize\minpack.py in _root_hybr(func, x0, args, jac, col_deriv, xtol, maxfev, band, eps, factor, diag, **unknown_options)
211 if not isinstance(args, tuple):
212 args = (args,)
--> 213 shape, dtype = _check_func('fsolve', 'func', func, x0, args, n, (n,))
214 if epsfcn is None:
215 epsfcn = finfo(dtype).eps
~\anaconda3\lib\site-packages\scipy\optimize\minpack.py in _check_func(checker, argname, thefunc, x0, args, numinputs, output_shape)
24 def _check_func(checker, argname, thefunc, x0, args, numinputs,
25 output_shape=None):
---> 26 res = atleast_1d(thefunc(*((x0[:numinputs],) + args)))
27 if (output_shape is not None) and (shape(res) != output_shape):
28 if (output_shape[0] != 1):
<ipython-input-366-6d13d246d69c> in func_2(x)
11
12 def func_2(x):
---> 13 return np.cos(x[0])-np.sin(x[1])+0.2
14
15 result = optimize.root(func_1, [1])
IndexError: index 1 is out of bounds for axis 0 with size 1
I'm trying to optimize this model:
minimize obj: sum(variables * a)
s.t: 1) sum(log(variables)*c) >= b
sum(variables) = 1
variables >= 0
where 'a' and 'b' are constant numbers and 'c' is a vector of type probability. I tried three different methods from scipy package but none of them worked. here is my code using 'trust-constr' method
def objective(x):
return np.sum(np.dot(x,v))
lincos = LinearConstraint( [1]*(len(f)), 1,1)
lincos1 = LinearConstraint( [1]*(len(f)), 0, np.inf)
nonlincos = NonlinearConstraint(constraint,0.01, np.inf)
result1 = minimize(objective, f, method='trust-constr', constraints=[lincos,lincos1 ,nonlincos ] )
this gives an value error below which I think it's related to my log constraint but I don't know how to get around this.
................................................................................
ValueError Traceback (most recent call last)
<ipython-input-97-18fbd5e75692> in <module>
----> 1 result1 = minimize(objective, f, method='trust-constr', constraints=[lincos,lincos1 ,nonlincos ] )
~\Anaconda3\lib\site-packages\scipy\optimize\_minimize.py in minimize(fun, x0, args, method, jac, hess, hessp, bounds, constraints, tol, callback, options)
620 return _minimize_trustregion_constr(fun, x0, args, jac, hess, hessp,
621 bounds, constraints,
--> 622 callback=callback, **options)
623 elif meth == 'dogleg':
624 return _minimize_dogleg(fun, x0, args, jac, hess,
~\Anaconda3\lib\site-packages\scipy\optimize\_trustregion_constr\minimize_trustregion_constr.py in _minimize_trustregion_constr(fun, x0, args, grad, hess, hessp, bounds, constraints, xtol, gtol, barrier_tol, sparse_jacobian, callback, maxiter, verbose, finite_diff_rel_step, initial_constr_penalty, initial_tr_radius, initial_barrier_parameter, initial_barrier_tolerance, factorization_method, disp)
517 initial_barrier_tolerance,
518 initial_constr_penalty, initial_tr_radius,
--> 519 factorization_method)
520
521 # Status 3 occurs when the callback function requests termination,
~\Anaconda3\lib\site-packages\scipy\optimize\_trustregion_constr\tr_interior_point.py in tr_interior_point(fun, grad, lagr_hess, n_vars, n_ineq, n_eq, constr, jac, x0, fun0, grad0, constr_ineq0, jac_ineq0, constr_eq0, jac_eq0, stop_criteria, enforce_feasibility, xtol, state, initial_barrier_parameter, initial_tolerance, initial_penalty, initial_trust_radius, factorization_method)
327 constr0_subprob, jac0_subprob, subprob.stop_criteria,
328 state, initial_penalty, trust_radius,
--> 329 factorization_method, trust_lb, trust_ub, subprob.scaling)
330 if subprob.terminate:
331 break
~\Anaconda3\lib\site-packages\scipy\optimize\_trustregion_constr\equality_constrained_sqp.py in equality_constrained_sqp(fun_and_constr, grad_and_jac, lagr_hess, x0, fun0, grad0, constr0, jac0, stop_criteria, state, initial_penalty, initial_trust_radius, factorization_method, trust_lb, trust_ub, scaling)
119 dt, cg_info = projected_cg(H, c_t, Z, Y, b_t,
120 trust_radius_t,
--> 121 lb_t, ub_t)
122
123 # Compute update (normal + tangential steps).
~\Anaconda3\lib\site-packages\scipy\optimize\_trustregion_constr\qp_subproblem.py in projected_cg(H, c, Z, Y, b, trust_radius, lb, ub, tol, max_iter, max_infeasible_iter, return_all)
497 # Initial Values
498 x = Y.dot(-b)
--> 499 r = Z.dot(H.dot(x) + c)
500 g = Z.dot(r)
501 p = -g
~\Anaconda3\lib\site-packages\scipy\sparse\linalg\interface.py in dot(self, x)
413
414 if x.ndim == 1 or x.ndim == 2 and x.shape[1] == 1:
--> 415 return self.matvec(x)
416 elif x.ndim == 2:
417 return self.matmat(x)
~\Anaconda3\lib\site-packages\scipy\sparse\linalg\interface.py in matvec(self, x)
227 raise ValueError('dimension mismatch')
228
--> 229 y = self._matvec(x)
230
231 if isinstance(x, np.matrix):
~\Anaconda3\lib\site-packages\scipy\sparse\linalg\interface.py in _matvec(self, x)
525
526 def _matvec(self, x):
--> 527 return self.__matvec_impl(x)
528
529 def _rmatvec(self, x):
~\Anaconda3\lib\site-packages\scipy\optimize\_trustregion_constr\projections.py in null_space(x)
191 # v = P inv(R) Q.T x
192 aux1 = Q.T.dot(x)
--> 193 aux2 = scipy.linalg.solve_triangular(R, aux1, lower=False)
194 v = np.zeros(m)
195 v[P] = aux2
~\Anaconda3\lib\site-packages\scipy\linalg\basic.py in solve_triangular(a, b, trans, lower, unit_diagonal, overwrite_b, debug, check_finite)
334
335 a1 = _asarray_validated(a, check_finite=check_finite)
--> 336 b1 = _asarray_validated(b, check_finite=check_finite)
337 if len(a1.shape) != 2 or a1.shape[0] != a1.shape[1]:
338 raise ValueError('expected square matrix')
~\Anaconda3\lib\site-packages\scipy\_lib\_util.py in _asarray_validated(a, check_finite, sparse_ok, objects_ok, mask_ok, as_inexact)
244 raise ValueError('masked arrays are not supported')
245 toarray = np.asarray_chkfinite if check_finite else np.asarray
--> 246 a = toarray(a)
247 if not objects_ok:
248 if a.dtype is np.dtype('O'):
~\Anaconda3\lib\site-packages\numpy\lib\function_base.py in asarray_chkfinite(a, dtype, order)
497 if a.dtype.char in typecodes['AllFloat'] and not np.isfinite(a).all():
498 raise ValueError(
--> 499 "array must not contain infs or NaNs")
500 return a
501
ValueError: array must not contain infs or NaNs
I have a function findMaxEval which I invoke in a following way:
eMax0,var0=findMaxEval(np.diag(eVal0),q,bWidth=.01)
where np.diag(eVal0) is an ndarray of shape (1000,), q is a number (10).
findMaxEval has the following definition:
def findMaxEval(eVal,q,bWidth):
out=minimize(lambda *x:errPDFs(*x),.5,args= (eVal,q,bWidth),bounds=((1E-5,1-1E-5),))
if out['success']:var=out['x'][0]
else:var=1
eMax=var*(1+(1./q)**.5)**2
return eMax,var
This funtion tries to minimize errPDFs which is defined as follows:
def errPDFs(var,eVal,q,bWidth,pts=1000):
pdf0=mpPDF(var,q,pts)
pdf1=fitKDE(eVal,bWidth,x=pdf0.index.values)
sse=np.sum((pdf1-pdf0)**2)
return sse
var is a number which I pass in the findMaxEval function in minimize, initial value is 0.5.
Further, mpPDF and fitKDE are defined:
def mpPDF(var,q,pts):
eMin,eMax=var*(1-(1./q)**.5)**2,var*(1+(1./q)**.5)**2
eVal=np.linspace(eMin,eMax,pts)
pdf=q/(2*np.pi*var*eVal)*((eMax-eVal)*(eVal-eMin))**.5
pdf=pd.Series(pdf,index=eVal)
return pdf
def fitKDE(obs,bWidth=.25,kernel='gaussian',x=None):
if len(obs.shape)==1:obs=obs.reshape(-1,1)
kde=KernelDensity(kernel=kernel,bandwidth=bWidth).fit(obs)
if x is None:x=np.unique(obs).reshape(-1,1)
if len(x.shape)==1:x=x.reshape(-1,1)
logProb=kde.score_samples(x) # log(density)
pdf=pd.Series(np.exp(logProb),index=x.flatten())
return pdf
When I invoke findMaxEval (first line in the description), I get the following error:
---------------------------------------------------------------------------
Exception Traceback (most recent call last)
<ipython-input-25-abd7cf64e843> in <module>
----> 1 eMax0,var0=findMaxEval(np.diag(eVal0),q,bWidth=.01)
2 nFacts0=eVal0.shape[0]-np.diag(eVal0)[::-1].searchsorted(eMax0)
<ipython-input-24-f44a1e9d84b1> in findMaxEval(eVal, q, bWidth)
1 def findMaxEval(eVal,q,bWidth):
2 # Find max random eVal by fitting Marcenko’s dist
----> 3 out=minimize(lambda *x:errPDFs(*x),.5,args= (eVal,q,bWidth),bounds=((1E-5,1-1E-5),))
4 if out['success']:var=out['x'][0]
5 else:var=1
/opt/anaconda3/lib/python3.7/site-packages/scipy/optimize/_minimize.py in minimize(fun, x0, args, method, jac, hess, hessp, bounds, constraints, tol, callback, options)
598 return _minimize_neldermead(fun, x0, args, callback, **options)
599 elif meth == 'powell':
--> 600 return _minimize_powell(fun, x0, args, callback, **options)
601 elif meth == 'cg':
602 return _minimize_cg(fun, x0, args, jac, callback, **options)
/opt/anaconda3/lib/python3.7/site-packages/scipy/optimize/lbfgsb.py in _minimize_lbfgsb(fun, x0, args, jac, bounds, disp, maxcor, ftol, gtol, eps, maxfun, maxiter, iprint, callback, maxls, **unknown_options)
333
334 while 1:
--> 335 # x, f, g, wa, iwa, task, csave, lsave, isave, dsave = \
336 _lbfgsb.setulb(m, x, low_bnd, upper_bnd, nbd, f, g, factr,
337 pgtol, wa, iwa, task, iprint, csave, lsave,
/opt/anaconda3/lib/python3.7/site-packages/scipy/optimize/lbfgsb.py in func_and_grad(x)
278 # unbounded variables must use None, not +-inf, for optimizer to work properly
279 bounds = [(None if l == -np.inf else l, None if u == np.inf else u) for l, u in bounds]
--> 280
281 if disp is not None:
282 if disp == 0:
/opt/anaconda3/lib/python3.7/site-packages/scipy/optimize/optimize.py in function_wrapper(*wrapper_args)
324
325 def function_wrapper(*wrapper_args):
--> 326 ncalls[0] += 1
327 return function(*(wrapper_args + args))
328
<ipython-input-24-f44a1e9d84b1> in <lambda>(*x)
1 def findMaxEval(eVal,q,bWidth):
2 # Find max random eVal by fitting Marcenko’s dist
----> 3 out=minimize(lambda *x:errPDFs(*x),.5,args= (eVal,q,bWidth),bounds=((1E-5,1-1E-5),))
4 if out['success']:var=out['x'][0]
5 else:var=1
<ipython-input-23-24070a331535> in errPDFs(var, eVal, q, bWidth, pts)
1 def errPDFs(var,eVal,q,bWidth,pts=1000):
2 # Fit error
----> 3 pdf0=mpPDF(var,q,pts) # theoretical pdf
4 pdf1=fitKDE(eVal,bWidth,x=pdf0.index.values) # empirical pdf
5 sse=np.sum((pdf1-pdf0)**2)
<ipython-input-17-565d70018af2> in mpPDF(var, q, pts)
10 eVal=np.linspace(eMin,eMax,pts)
11 pdf=q/(2*np.pi*var*eVal)*((eMax-eVal)*(eVal-eMin))**.5
---> 12 pdf=pd.Series(pdf,index=eVal)
13 return pdf
/opt/anaconda3/lib/python3.7/site-packages/pandas/core/series.py in __init__(self, data, index, dtype, name, copy, fastpath)
312
313 def _init_dict(self, data, index=None, dtype=None):
--> 314 """
315 Derive the "_data" and "index" attributes of a new Series from a
316 dictionary input.
/opt/anaconda3/lib/python3.7/site-packages/pandas/core/internals/construction.py in sanitize_array(data, index, dtype, copy, raise_cast_failure)
Exception: Data must be 1-dimensional
I don't understand what should be 1-Dimensional. np.diag(eVal0) is of shape (1000,).
I looked at all the other similar questions, but none seems to help me solve this.
Thanks.
The error is unrelated to bounds.
For some reason minimize() calls the custom function errPDFs() with the argument to be optimized - minimize() calls this for x0 - which is an array. So if you redefine the function errPDFs() to extract the first element of an array:
def errPDFs(var, eVal, q, bWidth, pts=1000):
print("var:"+var)
pdf0 = mpPDF(var[0], q, pts) #theoretical pdf
pdf1 = fitKDE(eVal, bWidth, x=pdf0.index.values) #empirical pdf
sse = np.sum((pdf1-pdf0)**2)
print("sse:"+str(sse))
return sse
It should work.
Sample output:
>>> out = minimize(lambda *x: errPDFs(*x), .5, args=(eVal, q, bWidth),bounds=
((1E-5, 1-1E-5),))
var:[0.5]
sse:743.6200749295413
var:[0.50000001]
sse:743.6199819531047
var:[0.99999]
sse:289.1462047531385
...
Updated 6/29 ... I got it to run this way, which is strange because it is the same thing, must be a bug in the library or casting explicitly like this gets it into the precise format desired:
import numpy as np
import pandas as pd
from scipy.optimize import minimize
from sklearn.neighbors import KernelDensity
def findMaxEval(eVal, q, bWidth):
bnds = ((float(1e5/10000000000), float(0.99999*-1)),)
print(bnds)
out = minimize(lambda *x: errPDFs(*x), .5, args=(eVal, q, bWidth), bounds=bnds)
if out['success']: var = out['x'][0]
else: var = 1
eMax = var*(1+(1./q)**.5)**2
return eMax, var
def errPDFs(var, eVal, q, bWidth, pts = 1000):
pdf0 = mpPDF(var, q, pts)
pdf1 = fitKDE(eVal, bWidth, x=pdf0.index.values)
sse=np.sum((pdf1-pdf0)**2)
return sse
def mpPDF(var, q, pts):
eMin, eMax=var*(1-(1./q)**.5)**2,var*(1+(1./q)**.5)**2
eVal = np.linspace(eMin, eMax, pts)
pdf = q/(2*np.pi*var*eVal)*((eMax-eVal)*(eVal-eMin))**.5
pdf = pd.Series(pdf, index=eVal)
return pdf
def fitKDE(obs, bWidth = .25, kernel='gaussian', x=None):
if len(obs.shape) == 1: obs = obs.reshape(-1, 1)
kde=KernelDensity(kernel=kernel, bandwidth=bWidth).fit(obs)
if x is None: x = np.unique(obs).reshape(-1, 1)
if len(x.shape) == 1: x = x.reshape(-1, 1)
logProb = kde.score_samples(x)
pdf=pd.Series(np.exp(logProb), index=x.flatten())
return pdf
eMax0, var0 = findMaxEval((1000,), 10, bWidth=.01)
print(eMax0)
print(var0)
Here is the updated output on Macbook in PyCharm Community, Python version 3.8.1:
I am using fmin_l_bfgs_b function while doing neural style transfer and keep getting
TypeError: 'numpy.float32' object is not callable
Details of the error block are:
TypeError Traceback (most recent call last)
<ipython-input-10-4699dceebbd9> in <module>()
----> 1 generate_art('/content/nst_images/5.jpg', '/content/nst_images/a.jpg',1, img_height=400)
4 frames
<ipython-input-9-67cbbed20548> in generate_art(content_image_path, style_image_path, iterations, img_height)
38 for i in range(iterations):
39
---> 40 x, min_val, info = fmin_l_bfgs_b(evaluator.loss(x,img_height,img_width,fetch_loss_and_grads),x, fprime=evaluator.grads, maxfun=20)
41 img = x.copy().reshape((img_height, img_width, 3))
42 img = deprocess_image(img)
/usr/local/lib/python3.6/dist-packages/scipy/optimize/lbfgsb.py in fmin_l_bfgs_b(func, x0, fprime, args, approx_grad, bounds, m, factr, pgtol, epsilon, iprint, maxfun, maxiter, disp, callback, maxls)
197
198 res = _minimize_lbfgsb(fun, x0, args=args, jac=jac, bounds=bounds,
--> 199 **opts)
200 d = {'grad': res['jac'],
201 'task': res['message'],
/usr/local/lib/python3.6/dist-packages/scipy/optimize/lbfgsb.py in _minimize_lbfgsb(fun, x0, args, jac, bounds, disp, maxcor, ftol, gtol, eps, maxfun, maxiter, iprint, callback, maxls, **unknown_options)
343 # until the completion of the current minimization iteration.
344 # Overwrite f and g:
--> 345 f, g = func_and_grad(x)
346 elif task_str.startswith(b'NEW_X'):
347 # new iteration
/usr/local/lib/python3.6/dist-packages/scipy/optimize/lbfgsb.py in func_and_grad(x)
293 else:
294 def func_and_grad(x):
--> 295 f = fun(x, *args)
296 g = jac(x, *args)
297 return f, g
/usr/local/lib/python3.6/dist-packages/scipy/optimize/optimize.py in function_wrapper(*wrapper_args)
325 def function_wrapper(*wrapper_args):
326 ncalls[0] += 1
--> 327 return function(*(wrapper_args + args))
328
329 return ncalls, function_wrapper
TypeError: 'numpy.float32' object is not callable
Any suggestions on how to troubleshoot this? thanks!
This was a silly error. for fmin_l_bfgs_b, arguments of the loss function have to be passed separately as args = ()