We Keep Coding

Python function calling with variable vs raw numbers

I am trying to implement a pso algorithm from Wikipedia https://en.wikipedia.org/wiki/Particle_swarm_optimization.
My problem is that when I am calling the cost function with a variable (Gbest), and then manually calling the cost function (with the Gbest data) I get a different output (cost) like the image bellow:
Code fault
I am new to python so thank you for any suggestions.
Here is the complete code:
import matplotlib.pyplot as plt
import numpy as np
from control.matlab import *
A = np.array([[0,0,1],[0,1,0],[1,2,-2]])
B = np.array( [[0],[1],[0]])
C = np.array([[0, 1,0]])
D = np.zeros([C.shape[0],B.shape[1]])
sys = ss(A,B,C,D)
sys_tf = tf(sys)
s = tf('s')
def cost(kp,ki):
global sys_tf, G, y, t, r
G = kp + ki/s
C = feedback(sys_tf*G, 1)
y, t = step(C, linspace(0,100))
r = np.ones(len(t))
return np.sum(y-r)**2
part = 100
ite = 10000
dim = 2
w = 0.001
wdamp = 0.99
phip = 0.9
phig = 0.1
blo, bup = -10,10
x = np.zeros([dim, part])
v = np.zeros([dim, part])
pbest = np.zeros([dim, part])
gbest = np.array([1000000,1000000])
for i in range(part):
for k in range(dim):
x[k][i] = pbest[k][i] = np.random.uniform(blo, bup)
v[k][i] = np.random.uniform(-np.abs(bup - blo), np.abs(bup - blo))
if cost(pbest[0][i], pbest[1][i]) < cost(gbest[0], gbest[1]):
gbest = np.array([pbest[0][i], pbest[1][i]])
for it in range(ite):
for i in range(part):
for k in range(dim):
rp = np.random.uniform(0,1)
rg = np.random.uniform(0,1)
v[k,:] = w*v[k,:] + phip*rp*(pbest[k,:] - x[k,:]) + phig*rg*(gbest[k] - x[k,:])
x[k,:] = x[k,:] + v[k,:]
w = w*wdamp
if cost(x[0][i], x[1][i]) < cost(pbest[0][i], pbest[1][i]):
pbest[:,i] = x[:,i]
if cost(pbest[0][i], pbest[1][i]) < cost(gbest[0], gbest[1]):
gbest = np.array([pbest[0][i], pbest[1][i]])
plt.plot(t, y, 'ro')
plt.plot(t, r, 'x')
plt.pause(0.005)
plt.title(gbest)
print([gbest, cost(gbest[0], gbest[1])])

GEKKO - How to fix Python Gekko Max Equation error - number of elements

I have developed a script using Gekko optimisation functions. The script below runs for a number of elements. I tested the optimisation algorithm for 20 and 47 cells (shapefile dataset) and the script achieves a solution. However, when I test for a bigger dataset, with 160 elements, for example, the following error message is shown:
“APM model error: string > 15000 characters
Consider breaking up the line into multiple equations”
I read some suggestions to fix this problem. I tried using m.sum, but the problem persists.
Please, could you help me fix this problem?
Please, find below the we transfer link to download the datasets with 47 cells and with 160 cells.
https://wetransfer.com/downloads/64cc631237adacc926c67f56124b327a20210928212223/d8a2d7
Thank you
Alexandre.
import geopandas as gpd
import time
import csv
from gekko import GEKKO
import numpy as np
import math
import pandas as pd
m = GEKKO()
A = -0.00000536
B = -0.0000291
E = 0.4040771
r = 0.085
input_path = 'D:/Alexandre/shapes/Threats/Prototype/BHO50k/Velhas_BHO50k1summ4_47cells.shp'
output_folder = 'D:/Alexandre/shapes/Threats/Prototype/Small_area/resultados'
input_layer = gpd.read_file(input_path)
input_layer = input_layer[
['cocursodag', 'cobacia', 'nuareacont', 'nudistbact', 'D0c', 'Ki0', 'Kj0', 'nuareamont', 'deltai', 'It',
'cost_op_BR', 'Ii_ub', 'Itj', 'cj', 'deltaj2']]
input_layer = input_layer.astype({'cobacia': 'string', 'cocursodag': 'string'})
count_input_feat = input_layer.shape[0]
row=count_input_feat
col=10
input_cobacia = {}
ubi = {}
numareacont = {}
Ki0 = {}
Kj0 = {}
X = {}
deltai2 = {}
ai = {}
aj = {}
D0 = {}
Itj = {}
It = {}
deltaj = {}
for row1 in input_layer.iterrows():
i = row1[0]
input_cobacia[i] = row1[1]['cobacia']
Ki0[i] = row1[1]['Ki0']+0.001
Kj0[i] = row1[1]['Kj0']
X[i] = row1[1]['nuareamont']
deltai2[i] = row1[1]['deltai']
ai[i] = 5423304*(pow(X[i],-0.1406852))
aj[i] = row1[1]['cj']*100 + row1[1]['cost_op_BR']*100
ubi[i] = row1[1]['Ii_ub']
numareacont[i] = row1[1]['nuareacont']
D0[i] = row1[1]['D0c']
It[i] = row1[1]['It']
Itj[i] = row1[1]['Itj']
if Itj[i]<1:
deltaj[i] = row1[1]['deltaj2'] * 0.0001
elif Itj[i]<2:
deltaj[i] = row1[1]['deltaj2'] * 0.0001
else:
deltaj[i] = row1[1]['deltaj2'] * 0.0001
Ii = m.Array(m.Var, (row, col))
Ij = m.Array(m.Var, (row, col))
for i in range(row):
for j in range(col):
if It[i] == 0:
Ii[i, j].value = 0
Ii[i, j].lower = 0
Ii[i, j].upper = 5
Ij[i,j].value = 0
Ij[i,j].lower = 0
Ij[i,j].upper = numareacont[i]*0.05*Itj[i]/3.704545
elif It[i] <= 2:
Ii[i, j].value = 0
Ii[i, j].lower = 0
Ii[i, j].upper = 10
Ij[i, j].value = 0
Ij[i, j].lower = 0
Ij[i, j].upper = numareacont[i]*0.05*Itj[i]/3.704545
elif It[i] <= 2.5:
Ii[i, j].value = 0
Ii[i, j].lower = 0
Ii[i, j].upper = 15
Ij[i, j].value = 0
Ij[i, j].lower = 0
Ij[i, j].upper = numareacont[i]*0.05*Itj[i]/3.704545
elif It[i] <= 3:
Ii[i, j].value = 0
Ii[i, j].lower = 0
Ii[i, j].upper = 15
Ij[i, j].value = 0
Ij[i, j].lower = 0
Ij[i, j].upper = numareacont[i]*0.05*Itj[i]/3.704545
else:
Ii[i,j].value = 0
Ii[i,j].lower = 0
Ii[i,j].upper = 20
Ij[i,j].value = 0
Ij[i,j].lower = 0
Ij[i,j].upper = numareacont[i]*0.05*Itj[i]/3.704545
Ki = m.Array(m.Var, (row, col))
Kj = m.Array(m.Var, (row, col))
indicator = m.Array(m.Var, (row, col))
p = 2
numerator = m.Array(m.Var, (row, col))
denominator = m.Array(m.Var, (row, col))
for row2 in input_layer.iterrows():
input_cobacia2 = row2[1]['cobacia']
input_cocursodag = row2[1]['cocursodag']
input_distance = row2[1]['nudistbact']
numerator = 0
denominator = 0
exp = f"cobacia > '{input_cobacia2}' and cocursodag.str.startswith('{input_cocursodag}')"
for j in range(col):
for target_feat in input_layer.query(exp).iterrows():
i=target_feat[0]
target_green_area = Ij[i,j]
target_distance = target_feat[1]['nudistbact']
distance = float(target_distance) - float(input_distance)
idw = 1 / (distance + 1) ** p
numerator += target_green_area * idw
denominator += idw
i=row2[0]
sum = Ij[i,j]
if denominator:
indicator[i,j] = numerator / denominator + sum
else:
indicator[i,j] = sum
D0F = m.Array(m.Var, (row, col))
for i in range(row):
def constraintD0(x):
return x - 0.2
for j in range(col):
if j == 0:
m.fix(Ki[i,j],val = Ki0[i])
Ki[i,j].lower = 0
Ki[i,j].upper = 500000
m.fix(Kj[i,j], val = Kj0[i])
Kj[i,j].lower = 0
Kj[i,j].upper = 100000
m.Equation(D0F[i, j] == A * Ki[i, j] + B * Kj[i, j] + E)
D0[i] = D0F[i, j]
else:
D0F[i,j].lower = 0
D0F[i, j].upper = 1
Ki[i,j].lower = 0
Ki[i,j].upper = 500000
Kj[i, j].lower = 0
Kj[i, j].upper = 100000
m.Equation(Ki[i,j] - Ki[i,j-1] == Ii[i,j] - deltai2[i] * Ki[i,j-1])
m.Equation(Kj[i,j] - Kj[i,j-1] == Ij[i,j] + deltaj[i] * Kj[i,j-1]+indicator[i,j])
m.Equation(D0F[i,j] == A*Ki[i,j] + B*Kj[i,j] + E)
m.Equation(D0F[i,j]<=D0[i])
dep = 1 / (1+r)
z1 = m.sum([m.sum([pow(dep, j)*(ai[i]*Ii[i,j]+aj[i]*Ij[i,j]) for i in range(row)]) for j in range(col)])
# Objective
m.Obj(z1)
m.options.IMODE = 3
m.options.SOLVER = 3
m.options.DIAGLEVEL = 1
m.options.REDUCE=3
try:
m.solve() # solve
# Outputs
output_Ki = pd.DataFrame(columns=['cobacia'] + list(range(col)))
output_Kj = pd.DataFrame(columns=['cobacia'] + list(range(col)))
output_Ii = pd.DataFrame(columns=['cobacia'] + list(range(col)))
output_Ij = pd.DataFrame(columns=['cobacia'] + list(range(col)))
output_D0 = pd.DataFrame(columns=['cobacia'] + list(range(col)))
output_ai = pd.DataFrame(columns=['cobacia'] + list(range(col)))
output_aj = pd.DataFrame(columns=['cobacia'] + list(range(col)))
for i in range(row):
for j in range(col):
print(Ki)
output_Ii.loc[i, 'cobacia'] = input_cobacia[i]
output_Ii.loc[i, j] = Ii[i,j].value[0]
output_Ij.loc[i, 'cobacia'] = input_cobacia[i]
output_Ij.loc[i, j] = Ij[i,j].value[0]
output_Ki.loc[i, 'cobacia'] = input_cobacia[i]
output_Ki.loc[i, j] = Ki[i,j].value[0]
output_Kj.loc[i, 'cobacia'] = input_cobacia[i]
output_Kj.loc[i, j] = Kj[i,j].value[0]
output_D0.loc[i, 'cobacia'] = input_cobacia[i]
output_D0.loc[i, j] = D0F[i, j].value[0]
output_ai.loc[i, 'cobacia'] = input_cobacia[i]
output_ai.loc[i, j] = ai[i]
output_aj.loc[i, 'cobacia'] = input_cobacia[i]
output_aj.loc[i, j] = aj[i]
df_outputIi = pd.DataFrame(output_Ii)
df_outputIj = pd.DataFrame(output_Ij)
df_outputKi = pd.DataFrame(output_Ki)
df_outputKj = pd.DataFrame(output_Kj)
df_outputD0 = pd.DataFrame(output_D0)
df_outputai = pd.DataFrame(output_ai)
df_outputaj = pd.DataFrame(output_aj)
with pd.ExcelWriter('output.xlsx') as writer:
df_outputIi.to_excel(writer, sheet_name="resultado Ii")
df_outputIj.to_excel(writer, sheet_name="resultado Ij")
df_outputKi.to_excel(writer, sheet_name="resultado Ki")
df_outputKj.to_excel(writer, sheet_name="resultado Kj")
df_outputD0.to_excel(writer, sheet_name="resultado D0")
df_outputai.to_excel(writer, sheet_name="ai")
df_outputaj.to_excel(writer, sheet_name="aj")
except:
print('Not successful')
from gekko.apm import get_file
print(m._server)
print(m._model_name)
f = get_file(m._server,m._model_name,'infeasibilities.txt')
f = f.decode().replace('\r','')
with open('infeasibilities.txt', 'w') as fl:
fl.write(str(f))
for i in range(row):
for j in range(col):
print(Ki[i,j].value)
print(Kj[i,j].value)
print(D0F[i,j].value)```

Look at the APMonitor model file gk0_model.apm in m.path by opening the run folder with m.open_folder(). Start with a smaller problem size and observe the equation size as the problem scales up. This limitation (<15,000 characters per equation) is built-in to encourage more efficient model expression and solution. An intermediate with m.Intermediate() can be used to more effectively express the model. Additional information on Intermediates is available in the documentation.

MPC with python and Error ValueError: `f0` passed has more than 1 dimension

I wrote a MPC with Python and it worked before. After a long time I want to use it again but I got this Error
f0 passed has more than 1 dimension.
But I didn't change anything on my code. It is some kind of strange.
Here is my code:
import numpy as np
import numpy.linalg as npl
import matplotlib.pyplot as plt
from scipy.optimize import minimize
def mpcAugment(Am, Bm, Cm ):
"Function for Augmented Model"
nx, nu = Bm.shape
ny = Cm.shape[0]
A = np.zeros((nx+ny,nx+ny))
A[0:nx,0:nx] = Am
A[nx:nx+ny,0:nx] = Cm#Am
A[nx:nx+ny,nx:nx+ny] = np.eye(ny)
B = np.zeros((nx+ny,nu))
B[0:nx,:nu] = Bm
B[nx:nx+ny,:nu] = Cm#Bm
C = np.zeros((ny,nx+ny))
C[:ny,nx:nx+ny] = np.eye(ny)
return A, B, C
'Define Parameters'
k = 0.4
AICB = 153.8
mcp = 8.8e4
vamb1 = 30
vamb2 = 45
a = -k*AICB/mcp
b = -1/mcp
Ts = 20
VICBref = -5.0
Am = np.array([[1+Ts*a]])
Bm = np.array([[Ts*b]])
Gm = np.array([[-Ts*a]])
Cm = np.array([[1]])
A, B, C = mpcAugment(Am,Bm,Cm)
A, G, C = mpcAugment(Am,Gm,Cm)
nx, nu = B.shape
ny = C.shape[0]
nd = G.shape[1]
Np = 20
Nu = 5
F = np.zeros((Np*ny,nx))
PHI = np.zeros((Np*ny,Nu*nu))
PHIw = np.zeros((Np*ny,Np*nd))
for i in range(0,Np):
Ai = npl.matrix_power(A, i+1)
F[i*ny:(i+1)*ny,:] = C#Ai
for j in range(0, Nu):
if j <= i:
Aij = np.linalg.matrix_power(A, i-j)
PHI[i*ny:(i+1)*ny, j*nu:(j+1)*nu] = C#Aij#B
for j in range(0, Np):
if j <= i:
Aij = np.linalg.matrix_power(A, i-j)
PHIw[i*ny:(i+1)*ny, j*nd:(j+1)*nd] = C#Aij#G
umax = 3100
umin = 0
Q = np.eye(Np*ny)
R = 1e-2*np.eye(Nu*nu)
Rs = VICBref*np.ones((Np*ny,1))
Ainq = np.zeros((2*Nu*nu,Nu*nu))
binq = np.zeros((2*Nu*nu,1))
cinq = np.zeros((2*Nu*nu,1))
for i in range(0,Nu):
binq[i*nu:(i+1)*nu] = umax
binq[(i+Nu)*nu:(Nu+i+1)*nu] = 1
cinq[i*nu:(i+1)*nu] = 1
cinq[(i+Nu)*nu:(Nu+i+1)*nu] = -1
for j in range(0,i+1):
Ainq[i*nu:(i+1)*nu,j*nu:(j+1)*nu] = np.eye(nu)
Ainq[(i+Nu)*nu:(Nu+i+1)*nu,j*nu:(j+1)*nu] = np.eye(nu)
u0 = 0
def objective(du):
dU = np.array(du).reshape((len(du),1))
Y = F#x + PHI#dU + PHIw#w
return np.transpose((Rs-Y))#(Rs-Y)+np.transpose(dU)#R#(dU)
def constraint1(du):
dU = np.array(du).reshape((len(du),1))
return (binq - Ainq#dU - cinq*u0)[0]
#print(objective([1,1,1]))
ulim = (umin, umax)
bnds = np.kron(np.ones((Nu,1)),ulim)
#print(bnds)
Um = np.ones((nu*Nu,1))
Tsim = 5e4
time = np.arange(0,Tsim,Ts)
Nt = len(time)
xm = np.zeros((Nt,1))
um = np.zeros((Nt,nu))
ym = np.zeros((Nt,ny))
xm[0] = 0
ym[0] = Cm.dot(xm[0])
w = np.zeros((Np*nd,1))
print('Am = ',Am)
print('Bm = ',Bm)
print('Cm = ',Cm)
x = np.zeros((nx,1))
x[1] = xm[0]
vamb = vamb1
Vamb = np.zeros((Nt,1))
Ns = int(np.floor(Nt/2))
Vamb[0:Ns] = vamb1*np.ones((Ns,1))
Vamb[Ns:Nt] = vamb2*np.ones((Nt-Ns,1))
Vref = VICBref*np.ones((Nt,1))
con = {'type':'ineq','fun':constraint1}
for i in range(0,Nt-1):
sol = minimize(objective, Um, method = 'SLSQP',constraints = con)
if sol.success == False:
print('Error Cant solve problem')
exit()
Um = sol.x
um[i+1] = um[i] + Um[0]
u0 = um[i+1]
xm[i+1] = Am.dot(xm[i])+Bm.dot(um[i+1])+Gm.dot(Vamb[i])
ym[i+1] = Cm.dot(xm[i+1])
for j in range(0,Np):
if i+j < Nt:
Rs[j] = Vref[i+j]
w[j] = Vamb[i+j]-Vamb[i+j-1]
else:
Rs[j] = Vref[Nt-1]
w[j] = 0
x[0] = xm[i+1] - xm[i]
x[1] = xm[i+1]
print('Q = ',um[i+1],' , VICB = ',xm[i+1], ' vamb = ', Vamb[i])
hour = 60*60
plt.figure()
plt.subplot(2,1,1)
plt.plot(time/hour,ym)
plt.plot(time/hour,Vref,'--')
plt.xlabel('time(hours)')
plt.xlim([0, Tsim/hour])
plt.subplot(2,1,2)
plt.plot(time/hour,um)
plt.xlim([0, Tsim/hour])
plt.show()
It about a controller, which control the temperature of a cool box.
Is that possible that anything changed in main simply code?
I think the problem is now in minimizations part.

I reinstalled all of my libraries and it worked

FloatingPointError: overflow encountered in double_scalars

I've set up numpy.seterr as follows:
np.seterr(invalid='raise', over ='raise', under='raise')
And I'm getting the following error:
c = beta[j,i] + oneminusbeta[j,i]
FloatingPointError: overflow encountered in double_scalars
I've checked what beta[j,i] and oneminusbeta[j,i] are at the point of crash, and these are their values:
beta: -131.340389182
oneminusbeta: 0.0
Please note, this line of addition (beta[j,i] + oneminusbeta[j,i]) has run for thousands of lines in a loop (that performs image classification) before crashing here at this point.
How can I deal with this? Is it necessary to change the type of the numpy arrays?
This is how I've initialized them:
beta = np.empty([m,n])
oneminusbeta = np.empty([m,n])
Is it possible to cast the individual values before adding them up? Rather than changing the entire array declarations? Or is this even a serious issue? Would it be safe to simply turn off the numpy.seterr configuration and let the calculations go ahead without raising the error?
Edit
Someone suggested below, and I suspected as well, that the values being added shouldn't cause an overflow. Then how can I find out where the overflow is really happening?
This is my code:
epthreshold = 709
enthreshold = -708
f.write("weights["+str(i)+", " + str(j)+"] = math.exp(beta: " +str(beta[j,i])+ " + oneminusbeta: " + str(oneminusbeta[j,i])+")\n" )
c = beta[j,i] + oneminusbeta[j,i]
weights[i,j] = math.exp(np.clip(c, enthreshold, epthreshold))
And when I check my log file, this is the line I get:
weights[5550, 13] = math.exp(beta: -131.340389182 + oneminusbeta: 0.0)
Edit 2
Here's the rest of my code, where variables n,m and H have already been initialized to integer values:
import numba
import numpy as np
import statsmodels.api as sm
weights = np.empty([n,m])
for curr_n in range(n):
for curr_m in range(m):
weights[curr_n,curr_m] = 1.0/(n)
beta = np.empty([m,n])
oneminusbeta = np.empty([m,n])
for curr_class in range(m):
for curr_sample in range(n):
beta[curr_class,curr_sample] = 1./m
epthreshold = 709 # positive exponential threshold
enthreshold = -708
for h in range(H):
print "Boosting round %d ... " % h
z = np.empty([n,m])
for j in range(m): # computing working responses and weights, Step 2(a)(i)
for i in range(no_samples):
i_class = y[i] #get the correct class for the current sample
if h == 0:
z[i,j] = (int(j==i_class) - beta[j,i])/((beta[j,i])*(1. - beta[j,i]))
weights[i,j] = beta[j,i]*(1. - beta[j,i])
else:
if j == i_class:
z[i,j] = math.exp(np.clip(-beta[j,i],enthreshold, epthreshold))
else:
z[i,j] = -math.exp(np.clip(oneminusbeta[j,i], enthreshold, epthreshold))
f.write("weights["+str(i)+", " + str(j)+"] = math.exp(beta: " +str(beta[j,i])+ " + oneminusbeta: " + str(oneminusbeta[j,i])+")\n" )
c = beta[j,i] + oneminusbeta[j,i]
weights[i,j] = math.exp(np.clip(c, enthreshold, epthreshold))
g_h = np.zeros([1,1])
j = 0
# Calculating regression coefficients per class
# building the parameters per j class
for y1_w in zip(z.T, weights.T):
y1, w = y1_w
temp_g = sm.WLS(y1, X, w).fit() # Step 2(a)(ii)
if np.allclose(g_h,0):
g_h = temp_g.params
else:
g_h = np.c_[g_h, temp_g.params]
j = j + 1
if np.allclose(g,0):
g = g_h
else:
g = g + g_h # Step(2)(a)(iii)
# now set g(x), function coefficients according to new formula, step (2)(b)
sum_g = g.sum(axis=1)
for j in range(m):
diff = (g[:,j] - ((1./m) * sum_g))
g[:,j] = ((m-1.)/m) * diff
g_per_round[h,:,j] = g[:,j]
#Now computing beta, Step 2(c)...."
Q = 0.
e = 0.
for j in range(m):
# Calculating beta and oneminusbeta for class j
aj = 0.0
for i in range(no_samples):
i_class = y[i]
X1 = X[i].reshape(1, no_features)
g1 = g[:,j].reshape(no_features, 1)
gc = g[:,i_class].reshape(no_features, 1)
dot = 1. + float(np.dot(X1, g1)) - float(np.dot(X1,gc))
aj = dot
sum_e = 0.
a_q = []
a_q.append(0.)
for j2 in range(m): # calculating sum of e's except for all j except where j=i_class
if j2 != i_class: # g based on j2, not necessarily g1?
g2 = g[:,j2].reshape(no_features, 1)
dot1 = 1. + float(np.dot(X1, g2)) - float(np.dot(X1,gc))
e2 = math.exp(np.clip(dot1,enthreshold, epthreshold))
sum_e = sum_e + e2
a_q.append(dot1)
if (int(j==i_class) == 1):
a_q_arr = np.array(a_q)
alpha = np.array(a_q_arr[1:])
Q = mylogsumexp(f,a_q_arr, 1, 0)
sumalpha = mylogsumexp(f,alpha, 1, 0)
beta[j,i] = -Q
oneminusbeta[j,i] = sumalpha - Q
else:
alpha = a_q
alpha = np.array(alpha[1:])
a_q_arr = np.array(a_q)
Q = mylogsumexp(f,a_q_arr, 0, aj)
sumalpha = log(math.exp(np.clip(Q, enthreshold, epthreshold)) - math.exp(np.clip(aj, enthreshold, epthreshold)))
beta[j,i] = aj - Q
oneminusbeta[j,i] = sumalpha - Q
and the function mylogsumexp is:
def mylogsumexp(f, a, is_class, maxaj, axis=None, b=None):
np.seterr(over="raise", under="raise", invalid="raise")
threshold = -sys.float_info.max
maxthreshold = sys.float_info.max
epthreshold = 709 # positive exponential threshold
enthreshold = -708
a = asarray(a)
if axis is None:
a = a.ravel()
else:
a = rollaxis(a, axis)
if is_class == 1:
a_max = a.max(axis=0)
else:
a_max = maxaj
#bnone = " none "
if b is not None:
a_max = maxaj
b = asarray(b)
if axis is None:
b = b.ravel()
else:
b = rollaxis(b, axis)
a = np.clip(a - a_max, enthreshold, epthreshold)
midout = np.sum(np.exp(a), axis=0)
midout = 1.0 + np.clip(midout - math.exp(a_max), threshold, maxthreshold)
out = np.log(midout)
else:
a = np.clip(a - a_max, enthreshold, epthreshold)
out = np.log(np.sum(np.exp(a)))
out += a_max
if out == float("inf"):
out = maxthreshold
if out == float("-inf"):
out = threshold
return out

RAM full in numpy sagemath

I wrote the next code. In 1-2 hours of execution time the RAM of my laptop (8gb) is filled and the sistem crash:
from scipy.stats import uniform
import numpy as np
cant_de_cadenas =[700,800,900]
cantidad_de_cadenas=np.array([])
for kkkkk in cant_de_cadenas:
cantidad_de_cadenas=np.append(cantidad_de_cadenas,kkkkk)
cantidad_de_cadenas=np.transpose(cantidad_de_cadenas)
b=10
h=b
Longitud=1
numero_experimentos=100
densidad_de_cadenas =cantidad_de_cadenas/(b**2)
prob_perc=np.array([])
tiempos=np.array([])
S_int=np.array([])
S_medio=np.array([])
desviacion_standard=np.array([])
desviacion_standard_nuevo=np.array([])
anisotropia_macroscopica_porcentual=np.array([])
componente_y=np.array([])
componente_x=np.array([])
import time
for N in cant_de_cadenas:
empieza=time.clock()
PERCOLACION=np.array([])
size_medio_intuitivo = np.array([])
size_medio_nuevo = np.array([])
std_dev_size_medio_intuitivo = np.array([])
std_dev_size_medio_nuevo = np.array([])
comp_y = np.array([])
comp_x = np.array([])
for u in xrange(numero_experimentos):
perco = False
array_x1=uniform.rvs(loc=-b/2, scale=b, size=N)
array_y1=uniform.rvs(loc=-h/2, scale=h, size=N)
array_angle=uniform.rvs(loc=-0.5*(np.pi), scale=np.pi, size=N)
array_pendiente_x=1./np.tan(array_angle)
random=uniform.rvs(loc=-1, scale=2, size=N)
lambda_sign=np.zeros([N])
for t in xrange(N):
if random[t]<0:
lambda_sign[t]=-1
else:
lambda_sign[t]=1
array_lambdas=(lambda_sign*Longitud)/np.sqrt(1+array_pendiente_x**2)
array_x2= array_x1 + array_lambdas*array_pendiente_x
array_y2= array_y1 + array_lambdas*1
array_x1 = np.append(array_x1, [-b/2, b/2, -b/2, -b/2])
array_y1 = np.append(array_y1, [-h/2, -h/2, -h/2, h/2])
array_x2 = np.append(array_x2, [-b/2, b/2, b/2, b/2])
array_y2 = np.append(array_y2, [h/2, h/2, -h/2, h/2])
M = np.zeros([N+4,N+4])
for j in xrange(N+4):
if j>0:
x_A1B1 = array_x2[j]-array_x1[j]
y_A1B1 = array_y2[j]-array_y1[j]
x_A1A2 = array_x1[0:j]-array_x1[j]
y_A1A2 = array_y1[0:j]-array_y1[j]
x_A2A1 = -1*x_A1A2
y_A2A1 = -1*y_A1A2
x_A2B2 = array_x2[0:j]-array_x1[0:j]
y_A2B2 = array_y2[0:j]-array_y1[0:j]
x_A1B2 = array_x2[0:j]-array_x1[j]
y_A1B2 = array_y2[0:j]-array_y1[j]
x_A2B1 = array_x2[j]-array_x1[0:j]
y_A2B1 = array_y2[j]-array_y1[0:j]
p1 = x_A1B1*y_A1A2 - y_A1B1*x_A1A2
p2 = x_A1B1*y_A1B2 - y_A1B1*x_A1B2
p3 = x_A2B2*y_A2B1 - y_A2B2*x_A2B1
p4 = x_A2B2*y_A2A1 - y_A2B2*x_A2A1
condicion_1=p1*p2
condicion_2=p3*p4
for k in xrange (j):
if condicion_1[k]<=0 and condicion_2[k]<=0:
M[j,k]=1
del condicion_1
del condicion_2
if j+1<N+4:
x_A1B1 = array_x2[j]-array_x1[j]
y_A1B1 = array_y2[j]-array_y1[j]
x_A1A2 = array_x1[j+1:]-array_x1[j]
y_A1A2 = array_y1[j+1:]-array_y1[j]
x_A2A1 = -1*x_A1A2
y_A2A1 = -1*y_A1A2
x_A2B2 = array_x2[j+1:]-array_x1[j+1:]
y_A2B2 = array_y2[j+1:]-array_y1[j+1:]
x_A1B2 = array_x2[j+1:]-array_x1[j]
y_A1B2 = array_y2[j+1:]-array_y1[j]
x_A2B1 = array_x2[j]-array_x1[j+1:]
y_A2B1 = array_y2[j]-array_y1[j+1:]
p1 = x_A1B1*y_A1A2 - y_A1B1*x_A1A2
p2 = x_A1B1*y_A1B2 - y_A1B1*x_A1B2
p3 = x_A2B2*y_A2B1 - y_A2B2*x_A2B1
p4 = x_A2B2*y_A2A1 - y_A2B2*x_A2A1
condicion_1=p1*p2
condicion_2=p3*p4
for k in xrange ((N+4)-j-1):
if condicion_1[k]<=0 and condicion_2[k]<=0:
M[j,k+j+1]=1
del condicion_1
del condicion_2
M[N,N+2]=0
M[N,N+3]=0
M[N+1,N+2]=0
M[N+1,N+3]=0
M[N+2,N]=0
M[N+2,N+1]=0
M[N+3,N]=0
M[N+3,N+1]=0
CD=np.array([])
POPOPO=[]
for g in xrange(N):
lala=0
r=False
while lala<=len(POPOPO)-1:
esta= g in POPOPO[lala]
if esta is True:
lala=len(POPOPO)
r=True
else:
lala=lala+1
if r is False:
L=np.array([g])
for s in xrange(N):
if M[g,s] != 0:
L=np.append(L,s)
x=0
while x<= N:
for l in xrange(N):
z= l in L
d=L[x]
if z is False and M[d,l] != 0:
L=np.append(L,l)
if x+1<len(L):
x+=1
else:
x=N+1.
q= len (L)
CD=np.append(CD, q)
POPOPO.append(L)
M_horizontal=M.copy()
M_horizontal[:,N+2] = np.zeros(N+4)
M_horizontal[:,N+3] = np.zeros(N+4)
M_horizontal[N+2] = np.zeros(N+4)
M_horizontal[N+3] = np.zeros(N+4)
L=np.array([N])
for s in xrange(N+4):
if M_horizontal[N,s] != 0:
L=np.append(L,s)
x=0
while x<= N+4:
for l in xrange(N+4):
z= l in L
d=L[x]
if z is False and M_horizontal[d,l] != 0:
L=np.append(L,l)
if x+1<len(L):
x+=1
else:
x=(N+4)+1.
LV1_in_L = N in L
LV2_in_L= (N+1) in L
if LV1_in_L is True and LV2_in_L is True:
perc_horiz=True
else:
perc_horiz=False
M_vertical=M.copy()
M_vertical[:,N] = np.zeros(N+4)
M_vertical[:,N+1] = np.zeros(N+4)
M_vertical[N] = np.zeros(N+4)
M_vertical[N+1] = np.zeros(N+4)
L=np.array([N+2])
for s in xrange(N+4):
if M_vertical[N+2,s] != 0:
L=np.append(L,s)
x=0
while x<= N+4:
for l in xrange(N+4):
z= l in L
d=L[x]
if z is False and M_vertical[d,l] != 0:
L=np.append(L,l)
if x+1<len(L):
x+=1
else:
x=(N+4)+1.
LH1_in_L = (N+2) in L
LH2_in_L= (N+3) in L
if LH1_in_L is True and LH2_in_L is True:
perc_ver = True
else:
perc_ver = False
if perc_ver is True or perc_horiz is True:
PERCOLACION=np.append(PERCOLACION,1)
perco=True
D = np.array([])
W = np.array([])
for c in xrange (int(min(CD)), int(max(CD)+1),1):
D=np.append(D,c)
frec = sum (CD == c)
W = np.append(W,frec)
if perco is True:
posicion=np.argmax(D)
D=np.delete(D,posicion)
W=np.delete(W,posicion)
if len(D) == 0 and len(W)==0:
S_medio_intuitivo_exp_u=0
S_medio_nuevo_exp_u = 0
std_dev_exp_u = 0
std_dev_nuevo_exp_u = 0
else:
S_medio_intuitivo_exp_u = np.average (D,weights=W)
peso_nuevo=D*W
S_medio_nuevo_exp_u = np.average (D,weights=peso_nuevo)
tipos=sum(W)
X=W*((D-S_medio_intuitivo_exp_u)**2)
S=sum(X)
std_dev_exp_u = np.sqrt(S/(tipos-1.))
tipos_nuevo=sum(peso_nuevo)
X_nuevo=peso_nuevo*((D-S_medio_nuevo_exp_u)**2)
S_nuevo=sum(X_nuevo)
std_dev_nuevo_exp_u = np.sqrt(S_nuevo/(tipos_nuevo-1.))
componente_longitudinal=Longitud*np.abs(np.cos(array_angle))
comp_y=np.append(comp_y, sum(componente_longitudinal)/N)
componente_transversal=Longitud*np.abs(np.sin(array_angle))
comp_x=np.append(comp_x, sum(componente_transversal)/N)
std_dev_size_medio_intuitivo=np.append(std_dev_size_medio_intuitivo, std_dev_exp_u)
std_dev_size_medio_nuevo=np.append(std_dev_size_medio_nuevo, std_dev_nuevo_exp_u)
size_medio_intuitivo=np.append(size_medio_intuitivo, S_medio_intuitivo_exp_u)
size_medio_nuevo=np.append(size_medio_nuevo, S_medio_nuevo_exp_u)
percolation_probability=sum(PERCOLACION)/numero_experimentos
prob_perc=np.append(prob_perc, percolation_probability)
S_int = np.append (S_int, sum(size_medio_intuitivo)/numero_experimentos)
S_medio=np.append (S_medio, sum(size_medio_nuevo)/numero_experimentos)
desviacion_standard = np.append (desviacion_standard, sum(std_dev_size_medio_intuitivo)/numero_experimentos)
desviacion_standard_nuevo=np.append (desviacion_standard_nuevo, sum(std_dev_size_medio_nuevo)/numero_experimentos)
tiempos=np.append(tiempos, time.clock()-empieza)
componente_y=np.append(componente_y, sum(comp_y)/numero_experimentos)
componente_x=np.append(componente_x, sum(comp_x)/numero_experimentos)
anisotropia_macroscopica_porcentual=100*(1-(componente_y/componente_x))
I tryed with gc and gc.collect() and 'del'command for deleting arrays after his use and nothing work!
What am I doing wrong? Why the memory becomes full while running (starts with 10% of RAM used and in 1-2hour is totally full used)?
Lets take an array M with size 300MB. When this is overwritten (for example, in each iteration), have we 300MB occupated in RAM memory or just 300MB? In the case that we have just 300MB, there should be no problem, so why have I this RAM issue? In the case of the RAM is acumulated, how can I do for free RAM memory occupated for the 'old' array?
Please help me, I'm totally stuck!
Thanks a lot!