Related
I have a function with different parameters that I want to optimize to fit some existing data.
The function runs fine on its own, but when I try to pass it through the scipy.optimize.curve_fit function, I get this error :
IndexError: invalid index to scalar variable.
I don't understand why the function would work on its own, and I would not get any errors.
What can I do ?
The original function used dictionnaries and I thought that might be the problem but I modified it and it still doesn't work.
This is the function I'm using :
def function_test(xy,X1,X2,X3,X4):
precip = xy\[0\]
potential_evap = xy\[1\]
nUH1 = int(math.ceil(X4))
nUH2 = int(math.ceil(2.0*X4))
uh1_ordinates = [0] * nUH1
uh2_ordinates = [0] * nUH2
UH1 = [0] * nUH1
UH2 = [0] * nUH2
for t in range(1, nUH1 + 1):
uh1_ordinates[t - 1] = s_curves1(t, X4) - s_curves1(t-1, X4)
for t in range(1, nUH2 + 1):
uh2_ordinates[t - 1] = s_curves2(t, X4) - s_curves2(t-1, X4)
production_store = X1*0.60# S
routing_store = X3*0.70# R
qsim = []
for j in range(2191):
if precip[j] > potential_evap[j]:
net_evap = 0
scaled_net_precip = (precip[j] - potential_evap[j])/X1
if scaled_net_precip > 13:
scaled_net_precip = 13.
tanh_scaled_net_precip = tanh(scaled_net_precip)
reservoir_production = (X1 * (1 - (production_store/X1)**2) * tanh_scaled_net_precip) / (1 + production_store/X1 * tanh_scaled_net_precip)
routing_pattern = precip[j]-potential_evap[j]-reservoir_production
else:
scaled_net_evap = (potential_evap[j] - precip[j])/X1
if scaled_net_evap > 13:
scaled_net_evap = 13.
tanh_scaled_net_evap = tanh(scaled_net_evap)
ps_div_x1 = (2 - production_store/X1) * tanh_scaled_net_evap
net_evap = production_store * (ps_div_x1) / \
(1 + (1 - production_store/X1) * tanh_scaled_net_evap)
reservoir_production = 0
routing_pattern = 0
production_store = production_store - net_evap + reservoir_production
percolation = production_store / (1 + (production_store/2.25/X1)**4)**0.25
routing_pattern = routing_pattern + (production_store-percolation)
production_store = percolation
for i in range(0, len(UH1) - 1):
UH1[i] = UH1[i+1] + uh1_ordinates[i]*routing_pattern
UH1[-1] = uh1_ordinates[-1] * routing_pattern
for j in range(0, len(UH2) - 1):
UH2[j] = UH2[j+1] + uh2_ordinates[j]*routing_pattern
UH2[-1] = uh2_ordinates[-1] * routing_pattern
groundwater_exchange = X2 * (routing_store / X3)**3.5
routing_store = max(0, routing_store + UH1[0] * 0.9 + groundwater_exchange)
R2 = routing_store / (1 + (routing_store / X3)**4)**0.25
QR = routing_store - R2
routing_store = R2
QD = max(0, UH2[0]*0.1+groundwater_exchange)
Q = QR + QD
qsim.append(Q)
return qsim
I am writing a SHA256 Hash function and it's mostly complete but when I run the code the hash is way to high and it's not correct. What is wrong with my code that's making it far off from the actual value to be? Now keep in mind I do not have a chunk loop so it might look a bit different. This also A GUI as well.
#VARIABLES
FF = (1 >> 32)-1
#Right rotate
def rr(a,b):
return((a >> b) | (a << (32 - b))) & FF
h0 = 0x6a09e667
h1 = 0xbb67ae85
h2 = 0x3c6ef372
h3 = 0xa54ff53a
h4 = 0x510e527f
h5 = 0x9b05688c
h6 = 0x1f83d9ab
h7 = 0x5be0cd19
#Messages
word_Hash = "1: Please write below what\n word you would like to hash."
words = []
playerinput = []
varwords = [word_Hash]
#FUNCTIONS
count = 0
def SHA256():
global playerinput, wordtohash, A, B, C, D, E, F, G, H
#get input
wordtohash = playerinput.pop(0)
#convert to binary
BinaryConversion = ''.join(format(ord(i), '08b') for i in wordtohash) + "1"
#pad helped by Dr.Glynn, Maple
if len(BinaryConversion) <= 512:
count = len(BinaryConversion)
while count <= 448:
BinaryConversion = BinaryConversion + "0"
count += 1
wordtohash = len(wordtohash)
endofpad = int(wordtohash) * 8
numberofzeros = 0
while numberofzeros < 63 - int(len(bin(endofpad)[2:])):
BinaryConversion = BinaryConversion + "0"
numberofzeros += 1
#BinaryConversion = (BinaryConversion) + str(endofpad)
BinaryConversion = str(BinaryConversion) + str(bin(endofpad)[2:])
#numbers = len(BinaryConversion)
#print(BinaryConversion)
#first 16 messages
w = [int('0b'+BinaryConversion[0:31], 2),
int('0b'+BinaryConversion[32:63], 2),
int('0b'+BinaryConversion[64:95],2),
int('0b'+BinaryConversion[96:127],2),
int('0b'+BinaryConversion[128:159],2),
int('0b'+BinaryConversion[160:191],2),
int('0b'+BinaryConversion[192:223],2),
int('0b'+BinaryConversion[224:255],2),
int('0b'+BinaryConversion[256:287],2),
int('0b'+BinaryConversion[288:319],2),
int('0b'+BinaryConversion[320:351],2),
int('0b'+BinaryConversion[352:383],2),
int('0b'+BinaryConversion[384:415],2),
int('0b'+BinaryConversion[416:447],2),
int('0b'+BinaryConversion[448:479],2),
int('0b'+BinaryConversion[480:511],2)]
#Message Scedule
#rest of the messages
for c in range(16,64):
S0 = rr(w[c-15], 7) ^ rr(w[c-15], 18) ^ (w[c-15] >> 3)
S1 = rr(w[c - 2], 17) ^ rr(w[c - 2], 19) ^ (w[c - 2] >> 10)
w.append((w[c - 16] + S0 + w[c-7] + S1) & FF)
print(w)
A = h0
B = h1
C = h2
D = h3
E = h4
F = h5
G = h6
H = h7
for i in range(64):
s1 = rr(E, 6) ^ rr(E, 11) ^ rr(E, 25)
ch = (E & F) ^ (~E & G)
temp1 = H + s1 + ch + K[i] + w[i]
s0 = rr(A, 2) ^ rr(A, 13) ^ rr(A, 22)
maj = (A & B) ^ (A & C) ^ (B & C)
temp2 = s0 + maj
H = G
G = F
F = E
E = D + temp1
D = C
C = B
B = A
A = temp1 + temp2
A = hex(A)[2:]
B = hex(B)[2:]
C = hex(C)[2:]
D = hex(D)[2:]
E = hex(E)[2:]
F = hex(F)[2:]
G = hex(G)[2:]
H = hex(H)[2:]
print(A)
def finish():
global count, varwords, playerinput, words
if count >= 1:
screenframe1.pack_forget()
frm_screen2.pack()
SHA256()
lbl_story["text"] = "Your word is {}\n Your hash value is {}{}{}{}{}{}{}{}".format(wordtohash,A,B,C,D,E,F,G,H)
I have developed a script using Gekko optimisation functions. The script below runs for a number of elements. I tested the optimisation algorithm for 20 and 47 cells (shapefile dataset) and the script achieves a solution. However, when I test for a bigger dataset, with 160 elements, for example, the following error message is shown:
“APM model error: string > 15000 characters
Consider breaking up the line into multiple equations”
I read some suggestions to fix this problem. I tried using m.sum, but the problem persists.
Please, could you help me fix this problem?
Please, find below the we transfer link to download the datasets with 47 cells and with 160 cells.
https://wetransfer.com/downloads/64cc631237adacc926c67f56124b327a20210928212223/d8a2d7
Thank you
Alexandre.
import geopandas as gpd
import time
import csv
from gekko import GEKKO
import numpy as np
import math
import pandas as pd
m = GEKKO()
A = -0.00000536
B = -0.0000291
E = 0.4040771
r = 0.085
input_path = 'D:/Alexandre/shapes/Threats/Prototype/BHO50k/Velhas_BHO50k1summ4_47cells.shp'
output_folder = 'D:/Alexandre/shapes/Threats/Prototype/Small_area/resultados'
input_layer = gpd.read_file(input_path)
input_layer = input_layer[
['cocursodag', 'cobacia', 'nuareacont', 'nudistbact', 'D0c', 'Ki0', 'Kj0', 'nuareamont', 'deltai', 'It',
'cost_op_BR', 'Ii_ub', 'Itj', 'cj', 'deltaj2']]
input_layer = input_layer.astype({'cobacia': 'string', 'cocursodag': 'string'})
count_input_feat = input_layer.shape[0]
row=count_input_feat
col=10
input_cobacia = {}
ubi = {}
numareacont = {}
Ki0 = {}
Kj0 = {}
X = {}
deltai2 = {}
ai = {}
aj = {}
D0 = {}
Itj = {}
It = {}
deltaj = {}
for row1 in input_layer.iterrows():
i = row1[0]
input_cobacia[i] = row1[1]['cobacia']
Ki0[i] = row1[1]['Ki0']+0.001
Kj0[i] = row1[1]['Kj0']
X[i] = row1[1]['nuareamont']
deltai2[i] = row1[1]['deltai']
ai[i] = 5423304*(pow(X[i],-0.1406852))
aj[i] = row1[1]['cj']*100 + row1[1]['cost_op_BR']*100
ubi[i] = row1[1]['Ii_ub']
numareacont[i] = row1[1]['nuareacont']
D0[i] = row1[1]['D0c']
It[i] = row1[1]['It']
Itj[i] = row1[1]['Itj']
if Itj[i]<1:
deltaj[i] = row1[1]['deltaj2'] * 0.0001
elif Itj[i]<2:
deltaj[i] = row1[1]['deltaj2'] * 0.0001
else:
deltaj[i] = row1[1]['deltaj2'] * 0.0001
Ii = m.Array(m.Var, (row, col))
Ij = m.Array(m.Var, (row, col))
for i in range(row):
for j in range(col):
if It[i] == 0:
Ii[i, j].value = 0
Ii[i, j].lower = 0
Ii[i, j].upper = 5
Ij[i,j].value = 0
Ij[i,j].lower = 0
Ij[i,j].upper = numareacont[i]*0.05*Itj[i]/3.704545
elif It[i] <= 2:
Ii[i, j].value = 0
Ii[i, j].lower = 0
Ii[i, j].upper = 10
Ij[i, j].value = 0
Ij[i, j].lower = 0
Ij[i, j].upper = numareacont[i]*0.05*Itj[i]/3.704545
elif It[i] <= 2.5:
Ii[i, j].value = 0
Ii[i, j].lower = 0
Ii[i, j].upper = 15
Ij[i, j].value = 0
Ij[i, j].lower = 0
Ij[i, j].upper = numareacont[i]*0.05*Itj[i]/3.704545
elif It[i] <= 3:
Ii[i, j].value = 0
Ii[i, j].lower = 0
Ii[i, j].upper = 15
Ij[i, j].value = 0
Ij[i, j].lower = 0
Ij[i, j].upper = numareacont[i]*0.05*Itj[i]/3.704545
else:
Ii[i,j].value = 0
Ii[i,j].lower = 0
Ii[i,j].upper = 20
Ij[i,j].value = 0
Ij[i,j].lower = 0
Ij[i,j].upper = numareacont[i]*0.05*Itj[i]/3.704545
Ki = m.Array(m.Var, (row, col))
Kj = m.Array(m.Var, (row, col))
indicator = m.Array(m.Var, (row, col))
p = 2
numerator = m.Array(m.Var, (row, col))
denominator = m.Array(m.Var, (row, col))
for row2 in input_layer.iterrows():
input_cobacia2 = row2[1]['cobacia']
input_cocursodag = row2[1]['cocursodag']
input_distance = row2[1]['nudistbact']
numerator = 0
denominator = 0
exp = f"cobacia > '{input_cobacia2}' and cocursodag.str.startswith('{input_cocursodag}')"
for j in range(col):
for target_feat in input_layer.query(exp).iterrows():
i=target_feat[0]
target_green_area = Ij[i,j]
target_distance = target_feat[1]['nudistbact']
distance = float(target_distance) - float(input_distance)
idw = 1 / (distance + 1) ** p
numerator += target_green_area * idw
denominator += idw
i=row2[0]
sum = Ij[i,j]
if denominator:
indicator[i,j] = numerator / denominator + sum
else:
indicator[i,j] = sum
D0F = m.Array(m.Var, (row, col))
for i in range(row):
def constraintD0(x):
return x - 0.2
for j in range(col):
if j == 0:
m.fix(Ki[i,j],val = Ki0[i])
Ki[i,j].lower = 0
Ki[i,j].upper = 500000
m.fix(Kj[i,j], val = Kj0[i])
Kj[i,j].lower = 0
Kj[i,j].upper = 100000
m.Equation(D0F[i, j] == A * Ki[i, j] + B * Kj[i, j] + E)
D0[i] = D0F[i, j]
else:
D0F[i,j].lower = 0
D0F[i, j].upper = 1
Ki[i,j].lower = 0
Ki[i,j].upper = 500000
Kj[i, j].lower = 0
Kj[i, j].upper = 100000
m.Equation(Ki[i,j] - Ki[i,j-1] == Ii[i,j] - deltai2[i] * Ki[i,j-1])
m.Equation(Kj[i,j] - Kj[i,j-1] == Ij[i,j] + deltaj[i] * Kj[i,j-1]+indicator[i,j])
m.Equation(D0F[i,j] == A*Ki[i,j] + B*Kj[i,j] + E)
m.Equation(D0F[i,j]<=D0[i])
dep = 1 / (1+r)
z1 = m.sum([m.sum([pow(dep, j)*(ai[i]*Ii[i,j]+aj[i]*Ij[i,j]) for i in range(row)]) for j in range(col)])
# Objective
m.Obj(z1)
m.options.IMODE = 3
m.options.SOLVER = 3
m.options.DIAGLEVEL = 1
m.options.REDUCE=3
try:
m.solve() # solve
# Outputs
output_Ki = pd.DataFrame(columns=['cobacia'] + list(range(col)))
output_Kj = pd.DataFrame(columns=['cobacia'] + list(range(col)))
output_Ii = pd.DataFrame(columns=['cobacia'] + list(range(col)))
output_Ij = pd.DataFrame(columns=['cobacia'] + list(range(col)))
output_D0 = pd.DataFrame(columns=['cobacia'] + list(range(col)))
output_ai = pd.DataFrame(columns=['cobacia'] + list(range(col)))
output_aj = pd.DataFrame(columns=['cobacia'] + list(range(col)))
for i in range(row):
for j in range(col):
print(Ki)
output_Ii.loc[i, 'cobacia'] = input_cobacia[i]
output_Ii.loc[i, j] = Ii[i,j].value[0]
output_Ij.loc[i, 'cobacia'] = input_cobacia[i]
output_Ij.loc[i, j] = Ij[i,j].value[0]
output_Ki.loc[i, 'cobacia'] = input_cobacia[i]
output_Ki.loc[i, j] = Ki[i,j].value[0]
output_Kj.loc[i, 'cobacia'] = input_cobacia[i]
output_Kj.loc[i, j] = Kj[i,j].value[0]
output_D0.loc[i, 'cobacia'] = input_cobacia[i]
output_D0.loc[i, j] = D0F[i, j].value[0]
output_ai.loc[i, 'cobacia'] = input_cobacia[i]
output_ai.loc[i, j] = ai[i]
output_aj.loc[i, 'cobacia'] = input_cobacia[i]
output_aj.loc[i, j] = aj[i]
df_outputIi = pd.DataFrame(output_Ii)
df_outputIj = pd.DataFrame(output_Ij)
df_outputKi = pd.DataFrame(output_Ki)
df_outputKj = pd.DataFrame(output_Kj)
df_outputD0 = pd.DataFrame(output_D0)
df_outputai = pd.DataFrame(output_ai)
df_outputaj = pd.DataFrame(output_aj)
with pd.ExcelWriter('output.xlsx') as writer:
df_outputIi.to_excel(writer, sheet_name="resultado Ii")
df_outputIj.to_excel(writer, sheet_name="resultado Ij")
df_outputKi.to_excel(writer, sheet_name="resultado Ki")
df_outputKj.to_excel(writer, sheet_name="resultado Kj")
df_outputD0.to_excel(writer, sheet_name="resultado D0")
df_outputai.to_excel(writer, sheet_name="ai")
df_outputaj.to_excel(writer, sheet_name="aj")
except:
print('Not successful')
from gekko.apm import get_file
print(m._server)
print(m._model_name)
f = get_file(m._server,m._model_name,'infeasibilities.txt')
f = f.decode().replace('\r','')
with open('infeasibilities.txt', 'w') as fl:
fl.write(str(f))
for i in range(row):
for j in range(col):
print(Ki[i,j].value)
print(Kj[i,j].value)
print(D0F[i,j].value)```
Look at the APMonitor model file gk0_model.apm in m.path by opening the run folder with m.open_folder(). Start with a smaller problem size and observe the equation size as the problem scales up. This limitation (<15,000 characters per equation) is built-in to encourage more efficient model expression and solution. An intermediate with m.Intermediate() can be used to more effectively express the model. Additional information on Intermediates is available in the documentation.
Doing a project for physics class. vPython documentation is not very good I think and it's a bit confusing. I need to move a ball, which currently is being moved by mouse, but I need to make this work with 2 players, so I need to use keyboard events.
Currently this is the code:
#GlowScript 2.7 VPython
scene.range = 30
scene.forward = vec(0,-0.25,-1)
#red side of the field
redField = box()
redField.size = vec(40,1,60)
redField.pos= vec(-20,0,0)
redField.color = color.red
#blue side of the field
redField = box()
redField.size = vec(40,1,60)
redField.pos= vec(20,0,0)
redField.color = color.blue
cueBall = sphere()
cueBall.radius = 1.5
cueBall.pos = vec(0,2,0)
cueBall.vel = vec(3,0,0) # velocity in m/s
cueBall.mass = 2.0
blueBall = sphere()
blueBall.radius = 1.5
blueBall.pos = vec(4,2,0)
blueBall.color = color.blue
blueBall.mass = 1.5
blueBall.vel = vec(0,0,0)
blueBall.force = arrow(axis = vec(0,0,0), color = color.blue)
rail1 = box()
rail1.texture = {'file': "https://i.imgur.com/ss5OcnL.jpg"}
rail1.size = vec(3,3,55)
rail1.pos = vec(-39,1,0)
rail2 = box()
rail2.texture = {'file': "https://i.imgur.com/ss5OcnL.jpg"}
rail2.size = vec(3,3,55)
rail2.pos = vec(39,1,0)
rail3 = box()
rail3.texture = {'file': "https://i.imgur.com/ss5OcnL.jpg"}
rail3.size = vec(80,3,3)
rail3.pos = vec(0,1,29)
rail4 = box()
rail4.texture = {'file': "https://i.imgur.com/ss5OcnL.jpg"}
rail4.size = vec(80,3,3)
rail4.pos = vec(0,1,-29)
redBall = sphere()
redBall.radius = 1.5
redBall.pos = vec(-3,2,0)
redBall.color = color.red
redBall.force = arrow(axis = vec(0,0,0), color = color.red)
redBall.vel = vec(0,0,0)
redBall.mass = 1.5
yBall = sphere()
yBall.radius = 1.5
yBall.pos = vec(-9,2,0)
yBall.color = color.yellow
yBall.vel = vec(0,0,0)
yBall.mass = 1.5
yBall.force = arrow(axis = vec(0,0,0), color = color.yellow)
rail = []
rail.append(rail1, rail2, rail3, rail4)
ball = []
ball.append(cueBall, blueBall, redBall, yBall)
ball[0].force = arrow(axis = vec(0,0,0), color = color.white)
drag = False
chosenObject = None
scene.bind("mousedown", down)
scene.bind("mousemove", move)
scene.bind("mouseup", up)
def down():
nonlocal drag, chosenObject
chosenObject = scene.mouse.pick()
drag = True
def move():
nonlocal drag, chosenObject
if (drag == True):
chosenObject.force.axis = scene.mouse.pos - chosenObject.pos
chosenObject.force.pos = chosenObject.pos
chosenObject.force.axis.y = 0
def up():
nonlocal drag, chosenObject
chosenObject.force.axis = vec(0,0,0)
chosenObject = None
drag = False
elasticConst = 500
dt = 0.01
time = 0
while (True):
rate(1/dt)
for i in range(len(ball)):
ball[i].vel = ball[i].vel + ( ball[i].force.axis * dt / ball[i].mass )
ball[i].force.pos = ball[i].pos
for i in range(len(ball)):
for j in range(len(ball)):
if (i == j): continue
separation = ball[i].pos - ball[j].pos
contactSeparation = separation.norm() * (ball[i].radius + ball[j].radius)
if (separation.mag < contactSeparation.mag):
elasticForce = - elasticConst * (separation - contactSeparation)
ball[i].vel = ball[i].vel + (elasticForce / ball[i].mass) * dt
for i in range(len(ball)):
if ( ball[i].pos.x < rail1.pos.x + 2):
ball[i].pos.x = rail1.pos.x + 2
ball[i].vel.x = - ball[i].vel.x
if ( ball[i].pos.x > rail2.pos.x - 2):
ball[i].pos.x = rail2.pos.x - 2
ball[i].vel.x = - ball[i].vel.x
if ( ball[i].pos.z > rail3.pos.z - 2):
ball[i].pos.z = rail3.pos.z - 2
ball[i].vel.z = - ball[i].vel.z
if ( ball[i].pos.z < rail4.pos.z + 2):
ball[i].pos.z = rail4.pos.z + 2
ball[i].vel.z = - ball[i].vel.z
## for i in range(len(ball)):
# for j in range(len(hole)):
# if ((ball[i].pos - hole[j].pos).mag < ball[i].radius + hole[j].radius):
# ball[i].visible = False
for i in range(len(ball)):
ball[i].pos = ball[i].pos + ball[i].vel * dt
If you could help me understand how I could increment the position variables on keyboard.
I'm working with RR peaks and want to derive the frequency domain measures for HRV to recreate the results from the native C package by Physionet (WFDB tools). Both signal processing and spectral analysis are new fields for me, but after a long week with trial an error I've hacked together some code based on the Astropy module after trying several other solutions.
from astropy.stats import LombScargle
import random
dy = 0.1 * random.randint(1,100)
t = drive01["time"].values
y = drive01["intervals"].values
frequency, power = LombScargle(t, y,dy).autopower(minimum_frequency=0.0,maximum_frequency=4)
plt.plot(frequency, power)
This creates a plot that looks quite similar to the plot from Physionets package.
Physionets HRV tools with the code get_hrv makes this plot
Then by calculating common frequency domain measures I get quite different results.
Pxx = np.nan_to_num(power)
Fxx = np.nan_to_num(frequency)
ulf = 0.003
vlf = 0.04
lf = 0.15
hf = 0.4
Fs = 15.5 # the sampling rate of the drive file
# find the indexes corresponding to the VLF, LF, and HF bands
vlf_freq_band = (Fxx >= ulf) & (Fxx <= vlf)
lf_freq_band = (Fxx >= vlf) & (Fxx <= lf)
hf_freq_band = (Fxx >= lf) & (Fxx <= hf)
tp_freq_band = (Fxx >= 0) & (Fxx <= hf)
# Calculate the area under the given frequency band
dy = 1.0 / Fs
VLF = np.trapz(y=abs(Pxx[vlf_freq_band]), x=None, dx=dy)
LF = np.trapz(y=abs(Pxx[lf_freq_band]), x=None, dx=dy)
HF = np.trapz(y=abs(Pxx[hf_freq_band]), x=None, dx=dy)
TP = np.trapz(y=abs(Pxx[tp_freq_band]), x=None, dx=dy)
LF_HF = float(LF) / HF
Python
'HF': 0.10918703853414605,
'LF': 0.050074418080717789,
'LF/HF': 0.45861137689028925,
'TP': 0.20150514290250854,
'VLF': 0.025953350304821571
From the Physionet package:
TOT PWR = 0.0185973
VLF PWR = 0.00372733
LF PWR = 0.00472635
HF PWR = 0.0101436
LF/HF = 0.465944
When comparing the results it looks like this:
Python Physionet
TP 0.201505143 0.0185973 Quite similar + decimal dif
HF 0.109187039 0.0101436 Quite similar + decimal dif
LF 0.050074418 0.00472635 Quite similar + decimal dif
VLF 0.02595335 0.00372733 Not similar
LF/HF 0.458611377 0.465944 Quite similar
The calculations in Python are based on the code from another Stackoverflow post but the fix he got from the respondent is based on a python module I'm not able to get working and he is not using the Lomb Periodgram. I'm very open for trying something else as well, as long as its working with uneven samples.
the data I'm working with is the drivedb from Physionet and I've used the Physionet packages to make a text file with RR peaks and time which is read into a Pandas DataFrame. The textfile can be found here
LombScargle based on the Astropy caculate power different with C package by Physionet (WFDB tools). I write lombscargle again in python and result the same with C package by Physionet (WFDB tools).
import numpy as np
import os
import math
import csv
from itertools import zip_longest
import time
DATA_PATH = '/home/quangpc/Desktop/Data/PhysionetData/mitdb/'
class FreqDomainClass:
#staticmethod
def power(freq, mag):
lo = [0, 0.0033, 0.04, 0.15]
hi = [0.0033, 0.04, 0.15, 0.4]
pr = np.zeros(4)
nbands = 4
for index in range(0, len(freq)):
pwr = np.power(mag[index], 2)
for n in range(0, nbands):
if (freq[index] >= lo[n]) and freq[index] <= hi[n]:
pr[n] += pwr
break
return pr
#staticmethod
def avevar(y):
var = 0.0
ep = 0.0
ave = np.mean(y)
for i in range(len(y)):
s = y[i] - ave
ep += s
var += s * s
var = (var - ep * ep / len(y)) / (len(y) - 1)
return var
def lomb(self, t, h, ofac, hifac):
period = max(t) - min(t)
z = h - np.mean(h)
f = np.arange(1 / (period * ofac), hifac * len(h) / (2 * period), 1 / (period * ofac))
f = f[:int(len(f) / 2) + 1]
f = np.reshape(f, (len(f), -1))
w = 2 * np.pi * f
lenght_t = len(t)
t = np.reshape(t, (lenght_t, -1))
t = np.transpose(t)
tau = np.arctan2(np.sum(np.sin(2 * w * t), axis=1), np.sum(np.cos(2 * w * t), axis=1)) / (2 * w)
tau = np.diag(tau)
tau = np.reshape(tau, (len(tau), -1))
tau = np.tile(tau, (1, lenght_t))
cos = np.cos(w * (t - tau))
sin = np.sin(w * (t - tau))
pc = np.power(np.sum(z * cos, axis=1), 2)
ps = np.power(np.sum(z * sin, axis=1), 2)
cs = pc / np.sum(np.power(cos, 2), axis=1)
ss = ps / np.sum(np.power(sin, 2), axis=1)
p = cs + ss
pwr = self.avevar(h)
nout = len(h)
p = p / (2 * pwr)
p = p / (nout / (2.0 * pwr))
return f, np.sqrt(p)
def lomb_for(self, t, h, ofac, hifac):
period = max(t) - min(t)
f = np.arange(1 / (period * ofac), hifac * len(h) / (2 * period), 1 / (period * ofac))
f = f[:int(len(f) / 2) + 1]
z = h - np.mean(h)
p = np.zeros(len(f))
for i in range(len(f)):
w = 2 * np.pi * f[i]
if w > 0:
twt = 2 * w * t
y = sum(np.sin(twt))
x = sum(np.cos(twt))
tau = math.atan2(y, x) / (2 * w)
wtmt = w * (t - tau)
cs = np.power(sum(np.multiply(z, np.cos(wtmt))), 2) / sum(np.power((np.cos(wtmt)), 2))
ss = np.power(sum(np.multiply(z, np.sin(wtmt))), 2) / sum(np.power((np.sin(wtmt)), 2))
p[i] = cs + ss
else:
p[i] = np.power(sum(np.multiply(z, t)), 1) / sum(np.power(t), 1)
pwr = self.avevar(h)
nout = len(h)
p = p / (2 * pwr)
p = p / (nout / (2.0 * pwr))
return f, np.sqrt(p)
def freq_domain(self, time, rr_intervals):
frequency, mag0 = self.lomb(time, rr_intervals, 4.0, 2.0)
frequency = np.round(frequency, 8)
mag0 = mag0 / 2.0
mag0 = np.round(mag0, 8)
result = self.power(frequency, mag0)
return result[0], result[1], result[2], result[3], result[0] + result[1] + result[2] + result[3], \
result[2] / result[3]
def time_domain(time, rr_intervals, ann):
sum_rr = 0.0
sum_rr2 = 0.0
rmssd = 0.0
totnn = 0
totnnn = 0
nrr = 1
totrr = 1
nnx = 0
nnn = 0
lastann = ann[0]
lastrr = int(rr_intervals[0])
lenght = 300
t = float(time[0])
end = t + lenght
i = 0
ratbuf = np.zeros(2400)
avbuf = np.zeros(2400)
sdbuf = np.zeros(2400)
for x in range(1, len(ann)):
t = float(time[x])
while t > (end+lenght):
i += 1
end += lenght
if t >= end:
if nnn > 1:
ratbuf[i] = nnn/nrr
sdbuf[i] = np.sqrt(((sdbuf[i] - avbuf[i]*avbuf[i]/nnn) / (nnn-1)))
avbuf[i] /= nnn
i += 1
nnn = nrr = 0
end += lenght
nrr += 1
totrr += 1
if ann[x] == 'N' and ann[x-1] == 'N':
rr_intervals[x] = int(rr_intervals[x])
nnn += 1
avbuf[i] += rr_intervals[x]
sdbuf[i] += (rr_intervals[x] * rr_intervals[x])
sum_rr += rr_intervals[x]
sum_rr2 += (rr_intervals[x] * rr_intervals[x])
totnn += 1
if lastann == 'N':
totnnn += 1
rmssd += (rr_intervals[x] - lastrr) * (rr_intervals[x] - lastrr)
# nndif[0] = NNDIF
if abs(rr_intervals[x] - lastrr) - 0.05 > (10 ** -10):
nnx += 1
lastann = ann[x-1]
lastrr = rr_intervals[x]
if totnn == 0:
return 0, 0, 0, 0
sdnn = np.sqrt((sum_rr2 - sum_rr * sum_rr / totnn) / (totnn - 1))
rmssd = np.sqrt(rmssd/totnnn)
pnn50 = nnx / totnnn
if nnn > 1:
ratbuf[i] = nnn / nrr
sdbuf[i] = np.sqrt((sdbuf[i] - avbuf[i] * avbuf[i] / nnn) / (nnn - 1))
avbuf[i] /= nnn
nb = i + 1
sum_rr = 0.0
sum_rr2 = 0.0
k = 0
h = 0
while k < nb:
if ratbuf[k] != 0:
h += 1
sum_rr += avbuf[k]
sum_rr2 += (avbuf[k] * avbuf[k])
k += 1
sdann = np.sqrt((sum_rr2 - sum_rr * sum_rr / h) / (h - 1))
return sdnn, sdann, rmssd, pnn50
def get_result_from_get_hrv(filename):
with open(filename, 'r') as f:
csv_reader = csv.reader(f, delimiter=',')
index = 0
for row in csv_reader:
if index > 0:
output = [s.strip() for s in row[0].split('=') if s]
# print('output = ', output)
if output[0] == 'SDNN':
sdnn = output[1]
if output[0] == 'SDANN':
sdann = output[1]
if output[0] == 'rMSSD':
rmssd = output[1]
if output[0] == 'pNN50':
pnn50 = output[1]
if output[0] == 'ULF PWR':
ulf = output[1]
if output[0] == 'VLF PWR':
vlf = output[1]
if output[0] == 'LF PWR':
lf = output[1]
if output[0] == 'HF PWR':
hf = output[1]
if output[0] == 'TOT PWR':
tp = output[1]
if output[0] == 'LF/HF':
ratio_lf_hf = output[1]
index += 1
return float(sdnn), float(sdann), float(rmssd), float(pnn50), float(ulf), float(vlf), \
float(lf), float(hf), float(tp), float(ratio_lf_hf)
def save_file():
extension = "atr"
result_all = []
file_process = ['File']
sdnn_l = ['sdnn']
sdann_l = ['sdann']
rmssd_l = ['rmssd']
pnn50_l = ['pnn50']
ulf_l = ['ulf']
vlf_l = ['vlf']
lf_l = ['lf']
hf_l = ['hf']
tp_l = ['tp']
ratio_lf_hf_l = ['ratio_lf_hf']
sdnn_l_p = ['sdnn']
sdann_l_p = ['sdann']
rmssd_l_p = ['rmssd']
pnn50_l_p = ['pnn50']
ulf_l_p = ['ulf']
vlf_l_p = ['vlf']
lf_l_p = ['lf']
hf_l_p = ['hf']
tp_l_p = ['tp']
ratio_lf_hf_l_p = ['ratio_lf_hf']
test_file = ['103', '113', '117', '121', '123', '200', '202', '210', '212', '213',
'219', '221', '213', '228', '231', '233', '234',
'101', '106', '108', '112', '114', '115', '116', '119', '122', '201', '203',
'205', '208', '209', '215', '220', '223', '230',
'105', '100']
file_dis = ['109', '111', '118', '124', '207', '214', '232']
for root, dirs, files in os.walk(DATA_PATH):
files = np.sort(files)
for name in files:
if extension in name:
if os.path.basename(name[:-4]) not in test_file:
continue
print('Processing file...', os.path.basename(name))
cur_dir = os.getcwd()
os.chdir(DATA_PATH)
os.system('rrlist {} {} -M -s >{}.rr'.format(extension, name.split('.')[0], name.split('.')[0]))
time_m = []
rr_intervals = []
ann = []
with open(name.split('.')[0] + '.rr', 'r') as rr_file:
for line in rr_file:
time_m.append(line.split(' ')[0])
rr_intervals.append(line.split(' ')[1])
ann.append(line.split(' ')[2].split('\n')[0])
time_m = np.asarray(time_m, dtype=float)
rr_intervals = np.asarray(rr_intervals, dtype=float)
sdnn, sdann, rmssd, pnn50 = time_domain(time_m, rr_intervals, ann)
if sdnn == 0 and sdann == 0 and rmssd == 0 and pnn50 == 0:
print('No result hrv')
file_dis.append(os.path.basename(name[:-4]))
continue
print('sdnn', sdnn)
print('rmssd', rmssd)
print('pnn50', pnn50)
print('sdann', sdann)
time_m = time_m - time_m[0]
time_m = np.round(time_m, 3)
time_nn = []
nn_intervals = []
for i in range(1, len(ann)):
if ann[i] == 'N' and ann[i - 1] == 'N':
nn_intervals.append(rr_intervals[i])
time_nn.append(time_m[i])
time_nn = np.asarray(time_nn, dtype=float)
nn_intervals = np.asarray(nn_intervals, dtype=float)
fc = FreqDomainClass()
ulf, vlf, lf, hf, tp, ratio_lf_hf = fc.freq_domain(time_nn, nn_intervals)
sdnn_l.append(sdnn)
sdann_l.append(sdann)
rmssd_l.append(rmssd)
pnn50_l.append(pnn50)
ulf_l.append(ulf)
vlf_l.append(vlf)
lf_l.append(lf)
hf_l.append(hf)
tp_l.append(tp)
ratio_lf_hf_l.append(ratio_lf_hf)
print('ULF PWR: ', ulf)
print('VLF PWR: ', vlf)
print('LF PWR: ', lf)
print('HF PWR: ', hf)
print('TOT PWR: ', tp)
print('LF/HF: ', ratio_lf_hf)
if os.path.exists('physionet_hrv.txt'):
os.remove('physionet_hrv.txt')
os.system('get_hrv -R ' + name.split('.')[0] + '.rr >> ' + 'physionet_hrv.txt')
sdnn, sdann, rmssd, pnn50, ulf, vlf, lf, hf, tp, ratio_lf_hf = \
get_result_from_get_hrv('physionet_hrv.txt')
file_process.append(os.path.basename(name[:-4]))
sdnn_l_p.append(sdnn)
sdann_l_p.append(sdann)
rmssd_l_p.append(rmssd)
pnn50_l_p.append(pnn50)
ulf_l_p.append(ulf)
vlf_l_p.append(vlf)
lf_l_p.append(lf)
hf_l_p.append(hf)
tp_l_p.append(tp)
ratio_lf_hf_l_p.append(ratio_lf_hf)
os.chdir(cur_dir)
result_all.append(file_process)
result_all.append(sdnn_l)
result_all.append(sdnn_l_p)
result_all.append(sdann_l)
result_all.append(sdann_l_p)
result_all.append(rmssd_l)
result_all.append(rmssd_l_p)
result_all.append(pnn50_l)
result_all.append(pnn50_l_p)
result_all.append(ulf_l)
result_all.append(ulf_l_p)
result_all.append(vlf_l)
result_all.append(vlf_l_p)
result_all.append(lf_l)
result_all.append(lf_l_p)
result_all.append(hf_l)
result_all.append(hf_l_p)
result_all.append(tp_l)
result_all.append(tp_l_p)
result_all.append(ratio_lf_hf_l)
result_all.append(ratio_lf_hf_l_p)
print(file_dis)
with open('hrv2.csv', 'w+') as f:
writer = csv.writer(f)
for values in zip_longest(*result_all):
writer.writerow(values)
def main():
extension = "atr"
for root, dirs, files in os.walk(DATA_PATH):
files = np.sort(files)
for name in files:
if extension in name:
if name not in ['101.atr']:
continue
cur_dir = os.getcwd()
os.chdir(DATA_PATH)
os.system('rrlist {} {} -M -s >{}.rr'.format(extension, name.split('.')[0], name.split('.')[0]))
time_m = []
rr_intervals = []
ann = []
with open(name.split('.')[0] + '.rr', 'r') as rr_file:
for line in rr_file:
time_m.append(line.split(' ')[0])
rr_intervals.append(line.split(' ')[1])
ann.append(line.split(' ')[2].split('\n')[0])
time_m = np.asarray(time_m, dtype=float)
rr_intervals = np.asarray(rr_intervals, dtype=float)
sdnn, sdann, rmssd, pnn50 = time_domain(time_m, rr_intervals, ann)
if sdnn == 0 and sdann == 0 and rmssd == 0 and pnn50 == 0:
print('No result hrv')
return 0
print('sdnn', sdnn)
print('rmssd', rmssd)
print('pnn50', pnn50)
print('sdann', sdann)
time_m = time_m - time_m[0]
time_m = np.round(time_m, 3)
time_nn = []
nn_intervals = []
for i in range(1, len(ann)):
if ann[i] == 'N' and ann[i - 1] == 'N':
nn_intervals.append(rr_intervals[i])
time_nn.append(time_m[i])
time_nn = np.asarray(time_nn, dtype=float)
nn_intervals = np.asarray(nn_intervals, dtype=float)
start = time.time()
fc = FreqDomainClass()
ulf, vlf, lf, hf, tp, ratio_lf_hf = fc.freq_domain(time_nn, nn_intervals)
end = time.time()
print('ULF PWR: ', ulf)
print('VLF PWR: ', vlf)
print('LF PWR: ', lf)
print('HF PWR: ', hf)
print('TOT PWR: ', tp)
print('LF/HF: ', ratio_lf_hf)
print('finish', end - start)
os.chdir(cur_dir)