We Keep Coding

Filtering and saving subset of pandas

I have a function that does the following:
Inserting class values 1,2,3 based on timestamps. This work as inspected and in the first iteration of the first for-loop i get the following class distribution:
mapping: {'Seizure': 1, 'Preictal': 2, 'Interictal': 3}
value counts:
3.0 3150000
2.0 450000
1.0 28000
Name: class, dtype:
So i have this number of rows for each class.
However in the second forloop i iterate through the same list of timestamps and want to subset the data between the timestamps and include some conditions based on the classes i inserted in first forloop.
This is the result of the same timestamps e.g. first iteration:
len sz: 28000
len prei: 450000
len pre int: 29700000
logging
len post int: 1485499
How the * does preint and post int (interictal class) get this high of a count? it doesn't at all correspond somewhat to the number interictal in the first?
here my function.
def insert_class_col(dataframe, sz_info_list, date_converter, save_filename, save_path, file_sample_rate, file_channel):
print(f"sz_info_list: {sz_info_list}")
if "class" not in dataframe.columns:
dataframe.insert(0, "class", np.nan)
file_channel.extend(['timestamp', 'class'])
dataframe = dataframe[file_channel]
# Insert class attributes to ensure that seizure, preictal, interictal does not overlap.
for index, container in enumerate(sz_info_list):
delay = container.delay * 1000
duration = container.duration * 1000
sz_start = date_converter(container.time_emu) + delay
sz_end = sz_start + duration
print(f"sz_start index = {sz_start}")
print(f"sz_end: {sz_end}")
preictal_start = sz_start - (15 * 60 * 1000)
interictal_start = sz_start - (1 * 60 * 60 * 1000)
interictal_end = sz_end + (1 * 60 * 60 * 1000)
dataframe['timestamp'] = pd.to_numeric(dataframe['timestamp'])
# hvis data er sezure tag seizure
# hvis data er preictal tag preictal/interictal, men ikke indenfor seizure data.
dataframe.loc[(dataframe['timestamp'] >= sz_start) & (dataframe['timestamp'] < sz_end), "class"] = class_mapping['Seizure']
dataframe.loc[(dataframe['class'] != class_mapping['Seizure']) & (dataframe['timestamp'] >= preictal_start) & (dataframe['timestamp'] < sz_start), "class"] = class_mapping['Preictal']
dataframe.loc[(dataframe['class'] != class_mapping['Seizure']) & (dataframe['class'] != class_mapping['Preictal']) & (dataframe['timestamp'] >= interictal_start) & (dataframe['timestamp'] < interictal_end), "class"] = class_mapping['Interictal']
print(f"mapping: {class_mapping} \n value counts: \n{dataframe['class'].value_counts()}")
print(f"Begginging current number of class in df {dataframe['class'].value_counts()}")
# Saving to csv
for index, container in enumerate(sz_info_list):
delay = container.delay * 1000
duration = container.duration * 1000
sz_start = date_converter(container.time_emu) + delay
sz_end = sz_start + duration
print(f"sz_start index = {sz_start}")
print(f"sz_end: {sz_end}")
preictal_start = sz_start - (15 * 60 * 1000)
interictal_start = sz_start - (1 * 60 * 60 * 1000)
interictal_end = sz_end + (1 * 60 * 60 * 1000)
dataframe['timestamp'] = pd.to_numeric(dataframe['timestamp'])
#INSERTING SEIZURE CLASS
sz_df = dataframe[(dataframe['timestamp'] >= sz_start) & (dataframe['timestamp'] < sz_end)].copy()
print(f"len sz: {len(sz_df)}")
#df_save_compress(f"Seizure_{index}_{save_filename}", save_path + "/Seizure", sz_df)
#logging_info_txt(f"Seizure_{index}_{save_filename}", save_path, file_sample_rate, file_channel)
#INSERTING PREICTAL
prei_df = dataframe[(dataframe['timestamp'] >= preictal_start) & (dataframe['timestamp'] < sz_start) & (dataframe['class'] != class_mapping["Seizure"])].copy()
print(f"len prei: {len(prei_df)}")
#df_save_compress(f"Preictal_{index}_{save_filename}", save_path + "/Preictal", prei_df)
#logging_info_txt(f"Preictal_{index}_{save_filename}", save_path, file_sample_rate, file_channel)
#INSERTING INTERICTAL
pre_int_df = dataframe[(dataframe['timestamp'] >= interictal_start) & (dataframe['timestamp'] < preictal_start) & (dataframe['class'] != class_mapping["Seizure"]) | (dataframe['class'] != class_mapping["Preictal"])].copy()
print(f"len pre int: {len(pre_int_df)}")
#df_save_compress(f"PreInt_{index}_{save_filename}", save_path + "/Interictal", pre_int_df)
logging_info_txt(f"PreInt_{index}_{save_filename}", save_path, file_sample_rate, file_channel)
post_int_df = dataframe[(dataframe['timestamp'] >= sz_end) & (dataframe['timestamp'] < interictal_end) & (dataframe['class'] != class_mapping["Seizure"]) & (dataframe['class'] != class_mapping["Preictal"])].copy()
print(f"len post int: {len(post_int_df)}")
#df_save_compress(f"PostInt_{index}_{save_filename}", save_path + "/Interictal", post_int_df)
logging_info_txt(f"PostInt_{index}_{save_filename}", save_path, file_sample_rate, file_channel)
#print(f"after = len df: {len(dataframe)} values class: \n {dataframe['class'].value_counts()}")
# clean up
del pre_int_df, post_int_df, sz_df, prei_df
gc.collect()
Notice that preint which is interictal is 29700000 while printing the classes i should be lower than 3150000.
Any ideas of this pandas behavior?

#richardec answered the question see comments.

How to fix "ZeroDivisionError: float division by zero"

I have calcularBeta1 method. When I run the program, I've got this error:
ZeroDivisionError: float division by zero
resultadoB1 = (sumaXY - ((sumaX * sumaY ) / totalElementos )) / (sumaXCuadrada - math.pow(sumaX, 2) / totalElementos)
Method calcularBeta1
def calcularBeta1(self, lista):
actual = lista.nodoInicio
sumaXY = 0
sumaX = 0
sumaY = 0
sumaXCuadrada = 0
totalElementos = 0
while actual != None:
dato1 = actual.dato1
dato2 = actual.dato2
sumaXY += dato1 * dato2
sumaX += dato1
sumaY += dato2
sumaXCuadrada += math.pow(dato1, 2)
totalElementos += 1
actual = actual.siguienteNodo
resultadoB1 = (sumaXY - ((sumaX * sumaY ) / totalElementos )) / (sumaXCuadrada - math.pow(sumaX, 2) / totalElementos)
return resultadoB1
LecturaArchivo class
class LecturaArchivo:
datosArchivo = ListaEnlazada()
operaciones = Operaciones()
xTemporal = 0
yTemporal = 0
nombreArchivo = input('Nombre del archivo: ')
archivo = open(nombreArchivo, "r")
lineas = archivo.read()
datos = lineas.split(',')
datoProxy = float(input('Proxy: '))
while lineas:
lineas = archivo.readlines()
xTemporal = datos[0]
yTemporal = datos[1]
datosArchivo.agregarNodoFinal(float(xTemporal), float(yTemporal))
print(datos)
sumaElementos = sum(datosArchivo.obtenerNodos())
mediaElementos = operaciones.media(sumaElementos, datosArchivo.tamano())
beta1 = operaciones.calcularBeta1(datosArchivo)
print('Beta1: ', beta1)
beta0 = operaciones.calcularBeta0(beta1, media)
print('Beta0: ', beta0)
yk = operaciones.calcularYK(beta0, beta1, datoProxy)
print('Regresión Líneal: ', yk)

The ZeroDivisionError happens when you try to divide a number by 0, which as you know is a mathematical impossibility, just change the value of the dividend.

Python is unable to divide numbers by 0. If you ever attempt to divide by 0, python will throw a ZeroDivisionError, which is what happened to you. The best way to fix it is to just not divide by zero. You can use an if statement to ensure that the values are not zero.

Don't divide by zero.
In the expression it complains about
resultadoB1 = (sumaXY - ((sumaX * sumaY ) / totalElementos )) / (sumaXCuadrada - math.pow(sumaX, 2) / totalElementos)
it will be either totalElementos or the results of sumaXCuadrada - math.pow(sumaX, 2) that are zero.
You'll need to add code to handle the possibility of those situations before trying to calculate that formula.

python to mysql query

im trying to convert this trending algorithm to mysql query
# Rewritten code from /r2/r2/lib/db/_sorts.pyx
from datetime import datetime, timedelta
from math import log
epoch = datetime(1970, 1, 1)
def epoch_seconds(date):
td = date - epoch
return td.days * 86400 + td.seconds + (float(td.microseconds) / 1000000)
def score(ups, downs):
return ups - downs
def hot(ups, downs, date):
s = score(ups, downs)
order = log(max(abs(s), 1), 10)
sign = 1 if s > 0 else -1 if s < 0 else 0
seconds = epoch_seconds(date) - 1134028003
return round(sign * order + seconds / 45000, 7)
mysql table have
up_count down_count created_date
is this possible ?
i found this ,but this have a bug if the down_vode is bigger then you have LOG10(Zero) and this will crash
ORDER BY
LOG10(ABS(thumbs_up - thumbs_down) + 1) * SIGN(thumbs_up - thumbs_down)
+ (UNIX_TIMESTAMP(created) / 300000) DESC
LIMIT 100

This is the MySql translation of you function:
SELECT ROUND((INTERVAL(up_count - down_count, -1, 0, 1) - 2) * LOG(GREATEST(ABS(up_count - down_count), 1), 10) + UNIX_TIMESTAMP(created_date) / 45000 , 7) FROM your_table

Add function names to a string in Python

I'm just beginning to learn Python on my own, and I'm trying to write a code that will calculate the end time of a jog.
My code so far looks as follows:
def elapsed(t):
t = raw_input('Enter time (hh:mm:ss): ')
th, tm, ts = t.split(':')
return int(ts) + int(tm) * 60 + int(th) * 3600
def mile(m):
m = raw_input('How many miles? ')
return int(m)
start = elapsed('start')
warmup = elapsed('warmup')
wmile = mile('wmile')
tempo = elapsed('tempo')
tmile = mile('tmile')
cooloff = elapsed('cooloff')
cmile = mile('cmile')
hour = (start + warmup * wmile + tempo * tmile + cooloff * cmile) // 3600
minute = (start + warmup * wmile + tempo * tmile + cooloff * cmile - hour * 3600) // 60
second = (start + warmup * wmile + tempo * tmile + cooloff * cmile - hour * 3600) % 60
print('Your run ended at %02d:%02d:%02d' % (hour, minute, second))
In this code, the time prompts are all the same: "Enter time (hh:mm:ss):" I want each prompt to refer to its variable name, e.g., "Enter start time (hh:mm:ss)" or "Enter time (hh:mm:ss): (warmup)". Is there a way to do this?
Note: While this may technically be a duplicate, I have examined the similar questions, but decided that both the questions and the answers provided were on the more unspecific side, and therefore decided to ask my question anyway.

Yes, use the input to your function elapsed(t).
Right now it's being overwritten with the return from raw_input()
def elapsed(t):
t1 = raw_input('Enter time (hh:mm:ss): ({0})'.format(t))
th, tm, ts = t1.split(':')
return int(ts) + int(tm) * 60 + int(th) * 3600
or
def elapsed(t):
t1 = raw_input('Enter time (hh:mm:ss): (%s)' % t))
th, tm, ts = t1.split(':')
return int(ts) + int(tm) * 60 + int(th) * 3600

def enter_time(specify_string):
print("Please enter the", specify_string, 'time in format hh:mm:ss > ', end='')
hh, mm, ss = input().split(':')
return (hh, mm, ss)
start_time = enter_time('start')
finish_time = enter_time('finish')
>>>Please enter the start time in format hh:mm:ss >13:32:34
>>>Please enter the finish time in format hh:mm:ss >12:21:21
>>>start_time
(13, 32, 34)
Now you can call the function with string arguments in the function call, and it will generalise the function for different requirements.
It is better to move the time between functions in a more readable format, such as a tuple. You can make more functions, eg:
- Test valid time entered
- Convert tuple to seconds for calculations
- Convert seconds back to tuple format
ect.
Let me know if I misunderstood you.

if logic breaks when calculating time over multiple days

My code below - yes, ugly. I am learning python. It works when the start/end dates are both the same day.
When start = 2017,6,15 6:00 & end = 2017,6,16 6:00
The if statement that gets applied is if start < 6am. Can someone explain why? It does recognize that 1 day is the duration.
from datetime import datetime, date, time
def babysitting():
(st_yr,st_mon,st_day) = [int(s) for s in input("Enter the start date in the format: YYYY,M,D: ").split(',')]
(st_hr,st_min) = [int(s) for s in input("Enter the start time in 24hr format -> HH:MM: ").split(":")]
(e_yr,e_mon,e_day) = [int(s) for s in input("Enter the end date in the format: YYYY,M,D: ").split(',')]
(e_hr,e_min) = [int(s) for s in input("Enter the end time in 24hr format -> HH:MM: ").split(":")]
st_date = date(st_yr,st_mon,st_day)
st_time = time(st_hr,st_min)
end_date= date(e_yr,e_mon,e_day)
end_time = time(e_hr,e_min)
start = datetime.combine(st_date,st_time)
end = datetime.combine(end_date,end_time)
ninePM = datetime.combine(end_date, time(21,00))
sixAM = datetime.combine(end_date, time(6,00))
if start >= sixAM:
if end <= ninePM:
normal_rate_duration = end - start
normal_rate_seconds = normal_rate_duration.total_seconds()
normal_rate_billing = round(((normal_rate_seconds / 60) / 60) * 2.5,2)
print("Cost for babysitting is: $", normal_rate_billing)
elif end > ninePM:
normal_rate_duration = ninePM - start
reduced_rate_pm_duration = end - ninePM
normal_rate_seconds = normal_rate_duration.total_seconds()
normal_rate_billing = round(((normal_rate_seconds / 60) / 60) * 2.5,2)
reduced_rate_pm_seconds = reduced_rate_pm_duration.total_seconds()
reduced_rate_pm_billing = round(((reduced_rate_pm_seconds / 60) / 60) * 1.75,2)
print("Cost for babysitting is: $", normal_rate_billing + reduced_rate_pm_billing)
if start < sixAM:
if end <= ninePM:
reduced_rate_am_duration = sixAM - start
reduced_rate_am_seconds = reduced_rate_am_duration.total_seconds()
reduced_rate_am_billing = round(((reduced_rate_am_seconds / 60) / 60) * 1.75,2)
normal_rate_duration = end - sixAM
normal_rate_seconds = normal_rate_duration.total_seconds()
normal_rate_billing = round(((normal_rate_seconds / 60) / 60) * 2.5,2)
print("Cost for babysitting is: $", normal_rate_billing + reduced_rate_am_billing)
elif end > ninePM:
reduced_rate_am_duration = sixAM - start
reduced_rate_am_seconds = reduced_rate_am_duration.total_seconds()
reduced_rate_am_billing = round(((reduced_rate_am_seconds / 60) / 60) * 1.75,2)
normal_rate_duration = ninePM - sixAM
normal_rate_seconds = normal_rate_duration.total_seconds()
normal_rate_billing = round(((normal_rate_seconds / 60) / 60) * 2.5,2)
reduced_rate_pm_duration = end - ninePM
reduced_rate_pm_seconds = reduced_rate_pm_duration.total_seconds()
reduced_rate_pm_billing =round(((reduced_rate_pm_seconds / 60) / 60) * 1.75,2)
print("Cost for babysitting is: $", normal_rate_billing + reduced_rate_am_billing + reduced_rate_pm_billing)
babysitting()

your if checks whether or not the start time was earlier than 6am on the end date, which of course it will be (it's some time the day before or something like that). you need to check if st_time is earlier than 6am (regardless of date, or on the st_date date).