I have a problem that involves collecting data continuously from multiple sources.
My setup as it is currently, writes each data entry from each source to a MySQL db, and then, with another python program, does Select's that bring all the data together. I need to make INSERT's at roughly 1000/second and as it is my SELECT's can take 15-20 seconds each.
The whole process takes so long the data is obsolete before I get to do anything useful with it.
I have created a toy example to try and demonstrate what I am looking for.
program 1 'generateClasses':
import time
import random
from datetime import datetime
class Race:
def __init__(self,name):
hist = {}
now = datetime.now()
self.name = name
self.now = now
hist[now] = 0
self.v = 0
self.hist = hist # example variable's.
def update(self,name,v):
now = datetime.now()
hist = self.hist
hist[now] = v
self.v = v
self.now - now
self.hist = hist
class Looper:
def __init__(self,name):
self.a = Race(name)
def loop(self,name):
# simulating the streaming API
while True:
v = self.a.v
v += 1
self.a.update(name,v)
print(a,datetime.now(),v) # can i access this stream using the location displayed with the print(a)?
time.sleep(0.1) # this should be more like time.sleep(0.001)
def pickData(self,name):
v = self.v
self.loop(name)
print('The state at {} {} = '.format(self.now,self.v))
return self.hist
if __name__ == "__main__":
x = 'Some_ID'
a = Looper(x)
a.loop(x)
program 2:
from generateClasses import Looper
from datetime import datetime
import time
start_time = int((datetime.now() - datetime(1970, 1, 1)).total_seconds())
print(start_time)
x = 'Some_orher_ID'
a = Looper(x)
print('this will print')
a.loop(x)
print('this wont ever print')
a.pickdata(x)
# this last section is the functionality i am looking for in this program, but, as it is, it will never run.
x = ‘Some_ID’
while True:
now_time = int((datetime.now() - datetime(1970, 1, 1)).total_seconds())
print(start_time)
if int(now_time-start_time) == 10:
a.pickData(x)
# b.pickData(x)
# c.pickData(x)
# d.pickData(x)
# make further actions.....
What happens currently in my examples is that it creates its own loop using the class structure from the first program.
What I want it to do is call the the pickData() method from program 2 at timely intervals of my choosing on a loop running in another program.
Is my best option picking a db located in memory and getting a faster computer?
Maybe something can be done with the object location shown when you print the instance name?
I have uploaded to github if anybody fancies it..
I would be grateful of any suggestions.
also, recommendations for further reading would be appreciated also.
Related
I'm doing project on RPi and came across problem, that I can't solve. I'm using multiprocess to calculate wind speed with anemometer, and servo based platform to move around pv panel. In the main function I declare processes and objects:
def main():
ane = Anemometer(pin=38, radius=65, anamometer_factor=1, action_time=5,
measurment_fraquency=0.01, max_wind_speed=40, mesurment_time=30)
prb = PhotoresistorBase(TR=1, TL=3, BR=0, BL=2, signifficant_diff=40)
BS = Servo(pin=12, min_val=10, max_val=50)
US = Servo(pin=18, min_val=29, max_val=43)
pv_based = Pv(6)
pv_servo = Pv(5)
...
calculate_wind_vel_p = Process(target=calculate_wind_vel, args=(ane, BS, US))
calculate_power_p = Process(target=calculate_power, args=(pv_based, pv_servo))
turn_platform_p = Process(target=turn_platform, args=(prb, BS, US))
calculate_wind_vel_p.start()
calculate_power_p.start()
turn_platform_p.start()
calculate_wind_vel_p.join()
calculate_power_p.join()
turn_platform_p.join()
Process calculate_power_p works great, problem appears in two other processes
Process calculate_wind_vel_p targets function calculate_wind_vel, which looks like this:
def calculate_wind_vel(ane, BS, US):
while True:
wind_speed = ane.calculate_mean_wind_velocity()
...
data = (datetime.datetime.now(), wind_speed)
insert_vel_data(conn, data)
time.sleep(1)
and Anemometer class, which contains calculate_mean_wind_velocity() looks like this:
class Anemometer:
def __init__(self, pin, radius, anamometer_factor, action_time,
measurment_fraquency, max_wind_speed, mesurment_time):
...
def calculate_wind_velocity(self) -> float:
rotations = 0
count = 0
endtime = time.time() + self.action_time
sensorstart = GPIO.input(self.pin)
circumference = (self.radius * 2 / 1000) * math.pi
while time.time() < endtime:
if GPIO.event_detected(self.pin):
count = count + 1
**print(count)**
if rotations==1 and sensorstart==1:
rotations = 0
rots_per_second = float(count/3/self.action_time)
wind_vel = float((rots_per_second)*circumference*self.anamometer_factor)
**print("Wind vel: " + str(wind_vel))**
return wind_vel
def calculate_mean_wind_velocity(self) -> float:
sum = 0
measure_count = int(self.mesurment_time/self.action_time)
for _ in range(measure_count):
sum += self.calculate_wind_velocity()
return float(sum/measure_count)
The Problem:
When process executes calculate_wind_vel() it jumps to ane.calculate_mean_wind_velocity(). The second function, executes calculate_wind_velocity e.g 4 times and calculate mean wind speed.
When calculate_wind_velocity() starts, program prints 'count' only once (first bolded print) and later count does not increment despite event is triggered. Than it calculate wind speed, with that one counted interruption. Later when calculate_wind_velocity() is execute again (process target has while True:) count is not incremented at all!
weirdest thing is that, when I run this code without process, just paste this into main():
ane = Anemometer(pin=38, radius=65, anamometer_factor=1, action_time=5,
measurment_fraquency=0.01, max_wind_speed=40, mesurment_time=30)
while True:
ret = ane.calculate_mean_wind_velocity()
print(ret)
time.sleep(2)
It works perfect - it calculates every single GPIO.event_detected(self.pin) and wind_speed, so it has smth to do with processes!.
Similar problem with moving platform - process executes functions from servo class, but it does not move, functions don't calculate/change state.
i'm stuck with this problem, i can't read the variables bpm and spo2 from the function run_sensor (i need these two variables in another script). If someone can help me, thanks.
class HeartRateMonitor(object):
LOOP_TIME = 0.10
def __init__(self, print_raw=False, print_result=False):
self.bpm = 0
if print_raw is True:
print('IR, Red')
self.print_raw = print_raw
self.print_result = print_result
def run_sensor(self):
sensor = MAX30102()
ir_data = []
red_data = []
bpms = []
# run until told to stop
while not self._thread.stopped:
# check if any data is available
num_bytes = sensor.get_data_present()
if num_bytes > 0:
# grab all the data and stash it into arrays
if len(ir_data) == 100:
bpm, valid_bpm, spo2, valid_spo2 = hrcalc.calc_hr_and_spo2(ir_data, red_data) # <-------------- here
if valid_bpm:
bpms.append(bpm)
while len(bpms) > 4:
bpms.pop(0)
self.bpm = np.mean(bpms)
if (np.mean(ir_data) < 50000 and np.mean(red_data) < 50000):
self.bpm = 0
if self.print_result:
print("Finger not detected")
if self.print_result:
print("BPM: {0}, SpO2: {1}".format(round(self.bpm), round(spo2)))
You need to use Global keyword for this issue. Write this -
global bpm
global spo2
under the function you need to use the variables in
You can't access class variables from another function. However, if you were to create an instance of the HeartRateMonitor class, then you can import it and get its bpm and spo2.
instance = HeartRateMonitor()
Then, in your other script:
from heartratemonitor.py import instance
instance.run_sensor() # I assume this is what you want to do
print(instance.bpm)
print(instance.spo2)
I'm having trouble with passing a value from one script to another, trying to take it a step at a time but the big picture would be to print the value obj1.get_predval to my Django view and wait for the users' input.
active_learner.obj1.get_predval in my beta.py script doesn't work, it just prints out the initial value which makes sense because it's not running the main.py but I'm not sure how I'd pass the value of obj1.set_predval(machine_prediction) from main.py. It properly outputs the obj1.get_predval in the main.py script.
I'm assuming I have a fundamental misunderstanding, for now, all I'm trying to return is the value of obj1.get_predval in function beta.py, when it gets to the line return value and wait for user input then continue.
main.py script below
obj1 = MachinePred()
def main():
model = load_model('model_new.h5')
DATAFILE = "c_user_model_data100000.csv"
dataset = loadtxt(DATAFILE, delimiter=",")
X_pool, Y = dataset[:, 0:5], dataset[:, 5:]
sc_x, sc_y = StandardScaler(), StandardScaler()
X_pool, Y = sc_x.fit_transform(X_pool), sc_y.fit_transform(Y)
learner = ActiveLearner(
estimator = model,
query_strategy = uncertainty_sampling
)
for i in range(3):
query_idx, query_inst = learner.query(X_pool)
print("The machine queried:\n{}\nat index {}".format(
sc_x.inverse_transform(query_inst),
query_idx
)
)
machine_prediction = learner.predict(X_pool[query_idx])
obj1.set_predval(machine_prediction)
print("predvalue:", (obj1.get_predval()))
ratings = []
cc_factor = ["delay", "speed", "missing_words", "paraphrasing"]
for f in cc_factor:
user_answer = input("How would you rate the quality of {} between [1-5]: ".format(f))
ratings.append(user_answer)
print(ratings, np.array([ratings]).reshape(1,-1))
if __name__ == '__main__':
main()
beta.py
This is the script I'm trying to pass the value to below
import active_learner
print(A is: ", active_learner.obj1.get_predval)
mac.py Simple python script using the get and set methods below.
class MachinePred:
predval = 0 # Default value of the 'x' configuration setting
def __init__(self):
self.predval = 0
def set_predval(self, val):
self.predval = val
def get_predval(self):
return self.predval
So the solution to this was very simple, from my understanding it could be done using generator-based coroutines or splitting it into two functions inside a class and use an OO design. The coroutine method would use"yield" which would force exit the function returning the value then re-enter the function but this would limit your ability to use non-generator based coroutines which I did need to await input from my front end.
Using a class though you could put the active learner model and data in an "init method" then split from the machine_prediction = learner.predict(X_pool[query_idx]) for the first function after returning the values and perform the rest in a second function.
I have a python 2.7 script which is getting pretty unweildly.
The script runs constantly, in each loop it checks a few things, compares them to the previous run and makes some decisions.
The biggest problem I have is with the variables, I have half a dozen and I am limited in how I use them due to scope.
eg;
import time
import os
LOG_FILE = "/var/log/sc.log"
CHECK_FILE_TMP = "/tmp/myfile"
CHECK_FILE_USR = "/home/snoppy/testfile"
CHECK_FILE_TMP_TIME_INTERVAL = 20
CHECK_FILE_USR_TIME_INTERVAL = 30
def main():
try:
last_file_tmp_size = 0
last_file_usr_size = 0
last_file_tmp_mtime = 0
last_file_usr_mtime = 0
last_file_tmp_check_time = 0
last_file_usr_check_time = 0
tmp_file_changed = False
usr_file_changed = False
loop_start_time = 0
print "Starting loop"
while True:
loop_start_time = time.time()
if (time.time() - last_file_tmp_check_time > CHECK_FILE_TMP_TIME_INTERVAL):
tmp_file_changed = checkFileChanged(CHECK_FILE_TMP, last_file_tmp_size, last_file_tmp_mtime)
last_file_tmp_size = getFileSize(CHECK_FILE_TMP)
last_file_tmp_mtime = getFileMTime(CHECK_FILE_TMP)
if(tmp_file_changed):
logChange(CHECK_FILE_TMP, last_file_tmp_size, last_file_tmp_mtime)
last_file_tmp_check_time = time.time()
....
....
sleep(1)
...
So thats sort of what I am dealing with.
I have local variables, which I seem to be stuck with, I have to pass them around into functions - I don't want to call them global.
Ideally.... if I could get a main() function like
try:
checkFile(CHECK_FILE_TMP)
checkFile(CHECK_FILE_USR)
sleep(0.1)
except:
...
as the main is so big! and I have to pass around the variables everywhere... it feels like the main function is so bloated!
Perhaps.... I might have to go for a class?
You need to identify the parts of your program that can be abstracted from their specific details and reused multiple times.
This is an art, don't expect it to be easy. You can find many books on Object oriented programming that explain the basics and give you some direction.
Here's just a quick example:
class FileChecker(object):
def __init__(self, path, interval):
self.path = path
self.interval = interval
self.last_size = None
self.last_mtime = None
self.last_check_time = None
def hasChanged(self):
...
def logChange(self):
...
class MultiFileChecker(object):
DELAY = 1
def __init__(self):
self.files = []
self.loop_start_time = 0
def addFile(self, f):
self.files.append(f)
def loop(self):
try:
print "Starting loop"
while True:
now = time.time()
self.loop_start_time = now
for f in self.files:
if now - f.last_check_time > f.interval:
if f.hasChanged():
f.logChange()
...
sleep(self.DELAY)
...
if __name__ == '__main__':
checker = MultiFileChecker()
checker.add(FileChecker("/var/log/sc.log", 10))
checker.add(FileChecker("/tmp/myfile", 20))
checker.add(FileChecker("/home/snoppy/testfile", 30))
checker.loop()
I have a temperature and pressure gauge that I want to use to track the temperature over time with. Since I may end up with multiple sensors over time, I want to be able to reference my BMP085 sensor as tp. In other words, I would like to call tp.temp or tp.pressure to obtain the current temperature, etc. The problem is that tp.temp or .pressure is not updating each time I call it. Suggestions?
#!/usr/bin/env python
#temperature logger for the BMP085 Temperature and Pressure Sensor on the Raspberry Pi
from Adafruit_BMP085 import BMP085
from time import sleep
import pickle, sys, os
class tps():
def __init__(self):
#temperature/pressure sensor setup
self.bmp = BMP085(0x77)
self.temp = self.bmp.readTemperature()*1.8+32
self.pressure = self.bmp.readPressure()*0.0002953
class data():
def __init__(self):
self.tp = tps()
self.savedata()
def savedata(self):
# if os.path.exists("data.dat")==True:
# if os.path.isfile("data.dat")==True:
# fileHandle = open ( 'data.dat' )
# olddata = pickle.load ( fileHandle )
# fileHandle.close()
print self.tp.temp, self.tp.pressure
sleep(4)
print self.tp.temp, self.tp.pressure
# newdata = [self.tp.temp, self.tp.pressure]
# self.datadump = [olddata]
# self.datadump.append(newdata)
# fileHandle = open ( 'data.dat', 'w' )
# pickle.dump ( self.datadump, fileHandle )
# fileHandle.close()
data()
That's because you only called the bmp.readTemperature() and bmp.readPressure() functions once - in tps.__init__. In your print statements at the end, you're just twice reading the values that those functions returned, rather than getting updated values.
Here's an example of how to get updated values:
class tps():
def __init__(self):
#temperature/pressure sensor setup
self.bmp = BMP085(0x77)
self.temp = None
self.pressure = None
# If you want to initialize your tps object with sensor data, you can call your updater method here.
self.updateTempAndPressure()
# Here's a function that you can call whenever you want updated data from the sensor
def updateTempAndPressure(self):
self.temp = self.bmp.readTemperature()*1.8+32
self.pressure = self.bmp.readPressure()*0.0002953
class data():
def __init__(self):
self.tp = tps()
self.savedata()
def savedata(self):
# Call the method that gets updated data from the sensor
self.tp.updateTempAndPressure()
print self.tp.temp, self.tp.pressure
sleep(4)
# Call the update method again
self.tp.updateTempAndPressure()
print self.tp.temp, self.tp.pressure
data()