We Keep Coding

How can I refresh the data in the background of a running flask app?

I have a simple flask app that queries a database to write a csv then pyplot to create a chart out of that.
I would like to refresh the data in the background every 10 minutes while the app is running. The page doesn't need to refresh the html automatically. It just needs to have fresh data when someone opens the page.
Can I do that in a single script? Or do I need to run a different script outside in crontab or something?
I would just kick over the container every 10 minutes but it takes about 5 minutes to get the query, so that's a 5 minute outage. Not a great idea. I'd prefer it to fetch in the background.
Here is what I'm working with:
import os
from datetime import date
import teradatasql
import pandas as pd
import matplotlib.pyplot as plt
from flask import Flask, render_template
import time
import multitasking
### variables
ausername = os.environ.get('dbuser')
apassword = os.environ.get('dbpassword')
ahost = os.environ.get('dbserver')
systems = ["prd1", "prd2", "frz1", "frz2", "devl"]
qgsystems = ["", "#Tera_Prd2_v2", "#Tera_Frz1_v2", "#Tera_Frz2_v2", "#Tera_Devl_v2"]
weeks = ["0", "7", "30"]
query = """{{fn teradata_write_csv({system}_{week}_output.csv)}}select (bdi.infodata) as sysname,
to_char (thedate, 'MM/DD' ) || ' ' || Cast (thetime as varchar(11)) as Logtime,
sum(drc.cpuuexec)/sum(drc.secs) (decimal(7,2)) as "User CPU",
sum(drc.cpuuserv)/sum(drc.secs) (decimal(7,2)) as "System CPU",
sum(drc.cpuiowait)/sum(drc.secs) (decimal(7,2)) as "CPU IO Wait"
from dbc.resusagescpu{qgsystem} as drc
left outer join boeing_tables.dbcinfotbl{qgsystem} as bdi
on bdi.infokey = 'sysname'
where drc.thedate >= (current_date - {week})
order by logtime asc
Group by sysname,logtime
;
"""
### functions
#multitasking.task
def fetch(system,qgsystem,week):
with teradatasql.connect (host=ahost, user=ausername, password=apassword) as con:
with con.cursor () as cur:
cur.execute (query.format(system=system, qgsystem=qgsystem, week=week))
[ print (row) for row in cur.fetchall () ]
#multitasking.task
def plot(system,week):
for week in weeks:
for system in systems:
df = pd.read_csv(system + "_" + week + "_output.csv")
df.pop('sysname')
df.plot.area(x="Logtime")
figure = plt.gcf()
figure.set_size_inches(12, 6)
plt.savefig( "/app/static/" + system + "_" + week + "_webchart.png", bbox_inches='tight', dpi=100)
### main
for week in weeks:
for system, qgsystem in zip(systems, qgsystems):
fetch(system,qgsystem,week)
for week in weeks:
for system in systems:
plot(system,week)
app = Flask(__name__,template_folder='templates')
#app.route('/')
def index():
return render_template("index.html")

How to automate control of Wemo light switch based on iPhone GPS

I'm writing a program to toggle the lights at my house based on my iPhone's GPS coordinates. Below is what I have so far. However, I feel like there must be a better way to do this. Is there a way to get GPS data without pinging my phone every five minutes?
So far I've tried the following with no joy:
Using Shortcuts and Scriptable I tried to write some JavaScript that would trigger when I got close to home. However, I could not figure out how to use await require('wemo-client') using scriptablify. I kept getting an error, "ReferenceError: Can't find variable: require".
IFTTT does not have a variable timed trigger so the lights won't turn off after 15 minutes. Also, I plan on adding a motion sensor trigger that is unsupported.
Pythonista is $10. Yes, I am that cheap.
Apple HomeKit does not support the model I'm using, Wemo Smart Light Switch F7C030.
The code below works, but I hate that I have to ping my phone every five minutes. I'd rather save battery life by firing this code once or twice a day, as needed.
Any suggestions would be greatly appreciated.
Code:
import sys
import time
import datetime
import os
from pyicloud import PyiCloudService
import pywemo
APPLE_ID = os.getenv('APPLE_ID') # Apple ID username
APPLE_ID_PASSWORD = os.getenv('APPLE_ID_PASSWORD') # Apple ID password
API = PyiCloudService(APPLE_ID, APPLE_ID_PASSWORD)
IPHONE = API.devices[1]
LOCATION = IPHONE.location()
FIVE = 300 # 5 * 60 seconds
FIFTEEN = 900 # 15 * 60 seconds
ONEMILE = 0.01449275362318840579710144927536 # one mile is 1/69 degrees lat or long
HOMELAT = # my home's latitude
HOMELONG = # my home's longitude
WEMOS = pywemo.discover_devices()
LEN_WEMOS = range(len(WEMOS))
# Two factor authentication to retrieve iPhone data
if API.requires_2fa:
import click
print("Two-step authentication required. Your trusted devices are:")
DEVICES = API.devices
for i, device in enumerate(DEVICES):
print(" %s: %s" % (i, device.get('deviceName', "SMS to %s" % device.get('phoneNumber'))))
DEF_DEVICE = click.prompt('Which device would you like to use?', default=0)
DEVICE = DEVICES[DEF_DEVICE]
if not API.send_verification_code(DEVICE):
print("Failed to send verification code")
sys.exit(1)
CODE = click.prompt('Please enter validation code')
if not API.validate_verification_code(DEVICE, CODE):
print("Failed to verify verification code")
sys.exit(1)
# Turn off the lights when I leave
def leavehome():
timenow = datetime.datetime.now()
print("Left home on {}".format(timenow.strftime("%B %d, %Y at %H:%M:%S")))
for wemo in LEN_WEMOS:
WEMOS[wemo].off()
# Turn on the lights for 15 minutes when I get home
def arrivehome():
timenow = datetime.datetime.now()
print("Arrived home on {}".format(timenow.strftime("%B %d, %Y at %H:%M:%S")))
# Loop through all Wemo devices
for wemo in LEN_WEMOS:
WEMOS[wemo].on()
time.sleep(FIFTEEN)
for wemo in LEN_WEMOS:
WEMOS[wemo].off()
# Automatically turn off the lights after 15 minutes - save electricity
def timeoff():
time.sleep(FIFTEEN)
for wemo in LEN_WEMOS:
WEMOS[wemo].off()
# Ping my phone for GPS data
def pingphone(prev):
mylat = LOCATION["latitude"]
mylong = LOCATION["longitude"]
logic(prev, mylat, mylong)
time.sleep(FIVE)
# Perform logic to determine if I'm home, out, arriving, or leaving
def logic(prev, lat, long):
inrange = (HOMELAT+ONEMILE >= lat >= HOMELAT-ONEMILE and HOMELONG+ONEMILE >= long >= HOMELONG-ONEMILE)
current = bool(inrange)
previous = prev
if current and not previous:
arrivehome()
elif previous and not current:
leavehome()
else:
timeoff()
pingphone(current)
# Run the script
pingphone(False)

Python Jupyter Notebook won't run my code, keeps reconnecting

How come this piece of code does not run properly on Jupyter Notebook.
It keeps reconnecting without any result. I try to make a database and scrape data as fast as possible from a webserver. I use threads to speed up the process and iterate over multiple url's (every different url represent a different day).
import pandas as pd
import datetime
import urllib
import requests
from pprint import pprint
import time
from io import StringIO
from multiprocessing import Process, Pool
symbols = ['AAP']
start = time.time()
dflist = []
def load(date):
if date is None:
return
url = "http://regsho.finra.org/FNYXshvol{}.txt".format(date)
try:
df = pd.read_csv(url,delimiter='|')
if any(df['Symbol'].isin(symbols)):
stocks = df[df['Symbol'].isin(symbols)]
print(stocks.to_string(index=False, header=False))
# Save stocks to mysql
else:
print(f'No stock found for {date}' )
except urllib.error.HTTPError:
pass
pool = []
numdays = 365
start_date = datetime.datetime(2019, 1, 15 ) #year - month - day
datelist = [
(start_date - datetime.timedelta(days=x)).strftime('%Y%m%d') for x in range(0, numdays)
]
pool = Pool(processes=16)
pool.map(load, datelist)
pool.close()
pool.join()
print(time.time() - start)
Would like to know how I can solve this and make it work

how to use the reqMktData from Ibpy properly?

Hi guys am just starting to work on an Ibpy algorithm and i would like to test it with paper trading first but i have a little understanding how to use the reqMktData to get the last price. I have no problem placing orders but this returns nothing for 25 seconds, Im thinking that it is only to be used during trading hours or maybe am just using it wrong any thoughts?
from ib.opt import ibConnection, message
from ib.ext.Contract import Contract
from time import sleep
def my_callback_handler(msg):
inside_mkt_bid = ''
inside_mkt_ask = ''
if msg.field == 1:
inside_mkt_bid = msg.price
print 'bid', inside_mkt_bid
elif msg.field == 2:
inside_mkt_ask = msg.price
print 'ask', inside_mkt_ask
tws = ibConnection()
tws.register(my_callback_handler, message.tickSize, message.tickPrice)
tws.connect()
c = Contract()
c.m_symbol = "DATA"
c.m_secType = "STK"
c.m_exchange = "SMART"
c.m_currency = "USD"
tws.reqMktData(788,c,"",False)
sleep(25)
print 'All done'
tws.disconnect()

I have try IbPy before and successfully get the data but now I have use Ibapi instead which is more difficult and still can't fully trade in it but it has an adjusted historical price.
So this is my code which you have to tailor what so you want.
1.Get the stock member form Excel
from ib.opt import ibConnection, message
from ib.ext.Contract import Contract
from ib.ext.Order import Order
from ib.ext.TickType import TickType as tt
from time import sleep, time, strftime
import datetime
from __future__ import print_function #I'm using 3.x style print
import pandas as pd
import numpy as np
from math import ceil
import re
xls_file = pd.ExcelFile('xxxx\\Interactive_Broker_trading\\SNP2.xlsx')
df = xls_file.parse('Sheet1')
Ticker = df.iloc[:,1]
all_data = pd.DataFrame(Ticker)
all_data.columns = ['ticker']
all_data['type'] = 'STK'
all_data['exchange'] = 'SMART'
all_data['curr'] = 'USD'
all_data['bidPrice'] =0
all_data['askPrice'] =0
all_data['lastPrice'] =0
all_data['HistoryPrice']=0
2.Get the Historical Price by using for loop since my account has a limit of 100 request per moment so I devide it into 8 multiple sessions for S&P 505. Then relogin at each 70 stocks. I can get the total of 505 in 2 minutes.
def error_handler(msg):
print(msg)
def my_callback_handler(msg):
if msg.field in [tt.BID,tt.ASK,tt.LAST]:
# from ib.ext.TickType import TickType as tt
#now we can just store the response in the data frame
all_data.loc[msg.tickerId,tt.getField(msg.field)] = msg.price
# if msg.field == tt.LAST:
# # print('a')
# print(all_data.loc[msg.tickerId,'ticker'],msg.price)
t = time()
max_amount_per_Iter = 70 #max number per iter to save cost
max_Iter = ceil(len(all_data)/max_amount_per_Iter)
for i in range (0,max_Iter):
print('====================for : ',i+1,'==========================')
sleep(1)
tws = ibConnection(clientId=11+i)
tws.register(my_callback_handler, message.tickPrice, message.tickSize)
tws.register(error_handler, 'Error')
tws.connect()
all_dum = all_data.iloc[i*max_amount_per_Iter:min((i+1)*max_amount_per_Iter,len(all_data)),:]
for index, row in all_dum.iterrows():
c = Contract()
c.m_symbol = row['ticker']
c.m_exchange = row['exchange']
c.m_currency = row['curr']
c.m_secType = row['type']
# the tickerId is just the index in but for some reason it needs str()
tws.reqMktData(str(index),c,'',False)
sleep(0.2)
sleep(2)
print('=========End round : ',i+1,'with time :',time() - t,'==============')
tws.disconnect()

I'm thinking it has something to do with the market data subscription within IB itself because I am having a similar problem. I get a TWS Time at connection... returned in the results along with the "market data farm connection" Msg.
Make sure you have a connection port & clientID established, i.e.:
tws = ibConnection(port=7496,clientId=100)
Note that 7496 is a common port, but the clientId is anything you wish to specify (within the IB account you are using under File->API->Settings).

how to write an if statement in python comparing time to a value

I'm trying to write a simple code in python to turn on a led light during certain hours of the day, then turn it off for the rest of the time. I tried to indicate the time frames in the if statement but everytime I run this code, only the "else" portion of the if statement works. Am I not allowed to compare my time format of HHMM to a flat value of 0745? If not how can I get the led to stay on from midnight to 7:45 in the morning? Any help or direction would be greatly appreciated!
import time
import RPi.GPIO as GPIO
GPIO.setwarnings(False)
GPIO.setmode(GPIO.BOARD)
GPIO.setup(3,GPIO.OUT)
print (time.strftime("%H%M"))
while True:
if time.strftime("%H%M") <= 0745:
GPIO.output(3,1)
else:
GPIO.output(3,0)

Try to use datetime.
import datetime
while True:
time = datetime.datetime.now()
morning = time.replace(hour=7, minute=45, second=0, microsecond=0)
if time <= morning:
print("turn on")
else:
print("turn off")

Python has a datetime module that provides data types to work with dates and times.
You could write the code from your question like this:
import datetime
import RPi.GPIO as GPIO
GPIO.setwarnings(False)
GPIO.setmode(GPIO.BOARD)
GPIO.setup(3,GPIO.OUT)
while True:
if datetime.datetime.now() <= datetime.datetime.combine(
datetime.date.today(),
datetime.time(7, 45)
):
GPIO.output(3,1)
else:
GPIO.output(3,0)
However, this still has the problem that the program will constantly spin in the while loop and suck up as much CPU cycles as it can get.
You could do something like this instead:
import time
import datetime
import RPi.GPIO as GPIO
GPIO.setwarnings(False)
GPIO.setmode(GPIO.BOARD)
GPIO.setup(3,GPIO.OUT)
schedule = [
(datetime.time(0, 0), 0),
(datetime.time(7, 45), 1),
(datetime.time(18, 0), 0)
]
def switchLight():
"""
Switch the light to the proper state according to the schedule.
Return the next time the light needs to be switched.
"""
today = datetime.date.today()
schedule_index = 0
while schedule_index < len(schedule) and \
datetime.datetime.now() > datetime.datetime.combine(today, schedule[schedule_index][0]):
schedule_index += 1
GPIO.output(3, schedule[schedule_index][1])
# calculate next switching time
schedule_index += 1
if schedule_index == len(schedule):
schedule_index = 0
today = today + datetime.timedelta(days=1)
return datetime.datetime.combine(today, schedule[schedule_index][0])
while True:
next_time = switchLight()
time.sleep((next_time - datetime.datetime.now()).seconds + 1)