We Keep Coding

Heroku Returned a H10 Error for Python App Deployment

I'm trying to deploy a python app on heroku, but heroku returned a H10 error. Here is my code python code:
import os
import requests
import random
from flask import Flask, jsonify, request
from flask_cors import CORS
from system.backend.data import Data
from system.backend.folder import Folder
from system.wallet.data_pool import DataPool
from system.wallet.exchange import Exchange
from system.pubsub import PubSub
app = Flask(__name__)
CORS(app, resources={ r'/*': { 'origins': 'http://localhost:8080' } })
data = Data()
data_pool = DataPool()
folder = Folder(data)
pubsub = PubSub(data, data_pool)
#app.route('/')
def default():
return 'Welcome to mypython app'
#app.route('/main')
def route_mainn():
return jsonify(data.to_json())
#app.route('/main/range')
def route_main_range():
start = int(request.args.get('start'))
end = int(request.args.get('end'))
return jsonify(data.to_json()[::-1][start:end])
#app.route('/main/datalength')
def route_main_datalength():
return jsonify(len(data.length))
#app.route('/myapp/data')
def route_myapp_data():
app_data = data_pool.app_data()
exchange_data.append(Exchange.reward_exchange(folder).to_json())
data.add_data(exchange_data)
folder = data.length[-1]
pubsub.broadcast_folder(folder)
data_pool.clear_data_exchange(data)
return jsonify(data.to_json())
#app.route('/folder/exchange', methods=['POST'])
def route_folder_exchange():
exchange_data = request.get_json()
exchange = data_pool.existing_exchange(folder.address)
if exchange:
exchange.update(
folder,
exchange_data['recipient'],
exchange_data['sender']
)
else:
exchange = Exchange(
folder,
exchange_data['recipient'],
exchange_data['sender']
)
pubsub.broadcast_exchange(exchange)
data_pool.set_exchange(exchange)
return jsonify(exchange.to_json())
#app.route('/folder/info')
def route_folder_info():
return jsonify({'address': folder.address, 'data': folder.balance})
#app.route('/known-addresses')
def route_known_addresses():
known_addresses = set()
for data in main.length:
for exchange in main.data:
exchange['output'].keys()
known_addresses.update(exchange['output'].keys())
return jsonify(list(known_addresses))
#app.route('/exchange')
def route_exchanges():
return jsonify(exchange_pool.exchange_data())
ROOT_PORT = 8080
PORT = ROOT_PORT
if os.environ.get('PEER') == 'True':
PORT = random.randint(8081, 9000)
result = requests.get(f'http://localhost:{ROOT_PORT}/main')
print(f'result.json(): {result.json()}')
result_main = Data.from_json(result.json())
try:
data.replace_length(result_data.length)
print('\n -- Successfully synchronized the local data')
except Exception as e:
print(f'\n -- Error synchronizing: {e}')
if os.environ.get('SEED_DATA') == 'True':
for i in range(10):
main.add_folder([
Exchange(Folder(), Folder().address, random.randint(500, 1000)).to_json(),
Exchange(Folder(), Folder().address, random.randint(500, 1000)).to_json()
])
for i in range(3):
data_pool.set_exchange(
Exchange(Folder(), Folder().address, random.randint(500, 1000))
)
app.run(port=PORT)
I also made a worker.py file with the code as follows:
import os
import redis
from rq import Worker, Queue, Connection
listen = ['high', 'default', 'low']
redis_url = os.getenv('REDIS_URL', 'redis://localhost:8080')
conn = redis.from_url(redis_url)
if __name__ == '__main__':
with Connection(conn):
worker = Worker(map(Queue, listen))
worker.work()
The mac terminal recommended to use waitress for python deployment, but I'm not sure on how to implement waitress within the code shown on the above.

To use waitress:
Just do pip install waitress
And add this snippet:
if __name__ == '__main__':
from waitress import serve
serve(app, host='0.0.0.0', port=8000)

Linux NoHup fails for Streaming API IG Markets where file is python

This is quite a specific question regarding nohup in linux, which runs a python file.
Back-story, I am trying to save down streaming data (from IG markets broadcast signal). And, as I am trying to run it via a remote-server (so I don't have to keep my own local desktop up 24/7),
somehow, the nohup will not engage when it 'listen's to a broadcast signal.
Below, is the example python code
#!/usr/bin/env python
#-*- coding:utf-8 -*-
"""
IG Markets Stream API sample with Python
"""
user_ = 'xxx'
password_ = 'xxx'
api_key_ = 'xxx' # this is the 1st api key
account_ = 'xxx'
acc_type_ = 'xxx'
fileLoc = 'marketdata_IG_spx_5min.csv'
list_ = ["CHART:IX.D.SPTRD.DAILY.IP:5MINUTE"]
fields_ = ["UTM", "LTV", "TTV", "BID_OPEN", "BID_HIGH", \
"BID_LOW", "BID_CLOSE",]
import time
import sys
import traceback
import logging
import warnings
warnings.filterwarnings('ignore')
from trading_ig import (IGService, IGStreamService)
from trading_ig.lightstreamer import Subscription
cols_ = ['timestamp', 'data']
# A simple function acting as a Subscription listener
def on_prices_update(item_update):
# print("price: %s " % item_update)
print("xxxxxxxx
))
# A simple function acting as a Subscription listener
def on_charts_update(item_update):
# print("price: %s " % item_update)
print(xxxxxx"\
.format(
stock_name=item_update["name"], **item_update["values"]
))
res_ = [xxxxx"\
.format(
stock_name=item_update["name"], **item_update["values"]
).split(' '))]
# display(pd.DataFrame(res_))
try:
data_ = pd.read_csv(fileLoc)[cols_]
data_ = data_.append(pd.DataFrame(res_, columns = cols_))
data_.to_csv(fileLoc)
print('there is data and we are reading it')
# display(data_)
except:
pd.DataFrame(res_, columns = cols_).to_csv(fileLoc)
print('there is no data and we are saving first time')
time.sleep(60) # sleep for 1 min
def main():
logging.basicConfig(level=logging.INFO)
# logging.basicConfig(level=logging.DEBUG)
ig_service = IGService(
user_, password_, api_key_, acc_type_
)
ig_stream_service = IGStreamService(ig_service)
ig_session = ig_stream_service.create_session()
accountId = account_
################ my code to set sleep function to sleep/read at only certain time intervals
s_time = time.time()
############################
# Making a new Subscription in MERGE mode
subscription_prices = Subscription(
mode="MERGE",
# make sure to put L1 in front of the instrument name
items= list_,
fields= fields_
)
# adapter="QUOTE_ADAPTER")
# Adding the "on_price_update" function to Subscription
subscription_prices.addlistener(on_charts_update)
# Registering the Subscription
sub_key_prices = ig_stream_service.ls_client.subscribe(subscription_prices)
print('this is the line here')
input("{0:-^80}\n".format("HIT CR TO UNSUBSCRIBE AND DISCONNECT FROM \
LIGHTSTREAMER"))
# Disconnecting
ig_stream_service.disconnect()
if __name__ == '__main__':
main()
#######
Then, I try to run it on linux using this command : nohup python marketdata.py
where marketdata.py is basically the python code above.
Somehow, the nohup will not engage....... Any experts/guru who might see what I am missing in my code?

Python Dash live update with dcc.Interval not working

I am trying to update a Dash chart opened in my web browser with datas coming from a websocket server I made.
I have no issue with the websocket part. My server sends to the client datas that I add to a list and convert this list to a pandas DataFrame object. Then this object is added to a multiprocess.Queue.
In my Dash callback from dcc.Interval, I get the data from the previous Queue (this part seems to work because I can print correctly the data) and update my Graph.
But the Graph in my browser is not updating. I do not see any error raised in my python code or in the browser console. (I tried with Firefox, Chrome and Brave)
My python console print dash-update-component HTTP/1.1" 200 - with my data at each callback interval and the tab name is Updating...
Worse, sometimes I can see one or 10 datas correctly displayed then nothing without changing the code, just by restarting my code...
Here the code (client side) :
import multiprocess as mp
import threading
import dash
from dash.dependencies import Output, Input
from dash import dcc
from dash import html
import plotly.graph_objs as go
from socket import *
import time
import json
import pandas as pd
from datetime import datetime
FLAG_QUIT = False
FLAG_ASK_DATA = False
MAX_SIZE = 10
queue = mp.Queue()
datas = []
app = dash.Dash(__name__)
app.layout = html.Div(
[
html.H4('Live Chart'),
dcc.Graph(id='live-update-graph', animate =True),
dcc.Interval(
id='interval-component',
interval=200,
n_intervals=0
)
]
)
#app.callback(Output('live-update-graph', 'figure'),
[Input('interval-component', 'n_intervals')])
def update_graph(n):
df = queue.get()
print(df)
fig = go.Scatter(x=df.index, y = df['Data'])
return fig
# function for receiving message from client
def send_to_server(clsock):
global FLAG_QUIT
global FLAG_ASK_DATA
while True:
try:
if FLAG_QUIT == True:
break
if FLAG_ASK_DATA:
time.sleep(0.5)
clsock.sendall("ok".encode())
except Exception as ex:
print(str(ex))
# function for receiving message from server
def recv_from_server(clsock):
global FLAG_QUIT
global FLAG_ASK_DATA
while True:
try:
data = clsock.recv(1024).decode()
if data == 'q':
print('Closing connection')
FLAG_QUIT = True
break
process_data(data)
FLAG_ASK_DATA = True
except Exception as ex:
print(str(ex))
def process_data(data):
global datas
global queue
process_websocket_data(data)
datas_dataframe = convert_data_to_dataframe(datas)
queue.put(datas_dataframe)
def convert_data_to_dataframe(data):
data_frame = pd.DataFrame(data)
data_frame = data_frame.drop_duplicates(0)
data_frame_date = data_frame[0]
final_date = []
for time in data_frame_date.unique():
readable = datetime.strptime(time, '%Y-%m-%d-%H-%M')
final_date.append(readable)
data_frame.pop(0)
dataframe_final_date = pd.DataFrame(final_date)
dataframe_final_date.columns = ['Date']
final_dataframe = data_frame.join(dataframe_final_date)
final_dataframe.set_index('Date', inplace = True)
final_dataframe.columns = ['Data']
return final_dataframe
def process_websocket_data(raw_data):
data_json = json.loads(raw_data)
data = [data_json['Date'],data_json['Data']]
datas.append(data)
while len(datas) > MAX_SIZE:
datas.pop(0)
def main():
threads = []
HOST = 'localhost'
PORT = 8765
clientSocket = socket(AF_INET, SOCK_STREAM)
connected = False
while not connected:
try:
print("Waiting connecting server...")
time.sleep(1)
clientSocket.connect((HOST, PORT))
connected = True
except :
pass
print('Client is connected to a chat sever!\n')
clientSocket.send('start'.encode())
t_send = threading.Thread(target=send_to_server, args=(clientSocket,))
t_rcv = threading.Thread(target=recv_from_server, args=(clientSocket,))
t_send.start()
t_rcv.start()
if __name__ == '__main__':
main()
app.run_server()
What did I code wrong or miss?
Edit :
This is the datas I get from the queue in the callback update_graph(n) method.
Data
Date
2019-09-16 10302.00
2019-09-23 8061.98
2019-09-30 8042.08
2019-10-07 7851.01
2019-10-14 8274.33
2019-10-21 8218.23
2019-10-28 9534.37
2019-11-04 9197.86
2019-11-11 9041.31
2019-11-18 8504.13
I also tried to replace my X axis by a list of int instead of a list of dates but the result is the same. No refreshing.
Thanks in advance.

I finally managed to find the problem and it seems to be the Queue in the callback. So I removed the queue mechanism and instead use a classic list with a lock.
app = dash.Dash(__name__)
app.layout = html.Div([
dcc.Graph(id="live_graph", animate=False),
dcc.Interval(id='interval_component',
interval=INTERVAL,
)])
#app.callback(Output('live_graph', 'figure'),
[Input('interval_component', 'n_intervals')])
def update_graph(num):
global datas_dataframe
lock.acquire()
if len(datas_dataframe) == 0:
print("No data")
lock.release()
return no_update
try:
df = datas_dataframe.pop(0)
fig = go.Figure(data=[go.Scatter(x=df.index,y=df['Data'])],
layout=go.Layout(yaxis=dict(tickfont=dict(size=22))))
lock.release()
return fig
except Exception as ex:
print(str(ex))
lock.release()
return no_update

Using HTML to output data from my RASPBERRY

heello
I build a raspberry temperature sensor. It works well. I can catch the temperature. I made a request on my raspberry file to communicate with my pc. But i got an internal server error :
I can't even go on my root page :
this i my code on my pc :
#app.route ('/')
def hello_world():
measures = []
query_sensor = models.Sensor.query.all ()
print (query_sensor)
for sensor in query_sensor:
query_mes = models.Measure.query.filter(models.Measure.sensor_id == sensor.id).all ()
measures += [{'serial':sensor.serial, 'measures':query_mes}]
print (measures)
return render_template('measures.html', data= query_mes)
#app.route('/data/measure', methods=['GET', 'POST'])
def page_data_measure ():
if request.method == 'POST':
data = request.form.to_dict()
print(data)
if 'serial' in data and 'value' in data and 'date' in data:
# recherche le capteur
sensor = None
query_sensor = models.Sensor.query.filter(models.Sensor.serial == data['serial']).first()
print (query_sensor)
if query_sensor is None:
sensor = models.Sensor (data['serial'])
database.db_session.add(sensor)
database.db_session.commit()
else:
sensor = query_sensor
mes = models.Measure (datetime.strptime(data['date'], '%Y-%m-%d %H:%M:%S.%f'), float(data['value']), sensor.id)
database.db_session.add (mes)
database.db_session.commit ()
else:
return 'Bad value', 422
return render_template('temperatures.html', data=mes)
this is the code with an api on the raspberry(writeme.py) :
import requests
import datetime
import time
from w1thermsensor import W1ThermSensor
sensor = W1ThermSensor()
print ("demarrer avec requete")
while True:
temperature = sensor.get_temperature()
data = {'date':datetime.datetime.now(), 'serial':'12345678', 'value':temperature}
r = requests.post("http://192.168.137.1:5000/data/measure", data=data)
print(r)
print("The temperature is %s celsius" % temperature)
time.sleep(1)
this is request result message :
the command that i run are : flask run -h 0.0.0.0
and on my raspberry are : python run writeme.py
thanks for your help

How to asynchrony get a real time data from redis by using python flask socket.io and threads?

My program collects data from sensors and sends this data to web interface, which shows it in real time graphs and sends some commands back.
The problem is that data sending has a big delay or sends old data. What i should change to send data async to both ways?
I simplified my code. Example 1, when data comes one time in a few minutes:
# System stuff
import os
import sys
import serial
# Multiprocessing
from multiprocessing import Process
from threading import Thread # for same memory
import timeit
from time import sleep
# database
import redis
# Web server
from flask import Flask, render_template
from flask_socketio import SocketIO, emit, send, join_room, leave_room, close_room, rooms, disconnect
# Config
DEBUG = True
db = redis.StrictRedis(host='localhost', port=6379, db=0)
#db vars
db.set('velocity', 0.0)
db.set('distance', 0.0)
def webServer(db):
app = Flask(__name__)
app.config['SECRET_KEY'] = 'secret!'
socketio = SocketIO(app)
#app.route('/')
def index():
return render_template('index.html')
def exchangeData():
prev_distance = 0.00
while(True):
distance = round(float(db.get('distance')), 2)
if(distance != prev_distance):
velocity = round(float(db.get('velocity')), 2)
distance = round(float(db.get('distance')), 2)
socketio.emit('exchangeData', {
'velocity': velocity,
'distance': distance,
})
prev_distance = distance
print("DATA sended: %s m/s" % velocity)
sleep(0.2)
#socketio.on('connect')
def test_connect():
t_exchangeData = Thread(target=exchangeData).start()
socketio.run(app, debug=True, host="0.0.0.0")
def newData(db):
c = 0.00
while(True):
db.set('velocity', c)
db.set('distance', c)
c += 1.00
sleep(1)
if __name__ == '__main__':
p_newData = Process(target=newData, args=(db,)).start()
p_webServer = Process(target=webServer, args=(db,)).start()
#p_checkConnection = Process(target=checkConnection, args=(db, HOSTNAME, pinglog)).start()
#p_calcVD = Process(target=calcVD, args=(db,)).start()
In browser, I get this with latency about 2 minutes:
Object {velocity: 218, distance: 218}
// two minutes later
Object {velocity: 306, distance: 306}
// two minutes later
Object {velocity: 306, distance: 306}
Example 2, when I do not use if statement and sleep:
# System stuff
import os
import sys
import serial
# Multiprocessing
from multiprocessing import Process
from threading import Thread # for same memory
import timeit
from time import sleep
# database
import redis
# Web server
from flask import Flask, render_template
from flask_socketio import SocketIO, emit, send, join_room, leave_room, close_room, rooms, disconnect
# Config
DEBUG = True
db = redis.StrictRedis(host='localhost', port=6379, db=0)
#db vars
db.set('velocity', 0.0)
db.set('distance', 0.0)
def webServer(db):
app = Flask(__name__)
app.config['SECRET_KEY'] = 'secret!'
socketio = SocketIO(app)
#app.route('/')
def index():
return render_template('index.html')
def exchangeData():
prev_distance = 0.00
while(True):
#distance = round(float(db.get('distance')), 2)
#if(distance != prev_distance):
velocity = round(float(db.get('velocity')), 2)
distance = round(float(db.get('distance')), 2)
socketio.emit('exchangeData', {
'velocity': velocity,
'distance': distance,
})
prev_distance = distance
print("DATA sended: %s m/s" % velocity)
#sleep(0.2)
#socketio.on('connect')
def test_connect():
t_exchangeData = Thread(target=exchangeData).start()
socketio.run(app, debug=True, host="0.0.0.0")
def newData(db):
c = 0.00
while(True):
db.set('velocity', c)
db.set('distance', c)
c += 1.00
sleep(1)
if __name__ == '__main__':
p_newData = Process(target=newData, args=(db,)).start()
p_webServer = Process(target=webServer, args=(db,)).start()
#p_checkConnection = Process(target=checkConnection, args=(db, HOSTNAME, pinglog)).start()
#p_calcVD = Process(target=calcVD, args=(db,)).start()
In this case, i get data in real time, but it is the same data, and changes only one time in few minutes:
Object {distance: 3, velocity: 3}
Object {distance: 3, velocity: 3}
Object {distance: 3, velocity: 3}
// repeating around 2 minutes
Object {distance: 357, velocity: 357}
// repeating again...

The main problem was that I did not add these two lines:
import eventlet
eventlet.monkey_patch()
Also, while I tried to find solution, made a mistake when I called data from redis out of while loop.