We Keep Coding

Data acquisition and plotting in separate threads

I have a class that does data acquisition, i.e. PySession.scope in a separate thread. I want to plot each scope in a ScopeGUI, which is based on PyQt5. Obviously, the GUI's need to be running in the main thread, however I want to be able to invoke another PySession.scope at any time, that is, the console has to be free to submit new commands.
To conclude: Let's say I start two scopes in parallel, while the data is being collected it should be visualized in two separate instances of the ScopeGUI. Simultaneously, the console should be free to start an additional scope. What's the best practice so achieve this behavior in Python?
Here's a reproducible example of what I have so far:
.
├── main.py
├── scope.py
├── alosaclient.py
main.py
import os
import time
import asyncio
from concurrent.futures import ThreadPoolExecutor
from alosaclient import PySession
import matplotlib.pyplot as plt
if __name__ == '__main__':
session = PySession(
"C:/ProjTMS/alosa1-client/test/config/linkcommand.toml")
variables = "speedcpu U_Z"
future1 = session.scope(
varnames=variables, nsamples=20, ndiv=1, realtime=True)
future2 = session.scope("m_1 m_u", nsamples=20, ndiv=1, realtime=True)
print("----------------------------------------------MAIN THREAD----------------------------------------------")
session.work()
result1 = future1.result()
print(result1.data[1].time)
print(result1.data[1].values)
result2 = future2.result()
print(result2.data[1].time)
print(result2.data[1].values)
scope.py
import numpy as np
import sys
from PyQt5 import QtWidgets
from matplotlib.backends.backend_qt5agg import (FigureCanvas,
NavigationToolbar2QT as
NavigationToolbar)
from matplotlib.figure import Figure
class ScopeGUI(QtWidgets.QMainWindow):
def __init__(self, varnames, nsamples):
super().__init__()
self.varnames = varnames
self.nsamples = nsamples
self.setEnabled(True)
self.setGeometry(0, 0, 800, 600)
self.setMinimumSize(800, 600)
self.main_widget = QtWidgets.QWidget()
self.setCentralWidget(self.main_widget)
self.main_layout = QtWidgets.QVBoxLayout()
self.main_widget.setLayout(self.main_layout)
self._fig = Figure(figsize=(8, 6))
self._canvas = FigureCanvas(self._fig)
self.main_layout.addWidget(self._canvas)
self._axes = self._fig.subplots()
self._axes.grid(True, which="both")
self._axes.set_xlabel('Time (s)')
self._axes.set_ylabel('Data (DSPu)')
self._axes.set_xlim(left=0, right=6.3)
self._axes.set_ylim(bottom=-1.5, top=1.5)
self.lines = []
self.initialize_lines()
self.addToolBar(NavigationToolbar(self._canvas, self))
self.show()
def initialize_lines(self):
variables = self.varnames.split()
for var in variables:
line, = self._axes.plot([], [])
line.set_marker('.')
self.lines.append(line)
def plot(self, scope_data):
print("plotting")
for signal, line in zip(scope_data, self.lines):
x = signal.time
y = signal.values
common_length = min(len(x), len(y))
line.set_xdata(x[:common_length])
line.set_ydata(y[:common_length])
self._canvas.draw()
self._canvas.flush_events()
alosaclient.py
import sys
import time
from concurrent.futures import ThreadPoolExecutor
import matplotlib
import matplotlib.pyplot as plt
import numpy as np
import random
from PyQt5 import QtWidgets
from scope import ScopeGUI
import asyncio
app = QtWidgets.QApplication(sys.argv)
class ScopeDataElement:
def __init__(self, variable_index, time, values):
self.variable_index = variable_index
self.time = time
self.values = values
class Scope:
def __init__(self, data):
self.data = data
class PySession:
def __init__(self, lcmdfile):
self.pool = ThreadPoolExecutor(8)
self.loop = asyncio.get_event_loop()
self.tasks = list()
self.scope_buffer = list()
def work(self):
self.loop.run_until_complete(asyncio.wait(self.tasks))
def scope(self, varnames, nsamples=1, ndiv=1, realtime=False):
future = self.pool.submit(
self.dummy_scope, varnames, nsamples, ndiv)
if realtime:
scope_gui = ScopeGUI(varnames, nsamples)
task = self.loop.create_task(self.update(scope_gui, varnames))
self.tasks.append(task)
return future
async def update(self, scope_gui, varnames):
variable_indices = [self.name_to_index(
var) for var in varnames.split()]
# find corresponding scope_buffer is it may potentially grow dynamically
scope_index = self.find_scope_index(variable_indices)
# as long as empty, wait
while not self.scope_buffer:
await asyncio.sleep(25e-3)
# while the data is not complete, update to GUI
while not all([len(signal.time) == scope_gui.nsamples for signal in self.scope_buffer[scope_index].data]):
scope_gui.plot(self.scope_buffer[scope_index].data)
await asyncio.sleep(25e-3)
#staticmethod
def name_to_index(varname):
# dummy cross reference table: get index from variable name
varnames = ["speedcpu", "U_Z", "m_1", "m_u"]
return varnames.index(varname)
def find_scope_index(self, variable_indices):
# get scope index from variable_indices, may change if scopes run parallel
result = filter(lambda pair: all([signal.variable_index == varindex for varindex, signal in zip(
variable_indices, pair[1].data)]), enumerate(self.scope_buffer))
index = list(result)[0][0]
return index
def find_data_index(self, scope, varname):
result = filter(lambda pair: self.name_to_index(varname) ==
pair[1].variable_index, enumerate(scope.data))
index = list(result)[0][0]
return index
def dummy_scope(self, varnames, nsamples, ndiv):
variables = varnames.split()
variable_indices = [self.name_to_index(
var) for var in variables]
content = [ScopeDataElement(self.name_to_index(
var), list(), list()) for var in variables]
scope = Scope(content)
self.scope_buffer.append(scope)
for var in variables:
scope_index = self.find_scope_index(variable_indices)
data_index = self.find_data_index(
self.scope_buffer[scope_index], var)
linspace = np.linspace(0, 2*np.pi, nsamples)
for arrayidx, point in enumerate(linspace):
print(f"scope index is {scope_index}")
print(f"data index is {data_index}")
self.scope_buffer[scope_index].data[data_index].time.append(
point)
self.scope_buffer[scope_index].data[data_index].values.append(
np.sin(point) + random.uniform(0, 0.2))
time.sleep(10e-3)
return self.scope_buffer[scope_index]
Essentially, this code does the following:
Start scope (PySession.scope)
trigger two dummy scope functions to gather some dummy data
push back according GUI update task into the event loop
Process event loop (PySession.work)
As soon as all desired scopes have been initialized, the event loop gets processes, i.e. the GUI's are updated
Issues:
As long as the event loop runs, the console is blocked and no more commands can be submitted.
IMPORTANT
I'm working with the Python interactive console, that's why there is no app.exec_() command. The reproducible example has the be started with python3 -i main.py to start on the same page as I.
I am fully aware that the way I'm trying to do this is probably dead wrong, that's why I'm asking you guys for help.
Thanks in advance!

Here's how it's done, no more asyncio needed:
The update function needs to be adapted as follows. Move the canvas.draw() from the GUI to the update routine to avoid flickering of the figure.
def update(self, scope_gui, varnames):
try:
variable_indices = [self.name_to_index(
var) for var in varnames.split()]
# find corresponding scope_buffer is it may potentially grow dynamically
scope_index = self.find_scope_index(variable_indices)
done = False
while not self.scope_buffer:
time.sleep(10e-3)
# while the data is not complete, update to GUI
while not all([len(signal.time) == scope_gui.nsamples for signal in self.scope_buffer[scope_index].data]):
scope_gui.plot(self.scope_buffer[scope_index].data)
scope_gui._canvas.draw()
time.sleep(25e-3)
except Exception as e:
print(f"EXCEPTION OCURRED: {e}")
raise(e)
Accordingly, the scope gets adapted as well.
def scope(self, varnames, nsamples=1, ndiv=1, realtime=False):
future = self.pool.submit(
self.dummy_scope, varnames, nsamples, ndiv)
if realtime:
scope_gui = ScopeGUI(varnames, nsamples)
self.pool.submit(self.update, scope_gui, varnames)
return future
This ensures that the GUI runs in the main thread, whereas the updating task is running in a separate thread and therefore does not block the console.

Accurate timer with PyQt

I'm using pyqtgraph to plot a huge number of data that I receive from sensors.
To do so, I made one thread that acquire the data and put in a queue. To plot the data, I check periodically with a timer if the queue is not empty.
The problem is that the accuracy of the timer (QTimer) seems to be really bad. I mean when the load is low (sleep for 1000/100 ms) in the measuring thread, the accuracy is pretty good but when the load increase (sleep for 10ms), my update function used to plot data is not called back with the same period.
Here is an example code:
import sys
import time
from queue import Queue
from random import random
import numpy as np
import pyqtgraph as pg
from pyqtgraph.Qt import QtCore, QtWidgets
data_queue = Queue()
class WorkerThread(QtCore.QThread):
def __init__(self, parent):
super(WorkerThread, self).__init__(parent=parent)
def run(self):
t_init = time.time()
while True:
# Generating random data
values = [(time.time()-t_init, random()) for _ in range(200)]
data_queue.put(values)
print("adding data")
self.msleep(10)
class GraphPlot(QtWidgets.QMainWindow):
def __init__(self, parent=None):
super(GraphPlot, self).__init__(parent)
self.mainbox = QtWidgets.QWidget()
self.setCentralWidget(self.mainbox)
self.mainbox.setLayout(QtWidgets.QVBoxLayout())
self.canvas = pg.GraphicsLayoutWidget()
self.mainbox.layout().addWidget(self.canvas)
self.analogPlot = self.canvas.addPlot(title='Real-time data')
self.drawplot = self.analogPlot.plot(pen='r')
numPoints = 20000
self.t = np.zeros(numPoints, dtype=int)
self.x = np.zeros(numPoints, dtype=int)
self.worker = WorkerThread(self)
self.worker.start()
self.timer = pg.QtCore.QTimer()
self.timer.setTimerType(QtCore.Qt.PreciseTimer)
self.timer.timeout.connect(self._update)
self.timer.start(1)
def _update(self):
print('start:', time.time())
size = data_queue.qsize()
if size > 0:
for _ in range(size):
values = data_queue.get()
for v in values:
self.t = np.append(self.t[1:], v[0])
self.x = np.append(self.x[1:], v[1])
self.drawplot.setData(self.t, self.x)
print('end:', time.time())
app = QtWidgets.QApplication(sys.argv)
plot = GraphPlot()
plot.show()
sys.exit(app.exec_())
An excerpt of the output:
start: 1572893919.9067862
adding data
end: 1572893919.9217482 <--
adding data
start: 1572893919.9586473 <-- there should be 1ms of difference with last 'end'
actually, there is around 37ms
I want the timer to be synchronous with the same period whatever the load on the measuring thread. I tried to decrease the priority of the former thread but it did not solve the problem.

QTimer documentation partially answers to your issue:
All timer types may time out later than expected if the system is busy
or unable to provide the requested accuracy. In such a case of timeout
overrun, Qt will emit timeout() only once, even if multiple timeouts
have expired, and then will resume the original interval.
The problem is that after you call _update from the timeout, Qt will need some time to process what happens after self.drawplot.setData(), which basically is computing graphical information and actually painting it on the screen.
You're not getting the 1ms delay because Qt just isn't able to work that fast.
Even if a QTimer can work in another thread ("asynchronously", but be careful about the meaning of this word), it always depend on the thread it is created or resides (a QTimer cannot be started or stopped from a thread different than its one). So, since you've created the timer in the window thread (the Qt main event loop), its timeout accuracy depends on the capacity of that loop to handle all its events, and since lot of events are related to GUI painting (which seems fast to our eyes, but is actually slow as it's very demanding for the CPU), you can easily understand why you'll never get that 1ms interval. And don't forget the fact that even if pyqtgraph is very fast we're still talking about Python.
While reaching a better accuracy for a 1ms QTimer is possible (creating a separate thread for it), you wouldn't get any advantage from it anyway: even with a very fast computer, what you're substantially requesting is to update the screen at 1000Hz, while most graphic hardware is not able to go much faster than 100-200Hz; this means that even if you own a high end system, you wouldn't get more than one update each 4ms.
If you need to update the plot each time new data is available, it is probably better to use signals and slots, which also avoids any unnecessary check on the queue and ensures to update the plot as much as needed:
class WorkerThread(QtCore.QThread):
newData = QtCore.pyqtSignal(object)
def __init__(self, parent):
super(WorkerThread, self).__init__(parent=parent)
def run(self):
t_init = time.time()
while True:
# Generating random data
values = [(time.time()-t_init, random()) for _ in range(200)]
print("adding data")
self.newData.emit(values)
self.msleep(10)
class GraphPlot(QtWidgets.QMainWindow):
def __init__(self, parent=None):
super(GraphPlot, self).__init__(parent)
self.mainbox = QtWidgets.QWidget()
self.setCentralWidget(self.mainbox)
self.mainbox.setLayout(QtWidgets.QVBoxLayout())
self.canvas = pg.GraphicsLayoutWidget()
self.mainbox.layout().addWidget(self.canvas)
self.analogPlot = self.canvas.addPlot(title='Real-time data')
self.drawplot = self.analogPlot.plot(pen='r')
numPoints = 20000
self.t = np.zeros(numPoints, dtype=int)
self.x = np.zeros(numPoints, dtype=int)
self.worker = WorkerThread(self)
self.worker.newData.connect(self.newData)
self.worker.start()
def newData(self, data):
print('start:', time.time())
for v in data:
self.t = np.append(self.t[1:], v[0])
self.x = np.append(self.x[1:], v[1])
self.drawplot.setData(self.t, self.x)
print('end:', time.time())
You won't get a 1ms update, but there would be no need for it anyway; also, remember that printing at that rate will always affect the performance in some way.
Finally, there is no advantage in setting PreciseTimer with a 1ms interval, since the timer accuracy is about 1ms on most platforms anyway (as explained at the beginning of the same paragraph in the documentation linked before), and setting the precision is only required for longer intervals (I'd say at least 25-50ms).
There's also an interesting answer about QTimer here, explaining what basically happens whenever you create one and it timeouts.

Using threading to plot data of a different class in the background of a command line interface (CLI)

I am trying to plot a variable battery voltage on the background while the command line code keeps running as the program is executed.
The plot begins when the user press plot and ends when exit is pressed.
from threading import Thread
import cmd
import matplotlib.pyplot as plt
import sys
from drawnow import *
plt.ion()
count=0
class myThreadPlot (Thread):
def __init__(self):
Thread.__init__(self)
self.state = test()
self.battVoltagePlot= []
self.flag = True
def run(self):
global count
while self.flag:
self.battVoltagePlot.append(self.state.batteryVoltage)
drawnow(self.plot_var,stop_on_close=True)
count+=1
if count>50: self.battVoltagePlot.pop(0) # to plot 50 values at
a time
plt.pause(1e-4)
plt.close()
def plot_var(self):
x =min(self.battVoltagePlot)
y = max(self.battVoltagePlot)
if x<0: plt.ylim(2*x,2*y)
else: plt.ylim(x/2,2*y)
plt.grid(True)
plt.ylabel('V')
plt.plot(self.battVoltagePlot, 'ro-', label='V')
class test(object):
def __init__(self):
self.batteryVoltage =30
class CLI(cmd.Cmd):
def __init__(self):
cmd.Cmd.__init__(self)
self.prompt = ">> "
self.intro = ("Welcome\n Press help to see the options.\n press
plot to plot battery voltage and press exit to exit ")
def do_plot(self,arg):
"calling myThreadPLot to plot a variable 'batteryVoltage' from
class 'test' in the background"
self.my = myThreadPlot()
self.my.start()
def do_exit(self, arg):
'''Exit this application'''
print("Bye")
self.my.flag = False
return True
if __name__ == '__main__':
CLI().cmdloop()
I am getting the following error as i press exit.
Exception RuntimeError: RuntimeError('main thread is not in main loop',) in
<bound method PhotoImage.__del__ of <Tkinter.PhotoImage instance at
0x7f38f5412320>> ignored
Tcl_AsyncDelete: async handler deleted by the wrong thread
Any suggestion on how can i use prevent this. Another issue is that
in the drawnow(self.plot_var,stop_on_close=True) i am allowing the user to close the figure by clicking the cancel button. but when i tried it again errors pops up.

GUI with pyqtgraph never refresh

I have been working on a GUI for the beagle bone black that launch a thread when a button is clicked and starts to get data through the SPI.
This function is inside a class called Scanner(QObject) and runs in a different thread when the Start button is clicked.
def scan (self):
thread_name = QThread.currentThread().objectName()
self.sig_msg.emit('Scanning '+thread_name)
for step in range(nsamples):
data = self.read_reg(reg[thread_name])
self.sig_data.emit(step, data)
QThread.currentThread().msleep(50)
app.processEvents()
if self.__abort:
self.sig_msg.emit('scan stopped by user')
break
self.sig_done.emit(thread_name)
sig_msg is a pyqtsignal connected to the following function inside the GUI thread.
#pyqtSlot(int, int)
def on_scaner_data(self, t: int, y: int):
app.processEvents()
self.debugBox.insertPlainText('t: '+str(t)+'y: '+str(y)+'\n')
self.debugBox.ensureCursorVisible()
self.MainGraph.update_fig(t,y)
And finally the MainGraph.update_fig() is called. Inside that function i have used setData(self.datat,self.datay) and app.processEvents() for update the graph, but nothing changes. If i run plot(self.datat,self.datay) instead it redraws the graph but causes a huge performance hit.
class DynamicPlotter(PlotWidget):
def __init__(self,parent=None):
PlotWidget.__init__(self)
self.setParent(parent)
# Use getPlotItem() to get the PlotItem inside PlotWidget.
self.pitem = self.getPlotItem()
#now pitem is our PlotItem
self.pitem.curve=self.pitem.plot()
#curve is a new PlotDataItem added by PlotItem.plot()
self.datat = [1,2]
self.datay = [1,2]
self.pitem.curve.setData(self.datat,self.datay)
#this graph works fine
self.datat = []
self.datay = []
def update_fig(self,t:int,y:int):
self.datat.append(t)
self.datay.append(y)
#it works
self.pitem.curve=self.pitem.plot(self.datat,self.datay)
#it doesn't
self.pitem.curve.setData(self.datat,self.datay)
app.processEvents()
print (self.datat)
log.debug(str(t)+str(y))
def reset_figure(self):
log.debug('clean graph')
self.clear()
I have been following this example from the pyqtplot and my idea was do something similar inside my GUI.
import initExample
from pyqtgraph.Qt import QtGui, QtCore
import numpy as np
import pyqtgraph as pg
from pyqtgraph.ptime import time
app = QtGui.QApplication([])
p = pg.plot()
p.setWindowTitle('pyqtgraph example: PlotSpeedTest')
p.setRange(QtCore.QRectF(0, -10, 5000, 20))
p.setLabel('bottom', 'Index', units='B')
curve = p.plot()
data = np.random.normal(size=(50,5000))
ptr = 0
lastTime = time()
fps = None
def update():
global curve, data, ptr, p, lastTime, fps
curve.setData(data[ptr%10])
ptr += 1
now = time()
dt = now - lastTime
lastTime = now
if fps is None:
fps = 1.0/dt
else:
s = np.clip(dt*3., 0, 1)
fps = fps * (1-s) + (1.0/dt) * s
p.setTitle('%0.2f fps' % fps)
app.processEvents() ## force complete redraw
timer = QtCore.QTimer()
timer.timeout.connect(update)
timer.start(0)
I have been reading the documentation and right know I'm not sure where is/are the problems. I bet for the threads or the event loop handler but i don't know.
Which are the critical points that i have to review?
Any clue?
Thank you.

After a while i have found the problem by myself.
I fixed the problem changing the way that i was using to reset the graph and stopping the scan thread until elements are plot.
Changes on reset function. self.clear() remove the traces on the PlotWidget and that wasn't what i needed.
def reset_figure(self):
log.debug('clean graph')
self.datat =[]
self.datay=[]
self.pitem.curve.setData(self.datat,self.datay)
Scan was modified to stop while the data is plotted in the other thread. sync_to_plot stop the thread execution until self._wait=False. This value is changed by wait_state.
def scan (self):
thread_name = QThread.currentThread().objectName()
#thread_id = str(QThread.currentThreadId())#review
self.sig_msg.emit('Scanning '+thread_name)
for step in range(nsamples):
data = self.read_reg(reg[thread_name])
self.sig_data.emit(step, data)
#pause while plot values
self.sync_to_plot()
if step % refrate == 0:
log.debug("%5d : %d" % (step, data) )
if self.__abort:
self.sig_msg.emit('scan stoped by user')
break
self.sig_done.emit(thread_name)
def sync_to_plot(self):
self._wait=True
while self._wait:
log.debug("waiting")
QThread.currentThread().msleep(1)
app.processEvents()
def wait_state(self, stat):
self._wait=stat
With that done the last change was on the on_scaner_data that unblocks the thread that is waiting on sync_to_plot.
#pyqtSlot(int, int)
def on_scaner_data(self, t: int, y: int):
app.processEvents()
self.debugBox.insertPlainText('t: '+str(t)+'y: '+str(y)+'\n')
self.debugBox.ensureCursorVisible()
self.MainGraph.update_fig(t,y)
for thread, scaner in self.__threads:
scaner.wait_state(False)
log.debug("scanner false")

Update pyqtgraph plot from RxPy observable

In the following script, a plot window is created and values are correctly passed to plot.update. However, the plot is not updating. What am I doing wrong?
import sys
import time
import numpy
from numpy import pi
import rx
from rx.concurrency import QtScheduler
import pyqtgraph as pg
from pyqtgraph.Qt import QtCore, QtGui
class DemoData:
def __init__(self, window, steps):
def signal(t):
omega = 5
return numpy.sin(2*pi*omega*t) + numpy.random.random()
self.stream = rx.Observable.range(
0, steps
).map(
signal
).buffer_with_count(
window, 1
)
class UpdatingPlot:
def __init__(self):
self.plot = pg.plot()
self.curve = self.plot.plot()
def update(self, values):
self.curve.setData(values)
def main():
app = QtGui.QApplication([])
scheduler = QtScheduler(QtCore)
window, steps = 50, 100
data = DemoData(window, steps)
plot = UpdatingPlot()
data.stream.subscribe_on(scheduler=scheduler).subscribe(plot.update)
sys.exit(app.exec_())
if __name__ == '__main__':
main()

This doesn't fix your issue, but it might fix your need. You can use the package joystick, install it using pip install joystick. It is designed to provide a real-time and interactive plotting framework.
Here is an example using it. It creates a graph-frame that 'listens' and displays your sine function as a function of time; a new data point is added every 0.2 sec.
import joystick as jk
import numpy as np
import time
class DemoData(jk.Joystick):
# initialize the infinite loop and callit decorators so they can auto-
# register methods they decorate
_infinite_loop = jk.deco_infinite_loop()
_callit = jk.deco_callit()
#_callit('before', 'init')
def _init_data(self, *args, **kwargs):
# Function automatically called at initialization, thanks to the
# decorator
self.tdata = np.array([]) # time x-axis
self.ydata = np.array([]) # data y-axis
self.omega = 5
#_callit('after', 'init')
def _build_frames(self, *args, **kwargs):
# Function automatically called at initialization, thanks to the
# decorator.
# Creates a graph frame
self.mygraph = self.add_frame(
jk.Graph(name="DemoData", size=(500, 500), pos=(50, 50),
fmt="go-", xnpts=50, freq_up=7, bgcol="w",
xylim=(None,None,-1.1,2.1), xlabel='t', ylabel='sine'))
#_callit('before', 'start')
def _set_t0(self):
# initialize t0 at start-up
self._t0 = time.time()
#_infinite_loop(wait_time=0.2)
def _get_data(self):
# This method will automatically be called with simulation start
# (t.start()), and looped every 0.2 in a separate thread as long as
# the simulation runs (running == True)
# It generates new data and pushes it to the frame.
# concatenate data on the time x-axis
timestamp = time.time() - self._t0
self.tdata = jk.add_datapoint(self.tdata,
timestamp,
xnptsmax=self.mygraph.xnptsmax)
new_y_data = np.sin(2*np.pi*self.omega*timestamp) + np.random.random()
self.ydata = jk.add_datapoint(self.ydata,
new_y_data,
xnptsmax=self.mygraph.xnptsmax)
# push new data to the graph
self.mygraph.set_xydata(np.round(self.tdata, 1), self.ydata)
d = DemoData()
d.start()
...
d.stop()