We Keep Coding

Question about translating different Ray workers into dask workers

I am currently working to translate Ray methods into dask and I wanted to know how to create five different "workers" for one class like the Ray program below does (run program in jupyter notebook):
So far i have a similar method in my dask version, but each "actor_" prints the same actor ID from the ping function. Could someone help with creating a similar product in Dask?
"""
#CURRENT VERSION
A simple test of Ray
This example uses placement_group API to spread work around
"""
import random
import os
import platform
import ray
import time
ray.init(ignore_reinit_error=True)
#ray.remote
class Actor():
def __init__(self, actor_id) -> None:
self.pid = os.getpid()
self.hostname = platform.node()
self.ip = ray._private.services.get_node_ip_address()
self.actor_id = actor_id
def ping(self):
print(f"{self.actor_id} {self.pid} {self.hostname} {self.ip} {time.time()} - ping")
time.sleep(random.randint(1,3))
return f"{self.actor_id}"
#ray.remote
def main():
# Get list of nodes to use
print(f"Found {len(actors)} Worker nodes in the Ray Cluster:")
# Setup one Actor per node
print(f"Setting up {len(actors)} Actors...")
time.sleep(1)
# Ping-Pong test
messages = [actors[a].ping.remote() for a in actors]
time.sleep(1)
for _ in range(20):
new_messages, messages = ray.wait(messages, num_returns=1)
for ray_message_id in new_messages:
pong = ray.get(ray_message_id)
print(pong, "- pong")
check = actors[pong].ping.remote()
time.sleep(1)
messages.append(check)
actors = {
"actor1" : Actor.remote(actor_id="actor1"),
"actor2" : Actor.remote(actor_id="actor2"),
"actor3" : Actor.remote(actor_id="actor3"),
"actor4" : Actor.remote(actor_id="actor4"),
"actor5" : Actor.remote(actor_id="actor5")
}
print(actors)
if __name__ == "__main__":
main.remote()

Reading higher frequency data in thread and plotting graph in real-time with Tkinter

In the last couple of weeks, I've been trying to make an application that can read EEG data from OpenBCI Cyton (#250Hz) and plot a graph in 'real-time'. What seems to work better here are threads. I applied the tips I found here 1 to communicate the thread with Tkinter, but the application still doesn't work (gives me the error RecursionError: maximum recursion depth exceeded while calling a Python object). Maybe I'm doing something wrong because I'm trying to use multiple .py files? See below the main parts of my code and a few more comments in context:
###FILE main.py
#!/usr/bin/env python
# -*- coding: utf-8 -*-
from AppWindow import *
window = AppWindow()
window.start()
###FILE AppWindow.py
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import tkinter as tk
from tkinter import ttk
from tkinter.scrolledtext import ScrolledText
import scroller as scrl
import logging
import requests
import matplotlib
import matplotlib.pyplot as plt
import matplotlib.animation as animation
from matplotlib.backends.backend_tkagg import FigureCanvasTkAgg
import random
from pandas import DataFrame
import stream_lsl_eeg as leeg
#Definitions
H = 720
W = 1280
#Color palette>> https://www.color-hex.com/color-palette/92077
bg_color = "#c4ac93"
sc_color = "#bba58e"
tx_color = "#313843"
dt_color = "#987f62"
wn_color = "#6b553b"
class AppWindow:
#Other Functions
def plotGraph(self, x, y):
self.ax.clear()
self.ax.plot(x,y, color = tx_color)
plt.subplots_adjust(bottom=0.31, left=0.136, top=0.9, right=0.99)
plt.ylabel('Magnitude', fontsize = 9, color = tx_color)
plt.xlabel('Freq', fontsize = 9, color = tx_color)
self.figure.canvas.draw()
def __init__(self):
self.root = tk.Tk() #start of application
self.root.wm_title("Hybrid BCI - SSVEP and Eye Tracker")
#Other Graphical Elements
#Button that calls function
self.btn_ReceiveEEG = tk.Button(self.EEG_frame, text = "Receive EEG signal", bg = bg_color, fg = tx_color, state = tk.DISABLED, command = lambda: leeg.getEEGstream(self))
self.btn_ReceiveEEG.place(anchor = 'nw', relx = 0.52, rely = 0.5, width = 196, height = 40)
#Other Graphical Elements
def start(self):
self.root.mainloop() #end of application
### FILE stream_lsl_eeg.py
from pylsl import StreamInlet, resolve_stream
import tkinter as tk
import AppWindow as app
import matplotlib.pyplot as plt
import matplotlib.animation as animation
import threading
import queue
import time
class myThread(threading.Thread):
def __init__(self, name, q, f):
threading.Thread.__init__(self)
self.name = name
self.q = q
self.f = f
def run(self):
print("Starting ", self.name)
pullSamples(self.q, self.f) #place where function is called
def getInlet(app): #this is triggered by another button and it's working fine
global inlet
app.logger.warn('Looking for an EEG strean...')
streams = resolve_stream('type', 'EEG')
inlet = StreamInlet(streams[0])
app.logger.warn('Connected')
app.btn_ReceiveEEG.config(state = tk.NORMAL)
def pullSamples(q):
i = 0
while i<1000:
sample, timestamp = inlet.pull_sample()
threadLock.acquire() #thread locks to put info in the queue
q.put([sample,timestamp]) #data is put in the queue for other threads to access
threadLock.release() #thread unlocks after info is in
i += 1
stopcollecting = 1
print("Exit flag on")
def plotSamples(app, kounter): #Outside Thread
if not stopcollecting: #testing if stream reception stopped
while dataqueue.qsize( ):
try:
kounter += 1
sample, timestamp = dataqueue.get(0)
samples.append(sample[0]) #getting just channel 1 (0)
timestamps.append(timestamp)
show_samples = samples[-250:]
show_timestamps = timestamps[-250:]
app.plotGraph(show_timestamps,show_samples)
print(counter) #this was just a control to count if the right amount of samples was coming out of the queue
except dataqueue.Empty:
pass #still not implemented, but will return to the main application
app.root.after(60, plotSamples(flag,app,kounter)) #60 chosen because plot should update every 15 samples (15/250 = 0,06s)
def getEEGstream(app): #function called by button
app.logger.warn('Starting thread...')
#
kounter = 0
start = time.perf_counter()
thread1.start()
##
plotSamples(flag, app, kounter)
##
thread1.join() #I don't know if I need this...
finish = time.perf_counter()
#
print(f'Sizes: Samples [{len(samples)}, {len(samples[0])}], {len(timestamps)} timestamps')
print(f'Sucessfully streamed in {round(finish-start,3)}s!')
###
threadLock = threading.Lock()
dataqueue = queue.Queue()
stopcollecting = 0
kounter = []
flag = queue.Queue() #secondary queue for flags not used at the moment
flag.put(0)
thread1 = myThread("Thread-1", dataqueue,flag)
samples,timestamps = [],[]
show_samples, show_timestamps = [],[]
As I found here 2, a function should not call itself, but it's basically what here 1 does. Also, I don't think I'm calling root.mainloop() multiple times like done in here 3.
After executing, python gives me the following error/output:
Exception in Tkinter callback
Traceback (most recent call last):
File "C:\Users\robotics\AppData\Local\Continuum\anaconda3\envs\psychopy\lib\tkinter\__init__.py", line 1705, in __call__
return self.func(*args)
File "C:\Users\robotics\Documents\gitDocuments\SSVEP_EyeGaze_py\AppWindow.py", line 109, in <lambda>
self.btn_ReceiveEEG = tk.Button(self.EEG_frame, text = "Receive EEG signal", bg = bg_color, fg = tx_color, state = tk.DISABLED, command = lambda: leeg.getEEGstream(self))
File "C:\Users\robotics\Documents\gitDocuments\SSVEP_EyeGaze_py\stream_lsl_eeg.py", line 118, in getEEGstream
plotSamples(flag, app, kounter)
File "C:\Users\robotics\Documents\gitDocuments\SSVEP_EyeGaze_py\stream_lsl_eeg.py", line 104, in plotSamples
app.root.after(60, plotSamples(flag,app,kounter))
File "C:\Users\robotics\Documents\gitDocuments\SSVEP_EyeGaze_py\stream_lsl_eeg.py", line 104, in plotSamples
app.root.after(60, plotSamples(flag,app,kounter))
File "C:\Users\robotics\Documents\gitDocuments\SSVEP_EyeGaze_py\stream_lsl_eeg.py", line 104, in plotSamples
app.root.after(60, plotSamples(flag,app,kounter))
[Previous line repeated 986 more times]
File "C:\Users\robotics\Documents\gitDocuments\SSVEP_EyeGaze_py\stream_lsl_eeg.py", line 92, in plotSamples
while dataqueue.qsize( ): # if not dataqueue.empty():
File "C:\Users\robotics\AppData\Local\Continuum\anaconda3\envs\psychopy\lib\queue.py", line 87, in qsize
with self.mutex:
RecursionError: maximum recursion depth exceeded while calling a Python object
Exit flag on
This means the thread is being successfully executed, apparently, but the plotSamples() is crashing.
Any advice??

after() (similar to button's command= and bind()) needs function's name without () and without argument - it is called callback - and after sends it to mainloop and mainloop later uses () to run it.
You use function with ()
app.root.after(60, plotSamples(flag,app,kounter))
so it runs it at once (it doesn't send it to mainloop) and this function runs at once again the same function which runs at once the same function, etc. - so you create recursion.
It works like
result = plotSamples(flag,app,kounter) # run at once
app.root.after(60, result)
If you have to use function with arguments then you can do
app.root.after(60, plotSamples, flag, app, kounter)
Eventually you can use lambda to create function without argument
app.root.after(60, lambda:plotSamples(flag,app,kounter) )

Serial Communication breaks down when threading

I am using a USB microwave source, which communicates via a virtual COM port.
Communication is done via python. Everything works fine, as long as I am executing the code blockingly.
However, as soon as any communication is done in a thread
I get a SerialException: Attempting to use a port that is not open. Is there an obvious reason, why this is happening? Nothing else, at least originating from my software, is trying to communicate with the port during that time.
The highest level script:
from serial import SerialException
import deer
import windfreak_usb
try:
deer.mw_targets = windfreak_usb.WindfreakSynthesizer()
except SerialException:
deer.mw_targets.port.close()
deer.mw_targets = windfreak_usb.WindfreakSynthesizer()
deer_mes = deer.DeerMeasurement(f_start=0.9e9,
f_end=1.2e9,
df=3e6,
f_nv=1.704e9,
seq=["[(['mw'],28),([],tau),(['mw', 'mwy'],56),([],tau),(['mw'],28),([],100)]",
"[(['mw'],28),([],tau),(['mw', 'mwy'],56),([],tau),(['mw'],84),([],100)]"],
power_nv=10,
power_targets=3,
tau=700
)
deer_mes.run(10e6) # <- this works perfectly, as it is the blocking version
# deer_mes.start(10e6) # <- raises the SerialException at the line indicated below
deer.mw_targets.port.close()
A reduced form of the microwave source (windfreak_usb.py):
import serial
import synthesizer
class WindfreakSynthesizer(synthesizer.Synthesizer):
def __init__(self):
synthesizer.Synthesizer.__init__(self)
self.port = serial.Serial(
port='COM14',
baudrate=9600,
timeout=10
)
self.off()
def __del__(self):
self.port.close()
def off(self):
self.port.write('o0')
def power(self, p):
p = int(p)
self.port.write('a{}'.format(p))
A reduced form of the measurement class (deer.py):
import threading
import time
import numpy
import os
from PyQt4 import QtCore
from PyQt4.QtCore import QObject
import matplotlib.pyplot as plt
import hardware
import matpickle
import pulsed
import pulser
import savepath
import synthesizer
if 'pg' not in globals():
pg = pulser.Pulser()
if 'mw_targets' not in globals():
mw_targets = synthesizer.Synthesizer()
timeout = 30
CurrentMeasurement = None # global variable pointing to the currently active measurement
class DeerMeasurement(QObject):
update = QtCore.pyqtSignal()
def __init__(self, f_start, f_end, df, f_nv, seq, power_nv, power_targets, tau, sweeps_per_iteration=50e3,
switching_time=300e-6):
super(QObject, self).__init__()
""" setting all parameters as self.parameter """
self.power_targets = power_targets
self.fs = numpy.arange(f_start, f_end + df, df)
self.abort = threading.Event()
self.save_deer()
def run(self, sweeps):
global CurrentMeasurement
if CurrentMeasurement is not None:
print('Deer Warning: cannot start measurement while another one is already running. Returning...')
return
CurrentMeasurement = self
# Start measurement
print('Deer measurement started.')
mw_targets.power(self.power_targets) # <- This causes the SerialException.
""" Here comes the actual measurement, that is never executed, as the line above kills the thread already with the SerialException. """
def start(self, sweeps, monitor=None):
"""Start Measurement in a thread."""
if monitor is not None:
monitor.register(self)
if not hasattr(self, 'mes_thread'):
# noinspection PyAttributeOutsideInit
self.mes_thread = threading.Thread(target=self.run, args=(sweeps,))
self.mes_thread.start()
else:
print('Already threading')
Any help is highly appreciated, as running the measurement outside a thread is not an option.
Best regards!

Read and plot real time live self updating csv file

So, I have a .csv file which updates itself. I would like to do some things with it and am not sure how to approach it, hope you can help me.
The data in the csv looks like this:
There is no headers. I can join the date and time to be in same column without a delimiter too.
07/12/2017,23:50,113.179,113.182,113.168,113.180,113.187,113.189,113.176,113.186,144
07/12/2017,23:51,113.180,113.190,113.180,113.187,113.186,113.196,113.186,113.193,175
07/12/2017,23:52,113.187,113.188,113.174,113.186,113.193,113.194,113.181,113.192,340
07/12/2017,23:53,113.186,113.192,113.175,113.181,113.192,113.199,113.182,113.188,282
07/12/2017,23:54,113.181,113.183,113.170,113.171,113.188,113.188,113.176,113.179,74
07/12/2017,23:55,113.171,113.181,113.170,113.179,113.179,113.188,113.176,113.186,329
07/12/2017,23:56,113.179,113.189,113.174,113.181,113.186,113.195,113.181,113.187,148
07/12/2017,23:57,113.181,113.181,113.169,113.169,113.187,113.187,113.175,113.175,55
07/12/2017,23:58,113.169,113.183,113.169,113.182,113.175,113.188,113.175,113.187,246
07/12/2017,23:59,113.182,113.210,113.175,113.203,113.187,113.215,113.181,113.209,378
08/12/2017,00:00,113.203,113.213,113.180,113.183,113.209,113.220,113.187,113.190,651
08/12/2017,00:01,113.183,113.190,113.164,113.167,113.190,113.196,113.171,113.174,333
08/12/2017,00:02,113.167,113.182,113.156,113.156,113.174,113.188,113.162,113.163,265
08/12/2017,00:03,113.156,113.165,113.151,113.163,113.163,113.172,113.158,113.170,222
08/12/2017,00:04,113.163,113.163,113.154,113.159,113.170,113.170,113.159,113.166,148
08/12/2017,00:05,113.159,113.163,113.153,113.154,113.166,113.168,113.159,113.162,162
For starters I would be interested in using just the first two (or 3 if date and time are separate) columns for this exercise. So for example:
07/12/2017,21:54,113.098
07/12/2017,21:55,113.096
07/12/2017,21:56,113.087
07/12/2017,21:57,113.075
07/12/2017,21:58,113.087
07/12/2017,21:59,113.079
New rows are being added with more recent date time every second or so.
I can do something like
df = pd.read_csv("C:\\Users\\xxx\\Desktop\\csvexport\\thefile.csv")
print(df[-1:])
To see the last row (tail) from the dataframe
Now, I can't see how to do the following and appreciate your help:
Update the dataframe so that I have the most recent version up to date available to make calculations on when new rows appear (without using sleep timer?)
Be able to plot the data with the newly updating data being reflected in the plot automatically as new data arrives (datetime on x axis, float on y)
The output I see in the command window from the program generating the .csv file is like this, if that matters
asset 08/12/2017 05:16:37 float:113.336 floattwo:113.328 digit:20
asset 08/12/2017 05:16:40 float:113.334 floattwo:113.328 digit:21
asset 08/12/2017 05:16:40 float:113.335 floattwo:113.323 digit:22
asset 08/12/2017 05:16:41 float:113.331 floattwo:113.328 digit:23
asset 08/12/2017 05:16:43 float:113.334 floattwo:113.327 digit:24
asset 08/12/2017 05:16:47 float:113.332 floattwo:113.328 digit:25
So you can see the updates are not exactly one second apart, they can have gaps, and can sometimes occur within the same second too (05:16:40 twice)
Therefore, what I would like to happen is keep the plot at equal time intervals actually (1 minute, or 5 minutes, etc) but keep changing the most recent point according to the float vlaue in the .csv belonging to that minute. When a row with the next minute arrives, only then should the plot move to the right (but constantly fluctuate in value as the float number is changing)... Hope you get the idea. I would like to use pyqtgraph for the plot.
I managed to code this much... but it is not the greatest example, excuse me. Of course the plot is not meant to look like this. Just illustrating what I would like to see. So the green bar should be changing value constantly until the next time step is added to the csv
import pyqtgraph as pg
from pyqtgraph import QtCore, QtGui
import pandas as pd
import datetime
x = pd.read_csv("C:\\Users\\xxx\\Desktop\\csvexport\\thefile.csv")
z = x[-1:]
def getlastrow():
for a in z.iterrows():
d = ((int(((a[1][0]).split("/")[0]))))
m = ((int(((a[1][0]).split("/")[1]))))
y = ((int(((a[1][0]).split("/")[2]))))
hh = ((int(((a[1][1]).split(":")[0]))))
mm = ((int(((a[1][1]).split(":")[1]))))
#ss = ((int(((a[1][1]).split(":")[2]))))
thedate = datetime.date(y, m, d)
thetime = datetime.time(hh, mm)
p = (a[1][2])
return ((thedate,thetime,p))
# print(str(getlastrow()[0]).replace("-",""))
# print(getlastrow()[1])
# print(getlastrow()[2])
class CandlestickItem(pg.GraphicsObject):
def __init__(self):
pg.GraphicsObject.__init__(self)
self.flagHasData = False
def set_data(self, data):
self.data = data
self.flagHasData = True
self.generatePicture()
self.informViewBoundsChanged()
def generatePicture(self):
self.picture = QtGui.QPicture()
p = QtGui.QPainter(self.picture)
p.setPen(pg.mkPen('w'))
w = (self.data[1][0] - self.data[0][0]) / 2.
for (t, open) in self.data:
p.drawLine(QtCore.QPointF(t, open), QtCore.QPointF(t, open))
p.setBrush(pg.mkBrush('r'))
if open > 122.8:
p.setBrush(pg.mkBrush('g'))
p.drawRect(QtCore.QRectF(t-w, open, w*2, open))
p.end()
def paint(self, p, *args):
if self.flagHasData:
p.drawPicture(0, 0, self.picture)
def boundingRect(self):
return QtCore.QRectF(self.picture.boundingRect())
app = QtGui.QApplication([])
data = [
[(int(str(getlastrow()[0]).replace("-",""))), (getlastrow()[2])],
[(int(str(getlastrow()[0]).replace("-","")))+1, (getlastrow()[2])+0.1],
[(int(str(getlastrow()[0]).replace("-","")))+2, (getlastrow()[2])+0.2],
]
item = CandlestickItem()
item.set_data(data)
plt = pg.plot()
plt.addItem(item)
plt.setWindowTitle('pyqtgraph example: customGraphicsItem')
def update():
global item, data
new_bar = (int(str(getlastrow()[0]).replace("-","")))+3, ((getlastrow()[2])+10)
data.append(new_bar)
item.set_data(data)
app.processEvents()
timer = QtCore.QTimer()
timer.timeout.connect(update)
timer.start(100)
if __name__ == '__main__':
import sys
if (sys.flags.interactive != 1) or not hasattr(QtCore, 'PYQT_VERSION'):
QtGui.QApplication.instance().exec_()

Hopefully the code below will help with point(1). I realise this is a partial answer. I tested using Linux. The code should be OS agnostic, but I have not tested this.
The code monitors the directory defined in TEST_DIR using the watchdog library. If the file defined in TEST_FILE is changed, then a message is sent from the event handling class called MyHandler to the main function. I put in some ugly time checking as each time a file is altered, multiple events are triggered. So only a single dispatch will be triggered for events occurring within THRESHOLD time. I set this to 0.01 s.
Add code to the dispatcher_receiver function to read in the updated file.
import ntpath
# pip3 install pydispatcher --user
from pydispatch import dispatcher
import sys
import time
from watchdog.observers import Observer
from watchdog.events import FileSystemEventHandler
MYHANDLER_SENDER = 'myhandler_sender'
MYHANDLER_SIGNAL = 'myhandler_signal'
TEST_FILE = 'test_data.csv'
TEST_DIR = '/home/bill/data/documents/infolab2/progs/jupyter_notebooks/pyqtgraph/test_data/'
THRESHOLD_TIME = 0.01
class MyHandler(FileSystemEventHandler):
''' handle events from the file system '''
def __init__(self):
self.start_time = time.time()
def on_modified(self, event):
now_time = time.time()
# filter out multiple modified events occuring for a single file operation
if (now_time - self.start_time) < THRESHOLD_TIME:
print('repeated event, not triggering')
return
changed_file = ntpath.basename(event.src_path)
if changed_file == TEST_FILE:
print('changed file: {}'.format(changed_file))
print('event type: {}'.format(event.event_type))
print('do something...')
# print(event)
message = '{} changed'.format(changed_file)
dispatcher.send(message=message, signal=MYHANDLER_SIGNAL, sender=MYHANDLER_SENDER)
self.start_time = now_time
def main():
dispatcher.connect(dispatcher_receive, signal=MYHANDLER_SIGNAL, sender=MYHANDLER_SENDER)
observer = Observer()
observer.schedule(event_handler, path=TEST_DIR, recursive=False)
observer.start()
try:
while True:
time.sleep(1)
except KeyboardInterrupt:
observer.stop()
observer.join()
def dispatcher_receive(message):
print('received dispatch: {}'.format(message))
# read in the altered file
if __name__ == "__main__":
event_handler = MyHandler()
main()

Access a variable that has been created in a different class. Python

I am not familiar with python classes. I need to pass the variable servicetime (which is located in the class simulation, under the function service) to the function customer and more specifically to the line which says priority=servicetime. Of course both servicetime's have to be equal and also equal to the specific instance of the class. How can this be done?
import simpy
import numpy as np
import random
import itertools
global meanqtime # average queue time
global meanstime # average server time
meanqtime=[]
meanstime=[]
class simulation(object):
"""
"""
def __init__(self,env,num_machines,servicerate):
self.env=env
self.machine=simpy.PriorityResource(env,num_machines)
self.servicerate=servicerate
def service (self,customer):
servicetime=random.expovariate(servicerate) #this
yield self.env.timeout(servicetime)
def customer(env, name, server):
"""The customer process arrives at the server
and requests to be serviced
"""
artime=env.now
#print('%s arrives at the desk at %.2f.' % (name, env.now))
with server.machine.request(priority=servicetime) as request: #to be here
yield request
#print('%s starts servicing at %.2f.' % (name, env.now))
qtime=env.now
meanqtime.append(qtime-artime)
yield env.process(server.service(name))
#print('%s leaves the desk at %.2f.' % (name, env.now))
stime=env.now
meanstime.append(stime-artime)
def setup(env, num_servers, servicerate, arrivalrate):
desk = simulation(env, num_servers, servicerate)
while True:
for i in itertools.count():
#t=np.random.poisson(arrivalrate)
t=random.expovariate(arrivalrate)
yield env.timeout(t)
env.process(customer(env, 'Person %d' % i, desk))
#yield env.timeout(t)
num_servers = 1 #number of servers
arrivalrate=0.1 #per hour
servicerate=0.125#hours
T=100
env = simpy.Environment()
env.process(setup(env, num_servers, servicerate, arrivalrate))
# Execute!
env.run(until=T)