We Keep Coding

updating the list store of gtk periodically in python

I'm new at phyton programming and developing gui interface has gtk framework and serial port. It has a treeview whose liststore model. I could the insert new rows easily.
I'm using the serialport recive callback in different thread from main gui thread avoid to not miss any data. After the received a new data, it should update the treeview. However, since the serialport is in different thread, I don't know how to update the list. Please help me to do this.
the gui class:
class MainGUI():
def __init__(self):
self.builder = Gtk.Builder()
self.builder.add_from_file("main.glade")
self.builder.connect_signals(MainGUI)
self.window = self.builder.get_object("window1")
self.mycombobox = self.builder.get_object('comboboxtext1')
self.toggle = self.builder.get_object('togglebutton1')
self.table = self.builder.get_object('treeview2')
self.list = self.builder.get_object('liststore1')
self.scroll_window = self.builder.get_object('scrolledwindow1')
def show(self):
print("App main thread number", format(threading.get_ident()))
self.window.show()
Gtk.main()
#staticmethod
def connect_toggled(_self):
if main.toggle.get_active():
main.toggle.set_label("Disconnect")
serial_port.connect(main.mycombobox.get_active_text())
t3 = threading.Thread(target=serial_port.read_from_port)
t3.start()
serial_port.disconnect()
def row_inserted_event(self, path, iter):
"""The actual scrolling method"""
adj = main.scroll_window.get_vadjustment()
adj.set_value(adj.get_upper() - adj.get_page_size())
def update_table(self):
# for i in range(256):
# main.list.append(['aaa', 'ddds', i])
# if len(main.list) > 50:
# main.list.remove(main.list.get_iter(0))
main.list.append(['aaa', 'ddds', 0])
if len(main.list) > 50:
main.list.remove(main.list.get_iter(0))
print(len(main.list))
if __name__ == "__main__":
serial_port = SerialPort()
ports = SerialPort().list_ports()
main = MainGUI()
for port in ports:
main.mycombobox.append_text(port)
main.mycombobox.set_active(0)
main.toggle.set_label("Connect")
main.update_table()
main.show()
the serial port class:
class SerialPort:
def __init__(self):
self.ser = serial.Serial()
self.baud_rate = 115200
def write(self, data):
self.ser.write(bytes(data))
print(data)
def connect(self, port):
print("serial port thread number = %d" % (threading.get_ident()))
print("connected the port = %s" % (port))
self.ser.port = port
self.ser.baudrate = self.baud_rate
self.ser.timeout = 0
if self.ser.isOpen():
print("already connected this port = %s" % (port))
else:
self.ser.open()
def disconnect(self):
if self.ser.isOpen():
self.ser.close()
print("disconnected port")
def read_from_port(self):
while True:
if self.ser.isOpen():
reading = self.ser.readline()
if len(reading) > 0:
self.received_callback(reading)
time.sleep(0.1)
def received_callback(self, data):
print(data)
def list_ports(self):
if sys.platform.startswith('win'):
ports = ['COM%s' % (i + 1) for i in range(256)]
elif sys.platform.startswith('linux') or sys.platform.startswith('cygwin'):
# this excludes your current terminal "/dev/tty"
# ports = glob.glob('/dev/tty[A-Za-z]*')
ports = ['/dev/pts/%s' % (i + 1) for i in range(256)]
elif sys.platform.startswith('darwin'):
ports = glob.glob('/dev/tty.*')
else:
raise EnvironmentError('Unsupported platform')
result = []
for port in ports:
try:
s = serial.Serial(port)
s.close()
result.append(port)
except (OSError, serial.SerialException):
pass
return result

I believe that your problem is more related to threading + GUI, than GTK.
As far as I know, when you modify the liststore that is the model for the treeview, the latter should be updated instantly. So, there should be no problem there.
A fundamental principle when working with threads and a GUI, is that you should only update the GUI from within its own thread (main loop). So what you need to do, is have your worker thread (serial port connection thread) send the update to the main GUI thread and let it update the treeview. The update can be scheduled with the GLib.idle_add function to let GTK do it when most convenient.
Now, to communicate between threads, you could use the queue module.
I don't quite understand your code. So I'll write a simple example (using gtk3 PyGObject, since you didn't specify).
import threading
import queue
import gi
gi.require_version('Gtk', '3.0')
gi.require_version('GLib', '2.0')
from gi.repository import Gtk, GLib
def do_work(com_queue):
# do some work
com_queue.put("update for your treeview")
# continue
class MainGUI(object):
def __init__(self):
self.com_queue = queue.Queue()
self.worker_thread = None
self.liststore = None
# more gui initialization...
def launch_worker_thread(self):
self.worker_thread = threading.Thread(target=do_work, args=(self.com_queue,))
self.worker_thread.start()
Glib.timeout_add(1000, self.check_queue) # run check_queue every 1 second
def check_queue(self):
if self.worker_thread.is_alive():
try:
update = self.com_queue.get()
GLib.idle_add(self.update_treeview, (update,)) # send tuple
except queue.Empty:
pass
return True # to keep timeout running
else:
return False # to end timeout
def update_treeview(self, update):
self.liststore.append(update) # here update the treeview model with tuple
if __name__ == "__main__":
gui = MainGUI()
Gtk.main()
I hope this helps.

Python: How do I lock an object until all threads have done a thing

I am writing a simple threaded server that will send a message to all clients. I have an object that is reset after posting the change message, however I am having a hard time figuring out how to reset that object only after all threads have posted the change message.
To add some context to the problem. I am building a multi user Tkinter python app which connects to a remote database to retrieve information and the application needs to know when data changes so that when a user updates data, all other running instances of the app will get the update. From what I understand, MySQL does not support asynchronous application updates. Instead of running a query every 5 seconds on the database to see if there is a change, I am putting this code server side so that it will send a message to a socket on the client that a change has occurred on the database.
The main loop is just a dummy that will simulate a change
Here is my code:
import socket, threading, time, select, os
class dbMonitor:
isDBAltered = False
def postChange(self):
self.isDBAltered = True
def __str__(self):
return str(self.isDBAltered)
class ThreadedServer(object):
def __init__(self, port,dbMon):
self.port = port
self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
self.sock.setblocking(0)
self.sock.bind((socket.gethostname(), self.port))
self.dbMon = dbMon
def listen(self):
self.sock.listen(100)
read_list = [self.sock]
while True:
read,write,error = select.select(read_list,[],[],1)
for s in read:
if s is self.sock:
client, address = self.sock.accept()
client.settimeout(60)
threading.Thread(target = self.listenToClient, args = (client,address)).start()
def listenToClient(self, client, address):
read_list = [client]
size = 1024
while True:
response = b'Ack'
if self.dbMon.isDBAltered:
response = b'CHANGE'
try:
client.send(response)
except:
client.close()
return False
self.dbMon.isDBAltered = False
read,write,error = select.select(read_list,[],[],1)
for s in read:
if s is client:
try:
data = client.recv(size)
print(data)
if data:
client.send(response)
else:
raise error('Client disconnected')
except:
client.close()
return False
def mainLoop():
while True:
time.sleep(15)
print(dbMon)
dbMon.postChange()
dbMon = dbMonitor()
server = ThreadedServer(5005,dbMon)
threading.Thread(target = mainLoop, args=()).start()
threading.Thread(target = server.listen(), args=()).start()
How do I get self.dbMon.isDBAltered = False to execute only after all threads have executed:
response = b'CHANGE'
try:
client.send(response)

You're trying to synchronize something that's asynchronous... This is massively more complicated than it should be. Your dbmon is only storing a boolean flag... why not just asynchronously modify the "database" instead? For example, if the "database" was a thread-safe buffer, you could just append to that buffer or modify that buffer without synchronizing each thread individually, pull the information written to that buffer and write it to the client socket they belong to in another event loop (this is pretty much what asyncore does)
That said, I have some (probably nonworking, but I hope you get the idea) reference modified code for you to go off of if you want to continue pursing this avenue.
Basically, dbmon will keep a mapping of thread ids to [creation time, modified flag]
Our predicate returns true iff all threads created before a certain threshold have ALL set the modified flag. We set the modified flag when we send the response in the data = client.recv(size) portion of your code. And then we wait on that condition in the server send. We keep notifying all waiting threads on each client receive so that when the condition is finally met, our waiting server threads will all unblock and send the subsequent response.
import socket, threading, time, select, os
import collections
class dbMonitor:
def __init__(self):
self.isDBAltered = {}
self.lock = threading.Lock()
def newThread(self, tid):
self.lock.acquire()
# time of creation, boolean whether that thread has sent response
self.isDBAltered[tid] = [time.time(), False]
self.lock.release()
def threadDone(self, tid):
self.lock.acquire()
self.isDBAltered.pop(tid, None)
self.lock.release()
def altered(self, tid):
self.lock.acquire()
self.isDBAltered[tid][1] = True
self.lock.release()
def reset(self, tid):
self.lock.acquire()
self.isDBAltered(tid)[1] = False
self.lock.release()
def __str__(self):
return str(self.isDBAltered)
class ThreadedServer(object):
def __init__(self, port,dbMon):
self.port = port
self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
self.sock.setblocking(0)
self.sock.bind((socket.gethostname(), self.port))
self.dbMon = dbMon
self.lock = threading.lock()
self.cv = threading.Condition()
self.thresh = 2000
def condition_pred(self):
# unblock if threads currently running for longer than self.thresh have all set their flags
return all([timecreate[1] if time.time() - timecreate[0] > self.thresh else True for tid,timecreate in self.dbMon.isDBAltered])
def listen(self):
self.sock.listen(100)
read_list = [self.sock]
while True:
read,write,error = select.select(read_list,[],[],1)
for s in read:
if s is self.sock:
self.lock.acquire()
client, address = self.sock.accept()
client.settimeout(60)
T = threading.Thread(target = self.listenToClient, args = (client,address)).start()
self.dbmon.newThread(T.ident)
self.lock.release()
def listenToClient(self, client, address):
read_list = [client]
size = 1024
while True:
response = b'Ack'
with self.cv:
self.cv.wait_for(self.condition_pred)
self.dbMon.reset(threading.get_ident())
response = b'CHANGE'
try:
client.send(response)
except:
client.close()
self.dbmon.threadDone(threading.get_ident())
return False
read,write,error = select.select(read_list,[],[],1)
for s in read:
if s is client:
with self.cv:
try:
data = client.recv(size)
print(data)
if data:
client.send(response)
self.dbMon.altered(threading.get_ident())
self.cv.notifyAll()
else:
raise error('Client disconnected')
except:
client.close()
self.dbmon.threadDone(threading.get_ident())
return False

python stop multithreaded echo server

I tried to create multithreaded echo server:
echomain.py:
#!/usr/bin/python
from echoserver import echoserver
server = echoserver()
print server.isRunning()
print server.port()
server.start()
print "Main program continues..."\\This part is not displayed(((
echoserver.py:
#!/usr/bin/python
import threading
import socket
class connection(threading.Thread):
def __init__(self, sock, addr):
self.sock = sock
self.addr = addr
threading.Thread.__init__(self)
def run (self):
while True:
buffer = self.sock.recv(1024)
if buffer == "disconnect\r\n":
self.sock.send("bye")
break
elif buffer:
self.sock.send(buffer)
self.sock.close()
class echoserver(object):
def __init__(self, port=12119):
self.running = False
self._port = port
self._socket = None
def isRunning(self):
return self.running
def port(self):
return self._port
def start(self):
self.running = True
self._socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self._socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
self._socket.bind(("0.0.0.0", self.port()))
self._socket.listen(5)
while True:
conn, addr = self._socket.accept()
connection(conn, addr).start()
def stop(self):
self._socket.close()
print "Server is closed..."
Could somebody help me in how I can launch echoserver class as a thread so it run simultaneously with main program so I could stop it with stop() method in echomain.py part?

Change your runner program to run the server as a thread:
echomain.py:
#!/usr/bin/python
from echoserver import echoserver
from threading import Thread
import time
server = echoserver()
print server.isRunning()
print server.port()
# server.start()
# run server in a different thread
serverThread = Thread(target=server.start)
serverThread.start()
print "main - server started"
# wait ten seconds before stopping
time.sleep(10)
server.stop()
print "main - server stopped"
print "Main program continues..."
This example simply stops the server after 10 seconds.

The simplest way is to have your echoserver itself be a Thread as proposed by Reut Sharabani, but IMHO, you should also implement a correct stop() method, ensuring that all children have ended.
Here is my implementation of your script :
#!/usr/bin/python
import threading
import socket
class connection(threading.Thread):
def __init__(self, sock, addr, parent):
self.sock = sock
self.addr = addr
self.parent = parent
threading.Thread.__init__(self)
self.sock.settimeout(None)
self.closed = False # will be set to True on thread end
def run (self):
while not self.parent._stopped:
buffer = self.sock.recv(1024)
if buffer == "disconnected\r\n":
self.sock.send("bye")
break
elif buffer:
self.sock.send(buffer)
self.sock.close()
self.closed = True
class echoserver(threading.Thread):
def __init__(self, port=12119):
threading.Thread.__init__(self)
self.running = False
self._port = port
self._socket = None
self._stopped = False
self._conns = [] # list of active connections
def isRunning(self):
return self.running
def port(self):
return self._port
def run(self):
self.running = True
self._socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self._socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
self._socket.bind(("0.0.0.0", self.port()))
self._socket.listen(5)
self._socket.settimeout(5) # use a timeout to respond to stop()
while not self._stopped:
try:
conn, addr = self._socket.accept()
c = connection(conn, addr, self)
self._conns.append(c) # add child the the list
c.start()
except Exception as e:
# print e # in debug
pass
self._conns = self.child_list() # remove closed child from list
self._socket.close()
print "Server is closing..."
for connect in self._conns: # join active children
connect.join()
print "Server is closed"
def stop(self):
self._stopped = True
def child_list(self):
l = []
for conn in self._conns:
if conn.closed:
conn.join()
else:
l.append(conn)
return l
Remarks :
you simply use it that way :
serv=echoserver()
serv.start()
... # sleep of do anything you want
serv.stop()
if no connection is active when you call stop() all stops at the end of the accept timeout and you get :
Server is closing...
Server is closed
if at least one connection is active when you call stop(), you get only Server is closing... at the end of the accept timeout. Then for each connection, it will end as soon as it receives a packet, and will be joined by echoserver. Then when all connection are over, you will get Server is closed and echoserver thread will terminate
that means that in you main thread you have only to do
serv.stop()
serv.join()
to be sure that all other threads are correctly terminated, and that all sockets are closed

Queue communication from serial thread to Tkinter GUI

I am doing simple gui for continuous reading data stream from serial COM port and display the results. I could not get up and running queue to push the data from reading thread to main (GUI) part. I think I am missing some part but cannot figure out what.
I can easily start thread and read data(printing them after) from COM port but it seems nothing is going to queue becase when I am printing output in GUI part is empty. Beside that reading and main thread is running ok.
Do you have an idea how to check if is something in the queue?
Here is part of the code if you could check it.
And also sorry for messed up indention :-P
Thank you
class MyGUI(Tkinter.Tk):
def __init__(self, parent):
Tkinter.Tk.__init__(self, parent)
self.parent = parent
self.queueData = Queue.Queue() # create queue
self.initialize()
def initialize(self):
"""Create the GUI"""
self.grid()
...gui code here...
def processQueueData(self):
try:
rawData = self.queueData.get()
except Queue.Empty:
pass
else:
print 'GUI:'+ rawData
#show result of the task
finally:
self.queueDataID = self.after(50, self.processQueueData)
And reading thread is here:
class MyProcess(threading.Thread):
def __init__(self, parent, queueData): #def __init__(self, startValue)
threading.Thread.__init__(self)
self._stop = False
self.parent = parent
self.queueData = queueData
try:
port = app.boxPort.get()
self.ser = serial.Serial(port, baudrate=19200, timeout=0, bytesize=serial.EIGHTBITS, parity=serial.PARITY_NONE, stopbits=serial.STOPBITS_ONE)
except serial.SerialException:
print ('Cannot open port')
self.stop()
else:
app.labelConStatus.set(u'Connected to ' + port)
def stop(self):
self._stop = True
try:
self.ser.close()
except serial.SerialException:
print ('Cannot close port')
else:
app.labelConStatus.set(u'Disconnected')
def run (self):
while self._stop == False:
data = self.ser.read(1)
n = self.ser.inWaiting()
if n:
data = data + self.ser.read(n)
self.queueData.put(data)
# print 'PORT:'+data

Using threads in the right way

I'm working on server written in python. When the client sends a cmd the server will call a function with unknown running time. So to avoid blocking I used threading. But when looking at the child process it seems that they're not terminating, causing a lot of memory usage.
EDIT : Here is the tree of the directory : http://pastebin.com/WZDxLquC
Following answers I found on stackoverflow I implemented a custom Thread class:
sThreads.py :
import threading
class Thread(threading.Thread):
def __init__(self, aFun, args = ()):
super(Thread, self).__init__(None, aFun, None, args)
self.stopped = threading.Event()
def stop(self):
self.stopped.set()
def isStopped(self):
return self.stopped.isSet()
Then here is the server's loop:
some where in mainServer.py:
def serve_forever(self, aCustomClass, aSize = 1024):
while True:
self.conn, self.addr = self.sock.accept()
msg = self.recvMSG(4096)
if(msg):
self.handShake(msg)
print 'Accepted !'
while True:
msg = self.recvMSG(aSize)
if(msg):
t = sThreads.Thread(self.handle, (aCustomClass,))
t.start()
self.currentThreads.append(t)
if(self.workers > 0):
tt = sThreads.Thread(self.respond)
tt.start()
if(self.workers == 0 and len(self.currentThreads) > 0):
for th in self.currentThreads:
th.stop()
Using a custom Thread class will not solve the issue and it still does not stop the terminated threads!
EDIT : added the handle() and respond() methods :
def handle(self, aClass):
self.workers += 1
self.queue.put(aClass._onRecieve(self.decodeStream()))
def respond(self):
while self.workers > 0:
msgToSend, wantToSend = self.queue.get()
self.workers -= 1
if(wantToSend):
print 'I want to send :', msgToSend
continue #Send is not yet implemented !

It seems that self.queue.get() was causing all the issue ...