I am using watchdog to get notified when a certain file is changed,
evertything worked as long as i am using no classes like so
class MyEventHandler(PatternMatchingEventHandler):
def on_modified(self, event):
super(MyEventHandler, self).on_modified(event)
logging.info("File %s was just modified" % event.src_path)
readConfigFile()
def readConfigFile():
# set certain values here
def main():
while True:
# using values from configFile here
if __name__ == '__main__':
main()
# do something else
but if i put some classes in place i don't know how to call the method then setup is like this now
class MyEventHandler(PatternMatchingEventHandler):
def on_modified(self, event):
super(MyEventHandler, self).on_modified(event)
logging.info("File %s was just modified" % event.src_path)
readConfigFile() #<- i want to call this method in class Configuration
class Configuration:
def readConfigFile(self):
# set certain values here
def provideValues(self):
# get certain values here
class MainClass:
def __init__(self):
self.conf = Configuration()
event_handler = MyEventHandler(patterns=patterns)
def main(self):
while True:
# using values from configFile here
bla = self.conf.provideValues()
if __name__ == '__main__':
mc = MainClass()
mc.main()
# do something else
There are a log of ways to do it; here's one:
class MyEventHandler(PatternMatchingEventHandler):
def hook(self, thing):
self.thing = thing
def on_modified(self, event):
super(MyEventHandler, self).on_modified(event)
logging.info("File %s was just modified" % event.src_path)
self.thing.readConfigFile() #<- i want to call this method in class Configuration
class MainClass:
def __init__(self):
self.conf = Configuration()
event_handler = MyEventHandler(patterns=patterns)
event_handler.hook(self.conf)
Related
I am implementing a system that uses callbacks for event handling. I currently register the callbacks in a dictionary in each module. For example, see below:
class Module(ABC):
topics = {}
_eventHandler = None
def __init__(self):
pass
def onNotify(self, event):
self.topics[event.topic](event.data)
class Reporter(Module):
def __init__(self):
self.topics = {'time':self.onMsg,
'data':self.onMsg}
def onMsg(self, msg):
print(f'[reporter] {msg}')
def update(self):
pass
A collection of Modules is registered in an EventHandler, as seen below:
class EventHandler(queue.Queue):
def __init__(self):
super().__init__()
self._modules = []
#property
def modules(self):
return self._modules
def attach(self, module):
self._modules.append(module)
module._eventHandler = self
def detach(self, module):
self._modules.append(module)
module._eventHandler = None
def notify(self, event):
for module in self.modules:
if event.topic in module.topics.keys():
module.onNotify(event)
def run(self):
idx = 0
while True:
while not self.empty():
event = self.get_nowait()
self.notify(event)
else:
self.modules[idx].update()
idx = (idx + 1) % len(self._modules)
I do not like that there is a separate list to denote callbacks. It seems that the Python developers agree, as PEP 318 states the following in support of decorators: "It also seems less than pythonic to name the function three times for what is conceptually a single declaration."
How can I use decorators to register functions as callbacks that are recognized by the EventHandler, without having a separate list to use as a directory. An example of my desired formatting for the modules is below:
class Module(ABC):
topics = {}
_eventHandler = None
def __init__(self):
pass
def onNotify(self, event):
'''Find the correct callback for event.topic, and direct the event'''
class Reporter(Module):
#callback(topic = ['time', 'data'])
def onMsg(self, msg):
print(f'[reporter] {msg}')
def update(self):
pass
and the EventHandler could use a method like:
def notify(self, event):
for module in self.modules:
if event.topic in module.callbacks:
module.onNotify(event)
I have determined that using decorators for this would add confusion, and is not a pythonic solution. See my new sample code below that shows an alternate solution.
import time
import threading
class Event():
def __init__(self, topic, payload):
self.topic = topic
self.payload = payload
class EventAggregator():
def __init__(self):
self.subscriber = {}
def publish(self, event):
if event.topic in self.subscriber.keys():
filtered_callbacks = self.subscriber[event.topic]
for callback in filtered_callbacks:
callback(event)
def subscribe(self, topic, callback):
if not topic in self.subscriber.keys():
self.subscriber[topic] = [callback]
else:
self.subscriber[topic].append(callback)
class Reporter():
def __init__(self, event_aggregator):
self._event_aggregator = event_aggregator
self._event_aggregator.subscribe('data', self.onData)
self._event_aggregator.subscribe('time', self.onTime)
def onData(self, event):
print(f'[reporter] (data):{event.payload}')
def onTime(self, event):
print(f'[reporter] (time):{event.payload}')
class Clock():
def __init__(self, event_aggregator):
self._event_aggregator = event_aggregator
threading.Thread(target=self.clockProcess).start()
def clockProcess(self):
while True:
t = time.time()
if round(t % 5) == 0:
event = Event('time', t)
self.publish(event)
time.sleep(1)
def publish(self, event):
self._event_aggregator.publish(event)
if __name__ == '__main__':
egg = EventAggregator()
reporter = Reporter(egg)
clock = Clock(egg)
event = Event('data', 5)
egg.publish(event)
I have a script that watches for changes in file and if occurred it should trigger some actions. Those actions come from two functions defined outside Class. In Class I have defined portion of code to look for changes in file. I cannot figure out how to pass two functions in Class arguments. Here is simplified portion of my script:
import time, os
watch_file = 'my_file.txt'
def first_action():
print('First action called')
def second_action():
print('Second action called')
class Watcher():
def __init__(self, watch_file, first_action=None, second_action=None):
self._cached_stamp = 0
self.filename = watch_file
self.first_action = first_action
self.second_action = second_action
# Look for changes in 'my_file.txt'
def look(self):
stamp = os.stat(self.filename).st_mtime
if stamp != self._cached_stamp:
self._cached_stamp = stamp
# File has changed, so do something...
print('File changed')
if self.first_action is not None:
print('Call first action')
self.first_action(self)
if self.second_action is not None:
print('Call second action')
self.second_action(self)
watcher = Watcher(watch_file, first_action(), second_action())
Doing it like above calls first_action() and second_action() but not inside Class. I know it because I dont see printed 'Call first action' or 'Call second action'
I was able to correctly trigger first action with below code:
watch_file = 'my_file.txt'
def first_action():
print('First action called')
def second_action():
print('Second action called')
class Watcher():
def __init__(self, watch_file, first_action=None, *args):
self._cached_stamp = 0
self.filename = watch_file
self.first_action = first_action
self.args = args
# Look for changes in 'my_file.txt'
def look(self):
stamp = os.stat(self.filename).st_mtime
if stamp != self._cached_stamp:
self._cached_stamp = stamp
# File has changed, so do something...
print('File changed')
if self.first_action is not None:
print('Call first action')
self.first_action(*self.args)
watcher = Watcher(watch_file, first_action)
For some reason I need to specify *args even for function which does not take any argument on call. Can someone explain why *args must be used?
You did it right in the init, but not in the call of "Watcher". To pass the function itself, instead of it's return value, you have to remove the braces.
watcher = Watcher(watch_file, first_action, second_action)
And you should be fine.
Background :
I have a script that allows me to make spatial queries on a PostgreSQL database via an API coming from a private editor (I can't directly query the database). This API is working with python 3.2. To summarize very quickly, this script is used to download elements of this database in the desired geographical footprint. Depending of the zone, you can obtain between 1 to over 100 elements, each of them having very different sizes (from Ko to Go).
The main window let you to set all options and then start the global process. When launched a console window appears letting you see what’s going on. Once an item has been downloaded a short “report” is displayed on the console. Currently everything is done sequentially one element at a time. As you can imagine, if this element is quite large, the console freezes while waiting for the end of the download process.
Code:
I'm not going to post the full script here, but through a very simple script I will try to show the main problem I'm trying to solve (i.e. avoid locking the user interface / have some sort of real-time output on what's going on).
So, in order to avoid these freezing problems, the use of threads seemed to me to be the best solution. To simulate the download process (see previous chapter) I used the url.request urlretrieve method with multiple urls (pointing to files of different sizes).
import os
import sys
import time
import urllib.request
from PyQt4 import QtCore, QtGui
url_1m = 'http://ipv4.sbg.proof.ovh.net/files/1Mio.dat'
url_10m = 'http://ipv4.sbg.proof.ovh.net/files/10Mio.dat'
url_100m = 'http://ipv4.sbg.proof.ovh.net/files/100Mio.dat'
url_1g = 'http://ipv4.sbg.proof.ovh.net/files/1Gio.dat'
url_10g = 'http://ipv4.sbg.proof.ovh.net/files/10Gio.dat'
urls = (url_1m, url_10m, url_100m, url_1g, url_10g)
# ---------------------------------------------------------------------------------
class DownloadWorkerSignals(QtCore.QObject):
"""
Defines the signals available from a running download worker thread.
"""
finished = QtCore.pyqtSignal(str)
# ---------------------------------------------------------------------------------
class DownloadWorker(QtCore.QRunnable):
"""
Worker thread
"""
def __init__(self, url, filepath, filename, index):
super(DownloadWorker, self).__init__()
self.url = url
self.file_path = filepath
self.filename = filename
self.index = index
self.signals = DownloadWorkerSignals()
#QtCore.pyqtSlot(str)
def run(self):
t = time.time()
message = 'Thread %d started\n' % self.index
try:
# The urlretrieve method will copy a network object to a local file
urllib.request.urlretrieve(url=self.url,
filename=os.path.join(self.file_path,
self.filename))
except IOError as error:
message += str(error) + '\n'
finally:
message += 'Thread %d ended %.2f s\n' % (self.index, time.time() - t)
self.signals.finished.emit(message) # Done
# ---------------------------------------------------------------------------------
class Main(QtGui.QMainWindow):
"""
Main window
"""
def __init__(self):
super(self.__class__, self).__init__()
self.resize(400, 200)
self.setWindowTitle("Main")
self.setWindowModality(QtCore.Qt.ApplicationModal)
self.centralwidget = QtGui.QWidget(self)
self.setCentralWidget(self.centralwidget)
# Ok / Close
# -------------------------------------------------------------------------
self.buttonBox = QtGui.QDialogButtonBox(self.centralwidget)
self.buttonBox.setStandardButtons(QtGui.QDialogButtonBox.Cancel |
QtGui.QDialogButtonBox.Ok)
self.buttonBox.setGeometry(QtCore.QRect(10, 160, 380, 20))
# Connect definition
# -------------------------------------------------------------------------
self.connect(self.buttonBox,
QtCore.SIGNAL('accepted()'),
self.button_ok_clicked)
self.connect(self.buttonBox,
QtCore.SIGNAL('rejected()'),
self.button_cancel_clicked)
# Connect functions
# -----------------------------------------------------------------------------
def button_cancel_clicked(self):
self.close()
def button_ok_clicked(self):
# Launch console
console = Console(parent=self)
console.exec_()
# ---------------------------------------------------------------------------------------------------------------
class Console(QtGui.QDialog):
"""
Console window
"""
def __init__(self, parent):
super(self.__class__, self).__init__()
self.parent = parent
self.resize(400, 200)
self.setWindowTitle("Console")
self.setModal(True)
self.verticalLayout = QtGui.QVBoxLayout(self)
# Text edit
# -------------------------------------------------------------------------
self.text_edit = QtGui.QPlainTextEdit(self)
self.text_edit.setReadOnly(True)
self.text_edit_cursor = QtGui.QTextCursor(self.text_edit.document())
self.verticalLayout.addWidget(self.text_edit)
# Ok / Close
# -------------------------------------------------------------------------
self.button_box = QtGui.QDialogButtonBox(self)
self.button_box.setStandardButtons(QtGui.QDialogButtonBox.Close)
self.verticalLayout.addWidget(self.button_box)
# Connect definition
# -------------------------------------------------------------------------
self.connect(self.button_box.button(QtGui.QDialogButtonBox.Close),
QtCore.SIGNAL('clicked()'),
self.button_cancel_clicked)
# Post initialization
# -------------------------------------------------------------------------
self.threadpool = QtCore.QThreadPool()
self.threadpool.setMaxThreadCount(2)
for index, url in enumerate(urls):
worker = DownloadWorker(url=url,
filepath='C:\\Users\\philippe\\Downloads',
filename='url_%d.txt' % index,
index=index)
worker.signals.finished.connect(self.write_message)
self.threadpool.start(worker)
'''
I have to wait for the end of the thread pool to make a post-processing.
If I use the waitForDone I don't see my console until the all work is done
'''
# self.threadpool.waitForDone()
# self.write_stram('Thread pool finished')
# Connect functions
# -----------------------------------------------------------------------------
def button_cancel_clicked(self):
if self.threadpool.activeThreadCount() != 0:
pass # How to interrupt the threadpool ?
self.close()
#QtCore.pyqtSlot(str)
def write_message(self, text):
self.text_edit.insertPlainText(text)
cursor = self.text_edit.textCursor()
self.text_edit.setTextCursor(cursor)
# ---------------------------------------------------------------------------------
if __name__ == '__main__':
app = QtGui.QApplication(sys.argv)
window = Main()
window.show()
app.exec_()
Questions:
Everything seems to work as expected but I encounter two difficulties:
At the end of the thread pool process I have to make some
post-processing. If I use the waitForDone method I don't see my
console until the all work is done and it’s not the type of behavior
wanted.
If the Cancel Button in the Console is clicked, I need to interrupt
the threadpool and I don’t know how to manage that.
I had another look at this problem (based largely on this : how-do-i-maintain-a-resposive-gui-using-qthread-with-pyqgis).
So I replaced the previous tandem QThreadPool/QRunnable, by Queue/QThread. The code below gives an overview.
import os
import sys
import time
import urllib.request
import queue
from PyQt4 import QtCore, QtGui
url_1m = 'http://ipv4.sbg.proof.ovh.net/files/1Mio.dat'
url_10m = 'http://ipv4.sbg.proof.ovh.net/files/10Mio.dat'
url_100m = 'http://ipv4.sbg.proof.ovh.net/files/100Mio.dat'
url_1g = 'http://ipv4.sbg.proof.ovh.net/files/1Gio.dat'
url_10g = 'http://ipv4.sbg.proof.ovh.net/files/10Gio.dat'
urls = (url_1m, url_10m, url_100m, url_1g, url_10g)
# ---------------------------------------------------------------------------------
class WorkerThread(QtCore.QThread):
"""
Worker thread
"""
def __init__(self, parent_thread):
QtCore.QThread.__init__(self, parent_thread)
def run(self):
self.running = True
success = self.do_work()
self.emit(QtCore.SIGNAL('jobFinished(PyQt_PyObject)'), success)
def stop(self):
self.running = False
pass
def do_work(self):
return True
def clean_up(self):
pass
# ---------------------------------------------------------------------------------
class LongRunningTask(WorkerThread):
def __init__(self, parent_thread, url, filepath, filename, index):
WorkerThread.__init__(self, parent_thread)
self.url = url
self.filepath = filepath
self.filename = filename
self.index = index
def do_work(self):
t = time.time()
self.emit(QtCore.SIGNAL('threadText(PyQt_PyObject)'), 'Thread %d started\n' % self.index)
try:
# The urlretrieve method will copy a network object to a local file
urllib.request.urlretrieve(url=self.url,
filename=os.path.join(self.filepath,
self.filename))
except IOError as error:
self.emit(QtCore.SIGNAL('threadText(PyQt_PyObject)'),
'Thread %d error - ' % self.index + str(error) + '\n')
finally:
self.emit(QtCore.SIGNAL('threadText(PyQt_PyObject)'),
'Thread %d ended %.2f s\n' % (self.index, time.time() - t))
return True
# ---------------------------------------------------------------------------------
class Console(QtGui.QDialog):
"""
Console window
"""
def __init__(self):
super(self.__class__, self).__init__()
self.resize(400, 200)
self.setWindowTitle("Console")
self.setModal(True)
self.setLayout(QtGui.QVBoxLayout())
# Text edit
# -------------------------------------------------------------------------
self.textEdit = QtGui.QPlainTextEdit(self)
self.textEdit.setReadOnly(True)
self.textEdit_cursor = QtGui.QTextCursor(self.textEdit.document())
self.layout().addWidget(self.textEdit)
# Ok / Close
# -------------------------------------------------------------------------
self.button_box = QtGui.QDialogButtonBox(self)
self.button_box.setStandardButtons(QtGui.QDialogButtonBox.Close)
self.button_box.button(QtGui.QDialogButtonBox.Close).setEnabled(False)
self.layout().addWidget(self.button_box)
# Connect definition
# -------------------------------------------------------------------------
self.connect(self.button_box.button(QtGui.QDialogButtonBox.Close),
QtCore.SIGNAL('clicked()'),
self.reject)
# Post-Initialization
# -------------------------------------------------------------------------
self.queue = queue.Queue()
# self.queue = queue.Queue(maxsize=2)
self.run_thread()
# Connect functions
# -----------------------------------------------------------------------------
def cancel_thread(self):
self.workerThread.stop()
def job_finished_from_thread(self, success):
self.workerThread.stop()
self.queue.get()
# Stop the pulsation
if self.queue.empty():
self.button_box.button(QtGui.QDialogButtonBox.Close).setEnabled(True)
self.emit(QtCore.SIGNAL('jobFinished(PyQt_PyObject)'), success)
def text_from_thread(self, value):
self.textEdit.insertPlainText(value)
cursor = self.textEdit.textCursor()
self.textEdit.setTextCursor(cursor)
def run_thread(self):
for index, url in enumerate(urls):
self.workerThread = LongRunningTask(parent_thread=self,
url=url,
filepath='C:\\Users\\philippe\\Downloads',
filename='url_%d.txt' % index,
index=index)
self.connect(self.workerThread,
QtCore.SIGNAL('jobFinished(PyQt_PyObject)'),
self.job_finished_from_thread)
self.connect(self.workerThread,
QtCore.SIGNAL('threadText(PyQt_PyObject)'),
self.text_from_thread)
self.queue.put(self.workerThread)
self.workerThread.start()
# If I set the queue to maxsize=2, how to manage it here
'''
while not self.queue.full():
self.queue.put(self.workerThread)
self.workerThread.start()
'''
# ---------------------------------------------------------------------------------
if __name__ == '__main__':
app = QtGui.QApplication(sys.argv)
window = Console()
window.show()
app.exec_()
Question:
Unfortunately, I encounter other types of difficulties. In reality, the queue can contain a large amount of threads (over 100). 1. How can I, like the QthreadPool and its setMaxThreadCount method, manage the number of threads running in parallel in order to prevent the system from collapsing completely ?
I will try to explain the problem I am facing with a small piece of code:
class MyHandler(PatternMatchingEventHandler):
patterns = ["*.csv","*.processing", "*.transforming","*.loading"]
def process(self, event):
eventFileName = event.src_path
eventType = event.event_type
if eventType == 'moved':
eventFileName = event.dest_path
fileNameWithPath, fileExtension = os.path.splitext(eventFileName)
if fileExtension == '.processing':
# Here some function is called to do something, and then appends ".loading" to the file name
testVariable = 75.3
if fileExtension == '.loading':
print testVariable
def on_moved(self, event):
self.process(event)
def on_created(self, event):
self.process(event)
if __name__ == '__main__':
observer = Observer()
observer.schedule(MyHandler(), path='.')
observer.start()
try:
while True:
time.sleep(1)
except KeyboardInterrupt:
observer.stop()
observer.join()
When I try to do the above I am getting this error: global name 'testVariable' is not defined which kinda makes sense but how do I make the above work? I tried to define "testVariable" globally and initiated to 0 and tried using it the way I showed in above code but it did not work as well.
I also tried initiating testVariable as testVariable=0 inside the class (right after "patterns = ....." line but I got this error: local variable "testVariable" referenced before assignment pointing towards print testVariable. So that didnot work as well.
"(...) how do I make the above work?"
By defining testVariable outside your conditional statements. E.g. here:
def process(self, event):
eventFileName = event.src_path
testVariable = 0
...
This will make it available within the process function. If you want it to be available throughout the class, you can define it here:
class MyHandler(PatternMatchingEventHandler):
patterns = ["*.csv","*.processing", "*.transforming","*.loading"]
testVariable = 0
But then you have to access it via the self object within functions like so:
def process(self, event):
...
if fileExtension == '.processing':
# Here some function is called to do something, and then appends ".loading" to the file name
self.testVariable = 75.3
testVariable only exists if you have the extension ".processing". If it's ".loading", the program tries to print a variable that hasn't been made to exist.
If statements do not create a garbage collecting scope in Python, so you don't have to "declare" it outside, so long as somewhere in your if-tree, tesVariable gets a value.
def process(self, event):
def extension():
eventFileName = event.src_path
eventType = event.event_type
if eventType == 'moved':
eventFileName = event.dest_path
return os.path.splitext(eventFileName)[1]
if extension() == '.processing':
...
if extension() == '.loading':
...
I have an annoying problem in Python concerning the heritage of class.
The code is the following:
import time
from threading import Thread
class Main:
def __init__(self):
self.var = 0
self.fils_on()
def fils_on(self):
self.temp = Fils()
self.temp.start()
def fils_off(self):
self.temp.stop()
del self.temp
class Fils(Thread, Main):
def __init__(self):
Main.__init__(self)
Thread.__init__(self)
self.encore = True
def run(self):
i = 0
while self.encore:
chaine = str(i)
print chaine
print "var: ", self.var
i += 1
time.sleep(1)
def stop(self):
self.encore = False
if __name__ == "__main__":
main = Main()
My problem is when I call "Main", I don't manage to print the self.var variable from Fils class.
Someone know why?
And how can I fix it?
I don't understand what you are trying to achieve but it seems you are trying to experiment with threads, The above implementation exceeds recursion depth because you have a cyclic dependency, __init__ from Main class depends on self.fils_on() and it construct Fils() which trigger Fils.__init__() which in turn calls Main.__init__(self) and again the whole process continues.
Here I tried to modify your code, in which I removed the inheritance between Fils and Mains and moved var inside Fils.
import time
from threading import Thread
class Main:
def __init__(self):
self.fils_on()
def fils_on(self):
self.temp = Fils()
self.temp.start()
def fils_off(self):
self.temp.stop()
del self.temp
class Fils(Thread):
def __init__(self):
Thread.__init__(self)
self.encore = True
self.var = 0
def run(self):
i = 0
while self.encore:
chaine = str(i)
print chaine
print "var: ", self.var
i += 1
time.sleep(1)
def stop(self):
self.encore = False
if __name__ == "__main__":
main = Main()