The following minimal program reproduces the problem.
import asyncio
import signal
class A:
def __init__(self):
self._event_loop = asyncio.new_event_loop()
def run(self):
print('starting event loop')
self._event_loop.run_forever()
print('event loop has stopped')
def stop(self):
print('stopping event loop')
self._event_loop.stop()
if __name__ == '__main__':
a = A()
def handle_term(*args):
a.stop()
signal.signal(signal.SIGTERM, handle_term)
a.run()
If you run the program and send a SIGTERM to the process, the print statement in line 16 (stopping event loop) is called but the programm does not terminate and the print statement in line 13 (event loop has stopped) is never called. So it seems that the event loop is never stopped and self._event_loop.run_forever() blocks indefinitely.
Why is this?
Note: A modified version of the program, where a.stop() is not called by a signal handler but by a seperate thread with a delay, works as expected. How can it make a difference how a.stop() is called?
Instead of signal.signal() use loop.add_signal_handler():
import asyncio
import signal
import os
class A:
def __init__(self):
self.loop = asyncio.new_event_loop()
self.loop.add_signal_handler(signal.SIGTERM, self.stop)
def stop(self):
print('stopping')
self.loop.stop()
def run(self, close=True):
print('starting loop')
try:
self.loop.run_forever()
print('loop stopped')
finally:
if close:
self.loop.close()
if __name__ == '__main__':
print("to stop run:\nkill -TERM {}".format(os.getpid()))
a = A()
a.run()
Related
I'm using python to create a script which runs and interacts with some processes simultaneously. For that I'm using asyncio to implement this parallelism. The main problem is how to run another cleanup routine when a KeyboardInterrupt or a SIGINT occurs.
Here's an example code I wrote to show the problem:
import asyncio
import logging
import signal
from time import sleep
class Process:
async def start(self, arguments):
self._process = await asyncio.create_subprocess_exec("/bin/bash", *arguments)
return await self._process.wait()
async def stop(self):
self._process.terminate()
class BackgroundTask:
async def start(self):
# Very important process which needs to run while process 2 is running
self._process1 = Process()
self._process1_task = asyncio.create_task(self._process1.start(["-c", "sleep 100"]))
self._process2 = Process()
self._process2_task = asyncio.create_task(self._process2.start(["-c", "sleep 50"]))
await asyncio.wait([self._process1_task, self._process2_task], return_when=asyncio.ALL_COMPLETED)
async def stop(self):
# Stop process
await self._process1.stop()
# Call a cleanup process which cleans up process 1
cleanup_process = Process()
await cleanup_process.start(["-c", "sleep 10"])
# After that we can stop our second process
await self._process2.stop()
backgroundTask = BackgroundTask()
async def main():
await asyncio.create_task(backgroundTask.start())
logging.basicConfig(level=logging.DEBUG)
asyncio.run(main(), debug=True)
This code creates a background task which starts two processes (in this example two bash sleep commands) and waits for them to finish. This works fine and both command are running in parallel.
The main problem is the stop routine. I'd like to run the stop method when the program receives a SIGINT or KeyboardInterrupt, which first stops the process1, then starts a cleanup method and stops process2 afterwards. This is necessary because the cleanup command depends on process2.
What I've tried (instead of the asyncio.run() and the async main):
def main():
try:
asyncio.get_event_loop().run_until_complete(backgroundTask.start())
except KeyboardInterrupt:
asyncio.get_event_loop().run_until_complete(backgroundTask.stop())
main()
This of course doens't work as expected, because as soon as an KeyboardInterrupt exception occours the backgroundTask.start Task is canceled and the backgroundTask.stop is started in the main loop, so my processes are canceled and can't stopped properly.
So is there a way to detect the KeyboardInterrupt without canceling the current main loop and run my backgroundTask.stop method instead?
You want to add a signal handler as shown in this example in the docs:
import asyncio
import functools
import os
import signal
def ask_exit(signame, loop):
print("got signal %s: exit" % signame)
loop.stop()
async def main():
loop = asyncio.get_running_loop()
for signame in {'SIGINT', 'SIGTERM'}:
loop.add_signal_handler(
getattr(signal, signame),
functools.partial(ask_exit, signame, loop))
await asyncio.sleep(3600)
print("Event loop running for 1 hour, press Ctrl+C to interrupt.")
print(f"pid {os.getpid()}: send SIGINT or SIGTERM to exit.")
asyncio.run(main())
That's a bit of an overcomplicated/outdated example though, consider it more like this (your coroutine code goes where the asyncio.sleep call is):
import asyncio
from signal import SIGINT, SIGTERM
async def main():
loop = asyncio.get_running_loop()
for signal_enum in [SIGINT, SIGTERM]:
loop.add_signal_handler(signal_enum, loop.stop)
await asyncio.sleep(3600) # Your code here
asyncio.run(main())
At this point a Ctrl + C will break the loop and raise a RuntimeError, which you can catch by putting the asyncio.run call in a try/except block like so:
try:
asyncio.run(main())
except RuntimeError as exc:
expected_msg = "Event loop stopped before Future completed."
if exc.args and exc.args[0] == expected_msg:
print("Bye")
else:
raise
That's not very satisfying though (what if something else caused the same error?), so I'd prefer to raise a distinct error. Also, if you're exiting on the command line, the proper thing to do is to return the proper exit code (in fact, the code in the example just uses the name, but it's actually an IntEnum with that numeric exit code in it!)
import asyncio
from functools import partial
from signal import SIGINT, SIGTERM
from sys import stderr
class SignalHaltError(SystemExit):
def __init__(self, signal_enum):
self.signal_enum = signal_enum
print(repr(self), file=stderr)
super().__init__(self.exit_code)
#property
def exit_code(self):
return self.signal_enum.value
def __repr__(self):
return f"\nExitted due to {self.signal_enum.name}"
def immediate_exit(signal_enum, loop):
loop.stop()
raise SignalHaltError(signal_enum=signal_enum)
async def main():
loop = asyncio.get_running_loop()
for signal_enum in [SIGINT, SIGTERM]:
exit_func = partial(immediate_exit, signal_enum=signal_enum, loop=loop)
loop.add_signal_handler(signal_enum, exit_func)
await asyncio.sleep(3600)
print("Event loop running for 1 hour, press Ctrl+C to interrupt.")
asyncio.run(main())
Which when Ctrl + C'd out of gives:
python cancelling_original.py
⇣
Event loop running for 1 hour, press Ctrl+C to interrupt.
^C
Exitted due to SIGINT
echo $?
⇣
2
Now there's some code I'd be happy to serve! :^)
P.S. here it is with type annotations:
from __future__ import annotations
import asyncio
from asyncio.events import AbstractEventLoop
from functools import partial
from signal import Signals, SIGINT, SIGTERM
from sys import stderr
from typing import Coroutine
class SignalHaltError(SystemExit):
def __init__(self, signal_enum: Signals):
self.signal_enum = signal_enum
print(repr(self), file=stderr)
super().__init__(self.exit_code)
#property
def exit_code(self) -> int:
return self.signal_enum.value
def __repr__(self) -> str:
return f"\nExitted due to {self.signal_enum.name}"
def immediate_exit(signal_enum: Signals, loop: AbstractEventLoop) -> None:
loop.stop()
raise SignalHaltError(signal_enum=signal_enum)
async def main() -> Coroutine:
loop = asyncio.get_running_loop()
for signal_enum in [SIGINT, SIGTERM]:
exit_func = partial(immediate_exit, signal_enum=signal_enum, loop=loop)
loop.add_signal_handler(signal_enum, exit_func)
return await asyncio.sleep(3600)
print("Event loop running for 1 hour, press Ctrl+C to interrupt.")
asyncio.run(main())
The advantage of a custom exception here is that you can then catch it specifically, and avoid the traceback being dumped to the screen
try:
asyncio.run(main())
except SignalHaltError as exc:
# log.debug(exc)
pass
else:
raise
I need to run a gstreamer pipeline to perform video streaming. The GStreamer pipeline requires a GObject.MainLoop object which has a run() method that does not terminate until quit() is called.
For this I create a process (P2) from my main application process (P1), which runs the GObject.MainLoop instance in its main thread. The problem is that loop goes on indefinitly within the process P2 and I'm unable to exit/quit it from the main application process (P1).
Following is the section of code that might help understanding the scenario.
'''
start() spawns a new process P2 that runs Mainloop within its main thread.
stop() is called from P1, but does not quit the Mainloop. This is probably because
processes do not have shared memory
'''
from multiprocessing import Process
import gi
from gi.repository import GObject
class Main:
def __init__(self):
self.process = None
self.loop = GObject.MainLoop()
def worker(self):
self.loop.run()
def start(self):
self.process=Process(target=self.worker, args=())
self.process.start()
def stop(self):
self.loop.quit()
Next, I tried using a multiprocessing Queue for sharing the 'loop' variable between the processes, but am still unable to quit the mainloop.
'''
start() spawns a new process and puts the loop object in a multiprocessing Queue
stop() calls get() from the loop and calls the quit() method, though it still does not quit the mainloop.
'''
from multiprocessing import Process, Queue
import gi
from gi.repository import GObject
class Main:
def __init__(self):
self.p=None
self.loop = GObject.MainLoop()
self.queue = Queue()
def worker(self):
self.queue.put(self.loop)
self.loop.run()
def start(self):
self.p=Process(target=self.worker, args=())
self.p.start()
def stop(self):
# receive loop instance shared by Child Process
loop=self.queue.get()
loop.quit()
How do I call the quit method for the MainLoop object which is only accessible within the child Process P2?
Ok firstly we need to be using threads not processes. Processes will be in a different address space.
What is the difference between a process and a thread?
Try passing the main loop object to a separate thread that does the actual work. This will make your main method in to nothing but a basic GLib event processing loop, but that is fine and the normal behavior in many GLib applciations.
Lastly, we need to handle the race condition of the child process finishing its work before the main loop activates. We do this with the while not loop.is_running() snippet.
from threading import Thread
import gi
from gi.repository import GObject
def worker(loop):
while not loop.is_running():
print("waiting for loop to run")
print("working")
loop.quit()
print("quitting")
class Main:
def __init__(self):
self.thread = None
self.loop = GObject.MainLoop()
def start(self):
self.thread=Thread(target=worker, args=(self.loop,))
self.thread.start()
self.loop.run()
def main():
GObject.threads_init()
m = Main()
m.start()
if __name__ =='__main__' : main()
I extended multiprocessing.Process module in my class Main and overridden its run() method to actually run the GObject.Mainloop instance inside another thread (T1) instead of its main thread. And then implemented a wait-notify mechanism which will make the main thread of Process (P2) to go under wait-notify loop and used multiprocessing.Queue to forward messages to the main thread of P2 and P2 will be notified at the same time. For eg, stop() method, which will send the quit message to P2 for which a handler is defined in the overridden run() method.
This module can be extended to parse any number of messages to the Child Process provided their handlers are to be defined also.
Following is the code snippet which I used.
from multiprocessing import Process, Condition, Queue
from threading import Thread
import gi
from gi.repository import GObject
loop=GObject.MainLoop()
def worker():
loop.run()
class Main(Process):
def __init__(self, target=None, args=()):
self.target=target
self.args=tuple(args)
print self.args
self.message_queue = Queue()
self.cond = Condition()
self.thread = None
self.loop = GObject.MainLoop()
Process.__init__(self)
def run(self):
if self.target:
self.thread = Thread(target=self.target, args=())
print "running target method"
self.thread.start()
while True:
with self.cond:
self.cond.wait()
msg = self.message_queue.get()
if msg == 'quit':
print loop.is_running()
loop.quit()
print loop.is_running()
break
else:
print 'message received', msg
def send_message(self, msg):
self.message_queue.put(msg)
with self.cond:
self.cond.notify_all()
def stop(self):
self.send_message("quit")
self.join()
def func1(self):
self.send_message("msg 1") # handler is defined in the overridden run method
# few others functions which will send unique messages to the process, and their handlers
# are defined in the overridden run method above
This method is working fine for my scenerio but suggestions are welcomed if there is a better way to do the same.
I have the following two threads:
myThread = threading.Thread(target=sender.mainloop.run, daemon=True)
myThread.start()
myThread2 = threading.Thread(target=receiver.mainloop.run, daemon=True)
myThread2.start()
The targets are GObject.Mainloop() methods.
Afterwards my main program is in an infinite loop.
My problem is that when the execution is terminated by CTRL-C, Keyboardexception is raised for both threads, but the main program does not terminate.
Any ideas how could both the main program and the two threads be terminated by CTRL-C?
ctrl-c issues a SIGINT signal, which you can capture in your main thread for a callback. You can then run whatever shutdown code you want in the callback, maybe a sender/receiver.mainloop.quit() or something.
import threading
import signal
import sys
def loop():
while True:
pass
def exit(signal, frame):
sys.exit(0)
myThread = threading.Thread(target=loop)
myThread.daemon = True
myThread.start()
myThread2 = threading.Thread(target=loop)
myThread2.daemon = True
myThread2.start()
signal.signal(signal.SIGINT, exit)
loop()
I try to stop a gobject.MainLoop() after a few seconds.
I don't know if it's possible to set a timeout to this kind of loop, it would be perfect but I have not found that.
So, I tried to workaround this with threading but unfortunately, the main loop block others threads.
Here my code (I'm working with python 2.7):
import MediaCenter_dbusConfig
import dbus
import gobject
from dbus.mainloop.glib import DBusGMainLoop
from time import sleep
from threading import Thread
mainloop=0
class Timeout(Thread):
global mainloop
def __init__(self):
Thread.__init__(self)
def run(self):
global mainloop
i = 0
while i < 30:
sleep(1)
i += 1
mainloop.quit()
class Loop(Thread):
global mainloop
def __init__(self):
Thread.__init__(self)
def run(self):
global mainloop
sleep(5)
mainloop.run()
def catchall_detectedDevicePopUp_from_willShowPopup_signals_handler(popup_string):
global mainloop
if(popup_string == "DetectedDevicePopUp.qml") :
print(popup_string)
mainloop.quit()
def detectedDevicePopUp_detector() :
global mainloop
dbus.mainloop.glib.DBusGMainLoop(set_as_default=True)
bus=MediaCenter_dbusConfig.init() # basicly do a dbus.bus.BusConnection()
bus.add_signal_receiver(catchall_detectedDevicePopUp_from_willShowPopup_signals_handler, dbus_interface = "com.orange.mediacenter.apimanager", signal_name = "willShowPopup")
mainloop = gobject.MainLoop()
thread1 = Timeout()
thread2 = Loop()
thread1.start()
thread2.start()
thread1.join()
thread2.join()
Here I call detectedDevicePopUp_detector(). I'm waiting for a signal named willShowPopup. If I received a signal, I want to stop the loop and continue my program, and after 30s, if I have not received any signal, I want the same thing (stop the loop and continue my program) but here it doesn't work, my Timeout thread is blocked by my Loop thread.
Clarification: I can not edit the signals sent (I test an application).
Any ideas ?
As I understood the question, threading is not really wanted. Below is an example that uses gobject.timeout_add to add a maximum time that the mainloop will run if there is no signal to stop it:
import gobject
import dbus
import dbus.service
from dbus.mainloop.glib import DBusGMainLoop
DBusGMainLoop(set_as_default=True)
OPATH = "/com/example/StopLoop"
IFACE = "com.example.StopLoop"
BUS_NAME = "com.example.StopLoop"
TIMEOUT = 30 * 1000
class Example(dbus.service.Object):
def __init__(self, loop):
self.loop = loop
bus = dbus.SessionBus()
bus.request_name(BUS_NAME)
bus_name = dbus.service.BusName(BUS_NAME, bus=bus)
dbus.service.Object.__init__(self, bus_name, OPATH)
# Add a timeout for how long to run mainloop
# if no signal is received
self.setup_timeout(TIMEOUT)
# Listen to the "Stop" signal
self.listen_for_signal(bus)
def setup_timeout(self, timeout):
gobject.timeout_add(timeout, self.handler)
def listen_for_signal(self, bus):
bus.add_signal_receiver(self.handler, "Stop")
def handler(self):
# This handler is used for both timeout and signal
self.loop.quit()
if __name__ == "__main__":
loop = gobject.MainLoop()
a = Example(loop)
loop.run()
print "Exited mainloop, continuing program..."
If the Stop signal is received e.g. by doing:
dbus-send --session --type=signal --dest=com.example.StopLoop /com/example/StopLoop com.example.StopLoop.Stop
The mainloop will exit and the code will continue from where loop.run() was called.
If no signal is received the mainloop will be stopped by the timeout (30 seconds in this case) and continue from where loop.run() was called.
In the following script, I get the "stop message received" output but the process never ends. Why is that? Is there another way to end a process besides terminate or os.kill that is along these lines?
from multiprocessing import Process
from time import sleep
class Test(Process):
def __init__(self):
Process.__init__(self)
self.stop = False
def run(self):
while self.stop == False:
print "running"
sleep(1.0)
def end(self):
print "stop message received"
self.stop = True
if __name__ == "__main__":
test = Test()
test.start()
sleep(1.0)
test.end()
test.join()
The start method has cloned the object into a separate process, where it executes run. The end method is nothing special, so it runs in the process that calls it -- the changes it performs to that object are not sent to the clone object.
So, use instead an appropriate means of interprocess communication, such as a multiprocessing.Event instance, e.g.:
from multiprocessing import Process, Event
from time import sleep
class Test(Process):
def __init__(self):
Process.__init__(self)
self.stop = Event()
def run(self):
while not self.stop.is_set():
print "running"
sleep(1.0)
def end(self):
print "stop message received"
self.stop.set()
if __name__ == "__main__":
test = Test()
test.start()
sleep(1.0)
test.end()
test.join()
As you see, the required changes are minimal.