We Keep Coding

Subprocess not receiving SIGINT on ubuntu: asyncio

I have written code using asyncio that runs a process and after a condition tries to safely interrupt it by sending sigint and waiting for it to close. I tested this on a oracle cloud server with the Oracle Linux 8 image and it worked as I expected. Once I moved over to my main server running Ubuntu 22.04 Everything worked the same except the process never receives the SIGINT signal. I've tried sending other signals and nothing reaches the process.
Code Example:
async def openProc(...):
...
proc = await asyncio.create_subprocess_shell(STARTCOMMAND,
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.STDOUT,
stdin=asyncio.subprocess.PIPE)
try:
await stop_request.wait()
proc.send_signal(signal.SIGINT) # The signal that works on one OS but not the other
finally:
await safe_quit.wait()
response = await server.sendData({"request":"STATE", "data":"STOPPED", "returncode":proc.returncode})
logger.info(response)
It just passes straight through the proc.send_signal(..) with no error message.

Callback function timeout/disruption in google Pub/Sub asynchronous pull subscriber

I have a subscriber application which pulls from a Google Cloud Pub/Sub asynchronously using the google-cloud-pubsub python library.
I am running into intermittent issues where my callback function doesnt finish running/is interrupted. Unfortunately I dont have any errors, I only know this is the case because it does not finish writing data to an external source.
for example, in the following sample code:
subscriber = pubsub_v1.SubscriberClient()
subscription_path = subscriber.subscription_path(
project, subscription_name)
def callback(message):
print('Received message: {}'.format(message))
message.ack()
# Limit the subscriber to only have ten outstanding messages at a time.
flow_control = pubsub_v1.types.FlowControl(max_messages=10)
subscriber.subscribe(
subscription_path, callback=callback, flow_control=flow_control)
# The subscriber is non-blocking, so we must keep the main thread from
# exiting to allow it to process messages in the background.
print('Listening for messages on {}'.format(subscription_path))
while True:
time.sleep(60)
the code in the callback function may sometimes take a while, and in some cases, I have noticed that it does not finish executing/seems to be disrupted by something.
Could that ever happen? is there a timeout on this function?

How to clean up connections after KeyboardInterrupt in python-trio

My class when is connected to the server should immediately send sign in string, afterwards when the session is over it should send out the sign out string and clean up the sockets. Below is my code.
import trio
class test:
_buffer = 8192
_max_retry = 4
def __init__(self, host='127.0.0.1', port=12345, usr='user', pwd='secret'):
self.host = str(host)
self.port = int(port)
self.usr = str(usr)
self.pwd = str(pwd)
self._nl = b'\r\n'
self._attempt = 0
self._queue = trio.Queue(30)
self._connected = trio.Event()
self._end_session = trio.Event()
#property
def connected(self):
return self._connected.is_set()
async def _sender(self, client_stream, nursery):
print('## sender: started!')
q = self._queue
while True:
cmd = await q.get()
print('## sending to the server:\n{!r}\n'.format(cmd))
if self._end_session.is_set():
nursery.cancel_scope.shield = True
with trio.move_on_after(1):
await client_stream.send_all(cmd)
nursery.cancel_scope.shield = False
await client_stream.send_all(cmd)
async def _receiver(self, client_stream, nursery):
print('## receiver: started!')
buff = self._buffer
while True:
data = await client_stream.receive_some(buff)
if not data:
print('## receiver: connection closed')
self._end_session.set()
break
print('## got data from the server:\n{!r}'.format(data))
async def _watchdog(self, nursery):
await self._end_session.wait()
await self._queue.put(self._logoff)
self._connected.clear()
nursery.cancel_scope.cancel()
#property
def _login(self, *a, **kw):
nl = self._nl
usr, pwd = self.usr, self.pwd
return nl.join(x.encode() for x in ['Login', usr,pwd]) + 2*nl
#property
def _logoff(self, *a, **kw):
nl = self._nl
return nl.join(x.encode() for x in ['Logoff']) + 2*nl
async def _connect(self):
host, port = self.host, self.port
print('## connecting to {}:{}'.format(host, port))
try:
client_stream = await trio.open_tcp_stream(host, port)
except OSError as err:
print('##', err)
else:
async with client_stream:
self._end_session.clear()
self._connected.set()
self._attempt = 0
# Sign in as soon as connected
await self._queue.put(self._login)
async with trio.open_nursery() as nursery:
print("## spawning watchdog...")
nursery.start_soon(self._watchdog, nursery)
print("## spawning sender...")
nursery.start_soon(self._sender, client_stream, nursery)
print("## spawning receiver...")
nursery.start_soon(self._receiver, client_stream, nursery)
def connect(self):
while self._attempt <= self._max_retry:
try:
trio.run(self._connect)
trio.run(trio.sleep, 1)
self._attempt += 1
except KeyboardInterrupt:
self._end_session.set()
print('Bye bye...')
break
tst = test()
tst.connect()
My logic doesn't quite work. Well it works if I kill the netcat listener, so then my session looks like the following:
## connecting to 127.0.0.1:12345
## spawning watchdog...
## spawning sender...
## spawning receiver...
## receiver: started!
## sender: started!
## sending to the server:
b'Login\r\nuser\r\nsecret\r\n\r\n'
## receiver: connection closed
## sending to the server:
b'Logoff\r\n\r\n'
Note that Logoff string has been sent out, although it doesn't make sense in here as connection is already broken by that time.
However my goal is to Logoff when user KeyboardInterrupt. In this case my session looks similar to this:
## connecting to 127.0.0.1:12345
## spawning watchdog...
## spawning sender...
## spawning receiver...
## receiver: started!
## sender: started!
## sending to the server:
b'Login\r\nuser\r\nsecret\r\n\r\n'
Bye bye...
Note that Logoff hasn't been sent off.
Any ideas?

Here your call tree looks something like:
connect
|
+- _connect*
|
+- _watchdog*
|
+- _sender*
|
+- _receiver*
The *s indicate the 4 trio tasks. The _connect task is sitting at the end of the nursery block, waiting for the child tasks to complete. The _watchdog task is blocked in await self._end_session.wait(), the _sender task is blocked in await q.get(), and the _receiver task is blocked in await client_stream.receive_some(...).
When you hit control-C, then the standard Python semantics are that whatever bit of Python code is running suddenly raises KeyboardInterrupt. In this case, you have 4 different tasks running, so one of those blocked operations gets picked at random [1], and raises a KeyboardInterrupt. This means a few different things might happen:
If _watchdog's wait call raises KeyboardInterrupt, then the _watchdog method immediately exits, so it never even tries to send logout. Then as part of unwinding the stack, trio cancels all the other tasks, and once they've exited then the KeyboardInterrupt keeps propagating up until it reaches your finally block in connect. At this point you try to notify the watchdog task using self._end_session.set(), but it's not running anymore, so it doesn't notice.
If _sender's q.get() call raises KeyboardInterrupt, then the _sender method immediately exits, so even if the _watchdog did ask it to send a logoff message, it won't be there to notice. And in any case, trio then proceeds to cancel the watchdog and receiver tasks anyway, and things proceed as above.
If _receiver's receive_all call raises KeyboardInterrupt... same thing happens.
Minor subtlety: _connect can also receive the KeyboardInterrupt, which does the same thing: cancels all the children, and then waits for them to stop before allowing the KeyboardInterrupt to keep propagating.
If you want to reliably catch control-C and then do something with it, then this business of it being raised at some random point is quite a nuisance. The simplest way to do this is to use Trio's support for catching signals to catch the signal.SIGINT signal, which is the thing that Python normally converts into a KeyboardInterrupt. (The "INT" stands for "interrupt".) Something like:
async def _control_c_watcher(self):
# This API is currently a little cumbersome, sorry, see
# https://github.com/python-trio/trio/issues/354
with trio.catch_signals({signal.SIGINT}) as batched_signal_aiter:
async for _ in batched_signal_aiter:
self._end_session.set()
# We exit the loop, restoring the normal behavior of
# control-C. This way hitting control-C once will try to
# do a polite shutdown, but if that gets stuck the user
# can hit control-C again to raise KeyboardInterrupt and
# force things to exit.
break
and then start this running alongside your other tasks.
You also have the problem that in your _watchdog method, it puts the logoff request into the queue – thus scheduling a message to be sent later, by the _sender task – and then immediately cancels all the tasks, so that the _sender task probably won't get a chance to see the message and react to it! In general, I find my code works nicer when I use tasks only when necessary. Instead of having a sender task and then putting messages in a queue when you want to send them, why not have the code that wants to send a message call stream.send_all directly? The one thing you have to watch out for is if you have multiple tasks that might send things simultaneously, you might want to use a trio.Lock() to make sure they don't bump into each other by calling send_all at the same time:
async def send_all(self, data):
async with self.send_lock:
await self.send_stream.send_all(data)
async def do_logoff(self):
# First send the message
await self.send_all(b"Logoff\r\n\r\n")
# And then, *after* the message has been sent, cancel the tasks
self.nursery.cancel()
If you do it this way, you might be able to get rid of the watchdog task and the _end_session event entirely.
A few other notes about your code while I'm here:
Calling trio.run multiple times like this is unusual. The normal style is to call it once at the top of your program, and put all your real code inside it. Once you exit trio.run, all of trio's state is lost, you're definitely not running any concurrent tasks (so there's no way anything could possibly be listening and notice your call to _end_session.set()!). And in general, almost all Trio functions assume that you're already inside a call to trio.run. It turns out that right now you can call trio.Queue() before starting trio without getting an exception, but that's basically just a coincidence.
The use of shielding inside _sender looks odd to me. Shielding is generally an advanced feature that you almost never want to use, and I don't think this is an exception.
Hope that helps! And if you want to talk more about style/design issues like this but are worried they might be too vague for stack overflow ("is this program designed well?"), then feel free to drop by the trio chat channel.
[1] Well, actually trio probably picks the main task for various reasons, but that's not guaranteed and in any case it doesn't make a difference here.

How to stop a websocket client without stopping reactor

I have an app similar to a chat-room writing in python that intends to do the following things:
A prompt for user to input websocket server address.
Then create a websocket client that connects to server and send/receive messages. Disable the ability to create a websocket client.
After receiving "close" from server (NOT a close frame), client should drop connecting and re-enable the app to create a client. Go back to 1.
If user exits the app, it exit the websocket client if there is one running.
My approach for this is using a main thread to deal with user input. When user hits enter, a thread is created for WebSocketClient using AutoBahn's twisted module and pass a Queue to it. Check if the reactor is running or not and start it if it's not.
Overwrite on message method to put a closing flag into the Queue when getting "close". The main thread will be busy checking the Queue until receiving the flag and go back to start. The code looks like following.
Main thread.
def main_thread():
while True:
text = raw_input("Input server url or exit")
if text == "exit":
if myreactor:
myreactor.stop()
break
msgq = Queue.Queue()
threading.Thread(target=wsthread, args=(text, msgq)).start()
is_close = False
while True:
if msgq.empty() is False:
msg = msgq.get()
if msg == "close":
is_close = True
else:
print msg
if is_close:
break
print 'Websocket client closed!'
Factory and Protocol.
class MyProtocol(WebSocketClientProtocol):
def onMessage(self, payload, isBinary):
msg = payload.decode('utf-8')
self.Factory.q.put(msg)
if msg == 'close':
self.dropConnection(abort=True)
class WebSocketClientFactoryWithQ(WebSocketClientFactory):
def __init__(self, *args, **kwargs):
self.queue = kwargs.pop('queue', None)
WebSocketClientFactory.__init__(self, *args, **kwargs)
Client thread.
def wsthread(url, q):
factory = WebSocketClientFactoryWithQ(url=url, queue=q)
factory.protocol = MyProtocol
connectWS(Factory)
if myreactor is None:
myreactor = reactor
reactor.run()
print 'Done'
Now I got a problem. It seems that my client thread never stops. Even if I receive "close", it seems still running and every time I try to recreate a new client, it creates a new thread. I understand the first thread won't stop since reactor.run() will run forever, but from the 2nd thread and on, it should be non-blocking since I'm not starting it anymore. How can I change that?
EDIT:
I end up solving it with
Adding stopFactory() after disconnect.
Make protocol functions with reactor.callFromThread().
Start the reactor in the first thread and put clients in other threads and use reactor.callInThread() to create them.

Your main_thread creates new threads running wsthread. wsthread uses Twisted APIs. The first wsthread becomes the reactor thread. All subsequent threads are different and it is undefined what happens if you use a Twisted API from them.
You should almost certainly remove the use of threads from your application. For dealing with console input in a Twisted-based application, take a look at twisted.conch.stdio (not the best documented part of Twisted, alas, but just what you want).

Trying to implement 2 "threads" using `asyncio` module

I've played around with threading before in Python, but decided to give the asyncio module a try, especially since you can cancel a running task, which seemed like a nice detail. However, for some reason, I can't wrap my head around it.
Here's what I wanted to implement (sorry if I'm using incorrect terminology):
a downloader thread that downloads the same file every x seconds, checks its hash against the previous download and saves it if it's different.
a webserver thread that runs in the background, allowing control (pause, list, stop) of the downloader thread.
I used aiohttp for the webserver.
This is what I have so far:
class aiotest():
def __init__(self):
self._dl = None # downloader future
self._webapp = None # web server future
self.init_server()
def init_server(self):
print('Setting up web interface')
app = web.Application()
app.router.add_route('GET', '/stop', self.stop)
print('added urls')
self._webapp = app
#asyncio.coroutine
def _downloader(self):
while True:
try:
print('Downloading and verifying file...')
# Dummy sleep - to be replaced by actual code
yield from asyncio.sleep(random.randint(3,10))
# Wait a predefined nr of seconds between downloads
yield from asyncio.sleep(30)
except asyncio.CancelledError:
break
#asyncio.coroutine
def _supervisor(self):
print('Starting downloader')
self._dl = asyncio.async(self._downloader())
def start(self):
loop = asyncio.get_event_loop()
loop.run_until_complete(self._supervisor())
loop.close()
#asyncio.coroutine
def stop(self):
print('Received STOP')
self._dl.cancel()
return web.Response(body=b"Stopping... ")
This class is called by:
t = aiotest()
t.start()
This doesn't work of course, and I feel that this is a horrible piece of code.
What's unclear to me:
I stop the downloader in the stop() method, but how would I go about stopping the webserver (e.g. in a shutdown() method)?
Does the downloader need a new event loop, or can I use the loop returned by asyncio.get_event_loop()?
Do I really need something like the supervisor for what I'm trying to implement? This seems so clunky. And how do I get supervisor to keep running instead of ending after a single execution as it does now?
One last, more general question: is asyncio supposed to replace the threading module (in the future)? Or does each have its own application?
I appreciate all the pointers, remarks and clarifications!

Why current code is not working:
You're running event loop until self._supervisor() is complete. self._supervisor() creates task (it happens immediately) and finishes immediately.
You're trying to run event loop until _supervisor complete, but how and when are you going start server? I think event loop should be running until server stopped. _supervisor or other stuff can be added as task (to same event loop). aiohttp already has function to start server and event loop - web.run_app, but we can do it manually.
Your questions:
Your server will run until you stop it. You can start/stop different
coroutines while your server working.
You need only one event loop for different coroutines.
I think you don't need supervisor.
More general question: asyncio helps you to run different
functions parallel in single thread in single process. That's why
asyncio is so cool and fast. Some of your sync code with threads you
can rewrite using asyncio and it's coroutines. Moreover: asyncio can
interact with threads and processes.
It can be useful in case you still need threads and processes: here's example.
Useful notes:
It's better to use term coroutine instead of thread while we talk about asyncio coroutines that are not threads
If you use Python 3.5, you can use async/await syntax
instead of coroutine/yield from
I rewrote your code to show you idea. How to check it: run program, see console, open http://localhost:8080/stop, see console, open http://localhost:8080/start, see console, type CTRL+C.
import asyncio
import random
from contextlib import suppress
from aiohttp import web
class aiotest():
def __init__(self):
self._webapp = None
self._d_task = None
self.init_server()
# SERVER:
def init_server(self):
app = web.Application()
app.router.add_route('GET', '/start', self.start)
app.router.add_route('GET', '/stop', self.stop)
app.router.add_route('GET', '/kill_server', self.kill_server)
self._webapp = app
def run_server(self):
# Create server:
loop = asyncio.get_event_loop()
handler = self._webapp.make_handler()
f = loop.create_server(handler, '0.0.0.0', 8080)
srv = loop.run_until_complete(f)
try:
# Start downloader at server start:
asyncio.async(self.start(None)) # I'm using controllers here and below to be short,
# but it's better to split controller and start func
# Start server:
loop.run_forever()
except KeyboardInterrupt:
pass
finally:
# Stop downloader when server stopped:
loop.run_until_complete(self.stop(None))
# Cleanup resources:
srv.close()
loop.run_until_complete(srv.wait_closed())
loop.run_until_complete(self._webapp.shutdown())
loop.run_until_complete(handler.finish_connections(60.0))
loop.run_until_complete(self._webapp.cleanup())
loop.close()
#asyncio.coroutine
def kill_server(self, request):
print('Server killing...')
loop = asyncio.get_event_loop()
loop.stop()
return web.Response(body=b"Server killed")
# DOWNLOADER
#asyncio.coroutine
def start(self, request):
if self._d_task is None:
print('Downloader starting...')
self._d_task = asyncio.async(self._downloader())
return web.Response(body=b"Downloader started")
else:
return web.Response(body=b"Downloader already started")
#asyncio.coroutine
def stop(self, request):
if (self._d_task is not None) and (not self._d_task.cancelled()):
print('Downloader stopping...')
self._d_task.cancel()
# cancel() just say task it should be cancelled
# to able task handle CancelledError await for it
with suppress(asyncio.CancelledError):
yield from self._d_task
self._d_task = None
return web.Response(body=b"Downloader stopped")
else:
return web.Response(body=b"Downloader already stopped or stopping")
#asyncio.coroutine
def _downloader(self):
while True:
print('Downloading and verifying file...')
# Dummy sleep - to be replaced by actual code
yield from asyncio.sleep(random.randint(1, 2))
# Wait a predefined nr of seconds between downloads
yield from asyncio.sleep(1)
if __name__ == '__main__':
t = aiotest()
t.run_server()