We Keep Coding

Close asyncio loop

import asyncio
import aiohttp
aut_token = ("token")
tasks = []
iter_flag = True
interval = 0
seq = 0
class WAPI:
async def receiver(WAPI_S):
async for msg in WAPI_S:
global interval
global seq
data = msg.json()
seq = data.get("s")
if data.get("op") == 10:
interval = data.get("d").get("heartbeat_interval") / 1000
if data.get("op") == 11:
pass
raise aiohttp.ClientError
async def heartbeating(WAPI_S):
while iter_flag:
await WAPI_S.send_json({
"op": 1,
"d": seq
})
await asyncio.sleep(interval)
async def event_manager():
loop = asyncio.get_running_loop()
try:
async with aiohttp.ClientSession() as session:
async with session.ws_connect("url") as WAPI_S:
task_receive = loop.create_task(WAPI.receiver(WAPI_S)); task_heartbeating = loop.create_task(WAPI.heartbeating(WAPI_S))
tasks.append(task_receive); tasks.append(task_heartbeating)
await asyncio.gather(*tasks)
except aiohttp.ClientError:
global iter_flag
iter_flag = False
await asyncio.sleep(interval)
for task in tasks:
task.cancel()
try:
loop.close()
except:
loop.stop()
asyncio.run(WAPI.event_manager())
I am trying to implement catching ClientError exception with loop closing, however, loop.close throws "RuntimeError: Event loop stopped before Future completed."
How to implement interception correctly?

You do not need to keep track of your tasks manually, you can simply use
asyncio.all_tasks():
Return a set of not yet finished Task objects run by the loop.
And then a:
pending = asyncio.all_tasks()
c in pending:
wait_for(c, timeout=5)
Also, you are trying to stop the loop while you are in it.
This is the pattern I use most of the times:
async def main():
<do some stuff>
loop = asyncio.get_event_loop()
try:
loop.run_until_complete(main())
except ExceptionYouWantToHandle:
<cleaning up>
finally:
loop.stop()
And in the event_manager you simply return once you get an execption or you pass the exception

Correct destruction process for async code running in a thread

Below is (working) code for a generic websocket streamer.
It creates a daemon thread from which performs asyncio.run(...).
The asyncio code spawns 2 tasks, which never complete.
How to correctly destroy this object?
One of the tasks is executing a keepalive 'ping', so I can easily exit that loop using a flag. But the other is blocking on a message from the websocket.
import json
import aiohttp
import asyncio
import gzip
import asyncio
from threading import Thread
class WebSocket:
KEEPALIVE_INTERVAL_S = 10
def __init__(self, url, on_connect, on_msg):
self.url = url
self.on_connect = on_connect
self.on_msg = on_msg
self.streams = {}
self.worker_thread = Thread(name='WebSocket', target=self.thread_func, daemon=True).start()
def thread_func(self):
asyncio.run(self.aio_run())
async def aio_run(self):
async with aiohttp.ClientSession() as session:
self.ws = await session.ws_connect(self.url)
await self.on_connect(self)
async def ping():
while True:
print('KEEPALIVE')
await self.ws.ping()
await asyncio.sleep(WebSocket.KEEPALIVE_INTERVAL_S)
async def main_loop():
async for msg in self.ws:
def extract_data(msg):
if msg.type == aiohttp.WSMsgType.BINARY:
as_bytes = gzip.decompress(msg.data)
as_string = as_bytes.decode('utf8')
as_json = json.loads(as_string)
return as_json
elif msg.type == aiohttp.WSMsgType.TEXT:
return json.loads(msg.data)
elif msg.type == aiohttp.WSMsgType.ERROR:
print('⛔️ aiohttp.WSMsgType.ERROR')
return msg.data
data = extract_data(msg)
self.on_msg(data)
# May want this approach if we want to handle graceful shutdown
# W.task_ping = asyncio.create_task(ping())
# W.task_main_loop = asyncio.create_task(main_loop())
await asyncio.gather(
ping(),
main_loop()
)
async def send_json(self, J):
await self.ws.send_json(J)

I'd suggest the use of asyncio.run_coroutine_threadsafe instead of asyncio.run. It returns a concurrent.futures.Future object which you can cancel:
def thread_func(self):
self.future = asyncio.run_coroutine_threadsafe(
self.aio_run(),
asyncio.get_event_loop()
)
# somewhere else
self.future.cancel()
Another approach would be to make ping and main_loop a task, and cancel them when necessary:
# inside `aio_run`
self.task_ping = asyncio.create_task(ping())
self.main_loop_task = asyncio.create_task(main_loop())
await asyncio.gather(
self.task_ping,
self.main_loop_task
return_exceptions=True
)
# somewhere else
self.task_ping.cancel()
self.main_loop_task.cancel()
This doesn't change the fact that aio_run should also be called with asyncio.run_coroutine_threadsafe. asyncio.run should be used as a main entry point for asyncio programs and should be only called once.

I would like to suggest one more variation of the solution. When finishing coroutines (tasks), I prefer minimizing the use of cancel() (but not excluding), since sometimes it can make it difficult to debug business logic (keep in mind that asyncio.CancelledError does not inherit from an Exception).
In your case, the code might look like this(only changes):
class WebSocket:
KEEPALIVE_INTERVAL_S = 10
def __init__(self, url, on_connect, on_msg):
# ...
self.worker_thread = Thread(name='WebSocket', target=self.thread_func)
self.worker_thread.start()
async def aio_run(self):
self._loop = asyncio.get_event_loop()
# ...
self._ping_task = asyncio.create_task(ping())
self._main_task = asyncio.create_task(main_loop())
await asyncio.gather(
self._ping_task,
self._main_task,
return_exceptions=True
)
# ...
async def stop_ping(self):
self._ping_task.cancel()
try:
await self._ping_task
except asyncio.CancelledError:
pass
async def _stop(self):
# wait ping end before socket closing
await self.stop_ping()
# lead to correct exit from `async for msg in self.ws`
await self.ws.close()
def stop(self):
# wait stopping ping and closing socket
asyncio.run_coroutine_threadsafe(
self._stop(), self._loop
).result()
self.worker_thread.join() # wait thread finish

Python - Cancel task in asyncio?

I have written code for async pool below. in __aexit__ i'm cancelling the _worker tasks after the tasks get finished. But when i run the code, the worker tasks are not getting cancelled and the code is running forever. This what the task looks like: <Task pending coro=<AsyncPool._worker() running at \async_pool.py:17> wait_for=<Future cancelled>>. The asyncio.wait_for is getting cancelled but not the worker tasks.
class AsyncPool:
def __init__(self,coroutine,no_of_workers,timeout):
self._loop = asyncio.get_event_loop()
self._queue = asyncio.Queue()
self._no_of_workers = no_of_workers
self._coroutine = coroutine
self._timeout = timeout
self._workers = None
async def _worker(self):
while True:
try:
ret = False
queue_item = await self._queue.get()
ret = True
result = await asyncio.wait_for(self._coroutine(queue_item), timeout = self._timeout,loop= self._loop)
except Exception as e:
print(e)
finally:
if ret:
self._queue.task_done()
async def push_to_queue(self,item):
self._queue.put_nowait(item)
async def __aenter__(self):
assert self._workers == None
self._workers = [asyncio.create_task(self._worker()) for _ in range(self._no_of_workers)]
return self
async def __aexit__(self,type,value,traceback):
await self._queue.join()
for worker in self._workers:
worker.cancel()
await asyncio.gather(*self._workers, loop=self._loop, return_exceptions =True)
To use the Asyncpool:
async def something(item):
print("got", item)
await asyncio.sleep(item)
async def main():
async with AsyncPool(something, 5, 2) as pool:
for i in range(10):
await pool.push_to_queue(i)
asyncio.run(main())
The Output in my terminal:

The problem is that your except Exception exception clause also catches cancellation, and ignores it. To add to the confusion, print(e) just prints an empty line in case of a CancelledError, which is where the empty lines in the output come from. (Changing it to print(type(e)) shows what's going on.)
To correct the issue, change except Exception to something more specific, like except asyncio.TimeoutError. This change is not needed in Python 3.8 where asyncio.CancelledError no longer derives from Exception, but from BaseException, so except Exception doesn't catch it.

When you have an asyncio task created and then cancelled, you still have the task alive that need to be "reclaimed". So you want to await worker for it. However, once you await such a cancelled task, as it will never give you back the expected return value, the asyncio.CancelledError will be raised and you need to catch it somewhere.
Because of this behavior, I don't think you should gather them but to await for each of the cancelled tasks, as they are supposed to return right away:
async def __aexit__(self,type,value,traceback):
await self._queue.join()
for worker in self._workers:
worker.cancel()
for worker in self._workers:
try:
await worker
except asyncio.CancelledError:
print("worker cancelled:", worker)

This appears to work. The event is a counting timer and when it expires it cancels the tasks.
import asyncio
from datetime import datetime as dt
from datetime import timedelta as td
import random
import time
class Program:
def __init__(self):
self.duration_in_seconds = 20
self.program_start = dt.now()
self.event_has_expired = False
self.canceled_success = False
async def on_start(self):
print("On Start Event Start! Applying Overrides!!!")
await asyncio.sleep(random.randint(3, 9))
async def on_end(self):
print("On End Releasing All Overrides!")
await asyncio.sleep(random.randint(3, 9))
async def get_sensor_readings(self):
print("getting sensor readings!!!")
await asyncio.sleep(random.randint(3, 9))
async def evauluate_data(self):
print("checking data!!!")
await asyncio.sleep(random.randint(3, 9))
async def check_time(self):
if (dt.now() - self.program_start > td(seconds = self.duration_in_seconds)):
self.event_has_expired = True
print("Event is DONE!!!")
else:
print("Event is not done! ",dt.now() - self.program_start)
async def main(self):
# script starts, do only once self.on_start()
await self.on_start()
print("On Start Done!")
while not self.canceled_success:
readings = asyncio.ensure_future(self.get_sensor_readings())
analysis = asyncio.ensure_future(self.evauluate_data())
checker = asyncio.ensure_future(self.check_time())
if not self.event_has_expired:
await readings
await analysis
await checker
else:
# close other tasks before final shutdown
readings.cancel()
analysis.cancel()
checker.cancel()
self.canceled_success = True
print("cancelled hit!")
# script ends, do only once self.on_end() when even is done
await self.on_end()
print('Done Deal!')
async def main():
program = Program()
await program.main()

Manage asyncio coroutines in dict

I want to manage some coroutines in a dict, while running a main coroutine.
Specifically I want to start endless coroutines, put the handler of them in a dict and cancel them via a dict call again. In my example I want to start 4 Coroutines and they shall cancel one by one again with the coroutine doomsday. I'm using Python 3.6.
import asyncio
import traceback
async def add_to_handler(node, func):
func_handler[node] = asyncio.ensure_future(func, loop=loop)
return
async def test_func1():
while True:
print("1 - HeNlO")
await asyncio.sleep(1)
async def test_func2():
while True:
print("2 - TestFunc2")
await asyncio.sleep(2)
async def test_func3():
while True:
print("3 - Tukan")
await asyncio.sleep(3)
async def test_func4():
while True:
print("4 - Do Coro!")
await asyncio.sleep(4)
async def doomsday():
# Cancel coroutine every 10 seconds
print("launch doomsday")
for i in range(len(func_handler)):
await asyncio.sleep(10)
print("start cancelling with {}".format(i))
func_handler[str(i + 1)].cancel()
return
async def main():
await add_to_handler("1", test_func1)
await add_to_handler("2", test_func2)
await add_to_handler("3", test_func3)
await add_to_handler("4", test_func4)
await doomsday()
while True:
print("z..z..Z..Z...Z")
print(func_handler)
await asyncio.sleep(5)
func_handler = {}
loop = asyncio.get_event_loop()
try:
loop.run_until_complete(main())
except KeyboardInterrupt:
print("stop loop")
loop.close()
I tried it with the .call_latermethod of AbstractEventLoop instead of an endless while loop, but it still doesn't want to work and it seems, that my coroutines are seen as function, but I don't know why. Where is my fault?

try to change this function:
async def add_to_handler(node, func):
func_handler[node] = asyncio.ensure_future(func(), loop=loop)
return None
pay attention on asyncio.ensure_future(func(),loop=loop)

two async tasks - one depends on another in python

I need to write a code where i need to to check in real time a status of some variable. I decited to use asyncio to create two async def functions
import asyncio
async def one():
global flag
flag = True
while flag == True:
await asyncio.sleep(0.2)
print("Doing one")
async def two():
await asyncio.sleep(2)
global flag
flag = False
async def main():
tasks = []
tasks.append(one())
tasks.append(two())
await asyncio.gather(*tasks)
loop = asyncio.get_event_loop()
try:
loop.run_until_complete(main())
finally:
loop.close()
print("Loop ended")
When loop starts, all tasks has been lauched and after 2 seconds def two() sets flag=False, which stops def one(). It's good but i want def one() to perform while loop without await asyncio.sleep(0.2) becouse i dont want to have real live update so i set await asyncio.sleep(0.0).
Is it a good practice?

Using a global variable is indeed bad practice. What you are looking for is asyncio's primitives, specifically the asyncio.Event primitive. Here is what you are doing, but with asyncio.Event:
import asyncio
async def one(event):
while event.is_set() == False:
await asyncio.sleep(0.5)
print("Hello World!")
async def two(event):
await asyncio.sleep(2)
event.set()
async def main():
event = asyncio.Event()
await asyncio.gather(*[one(event), two(event)])
asyncio.run(main())