I need to shield a task been fired with create_task within an aiohttp handler process like
async def handler(request):
asyncio.create_task(long_process())
return {}, 200
So the code is
import asyncio
async def shielded(coro):
try:
await asyncio.shield(coro)
except asyncio.CancelledError:
await coro
raise
def create_task(coro) -> asyncio.Task:
task = asyncio.create_task(shielded(coro))
return task
But this test
async def test_create_task_cancel():
async def coro():
await asyncio.sleep(1)
return None
task = create_task(coro())
await asyncio.sleep(0.1)
task.cancel()
await asyncio.sleep(1)
assert task.done() and task.result() is None
with
RuntimeError: coroutine is being awaited already
aiohttp manual suggests to use aiojobs.Scheduler, but it doesn't work as expected
https://github.com/aio-libs/aiojobs/issues/148
https://github.com/aio-libs/aiojobs/issues/72
Working shield function
async def shielded(coro):
ft = asyncio.ensure_future(coro)
shielded_ft = asyncio.shield(ft)
try:
await shielded_ft
except asyncio.CancelledError:
await ft
raise
Related
Hello everyone, I'm writing tests for a game that is written in the python websockets library. When running the test test_game_started an error appears that asyncio loop is not running, but the test passes.
I provide the entire test code and the error below:
Code of HelpTest class:
class HelpTest:
def __init__(self, loop):
self.loop = loop
self.ws1 = None
self.ws2 = None
self.msg1 = None
self.msg2 = None
self.tasks = []
async def listen1(self):
async with websockets.connect(f"ws://localhost:27000") as websocket:
self.ws1 = websocket
while True:
try:
self.msg1 = await websocket.recv()
await asyncio.sleep(0.1)
except RuntimeError:
break
async def listen2(self):
async with websockets.connect(f"ws://localhost:27000") as websocket:
self.ws2 = websocket
while True:
try:
self.msg2 = await websocket.recv()
await asyncio.sleep(0.1)
except RuntimeError:
break
async def in_game_helper(self, mapData1, mapData2):
self.tasks.extend([self.loop.create_task(self.listen1()), self.loop.create_task(self.listen2())])
await asyncio.wait(self.tasks)
await asyncio.sleep(1)
await self.ws1.send(generate_payload("knock-knock", {"nick": 'a'}))
await asyncio.sleep(0.4)
await self.ws2.send(generate_payload("knock-knock", {"nick": 'b'}))
await asyncio.sleep(0.4)
await self.ws1.send(json.dumps({"header": "send_map", 'data': mapData1}))
await asyncio.sleep(0.4)
await self.ws1.send(json.dumps({"header": "send_map", 'data': mapData2}))
await asyncio.sleep(3)
#for task in self.tasks:
# print(task)
# task.cancel()
#self.loop.stop()
return json.loads(self.msg1)
async def kill_helper(self, coords):
pass
async def miss_helper(self, coords):
pass
Code of TestServer class with tests:
class TestServer:
# main()
def setup_class(self):
self.loop = asyncio.new_event_loop()
self.ws1 = None
self.ws2 = None
self.msg1 = None
self.msg2 = None
self.tasks = []
#pytest.fixture
def mapData1(self):
f = open("map1.json", 'r').read()
data = json.loads(f)
return data
#pytest.fixture
def mapData2(self):
f = open("map2.json", 'r').read()
data = json.loads(f)
return data
#pytest.fixture
def event_loop(self):
loop = asyncio.get_event_loop()
yield loop
loop.close()
#pytest.mark.asyncio
async def test_knock(self):
async with websockets.connect(f"ws://localhost:27000") as websocket:
await websocket.send(generate_payload("knock-knock", {"nick": 'a'}))
for i in range(3):
msg = json.loads(await websocket.recv())
await asyncio.sleep(0.5)
assert msg['header'] == 'registered'
#pytest.mark.asyncio
async def test_send1(self, mapData1):
async with websockets.connect(f"ws://localhost:27000") as websocket:
await websocket.send(generate_payload("knock-knock", {"nick": 'a'}))
for i in range(3):
await websocket.recv()
await asyncio.sleep(0.5)
await websocket.send(json.dumps({"header": "send_map", 'data': mapData1}))
msg = json.loads(await websocket.recv())
assert msg['header'] == 'ready'
await websocket.close()
#pytest.mark.asyncio
async def test_send2(self, mapData2):
async with websockets.connect(f"ws://localhost:27000") as websocket:
await websocket.send(generate_payload("knock-knock", {"nick": 'b'}))
for i in range(4):
await websocket.recv()
await asyncio.sleep(0.5)
await websocket.send(json.dumps({"header": "send_map", 'data': mapData2}))
msg = json.loads(await websocket.recv())
assert msg['header'] == 'ready'
await websocket.close()
#pytest.mark.asyncio
async def test_game_started(self, mapData1, mapData2, event_loop):
helper = HelpTest(event_loop)
answer = await helper.in_game_helper(mapData1, mapData2)
print(answer)
assert answer['header'] == "in_game!!!"
Traceback of runned test:
Traceback (most recent call last):
File "tests.py", line 40, in listen2
break
File "SeaBattle\venv\lib\site-packages\websockets\legacy\client.py", line 650, in __aexit__
await self.protocol.close()
File "SeaBattle\venv\lib\site-packages\websockets\legacy\protocol.py", line 768, in close
await asyncio.wait_for(
File "C:\Users\zayyc\AppData\Local\Programs\Python\Python39\lib\asyncio\tasks.py", line 435, in wait_for
loop = events.get_running_loop()
RuntimeError: no running event loop
I've tried to use pytest,mark.asyncio loop, but it did not help. I think the problem is that the tasks listen1() and listen2() keep running after the loop is closed, I tried to stop them, but the error still persists. I will be very happy with your decisions.
I am writing a personal telegram bot, one of its functions is to show the balance of my accounts market.csgo.com. My code:
import asyncio
import aiogram
import aiohttp
...
async def get_balance(session, profiles_dict, message):
async with session.get(f'https://market.csgo.com/api/v2/get-money?key={profiles_dict[1][1]}') as resp:
html = await resp.json()
each_wallet = int(html['money'])
await bot.send_message(message.from_user.id,
f'🟢 {profiles_dict[0]} : <i>{each_wallet}</i>',
disable_web_page_preview=True, parse_mode=types.ParseMode.HTML)
...
#dp.message_handler(content_types=['text'])
async def main(message):
profiles = users()
async with aiohttp.ClientSession(trust_env=True) as session:
tasks = []
if message.text == 'Balance 💸':
await bot.send_message(message.from_user.id, 'Information request. Wait..')
for i in profiles.items():
task = asyncio.ensure_future(get_balance(session, i, message, stats))
tasks.append(task)
await asyncio.gather(*tasks)
if message.text == 'On Sale 💰':
...
if message.text == 'Timeout Items ⌛':
...
executor.start_polling(dp, skip_updates=False)
get_balance() works in async mode, sends aiohttp requests to the API and outputs information await bot.send_message(). Result:
Now the launch of the function is implemented through the keyboard button, but how to make the function run every hour? I am aware of the existence of asynchronous task scheduler aioschedule and have seen this example. But they run a function without arguments, but I have as many as 3 of them async def get_balance(session, profiles_dict, message). I tried to do this:
import asyncio
import aioschedule
async def scheduler(session, profiles_dict, message):
aioschedule.every().hour.do(get_balance(session, profiles_dict, message))
while True:
await aioschedule.run_pending()
await asyncio.sleep(1)
async def on_startup(session, profiles_dict, message):
asyncio.create_task(scheduler(session, profiles_dict, message))
if __name__ == '__main__':
executor.start_polling(dp, skip_updates=False, on_startup=on_startup(session, profiles_dict, message))
Obviously it doesn't work that way.
My question is:
How to run an async function with arguments that sends aiohttp requests through task scheduling aioschedule and display the result through telegram aiogram?
Solution:
import aiogram
import asyncio
import aiohttp
import aioschedule
...
async def get_balance(session, profiles_dict):
async with session.get(f'https://market.csgo.com/api/v2/get-money?key={profiles_dict[1][1]}') as resp:
html = await resp.json()
each_wallet = int(html['money'])
await bot.send_message(MY_TELEGRAM_ID,
f'🟢 {profiles_dict[0]} : <i>{each_wallet}</i>',
disable_web_page_preview=True, parse_mode=types.ParseMode.HTML)
...
#dp.message_handler(content_types=['text'])
async def main(message):
profiles = users()
async with aiohttp.ClientSession(trust_env=True) as session:
tasks = []
if message.text == 'Balance 💸':
await bot.send_message(message.from_user.id, 'Information request. Wait..')
for i in profiles.items():
task = asyncio.ensure_future(get_balance(session, i))
tasks.append(task)
await asyncio.gather(*tasks)
if message.text == 'On Sale 💰':
...
if message.text == 'Timeout Items ⌛':
...
# Client session get_balance function
async def session_get_balance():
profiles = users()
async with aiohttp.ClientSession(trust_env=True) as session:
tasks = []
for i in profiles.items():
task = asyncio.ensure_future(get_balance(session, i))
tasks.append(task)
await asyncio.gather(*tasks)
# Schedule functions by time
async def scheduler():
aioschedule.every().hour.do(session_get_balance)
while True:
await aioschedule.run_pending()
await asyncio.sleep(1)
# Function at start
async def on_startup(_):
asyncio.create_task(scheduler())
# Launch telegram bot
if __name__ == "__main__":
executor.start_polling(dp, skip_updates=True, on_startup=on_startup)
Since this is my personal bot, instead of message.from_user.id I specified my MY_TELEGRAM_ID.
await bot.send_message(MY_TELEGRAM_ID,
f'🟢 {profiles_dict[0]} : <i>{each_wallet}</i>',
disable_web_page_preview=True, parse_mode=types.ParseMode.HTML)
There is the following code:
import asyncio
import aiohttp
aut_token = ("token")
tasks = []
iter_flag = False
class WAPI:
async def receiver(WAPI_S):
async for msg in WAPI_S:
data = msg.json()
raise aiohttp.ClientError #test
async def heartbeating(WAPI_S):
while iter_flag:
await WAPI_S.send_json({
"op": 1,
"d": None
})
await asyncio.sleep(42.5)
async def event_manager():
loop = asyncio.get_running_loop()
try:
async with aiohttp.ClientSession().ws_connect("url") as WAPI_S:
task_receive = loop.create_task(WAPI.receiver(WAPI_S)); task_heartbeating = loop.create_task(WAPI.heartbeating(WAPI_S))
tasks.append(task_receive); tasks.append(task_heartbeating)
await asyncio.gather(*tasks)
except aiohttp.ClientError:
global iter_flag
iter_flag = False
await asyncio.sleep(44)
[task.cancel() for task in tasks]
try:
loop.close()
except:
loop.stop()
asyncio.run(WAPI.event_manager())
I want to correctly shutdown the client when the exception is raised. My implementation throws "RuntimeError: Event loop stopped before Future completed" exception while executing. How to do it right?
In method event_manager, the statement:
async with aiohttp.ClientSession().ws_connect("url") as WAPI_S:
needs to be replaced with:
async with aiohttp.ClientSession() as session:
async with session.ws_connect("url") as WAPI_S:
Also, it is considered anti-Pythonic to use a list comprehension for its side effects. See Is it Pythonic to use list comprehensions for just side effects? So you really should replace:
[task.cancel() for task in tasks]
with:
for task in tasks:
task.cancel()
Putting this all together:
async def event_manager():
loop = asyncio.get_running_loop()
try:
async with aiohttp.ClientSession() as session:
async with session.ws_connect("url") as WAPI_S:
task_receive = loop.create_task(WAPI.receiver(WAPI_S)); task_heartbeating = loop.create_task(WAPI.heartbeating(WAPI_S))
tasks.append(task_receive); tasks.append(task_heartbeating)
await asyncio.gather(*tasks)
except aiohttp.ClientError:
global iter_flag
iter_flag = False
await asyncio.sleep(44)
for task in tasks:
task.cancel()
try:
loop.close()
except:
loop.stop()
I have written code for async pool below. in __aexit__ i'm cancelling the _worker tasks after the tasks get finished. But when i run the code, the worker tasks are not getting cancelled and the code is running forever. This what the task looks like: <Task pending coro=<AsyncPool._worker() running at \async_pool.py:17> wait_for=<Future cancelled>>. The asyncio.wait_for is getting cancelled but not the worker tasks.
class AsyncPool:
def __init__(self,coroutine,no_of_workers,timeout):
self._loop = asyncio.get_event_loop()
self._queue = asyncio.Queue()
self._no_of_workers = no_of_workers
self._coroutine = coroutine
self._timeout = timeout
self._workers = None
async def _worker(self):
while True:
try:
ret = False
queue_item = await self._queue.get()
ret = True
result = await asyncio.wait_for(self._coroutine(queue_item), timeout = self._timeout,loop= self._loop)
except Exception as e:
print(e)
finally:
if ret:
self._queue.task_done()
async def push_to_queue(self,item):
self._queue.put_nowait(item)
async def __aenter__(self):
assert self._workers == None
self._workers = [asyncio.create_task(self._worker()) for _ in range(self._no_of_workers)]
return self
async def __aexit__(self,type,value,traceback):
await self._queue.join()
for worker in self._workers:
worker.cancel()
await asyncio.gather(*self._workers, loop=self._loop, return_exceptions =True)
To use the Asyncpool:
async def something(item):
print("got", item)
await asyncio.sleep(item)
async def main():
async with AsyncPool(something, 5, 2) as pool:
for i in range(10):
await pool.push_to_queue(i)
asyncio.run(main())
The Output in my terminal:
The problem is that your except Exception exception clause also catches cancellation, and ignores it. To add to the confusion, print(e) just prints an empty line in case of a CancelledError, which is where the empty lines in the output come from. (Changing it to print(type(e)) shows what's going on.)
To correct the issue, change except Exception to something more specific, like except asyncio.TimeoutError. This change is not needed in Python 3.8 where asyncio.CancelledError no longer derives from Exception, but from BaseException, so except Exception doesn't catch it.
When you have an asyncio task created and then cancelled, you still have the task alive that need to be "reclaimed". So you want to await worker for it. However, once you await such a cancelled task, as it will never give you back the expected return value, the asyncio.CancelledError will be raised and you need to catch it somewhere.
Because of this behavior, I don't think you should gather them but to await for each of the cancelled tasks, as they are supposed to return right away:
async def __aexit__(self,type,value,traceback):
await self._queue.join()
for worker in self._workers:
worker.cancel()
for worker in self._workers:
try:
await worker
except asyncio.CancelledError:
print("worker cancelled:", worker)
This appears to work. The event is a counting timer and when it expires it cancels the tasks.
import asyncio
from datetime import datetime as dt
from datetime import timedelta as td
import random
import time
class Program:
def __init__(self):
self.duration_in_seconds = 20
self.program_start = dt.now()
self.event_has_expired = False
self.canceled_success = False
async def on_start(self):
print("On Start Event Start! Applying Overrides!!!")
await asyncio.sleep(random.randint(3, 9))
async def on_end(self):
print("On End Releasing All Overrides!")
await asyncio.sleep(random.randint(3, 9))
async def get_sensor_readings(self):
print("getting sensor readings!!!")
await asyncio.sleep(random.randint(3, 9))
async def evauluate_data(self):
print("checking data!!!")
await asyncio.sleep(random.randint(3, 9))
async def check_time(self):
if (dt.now() - self.program_start > td(seconds = self.duration_in_seconds)):
self.event_has_expired = True
print("Event is DONE!!!")
else:
print("Event is not done! ",dt.now() - self.program_start)
async def main(self):
# script starts, do only once self.on_start()
await self.on_start()
print("On Start Done!")
while not self.canceled_success:
readings = asyncio.ensure_future(self.get_sensor_readings())
analysis = asyncio.ensure_future(self.evauluate_data())
checker = asyncio.ensure_future(self.check_time())
if not self.event_has_expired:
await readings
await analysis
await checker
else:
# close other tasks before final shutdown
readings.cancel()
analysis.cancel()
checker.cancel()
self.canceled_success = True
print("cancelled hit!")
# script ends, do only once self.on_end() when even is done
await self.on_end()
print('Done Deal!')
async def main():
program = Program()
await program.main()
I want to manage some coroutines in a dict, while running a main coroutine.
Specifically I want to start endless coroutines, put the handler of them in a dict and cancel them via a dict call again. In my example I want to start 4 Coroutines and they shall cancel one by one again with the coroutine doomsday. I'm using Python 3.6.
import asyncio
import traceback
async def add_to_handler(node, func):
func_handler[node] = asyncio.ensure_future(func, loop=loop)
return
async def test_func1():
while True:
print("1 - HeNlO")
await asyncio.sleep(1)
async def test_func2():
while True:
print("2 - TestFunc2")
await asyncio.sleep(2)
async def test_func3():
while True:
print("3 - Tukan")
await asyncio.sleep(3)
async def test_func4():
while True:
print("4 - Do Coro!")
await asyncio.sleep(4)
async def doomsday():
# Cancel coroutine every 10 seconds
print("launch doomsday")
for i in range(len(func_handler)):
await asyncio.sleep(10)
print("start cancelling with {}".format(i))
func_handler[str(i + 1)].cancel()
return
async def main():
await add_to_handler("1", test_func1)
await add_to_handler("2", test_func2)
await add_to_handler("3", test_func3)
await add_to_handler("4", test_func4)
await doomsday()
while True:
print("z..z..Z..Z...Z")
print(func_handler)
await asyncio.sleep(5)
func_handler = {}
loop = asyncio.get_event_loop()
try:
loop.run_until_complete(main())
except KeyboardInterrupt:
print("stop loop")
loop.close()
I tried it with the .call_latermethod of AbstractEventLoop instead of an endless while loop, but it still doesn't want to work and it seems, that my coroutines are seen as function, but I don't know why. Where is my fault?
try to change this function:
async def add_to_handler(node, func):
func_handler[node] = asyncio.ensure_future(func(), loop=loop)
return None
pay attention on asyncio.ensure_future(func(),loop=loop)