I have an websocket server up and running fine using FastAPI.
However, when i am using those "await", i am getting some latency issues. At first, i though this had something to do with internet connection, or perhaps my linux server. But it appears to be that asyncio waits for other tasks.
Here is my code:
import asyncio
from pydantic import BaseModel
class UserClientWebSocket(BaseModel):
id: str
ws: WebSocket
class Config:
arbitrary_types_allowed = True
class ConnectionManager:
def __init__(self):
self.active_user_client_connections: List[UserClientWebSocket] = []
self.collect_user_IDs = []
async def connect_the_user_client(self, websocket: WebSocket, THE_USER_ID):
await websocket.accept()
await self.send_message_to_absolutely_everybody(f"User: {THE_USER_ID} connected to server!")
print("User: {} ".format(THE_USER_ID) + " Connected")
if THE_USER_ID not in self.collect_user_IDs:
self.collect_user_IDs.append(THE_USER_ID)
else:
await self.send_message_to_absolutely_everybody(f"Somebody connected with the same ID as client: {THE_USER_ID}")
await self.send_message_to_absolutely_everybody("but Vlori is a nice and kind guy, so he wil not get kicked :)")
self.collect_user_IDs.append(THE_USER_ID)
self.active_user_client_connections.append(UserClientWebSocket(ws=websocket, id=THE_USER_ID))
await self.show_number_of_clients()
async def function_one_send_message_to_absolutely_everybody(self, message: str):
try:
await asyncio.sleep(2)
await asyncio.gather(*(conn.ws.send_text(message) for conn in self.active_webpage_client_connections))
await asyncio.gather(*(conn.ws.send_text(message) for conn in self.active_user_client_connections))
except:
print("waiting")
async def function_two_send_personal_message_to_user(self, message: str, websocket: WebSocket):
try:
await websocket.send_text(message)
except:
print("waiting for task..")
...
...
...
...
...
and further down is the channel in which client connects:
#app.websocket("/ws/testchannel")
async def websocket_endpoint(websocket: WebSocket):
await websocket.accept()
try:
while True:
data = await websocket.receive_text()
print_result = print("Received data: {} ".format(data))
send_data_back_to_user = await websocket.send_text(f"you sent message: {data}")
except WebSocketDisconnect as e:
print("client left chat, error = ", e)
The code as it stands now works perfectly, and the performance is good! However, if i add an async def function under the "send_data_back_to_user" line such as this:
await connection_manager.function_one_send_message_to_absolutely_everybody(data)
Then there is a huge latency! why is that?
I am playing around and tried this:
#app.websocket("/ws/testchannel")
async def websocket_endpoint(websocket: WebSocket):
await websocket.accept()
try:
while True:
data = await websocket.receive_text()
print_result = print("Received data: {} ".format(data))
send_data_back_to_user = await websocket.send_text(f"you sent message: {data}") # if i comment this line and the line underneath, and the speed is extremely fast!
the_asyncio_loop = asyncio.get_event_loop()
print_data_on_terminal = asyncio.gather(print_result)
return_data_back_to_user = asyncio.gather(send_data_back_to_user)
broadcast_msg = asyncio.gather(connection_manager.function_one_send_message_to_absolutely_everybody(data))
run_all_loops_together = asyncio.gather(print_data_on_terminal, return_data_back_to_user, broadcast_msg)
results = the_asyncio_loop.run_until_complete(run_all_loops_together)
print(results)
except WebSocketDisconnect as e:
print("client left chat, error = ", e)
but gives me the error:
TypeError: An asyncio.Future, a coroutine or an awaitable is required
could someone help me with this?
thanks.
Related
Hello everyone, I'm writing tests for a game that is written in the python websockets library. When running the test test_game_started an error appears that asyncio loop is not running, but the test passes.
I provide the entire test code and the error below:
Code of HelpTest class:
class HelpTest:
def __init__(self, loop):
self.loop = loop
self.ws1 = None
self.ws2 = None
self.msg1 = None
self.msg2 = None
self.tasks = []
async def listen1(self):
async with websockets.connect(f"ws://localhost:27000") as websocket:
self.ws1 = websocket
while True:
try:
self.msg1 = await websocket.recv()
await asyncio.sleep(0.1)
except RuntimeError:
break
async def listen2(self):
async with websockets.connect(f"ws://localhost:27000") as websocket:
self.ws2 = websocket
while True:
try:
self.msg2 = await websocket.recv()
await asyncio.sleep(0.1)
except RuntimeError:
break
async def in_game_helper(self, mapData1, mapData2):
self.tasks.extend([self.loop.create_task(self.listen1()), self.loop.create_task(self.listen2())])
await asyncio.wait(self.tasks)
await asyncio.sleep(1)
await self.ws1.send(generate_payload("knock-knock", {"nick": 'a'}))
await asyncio.sleep(0.4)
await self.ws2.send(generate_payload("knock-knock", {"nick": 'b'}))
await asyncio.sleep(0.4)
await self.ws1.send(json.dumps({"header": "send_map", 'data': mapData1}))
await asyncio.sleep(0.4)
await self.ws1.send(json.dumps({"header": "send_map", 'data': mapData2}))
await asyncio.sleep(3)
#for task in self.tasks:
# print(task)
# task.cancel()
#self.loop.stop()
return json.loads(self.msg1)
async def kill_helper(self, coords):
pass
async def miss_helper(self, coords):
pass
Code of TestServer class with tests:
class TestServer:
# main()
def setup_class(self):
self.loop = asyncio.new_event_loop()
self.ws1 = None
self.ws2 = None
self.msg1 = None
self.msg2 = None
self.tasks = []
#pytest.fixture
def mapData1(self):
f = open("map1.json", 'r').read()
data = json.loads(f)
return data
#pytest.fixture
def mapData2(self):
f = open("map2.json", 'r').read()
data = json.loads(f)
return data
#pytest.fixture
def event_loop(self):
loop = asyncio.get_event_loop()
yield loop
loop.close()
#pytest.mark.asyncio
async def test_knock(self):
async with websockets.connect(f"ws://localhost:27000") as websocket:
await websocket.send(generate_payload("knock-knock", {"nick": 'a'}))
for i in range(3):
msg = json.loads(await websocket.recv())
await asyncio.sleep(0.5)
assert msg['header'] == 'registered'
#pytest.mark.asyncio
async def test_send1(self, mapData1):
async with websockets.connect(f"ws://localhost:27000") as websocket:
await websocket.send(generate_payload("knock-knock", {"nick": 'a'}))
for i in range(3):
await websocket.recv()
await asyncio.sleep(0.5)
await websocket.send(json.dumps({"header": "send_map", 'data': mapData1}))
msg = json.loads(await websocket.recv())
assert msg['header'] == 'ready'
await websocket.close()
#pytest.mark.asyncio
async def test_send2(self, mapData2):
async with websockets.connect(f"ws://localhost:27000") as websocket:
await websocket.send(generate_payload("knock-knock", {"nick": 'b'}))
for i in range(4):
await websocket.recv()
await asyncio.sleep(0.5)
await websocket.send(json.dumps({"header": "send_map", 'data': mapData2}))
msg = json.loads(await websocket.recv())
assert msg['header'] == 'ready'
await websocket.close()
#pytest.mark.asyncio
async def test_game_started(self, mapData1, mapData2, event_loop):
helper = HelpTest(event_loop)
answer = await helper.in_game_helper(mapData1, mapData2)
print(answer)
assert answer['header'] == "in_game!!!"
Traceback of runned test:
Traceback (most recent call last):
File "tests.py", line 40, in listen2
break
File "SeaBattle\venv\lib\site-packages\websockets\legacy\client.py", line 650, in __aexit__
await self.protocol.close()
File "SeaBattle\venv\lib\site-packages\websockets\legacy\protocol.py", line 768, in close
await asyncio.wait_for(
File "C:\Users\zayyc\AppData\Local\Programs\Python\Python39\lib\asyncio\tasks.py", line 435, in wait_for
loop = events.get_running_loop()
RuntimeError: no running event loop
I've tried to use pytest,mark.asyncio loop, but it did not help. I think the problem is that the tasks listen1() and listen2() keep running after the loop is closed, I tried to stop them, but the error still persists. I will be very happy with your decisions.
I write a telegram bot on aiogram that gives me information about my accounts market.csgo.com. The meaning of the script is simple - I click on the button, it displays the text and and the function is run.
My functions send async requests and work fine, but I don't know how to get aiohttp and aiogram to work together.
from aiogram import Bot, types
from aiogram.dispatcher import Dispatcher
from aiogram.utils import executor
from auth import *
import asyncio
import aiohttp
bot = Bot(token=token)
dp = Dispatcher(bot)
def users():
***Data of my accounts from txt to dict***
async def get_info(session, dictt, message):
total_wallet = 0
async with session.get(f'https://market.csgo.com/api/v2/get-money?key={dictt[1][1]}') as resp:
html = await resp.json()
total_wallet += int(html['money'])
#await bot.send_message(message.from_user.id, f'{total_wallet}')
async def get_on_sale(session, dictt, message):
sale_total_sum = 0
async with session.get(f'https://market.csgo.com/api/v2/items?key={dictt[1][1]}') as resp:
html = await resp.json()
for i in html['items']:
sale_total_sum += i['price']
#await bot.send_message(message.from_user.id, f'{sale_total_sum}')
#dp.message_handler(content_types=['text'])
async def Main():
try:
profiles = users()
async with aiohttp.ClientSession(trust_env=True) as session:
tasks = []
if message.text == 'info 📊':
await bot.send_message(message.from_user.id, 'Wait for information..')
for i in profiles.items():
task = asyncio.ensure_future(get_info(session, i))
tasks.append(task)
await asyncio.gather(*tasks)
if message.text == 'on sale 💰':
await bot.send_message(message.from_user.id, 'Wait for information..')
for i in profiles.items():
task = asyncio.ensure_future(get_on_sale(session, i))
tasks.append(task)
await asyncio.gather(*tasks)
except Exception as ex:
print(f'Error {ex}')
loop = asyncio.get_event_loop()
loop.run_until_complete(Main())
executor.start_polling(dp, skip_updates=True)
My problem is that I don't know how to properly pass the message argument to the Main function
#dp.message_handler(content_types=['text'])
async def Main(): #async def Main(message)
And run aiogram along with aiohttp.
loop.run_until_complete(Main()) #loop.run_until_complete(Main(message))
If I do like this: async def Main(message) and loop.run_until_complete(Main(message)) Then I get an error:
loop.run_until_complete(Main(message))
NameError: name 'message' is not defined
or if I use only async def Main(message) get this:
loop.run_until_complete(Main())
TypeError: Main() missing 1 required positional argument: 'message'
Solution:
async def loop_on(message):
loop = asyncio.get_event_loop()
loop.run_until_complete(Main(message))
In Spring, to set the concurrency for rabbitmq-consumer is so easy. Like:
container.setConcurrentConsumers(consumerSize);
container.setMaxConcurrentConsumers(consumerMaxSize);
Is it possible in python?
My python code looks like:
async def handle_message(loop):
connection = await connect(SETTINGS.cloudamqp_url, loop = loop)
channel = await connection.channel()
queue = await channel.declare_queue(SETTINGS.request_queue, durable=True)
await queue.consume(on_message, no_ack = True)
I solved my problem with using Thread:
My code looks like:
import threading
from aio_pika import connect, IncomingMessage, Message
import json
class QueueWorker(threading.Thread):
def __init__(self):
threading.Thread.__init__(self)
self.connection = None
self.channel = None
self.queue = None
async def init(self, loop):
self.connection = await connect(cloudamqp_url, loop=loop)
self.channel = await self.connection.channel()
await self.channel.set_qos(prefetch_count=1)
self.queue = await self.channel.declare_queue(queue, durable=True)
await self.queue.consume(self.callback, no_ack=False)
async def callback(self, message: IncomingMessage):
request = json.loads(message.body.decode("utf-8"))
try:
handle(request)
except Exception as e:
handleException...
finally:
await message.ack()
Consume queue with concurrency:
async def listen_queue(loop):
for _ in range(consumer_count):
td = QueueWorker()
await td.init(loop)
Note: Inspired from Consuming rabbitmq queue from inside python threads
i am handling data from 2 websocket servers and i would like to know whats the fastest way to handle both connections in the same time given that the 1st connection would send data every 0.1-10ms.
what i am doing so far is:
import json
import websockets
async def run():
async with websockets.connect("ws://localhost:8546/") as ws1:
async with websockets.connect(uri="wss://api.blxrbdn.com/ws", extra_headers = {"Authorization": "apikey") as ws2:
sub1 = await ws1.send("subscription 1")
sub2 = await ws2.send("subscription 2")
while True:
try:
msg1 = await ws1.recv()
msg1 = json.loads(msg1)
msg2 = await ws2.recv()
msg2 = json.loads(msg2)
# process msg1 & msg2
except Exception as e:
print(e, flush=True)
asyncio.run(run())
As stated in the comments, try to handle each connection in its own coroutine. Here is small example:
import asyncio
import websockets
async def worker(ws, msg, t):
while True:
sub = await ws.send(msg)
print("Received from the server:", await ws.recv())
await asyncio.sleep(t)
async def run():
url1 = "ws://localhost:8765/"
url2 = "ws://something_different:8765/"
async with websockets.connect(url1) as ws1, websockets.connect(url2) as ws2:
await asyncio.gather(worker(ws1, "sub1", 1), worker(ws2, "sub2", 2))
asyncio.run(run())
There is the following code:
import asyncio
import aiohttp
aut_token = ("token")
tasks = []
iter_flag = False
class WAPI:
async def receiver(WAPI_S):
async for msg in WAPI_S:
data = msg.json()
raise aiohttp.ClientError #test
async def heartbeating(WAPI_S):
while iter_flag:
await WAPI_S.send_json({
"op": 1,
"d": None
})
await asyncio.sleep(42.5)
async def event_manager():
loop = asyncio.get_running_loop()
try:
async with aiohttp.ClientSession().ws_connect("url") as WAPI_S:
task_receive = loop.create_task(WAPI.receiver(WAPI_S)); task_heartbeating = loop.create_task(WAPI.heartbeating(WAPI_S))
tasks.append(task_receive); tasks.append(task_heartbeating)
await asyncio.gather(*tasks)
except aiohttp.ClientError:
global iter_flag
iter_flag = False
await asyncio.sleep(44)
[task.cancel() for task in tasks]
try:
loop.close()
except:
loop.stop()
asyncio.run(WAPI.event_manager())
I want to correctly shutdown the client when the exception is raised. My implementation throws "RuntimeError: Event loop stopped before Future completed" exception while executing. How to do it right?
In method event_manager, the statement:
async with aiohttp.ClientSession().ws_connect("url") as WAPI_S:
needs to be replaced with:
async with aiohttp.ClientSession() as session:
async with session.ws_connect("url") as WAPI_S:
Also, it is considered anti-Pythonic to use a list comprehension for its side effects. See Is it Pythonic to use list comprehensions for just side effects? So you really should replace:
[task.cancel() for task in tasks]
with:
for task in tasks:
task.cancel()
Putting this all together:
async def event_manager():
loop = asyncio.get_running_loop()
try:
async with aiohttp.ClientSession() as session:
async with session.ws_connect("url") as WAPI_S:
task_receive = loop.create_task(WAPI.receiver(WAPI_S)); task_heartbeating = loop.create_task(WAPI.heartbeating(WAPI_S))
tasks.append(task_receive); tasks.append(task_heartbeating)
await asyncio.gather(*tasks)
except aiohttp.ClientError:
global iter_flag
iter_flag = False
await asyncio.sleep(44)
for task in tasks:
task.cancel()
try:
loop.close()
except:
loop.stop()