What's wrong with implementation of passing class instance as a dependency in FastAPI router or is it a bug?
1) I have defined router with dependency
app = FastAPI()
dbconnector_is = AsyncDBPool(conn=is_cnx, loop=None)
app.include_router(test_route.router, dependencies=[Depends(dbconnector_is)])
#app.on_event('startup')
async def startup():
app.logger = await AsyncLogger().getlogger(log)
await app.logger.warning('Webservice is starting up...')
await app.logger.info("Establishing RDBS Integration Pool Connection...")
await dbconnector_is.connect()
#app.on_event('startup')
async def startup():
await app.logger.getlogger(log)
await app.logger.warning('Webservice is starting up...')
await dbconnector_is.connect()
router
#router.get('/test')
async def test():
data = await dbconnector_is.callproc('is_processes_get', rows=-1, values=[None, None])
return Response(json.dumps(data, default=str))
custom class for passing it's instance as callable.
class AsyncDBPool:
def __init__(self, conn: dict, loop: None):
self.conn = conn
self.pool = None
self.connected = False
self.loop = loop
def __call__(self):
return self
async def connect(self):
while self.pool is None:
try:
self.pool = await aiomysql.create_pool(**self.conn, loop=self.loop, autocommit=True)
except aiomysql.OperationalError as e:
await asyncio.sleep(1)
continue
else:
return self.pool
And when I send request I receive this error.
data = await dbconnector_is.callproc('is_processes_get', rows=-1, values=[None, None])
NameError: name 'dbconnector_is' is not defined
Related
Hello everyone, I'm writing tests for a game that is written in the python websockets library. When running the test test_game_started an error appears that asyncio loop is not running, but the test passes.
I provide the entire test code and the error below:
Code of HelpTest class:
class HelpTest:
def __init__(self, loop):
self.loop = loop
self.ws1 = None
self.ws2 = None
self.msg1 = None
self.msg2 = None
self.tasks = []
async def listen1(self):
async with websockets.connect(f"ws://localhost:27000") as websocket:
self.ws1 = websocket
while True:
try:
self.msg1 = await websocket.recv()
await asyncio.sleep(0.1)
except RuntimeError:
break
async def listen2(self):
async with websockets.connect(f"ws://localhost:27000") as websocket:
self.ws2 = websocket
while True:
try:
self.msg2 = await websocket.recv()
await asyncio.sleep(0.1)
except RuntimeError:
break
async def in_game_helper(self, mapData1, mapData2):
self.tasks.extend([self.loop.create_task(self.listen1()), self.loop.create_task(self.listen2())])
await asyncio.wait(self.tasks)
await asyncio.sleep(1)
await self.ws1.send(generate_payload("knock-knock", {"nick": 'a'}))
await asyncio.sleep(0.4)
await self.ws2.send(generate_payload("knock-knock", {"nick": 'b'}))
await asyncio.sleep(0.4)
await self.ws1.send(json.dumps({"header": "send_map", 'data': mapData1}))
await asyncio.sleep(0.4)
await self.ws1.send(json.dumps({"header": "send_map", 'data': mapData2}))
await asyncio.sleep(3)
#for task in self.tasks:
# print(task)
# task.cancel()
#self.loop.stop()
return json.loads(self.msg1)
async def kill_helper(self, coords):
pass
async def miss_helper(self, coords):
pass
Code of TestServer class with tests:
class TestServer:
# main()
def setup_class(self):
self.loop = asyncio.new_event_loop()
self.ws1 = None
self.ws2 = None
self.msg1 = None
self.msg2 = None
self.tasks = []
#pytest.fixture
def mapData1(self):
f = open("map1.json", 'r').read()
data = json.loads(f)
return data
#pytest.fixture
def mapData2(self):
f = open("map2.json", 'r').read()
data = json.loads(f)
return data
#pytest.fixture
def event_loop(self):
loop = asyncio.get_event_loop()
yield loop
loop.close()
#pytest.mark.asyncio
async def test_knock(self):
async with websockets.connect(f"ws://localhost:27000") as websocket:
await websocket.send(generate_payload("knock-knock", {"nick": 'a'}))
for i in range(3):
msg = json.loads(await websocket.recv())
await asyncio.sleep(0.5)
assert msg['header'] == 'registered'
#pytest.mark.asyncio
async def test_send1(self, mapData1):
async with websockets.connect(f"ws://localhost:27000") as websocket:
await websocket.send(generate_payload("knock-knock", {"nick": 'a'}))
for i in range(3):
await websocket.recv()
await asyncio.sleep(0.5)
await websocket.send(json.dumps({"header": "send_map", 'data': mapData1}))
msg = json.loads(await websocket.recv())
assert msg['header'] == 'ready'
await websocket.close()
#pytest.mark.asyncio
async def test_send2(self, mapData2):
async with websockets.connect(f"ws://localhost:27000") as websocket:
await websocket.send(generate_payload("knock-knock", {"nick": 'b'}))
for i in range(4):
await websocket.recv()
await asyncio.sleep(0.5)
await websocket.send(json.dumps({"header": "send_map", 'data': mapData2}))
msg = json.loads(await websocket.recv())
assert msg['header'] == 'ready'
await websocket.close()
#pytest.mark.asyncio
async def test_game_started(self, mapData1, mapData2, event_loop):
helper = HelpTest(event_loop)
answer = await helper.in_game_helper(mapData1, mapData2)
print(answer)
assert answer['header'] == "in_game!!!"
Traceback of runned test:
Traceback (most recent call last):
File "tests.py", line 40, in listen2
break
File "SeaBattle\venv\lib\site-packages\websockets\legacy\client.py", line 650, in __aexit__
await self.protocol.close()
File "SeaBattle\venv\lib\site-packages\websockets\legacy\protocol.py", line 768, in close
await asyncio.wait_for(
File "C:\Users\zayyc\AppData\Local\Programs\Python\Python39\lib\asyncio\tasks.py", line 435, in wait_for
loop = events.get_running_loop()
RuntimeError: no running event loop
I've tried to use pytest,mark.asyncio loop, but it did not help. I think the problem is that the tasks listen1() and listen2() keep running after the loop is closed, I tried to stop them, but the error still persists. I will be very happy with your decisions.
In Spring, to set the concurrency for rabbitmq-consumer is so easy. Like:
container.setConcurrentConsumers(consumerSize);
container.setMaxConcurrentConsumers(consumerMaxSize);
Is it possible in python?
My python code looks like:
async def handle_message(loop):
connection = await connect(SETTINGS.cloudamqp_url, loop = loop)
channel = await connection.channel()
queue = await channel.declare_queue(SETTINGS.request_queue, durable=True)
await queue.consume(on_message, no_ack = True)
I solved my problem with using Thread:
My code looks like:
import threading
from aio_pika import connect, IncomingMessage, Message
import json
class QueueWorker(threading.Thread):
def __init__(self):
threading.Thread.__init__(self)
self.connection = None
self.channel = None
self.queue = None
async def init(self, loop):
self.connection = await connect(cloudamqp_url, loop=loop)
self.channel = await self.connection.channel()
await self.channel.set_qos(prefetch_count=1)
self.queue = await self.channel.declare_queue(queue, durable=True)
await self.queue.consume(self.callback, no_ack=False)
async def callback(self, message: IncomingMessage):
request = json.loads(message.body.decode("utf-8"))
try:
handle(request)
except Exception as e:
handleException...
finally:
await message.ack()
Consume queue with concurrency:
async def listen_queue(loop):
for _ in range(consumer_count):
td = QueueWorker()
await td.init(loop)
Note: Inspired from Consuming rabbitmq queue from inside python threads
I have an websocket server up and running fine using FastAPI.
However, when i am using those "await", i am getting some latency issues. At first, i though this had something to do with internet connection, or perhaps my linux server. But it appears to be that asyncio waits for other tasks.
Here is my code:
import asyncio
from pydantic import BaseModel
class UserClientWebSocket(BaseModel):
id: str
ws: WebSocket
class Config:
arbitrary_types_allowed = True
class ConnectionManager:
def __init__(self):
self.active_user_client_connections: List[UserClientWebSocket] = []
self.collect_user_IDs = []
async def connect_the_user_client(self, websocket: WebSocket, THE_USER_ID):
await websocket.accept()
await self.send_message_to_absolutely_everybody(f"User: {THE_USER_ID} connected to server!")
print("User: {} ".format(THE_USER_ID) + " Connected")
if THE_USER_ID not in self.collect_user_IDs:
self.collect_user_IDs.append(THE_USER_ID)
else:
await self.send_message_to_absolutely_everybody(f"Somebody connected with the same ID as client: {THE_USER_ID}")
await self.send_message_to_absolutely_everybody("but Vlori is a nice and kind guy, so he wil not get kicked :)")
self.collect_user_IDs.append(THE_USER_ID)
self.active_user_client_connections.append(UserClientWebSocket(ws=websocket, id=THE_USER_ID))
await self.show_number_of_clients()
async def function_one_send_message_to_absolutely_everybody(self, message: str):
try:
await asyncio.sleep(2)
await asyncio.gather(*(conn.ws.send_text(message) for conn in self.active_webpage_client_connections))
await asyncio.gather(*(conn.ws.send_text(message) for conn in self.active_user_client_connections))
except:
print("waiting")
async def function_two_send_personal_message_to_user(self, message: str, websocket: WebSocket):
try:
await websocket.send_text(message)
except:
print("waiting for task..")
...
...
...
...
...
and further down is the channel in which client connects:
#app.websocket("/ws/testchannel")
async def websocket_endpoint(websocket: WebSocket):
await websocket.accept()
try:
while True:
data = await websocket.receive_text()
print_result = print("Received data: {} ".format(data))
send_data_back_to_user = await websocket.send_text(f"you sent message: {data}")
except WebSocketDisconnect as e:
print("client left chat, error = ", e)
The code as it stands now works perfectly, and the performance is good! However, if i add an async def function under the "send_data_back_to_user" line such as this:
await connection_manager.function_one_send_message_to_absolutely_everybody(data)
Then there is a huge latency! why is that?
I am playing around and tried this:
#app.websocket("/ws/testchannel")
async def websocket_endpoint(websocket: WebSocket):
await websocket.accept()
try:
while True:
data = await websocket.receive_text()
print_result = print("Received data: {} ".format(data))
send_data_back_to_user = await websocket.send_text(f"you sent message: {data}") # if i comment this line and the line underneath, and the speed is extremely fast!
the_asyncio_loop = asyncio.get_event_loop()
print_data_on_terminal = asyncio.gather(print_result)
return_data_back_to_user = asyncio.gather(send_data_back_to_user)
broadcast_msg = asyncio.gather(connection_manager.function_one_send_message_to_absolutely_everybody(data))
run_all_loops_together = asyncio.gather(print_data_on_terminal, return_data_back_to_user, broadcast_msg)
results = the_asyncio_loop.run_until_complete(run_all_loops_together)
print(results)
except WebSocketDisconnect as e:
print("client left chat, error = ", e)
but gives me the error:
TypeError: An asyncio.Future, a coroutine or an awaitable is required
could someone help me with this?
thanks.
in my class I have a method that fetches the website (visible below).
I've noticed that other methods that use this method, can lead to the opening of multiple requests to one site (when one request is pending self._page is still none).
How can I avoid it?
I mean when there is another call to _get_page but one is pending, just return a future from the first one and don't repeat a page request
async def _get_page(self) -> HtmlElement:
if self._page is None:
async with self._get_session().get(self._url) as page:
self._page = lxml.html.document_fromstring(await page.text())
return self._page
How can I avoid [multiple requests]?
You could use an asyncio.Lock:
saync def __init__(self, ...):
...
self._page_lock = asyncio.Lock()
async def _get_page(self) -> HtmlElement:
async with self._page_lock:
if self._page is None:
async with self._get_session().get(self._url) as page:
self._page = lxml.html.document_fromstring(await page.text())
return self._page
Update for Python 3.8 and jupyter notebook
import asyncio
import aiohttp
from lxml import html
class MyClass:
def __init__(self):
self._url = 'https://www.google.com'
self._page = None
self._futures = []
self._working = False
self._session = aiohttp.ClientSession()
async def _close(self):
if self._session:
session = self._session
self._session = None
await session.close()
def _get_session(self):
return self._session
async def _get_page(self):
if self._page is None:
if self._working:
print('will await current page request')
loop = asyncio.get_event_loop()
future = loop.create_future()
self._futures.append(future)
return await future
else:
self._working = True
session = self._get_session()
print('making url request')
async with session.get(self._url) as page:
print('status =', page.status)
print('making page request')
self._page = html.document_fromstring(await page.text())
print('Got page text')
for future in self._futures:
print('setting result to awaiting request')
future.set_result(self._page)
self._futures = []
self._working = False
return self._page
async def main():
futures = []
m = MyClass()
futures.append(asyncio.ensure_future(m._get_page()))
futures.append(asyncio.ensure_future(m._get_page()))
futures.append(asyncio.ensure_future(m._get_page()))
results = await asyncio.gather(*futures)
for result in results:
print(result[0:80])
await m._close()
if __name__ == '__main__':
asyncio.run(main())
#await main() # In jupyter notebook and iPython
Note that on Windows 10 I have seen at termination:
RuntimeError: Event loop is closed
See https://github.com/aio-libs/aiohttp/issues/4324
The toy script shows an application using a class that is dependent on an implementation that is not asyncio-aware, and obviously doesn't work.
How would the fetch method of MyFetcher be implemented, using the asyncio-aware client, while still maintaining the contract with the _internal_validator method of FetcherApp? To be very clear, FetcherApp and AbstractFetcher cannot be modified.
To use async fetch_data function inside fetch both fetch and is_fetched_data_valid functions should be async too. You can change them in child classes without modify parent:
import asyncio
class AsyncFetcherApp(FetcherApp):
async def is_fetched_data_valid(self): # async here
data = await self.fetcher_implementation.fetch() # await here
return self._internal_validator(data)
class AsyncMyFetcher(AbstractFetcher):
def __init__(self, client):
super().__init__()
self.client = client
async def fetch(self): # async here
result = await self.client.fetch_data() # await here
return result
class AsyncClient:
async def fetch_data(self):
await asyncio.sleep(1) # Just to sure it works
return 1
async def main():
async_client = AsyncClient()
my_fetcher = AsyncMyFetcher(async_client)
fetcherApp = AsyncFetcherApp(my_fetcher)
# ...
is_valid = await fetcherApp.is_fetched_data_valid() # await here
print(repr(is_valid))
if __name__ == "__main__":
loop = asyncio.get_event_loop()
loop.run_until_complete(main())