I am not able to run this test, i always have the same error
RuntimeError: Event loop is closed
What i need to add to this code?
from motor.motor_asyncio import AsyncIOMotorClient
import pytest
import asyncio
client = AsyncIOMotorClient("mongodb://mongo:mongo#192.168.0.11:27017/admin?retryWrites=false")
db = client['app']
aux = db['users']
async def create_user_db(a: dict):
x = await aux.insert_one(a)
return x
#pytest.mark.asyncio
async def test_create():
form = {'username': 'c3', 'password': 'c3'}
res = await create_user_db(form)
assert res != None
This is the error
In your example, your opening the database during "import" time, but we still have no eventloop. The event loop is created when the test case runs.
You could define your database as fixture and provide it to the testing functions, e.g.:
#pytest.fixture
def client():
return AsyncIOMotorClient("mongodb://localhost:27017/")
#pytest.fixture
def db(client):
return client['test']
#pytest.fixture
def collection(db):
return db['test']
async def create_user_db(collection, a: dict):
x = await collection.insert_one(a)
return x
#pytest.mark.asyncio
async def test_create(collection):
form = {'username': 'c3', 'password': 'c3'}
res = await create_user_db(collection, form)
assert res != None
Related
I'm trying to implement a faust agent using take to process multiple messages at the same time.
app = faust.App('vectors-stream')
vector_topic = app.topic('vector', value_type=VectorRecord)
#app.agent(vector_topic)
async def process_entities(stream: faust.streams.Stream):
async for records in stream.take(max_=500, within=timedelta(seconds=5)):
# yield await update_company_partition(records=records)
yield print(len(records))
now, I'm trying to write a test, just to see that the behaviour is as I except.
import asyncio
import random
from unittest import IsolatedAsyncioTestCase
import pytest
from app.data.kafka.consumer import process_entities, VectorRecord, app
class TestKafkaStream(IsolatedAsyncioTestCase):
async def asyncSetUp(self) -> None:
app.conf.store = 'memory://'
def generate_vector(self, dim: int):
return [random.uniform(0.001, 1) for i in range(dim)]
#pytest.mark.asyncio()
async def test_vectors_kafka_stream(self):
async with process_entities.test_context() as agent:
companies = ['se', 'spotify']
for company in companies:
for i in range(10):
_type = random.choice(['JobCascaded', 'UserHistory'])
msg = VectorRecord(company_slug=company, operation='upsert',
vector=self.generate_vector(16), vector_type=_type, id=i)
await agent.put(msg)
But when I put a break point on the yield print(len(records)) row, it prints that the len of records is just 1.
in my class I have a method that fetches the website (visible below).
I've noticed that other methods that use this method, can lead to the opening of multiple requests to one site (when one request is pending self._page is still none).
How can I avoid it?
I mean when there is another call to _get_page but one is pending, just return a future from the first one and don't repeat a page request
async def _get_page(self) -> HtmlElement:
if self._page is None:
async with self._get_session().get(self._url) as page:
self._page = lxml.html.document_fromstring(await page.text())
return self._page
How can I avoid [multiple requests]?
You could use an asyncio.Lock:
saync def __init__(self, ...):
...
self._page_lock = asyncio.Lock()
async def _get_page(self) -> HtmlElement:
async with self._page_lock:
if self._page is None:
async with self._get_session().get(self._url) as page:
self._page = lxml.html.document_fromstring(await page.text())
return self._page
Update for Python 3.8 and jupyter notebook
import asyncio
import aiohttp
from lxml import html
class MyClass:
def __init__(self):
self._url = 'https://www.google.com'
self._page = None
self._futures = []
self._working = False
self._session = aiohttp.ClientSession()
async def _close(self):
if self._session:
session = self._session
self._session = None
await session.close()
def _get_session(self):
return self._session
async def _get_page(self):
if self._page is None:
if self._working:
print('will await current page request')
loop = asyncio.get_event_loop()
future = loop.create_future()
self._futures.append(future)
return await future
else:
self._working = True
session = self._get_session()
print('making url request')
async with session.get(self._url) as page:
print('status =', page.status)
print('making page request')
self._page = html.document_fromstring(await page.text())
print('Got page text')
for future in self._futures:
print('setting result to awaiting request')
future.set_result(self._page)
self._futures = []
self._working = False
return self._page
async def main():
futures = []
m = MyClass()
futures.append(asyncio.ensure_future(m._get_page()))
futures.append(asyncio.ensure_future(m._get_page()))
futures.append(asyncio.ensure_future(m._get_page()))
results = await asyncio.gather(*futures)
for result in results:
print(result[0:80])
await m._close()
if __name__ == '__main__':
asyncio.run(main())
#await main() # In jupyter notebook and iPython
Note that on Windows 10 I have seen at termination:
RuntimeError: Event loop is closed
See https://github.com/aio-libs/aiohttp/issues/4324
I am connecting to aioredis from __init__ (I do not want to move it out since this means I have to some extra major changes). How can I wait for aioredis connection task in below __init__ example code and have it print self.sub and self.pub object? Currently it gives an error saying
abc.py:42> exception=AttributeError("'S' object has no attribute
'pub'")
I do see redis connections created and coro create_connetion done.
Is there a way to call blocking asyncio calls from __init__. If I replace asyncio.wait with asyncio.run_until_complete I get an error that roughly says
loop is already running.
asyncio.gather is
import sys, json
from addict import Dict
import asyncio
import aioredis
class S():
def __init__(self, opts):
print(asyncio.Task.all_tasks())
task = asyncio.wait(asyncio.create_task(self.create_connection()), return_when="ALL_COMPLETED")
print(asyncio.Task.all_tasks())
print(task)
print(self.pub, self.sub)
async def receive_message(self, channel):
while await channel.wait_message():
message = await channel.get_json()
await asyncio.create_task(self.callback_loop(Dict(json.loads(message))))
async def run_s(self):
asyncio.create_task(self.listen())
async def callback_loop(msg):
print(msg)
self.callback_loop = callback_loop
async def create_connection(self):
self.pub = await aioredis.create_redis("redis://c8:7070/0", password="abc")
self.sub = await aioredis.create_redis("redis://c8:7070/0", password="abc")
self.db = await aioredis.create_redis("redis://c8:7070/0", password="abc")
self.listener = await self.sub.subscribe(f"abc")
async def listen(self):
self.tsk = asyncio.ensure_future(self.receive_message(self.listener[0]))
await self.tsk
async def periodic(): #test function to show current tasks
number = 5
while True:
await asyncio.sleep(number)
print(asyncio.Task.all_tasks())
async def main(opts):
loop.create_task(periodic())
s = S(opts)
print(s.pub, s.sub)
loop.create_task(s.run_s())
if __name__ == "__main__":
loop = asyncio.get_event_loop()
main_task = loop.create_task(main(sys.argv[1:]))
loop.run_forever() #I DONT WANT TO MOVE THIS UNLESS IT IS NECESSARY
I think what you want to do is to make sure the function create_connections runs to completion BEFORE the S constructor. A way to do that is to rearrange your code a little bit. Move the create_connections function outside the class:
async def create_connection():
pub = await aioredis.create_redis("redis://c8:7070/0", password="abc")
sub = await aioredis.create_redis("redis://c8:7070/0", password="abc")
db = await aioredis.create_redis("redis://c8:7070/0", password="abc")
listener = await self.sub.subscribe(f"abc")
return pub, sub, db, listener
Now await that function before constructing S. So your main function becomes:
async def main(opts):
loop.create_task(periodic())
x = await create_connections()
s = S(opts, x) # pass the result of create_connections to S
print(s.pub, s.sub)
loop.create_task(s.run_s())
Now modify the S constructor to receive the objects created:
def __init__(self, opts, x):
self.pub, self.sub, self.db, self.listener = x
I'm not sure what you're trying to do with the return_when argument and the call to asyncio.wait. The create_connections function doesn't launch a set of parallel tasks, but rather awaits each of the calls before moving on to the next one. Perhaps you could improve performance by running the four calls in parallel but that's a different question.
I want to mock the json() coroutine from the aiohttp.ClientSession.get method. It looks to return an async generator object, which is where I'm confused on how to mock in my example. Here is my code:
async def get_access_token():
async with aiohttp.ClientSession(auth=auth_credentials) as client:
async with client.get(auth_path, params={'grant_type': 'client_credentials'}) as auth_response:
assert auth_response.status == 200
auth_json = await auth_response.json()
return auth_json['access_token']
This is my test case to mock the get method:
json_data = [{
'access_token': 'HSG9hsf328bJSWO82sl',
'expires_in': 86399,
'token_type': 'bearer'
}]
class AsyncMock:
async def __aenter__(self):
return self
async def __aexit__(self, *error_info):
return self
#pytest.mark.asyncio
async def test_wow_api_invalid_credentials(monkeypatch, mocker):
def mock_client_get(self, auth_path, params):
mock_response = AsyncMock()
mock_response.status = 200
mock_response.json = mocker.MagicMock(return_value=json_data)
return mock_response
monkeypatch.setattr('wow.aiohttp.ClientSession.get', mock_client_get)
result = await wow.get_access_token()
assert result == 'HSG9hsf328bJSWO82sl'
I think the problem might be that mock_response.json() is not awaitable. In my example I can't call await from a non async function so I'm confused on how I would do that. I would like to keep the test libraries to a minimum which is pytest and pytest-asyncio for the learning experiencing and to rely less on 3rd party libraries.
I was making it more complicated than it needed to be. I simply defined json as an awaitable attribute of AsyncMock which returns the json_data. The complete code looks like this:
json_data = {
'access_token': 'HSG9hsf328bJSWO82sl',
'expires_in': 86399,
'token_type': 'bearer'
}
class AsyncMock:
async def __aenter__(self):
return self
async def __aexit__(self, *error_info):
return self
async def json(self):
return json_data
#pytest.mark.asyncio
async def test_wow_api_invalid_credentials(monkeypatch):
def mock_client_get(self, auth_path, params):
mock_response = AsyncMock()
mock_response.status = 200
return mock_response
monkeypatch.setattr('wow.aiohttp.ClientSession.get', mock_client_get)
result = await wow.get_access_token()
assert result == 'HSG9hsf328bJSWO82sl'
This is part 1, but i suggest you watch part2.
Im not sure i understand your question totally, because using async def or #asyncio.coroutine can help you do this. Actually, i want to write it as comment, however there are so many differences that i can't put it into comment.
import asyncio
json_ = [{
'access_token': 'HSG9hsf328bJSWO82sl',
'expires_in': 86399,
'token_type': 'bearer'
}]
async def response_from_sun():
return json_
class AsyncMock:
async def specify(self):
return self.json[0].get("access_token")
async def __aenter__(self):
return self
async def __aexit__(self, *error_info):
return self
async def mock_client_get():
mock_response = AsyncMock()
mock_response.status = 200
mock_response.json = await response_from_sun()
return mock_response
async def go():
resp = await mock_client_get()
result = await resp.specify()
assert result == 'HSG9hsf328bJSWO82sl'
asyncio.get_event_loop().run_until_complete(go())
PART2
After adding my answer, i found there is a problem about your mock_response content. Becausemock_response does not contain variable and function which ClientResponse have.
Edit: I try many times and watch ClientSession's code, then i found you can specify a new response class by its parameter. Note: connector=aiohttp.TCPConnector(verify_ssl=False) is unnecessary
import asyncio
import aiohttp
class Mock(aiohttp.ClientResponse):
print("Mock")
async def specify(self):
json_ = (await self.json()).get("hello")
return json_
async def go():
async with aiohttp.ClientSession(connector=aiohttp.TCPConnector(verify_ssl=False),response_class=Mock) as session:
resp = await session.get("https://www.mocky.io/v2/5185415ba171ea3a00704eed")
result = await resp.specify()
print(result)
assert result == 'world'
asyncio.get_event_loop().run_until_complete(go())
The toy script shows an application using a class that is dependent on an implementation that is not asyncio-aware, and obviously doesn't work.
How would the fetch method of MyFetcher be implemented, using the asyncio-aware client, while still maintaining the contract with the _internal_validator method of FetcherApp? To be very clear, FetcherApp and AbstractFetcher cannot be modified.
To use async fetch_data function inside fetch both fetch and is_fetched_data_valid functions should be async too. You can change them in child classes without modify parent:
import asyncio
class AsyncFetcherApp(FetcherApp):
async def is_fetched_data_valid(self): # async here
data = await self.fetcher_implementation.fetch() # await here
return self._internal_validator(data)
class AsyncMyFetcher(AbstractFetcher):
def __init__(self, client):
super().__init__()
self.client = client
async def fetch(self): # async here
result = await self.client.fetch_data() # await here
return result
class AsyncClient:
async def fetch_data(self):
await asyncio.sleep(1) # Just to sure it works
return 1
async def main():
async_client = AsyncClient()
my_fetcher = AsyncMyFetcher(async_client)
fetcherApp = AsyncFetcherApp(my_fetcher)
# ...
is_valid = await fetcherApp.is_fetched_data_valid() # await here
print(repr(is_valid))
if __name__ == "__main__":
loop = asyncio.get_event_loop()
loop.run_until_complete(main())