I'm using AioHttp to implement a service at work, and during my tests, I'm mocking a method, the issue is there the call for the method is calling the real instead of the mocked method.
#unittest_run_loop
#patch('export_api.main.add_job_to_db')
async def test_view_job(self, mocked_method):
json = {
"edl": "somedata"
}
response = await self.client.request("PUT", "/v1/job", json=json)
mocked_method.assert_called_once_with()
assert response.status == 200
So I get this error on the assertion of the mock:
msg = "Expected 'add_job_to_db' to be called once. Called 0 times."
My method on the main.py:
async def __call__(self, request):
"""Faz post do Job na fila do Render"""
data = await request.json()
job_id = uuid.uuid4()
job = Jobs(
job_id=str(job_id),
body=data
)
try:
add_job_to_db(self.app['db'], job)
return web.Response(status=200)
except DatabaseError as e:
print(e)
return web.Response(status=500)
Yes, is a callable method inside a class. The test work fine without the mocking. But I need to mock the call for the db, and I'm not having luck so far.
Any ideas?
Checklist:
Use asynctest package
Don't forget to use #asyncio.coroutine decorator around the unit test
Patch the object where it is used, in this case it export_api.main.add_job_to_db
I'm the author of mocket and few days ago I released the version 2.0.0 which fully supports asyncio/aiohttp.
Here is the same example of code mocking a URL on HTTP and on HTTPS:
import aiohttp
import asyncio
import async_timeout
from unittest import TestCase
from mocket.mocket import mocketize
from mocket.mockhttp import Entry
class AioHttpEntryTestCase(TestCase):
#mocketize
def test_http_session(self):
url = 'http://httpbin.org/ip'
body = "asd" * 100
Entry.single_register(Entry.GET, url, body=body, status=404)
Entry.single_register(Entry.POST, url, body=body*2, status=201)
async def main(l):
async with aiohttp.ClientSession(loop=l) as session:
with async_timeout.timeout(3):
async with session.get(url) as get_response:
assert get_response.status == 404
assert await get_response.text() == body
with async_timeout.timeout(3):
async with session.post(url, data=body * 6) as post_response:
assert post_response.status == 201
assert await post_response.text() == body * 2
loop = asyncio.get_event_loop()
loop.set_debug(True)
loop.run_until_complete(main(loop))
#mocketize
def test_https_session(self):
url = 'https://httpbin.org/ip'
body = "asd" * 100
Entry.single_register(Entry.GET, url, body=body, status=404)
Entry.single_register(Entry.POST, url, body=body*2, status=201)
async def main(l):
async with aiohttp.ClientSession(loop=l) as session:
with async_timeout.timeout(3):
async with session.get(url) as get_response:
assert get_response.status == 404
assert await get_response.text() == body
with async_timeout.timeout(3):
async with session.post(url, data=body * 6) as post_response:
assert post_response.status == 201
assert await post_response.text() == body * 2
loop = asyncio.get_event_loop()
loop.set_debug(True)
loop.run_until_complete(main(loop))
Source: https://github.com/mindflayer/python-mocket/blob/master/tests/tests35/test_http_aiohttp.py
Related
Basically, what it does, is to do 20 requests async to google.
If I launch it without using PyTest, just a snip of code, like this, it works:
import asyncio
import aiohttp
async def get(
session: aiohttp.ClientSession,
) -> dict:
url = f"https://www.google.com/"
resp = await session.request('GET', url=url)
data = await resp.json()
return data
async def sessions():
async with aiohttp.ClientSession() as session:
tasks = []
for i in range(20):
tasks.append(get(session=session))
return await asyncio.gather(*tasks, return_exceptions=True)
def main():
loop = asyncio.new_event_loop()
try:
asyncio.set_event_loop(loop)
htmls = loop.run_until_complete(sessions())
finally:
loop.close()
print(htmls)
But when I use PyTest, in spite of being the same code (almost), the "htmls" variable at the end is not assignated any value
import aiohttp
import asyncio
async def get(
session: aiohttp.ClientSession,
) -> dict:
url = f"https://www.google.com/"
resp = await session.request('GET', url=url)
data = await resp.json()
return data
async def sessions(self):
async with aiohttp.ClientSession() as session:
tasks = []
for i in range(20):
tasks.append(self.get(session=session))
return await asyncio.gather(*tasks, return_exceptions=True)
def test_example(self):
loop = asyncio.new_event_loop()
try:
asyncio.set_event_loop(loop)
htmls = loop.run_until_complete(self.sessions())
finally:
loop.close()
print(htmls)
Why is this? It is like loop.run_until_complete(self.sessions()) is not waiting for it to finish.
It is resolved. It needed a self as first parameter for the get() method :S
in my class I have a method that fetches the website (visible below).
I've noticed that other methods that use this method, can lead to the opening of multiple requests to one site (when one request is pending self._page is still none).
How can I avoid it?
I mean when there is another call to _get_page but one is pending, just return a future from the first one and don't repeat a page request
async def _get_page(self) -> HtmlElement:
if self._page is None:
async with self._get_session().get(self._url) as page:
self._page = lxml.html.document_fromstring(await page.text())
return self._page
How can I avoid [multiple requests]?
You could use an asyncio.Lock:
saync def __init__(self, ...):
...
self._page_lock = asyncio.Lock()
async def _get_page(self) -> HtmlElement:
async with self._page_lock:
if self._page is None:
async with self._get_session().get(self._url) as page:
self._page = lxml.html.document_fromstring(await page.text())
return self._page
Update for Python 3.8 and jupyter notebook
import asyncio
import aiohttp
from lxml import html
class MyClass:
def __init__(self):
self._url = 'https://www.google.com'
self._page = None
self._futures = []
self._working = False
self._session = aiohttp.ClientSession()
async def _close(self):
if self._session:
session = self._session
self._session = None
await session.close()
def _get_session(self):
return self._session
async def _get_page(self):
if self._page is None:
if self._working:
print('will await current page request')
loop = asyncio.get_event_loop()
future = loop.create_future()
self._futures.append(future)
return await future
else:
self._working = True
session = self._get_session()
print('making url request')
async with session.get(self._url) as page:
print('status =', page.status)
print('making page request')
self._page = html.document_fromstring(await page.text())
print('Got page text')
for future in self._futures:
print('setting result to awaiting request')
future.set_result(self._page)
self._futures = []
self._working = False
return self._page
async def main():
futures = []
m = MyClass()
futures.append(asyncio.ensure_future(m._get_page()))
futures.append(asyncio.ensure_future(m._get_page()))
futures.append(asyncio.ensure_future(m._get_page()))
results = await asyncio.gather(*futures)
for result in results:
print(result[0:80])
await m._close()
if __name__ == '__main__':
asyncio.run(main())
#await main() # In jupyter notebook and iPython
Note that on Windows 10 I have seen at termination:
RuntimeError: Event loop is closed
See https://github.com/aio-libs/aiohttp/issues/4324
I have a test. It sends a get request to a list of urls and checks that the response is not 500.
#pytest.mark.asyncio
#pytest.mark.parametrize('url_test_list', get_all_url_list(HOST_FOR_TEST))
async def test_check_status_urls(self, url_test_list):
returned_status = await get(url_test_list)
assert returned_status < 500
and this is my "get" function
async def get(url):
async with aiohttp.ClientSession() as session:
async with session.get(url) as response:
response_status = response.status
return response_status
It works, but it is slow. It takes about 3 minutes to complete.
But when I use this test without #parametrize and my "get" function takes the url_list - it runs in about 1 minute. My code in second case:
#pytest.mark.asyncio
async def test_check_status_urls(self):
url_list = make_url_list()
returned_status = await get(url_list)
assert all(returned_status) > 500
async def get(urls):
good_list = []
async with aiohttp.ClientSession() as session:
for url in urls:
async with session.get(url) as response:
response_status = response.status
good_list.append(response_status)
return good_list
I would like to have the best of both worlds here. Is there a way I can have the tests run quickly, but also run as individual units?
I want to mock the json() coroutine from the aiohttp.ClientSession.get method. It looks to return an async generator object, which is where I'm confused on how to mock in my example. Here is my code:
async def get_access_token():
async with aiohttp.ClientSession(auth=auth_credentials) as client:
async with client.get(auth_path, params={'grant_type': 'client_credentials'}) as auth_response:
assert auth_response.status == 200
auth_json = await auth_response.json()
return auth_json['access_token']
This is my test case to mock the get method:
json_data = [{
'access_token': 'HSG9hsf328bJSWO82sl',
'expires_in': 86399,
'token_type': 'bearer'
}]
class AsyncMock:
async def __aenter__(self):
return self
async def __aexit__(self, *error_info):
return self
#pytest.mark.asyncio
async def test_wow_api_invalid_credentials(monkeypatch, mocker):
def mock_client_get(self, auth_path, params):
mock_response = AsyncMock()
mock_response.status = 200
mock_response.json = mocker.MagicMock(return_value=json_data)
return mock_response
monkeypatch.setattr('wow.aiohttp.ClientSession.get', mock_client_get)
result = await wow.get_access_token()
assert result == 'HSG9hsf328bJSWO82sl'
I think the problem might be that mock_response.json() is not awaitable. In my example I can't call await from a non async function so I'm confused on how I would do that. I would like to keep the test libraries to a minimum which is pytest and pytest-asyncio for the learning experiencing and to rely less on 3rd party libraries.
I was making it more complicated than it needed to be. I simply defined json as an awaitable attribute of AsyncMock which returns the json_data. The complete code looks like this:
json_data = {
'access_token': 'HSG9hsf328bJSWO82sl',
'expires_in': 86399,
'token_type': 'bearer'
}
class AsyncMock:
async def __aenter__(self):
return self
async def __aexit__(self, *error_info):
return self
async def json(self):
return json_data
#pytest.mark.asyncio
async def test_wow_api_invalid_credentials(monkeypatch):
def mock_client_get(self, auth_path, params):
mock_response = AsyncMock()
mock_response.status = 200
return mock_response
monkeypatch.setattr('wow.aiohttp.ClientSession.get', mock_client_get)
result = await wow.get_access_token()
assert result == 'HSG9hsf328bJSWO82sl'
This is part 1, but i suggest you watch part2.
Im not sure i understand your question totally, because using async def or #asyncio.coroutine can help you do this. Actually, i want to write it as comment, however there are so many differences that i can't put it into comment.
import asyncio
json_ = [{
'access_token': 'HSG9hsf328bJSWO82sl',
'expires_in': 86399,
'token_type': 'bearer'
}]
async def response_from_sun():
return json_
class AsyncMock:
async def specify(self):
return self.json[0].get("access_token")
async def __aenter__(self):
return self
async def __aexit__(self, *error_info):
return self
async def mock_client_get():
mock_response = AsyncMock()
mock_response.status = 200
mock_response.json = await response_from_sun()
return mock_response
async def go():
resp = await mock_client_get()
result = await resp.specify()
assert result == 'HSG9hsf328bJSWO82sl'
asyncio.get_event_loop().run_until_complete(go())
PART2
After adding my answer, i found there is a problem about your mock_response content. Becausemock_response does not contain variable and function which ClientResponse have.
Edit: I try many times and watch ClientSession's code, then i found you can specify a new response class by its parameter. Note: connector=aiohttp.TCPConnector(verify_ssl=False) is unnecessary
import asyncio
import aiohttp
class Mock(aiohttp.ClientResponse):
print("Mock")
async def specify(self):
json_ = (await self.json()).get("hello")
return json_
async def go():
async with aiohttp.ClientSession(connector=aiohttp.TCPConnector(verify_ssl=False),response_class=Mock) as session:
resp = await session.get("https://www.mocky.io/v2/5185415ba171ea3a00704eed")
result = await resp.specify()
print(result)
assert result == 'world'
asyncio.get_event_loop().run_until_complete(go())
I have this 3.6 async code:
async def send(command,userPath,token):
async with websockets.connect('wss://127.0.0.1:7000',ssl=ssl.SSLContext(protocol=ssl.PROTOCOL_TLS)) as websocket:
data = json.dumps({"api_command":"session","body":command,"headers": {'X-User-Path': userPath, 'X-User-Token': token}})
await websocket.send(data)
response = await websocket.recv()
response = json.loads(response)
if 'command' in response:
if response['command'] == 'ACK_COMMAND' or response['command'] == 'ACK_INITIALIZATION':
return (response['message'],200)
else:
return(response,400)
which I converted to this 3.4 async code
#asyncio.coroutine
def send(command,userPath,token):
with websockets.connect('wss://127.0.0.1:7000',ssl=ssl.SSLContext(protocol=ssl.PROTOCOL_TLS)) as websocket:
data = json.dumps({"api_command":"session","body":command,"headers": {'X-User-Path': userPath, 'X-User-Token': token}})
yield from websocket.send(data)
response = yield from websocket.recv()
response = json.loads(response)
if 'command' in response:
if response['command'] == 'ACK_COMMAND' or response['command'] == 'ACK_INITIALIZATION':
return (response['message'],200)
else:
return(response,400)
Although the interpreter runs the conversion, when I call the function this error occurs:
with websockets.connect('wss://127.0.0.1:7000',ssl=ssl.SSLContext(protocol=ssl.PROTOCOL_TLS)) as websocket:
AttributeError: __enter__
I feel like there is more stuff to convert, but I don't know what. How can I make the 3.4 code work?
Note: I run the 3.4 code with a 3.6 python
As can be found here of async with websockets.connect you should do:
websocket = yield from websockets.connect('ws://localhost:8765/')
try:
# your stuff
finally:
yield from websocket.close()
In your case it would be:
#asyncio.coroutine
def send(command,userPath,token):
websocket = yield from websockets.connect('wss://127.0.0.1:7000',ssl=ssl.SSLContext(protocol=ssl.PROTOCOL_TLS))
try:
data = json.dumps({"api_command":"session","body":command,"headers": {'X-User-Path': userPath, 'X-User-Token': token}})
yield from websocket.send(data)
response = yield from websocket.recv()
response = json.loads(response)
if 'command' in response:
if response['command'] == 'ACK_COMMAND' or response['command'] == 'ACK_INITIALIZATION':
return (response['message'],200)
else:
return(response,400)
finally:
yield from websocket.close()