I am testing an async function that might get deadlocked. I tried to add a fixture to limit the function to only run for 5 seconds before raising a failure, but it hasn't worked so far.
Setup:
pipenv --python==3.6
pipenv install pytest==4.4.1
pipenv install pytest-asyncio==0.10.0
Code:
import asyncio
import pytest
#pytest.fixture
def my_fixture():
# attempt to start a timer that will stop the test somehow
asyncio.ensure_future(time_limit())
yield 'eggs'
async def time_limit():
await asyncio.sleep(5)
print('time limit reached') # this isn't printed
raise AssertionError
#pytest.mark.asyncio
async def test(my_fixture):
assert my_fixture == 'eggs'
await asyncio.sleep(10)
print('this should not print') # this is printed
assert 0
--
Edit: Mikhail's solution works fine. I can't find a way to incorporate it into a fixture, though.
Convenient way to limit function (or block of code) with timeout is to use async-timeout module. You can use it inside your test function or, for example, create a decorator. Unlike with fixture it'll allow to specify concrete time for each test:
import asyncio
import pytest
from async_timeout import timeout
def with_timeout(t):
def wrapper(corofunc):
async def run(*args, **kwargs):
with timeout(t):
return await corofunc(*args, **kwargs)
return run
return wrapper
#pytest.mark.asyncio
#with_timeout(2)
async def test_sleep_1():
await asyncio.sleep(1)
assert 1 == 1
#pytest.mark.asyncio
#with_timeout(2)
async def test_sleep_3():
await asyncio.sleep(3)
assert 1 == 1
It's not hard to create decorator for concrete time (with_timeout_5 = partial(with_timeout, 5)).
I don't know how to create texture (if you really need fixture), but code above can provide starting point. Also not sure if there's a common way to achieve goal better.
There is a way to use fixtures for timeout, one just needs to add the following hook into conftest.py.
Any fixture prefixed with timeout must return a number of seconds(int, float) the test can run.
The closest fixture w.r.t scope is chosen. autouse fixtures have lesser priority than explicitly chosen ones. Later one is preferred. Unfortunately order in the function argument list does NOT matter.
If there is no such fixture, the test is not restricted and will run indefinitely as usual.
The test must be marked with pytest.mark.asyncio too, but that is needed anyway.
# Add to conftest.py
import asyncio
import pytest
_TIMEOUT_FIXTURE_PREFIX = "timeout"
#pytest.hookimpl(tryfirst=True, hookwrapper=True)
def pytest_runtest_setup(item: pytest.Item):
"""Wrap all tests marked with pytest.mark.asyncio with their specified timeout.
Must run as early as possible.
Parameters
----------
item : pytest.Item
Test to wrap
"""
yield
orig_obj = item.obj
timeouts = [n for n in item.funcargs if n.startswith(_TIMEOUT_FIXTURE_PREFIX)]
# Picks the closest timeout fixture if there are multiple
tname = None if len(timeouts) == 0 else timeouts[-1]
# Only pick marked functions
if item.get_closest_marker("asyncio") is not None and tname is not None:
async def new_obj(*args, **kwargs):
"""Timed wrapper around the test function."""
try:
return await asyncio.wait_for(
orig_obj(*args, **kwargs), timeout=item.funcargs[tname]
)
except Exception as e:
pytest.fail(f"Test {item.name} did not finish in time.")
item.obj = new_obj
Example:
#pytest.fixture
def timeout_2s():
return 2
#pytest.fixture(scope="module", autouse=True)
def timeout_5s():
# You can do whatever you need here, just return/yield a number
return 5
async def test_timeout_1():
# Uses timeout_5s fixture by default
await aio.sleep(0) # Passes
return 1
async def test_timeout_2(timeout_2s):
# Uses timeout_2s because it is closest
await aio.sleep(5) # Timeouts
WARNING
Might not work with some other plugins, I have only tested it with pytest-asyncio, it definitely won't work if item is redefined by some hook.
I just loved Quimby's approach of marking tests with timeouts. Here's my attempt to improve it, using pytest marks:
# tests/conftest.py
import asyncio
#pytest.hookimpl(tryfirst=True, hookwrapper=True)
def pytest_pyfunc_call(pyfuncitem: pytest.Function):
"""
Wrap all tests marked with pytest.mark.async_timeout with their specified timeout.
"""
orig_obj = pyfuncitem.obj
if marker := pyfuncitem.get_closest_marker("async_timeout"):
async def new_obj(*args, **kwargs):
"""Timed wrapper around the test function."""
try:
return await asyncio.wait_for(orig_obj(*args, **kwargs), timeout=marker.args[0])
except (asyncio.CancelledError, asyncio.TimeoutError):
pytest.fail(f"Test {pyfuncitem.name} did not finish in time.")
pyfuncitem.obj = new_obj
yield
def pytest_configure(config: pytest.Config):
config.addinivalue_line("markers", "async_timeout(timeout): cancels the test execution after the specified amount of seconds")
Usage:
#pytest.mark.asyncio
#pytest.mark.async_timeout(10)
async def potentially_hanging_function():
await asyncio.sleep(20)
It should not be hard to include this to the asyncio mark on pytest-asyncio, so we can get a syntax like:
#pytest.mark.asyncio(timeout=10)
async def potentially_hanging_function():
await asyncio.sleep(20)
EDIT: looks like there's already a PR for that.
Related
Im using unittest.mock for building tests for my python code. I have a method that I am trying to test that contains a async call to another function. I want to patch that async call so that I can just have Mock return a testing value for the asset id, and not actually call the async method. I have tried many things I've found online, but none have worked thus far.
Simplified example below:
test.py
import pytest
from app.create.creations import generate_new_asset
from app.fakeapi.utils import create_asset
from unittest.mock import Mock, patch
#patch("app.fakeapi.utils.create_asset")
#pytest.mark.anyio
async def test_generate_new_asset(mock_create):
mock_create.return_value = 12345678
await generate_new_asset()
...
creations.py
from app.fakeapi.utils import create_asset
...
async def generate_new_asset()
...
# When I run tests this does not return the 12345678 value, but actually calls the `create_asset` method.
return await create_asset(...)
Testing async code is bit tricky. If you are using python3.8 or higher AsyncMock is available.
Note: it will work only for Python > 3.8
I think in your case event loop is missing. Here is the code which should work, you may need to do few tweaks. You may also need to install pytest-mock. Having it as fixture will allow you to have mock different values for testing for different scenarios.
import asyncio
from unittest.mock import AsyncMock, Mock
#pytest.fixture(scope="module")
def mock_create_asset(mocker):
async_mock = AsyncMock()
mocker.patch('app.fakeapi.utils.create_asset', side_effect=async_mock)
return async_mock
#pytest.fixture(scope="module")
def event_loop():
return asyncio.get_event_loop()
#pytest.mark.asyncio
async def test_generate_new_asset(mock_create_asset):
mock_create_asset.return_value = 12345678
await generate_new_asset()
The following code works as expected for the synchronous part but gives me a TypeError for the async call (TypeError: object NoneType can't be used in 'await' expression), presumably because the mock constructor can't properly deal with the spec. How do I properly tell Mockito that it needs to set up an asynchronous mock for async_method ?
class MockedClass():
def sync_method(self):
pass
async def async_method(self):
pass
class TestedClass():
def do_something_sync(self, mocked: MockedClass):
mocked.sync_method()
async def do_something_async(self, mocked: MockedClass):
await mocked.async_method()
#pytest.mark.asyncio
async def test():
my_mock = mock(spec=MockedClass)
tested_class = TestedClass()
tested_class.do_something_sync(my_mock)
verify(my_mock).sync_method()
await tested_class.do_something_async(my_mock) # <- Fails here
verify(my_mock).async_method()
Edit:
For reference, this is how it works with the standard mocks (the behavior that I expect):
In mockito my_mock.async_method() would not return anything useful by default and without further configuration. (T.i. it returns None which is not awaitable.)
What I did in the past:
# a helper function
def future(value=None):
f = asyncio.Future()
f.set_result(value)
return f
# your code
#pytest.mark.asyncio
async def test():
my_mock = mock(spec=MockedClass)
when(my_mock).async_method().thenReturn(future(None)) # fill in whatever you expect the method to return
# ....
Im using pytest-asyncio in a project that I'm currently working on.
In this project Im implementing the repository pattern and for tests I code a simple "In memory repository" (ie: dict with pk on keys and entities on values ). This repository is a class with async methods and have the following method:
async def update(self, entity: IEntity) -> IEntity:
try:
if entity.id not in self._storage:
raise KeyError()
self._storage[entity.id] = entity
except KeyError:
raise EntityNotFound(
f'Cant found {self._entity_type} with id: {entity.id}',
_id=entity.id,
)
return entity
And I have the following test:
#pytest.mak.asyncio
async def test_delete_nonexistent_sale(self):
with pytest.raises(EntityNotFound) as e:
await self.service.handle({
'sale_id': '93939393939393', 'salesman_id': self.salesman.id,
})
assert 1 == 2
# Ignore this assert for now, Youll understand soon
where service.handle is another async function that has a await repository.update(pk) on the first line and has not try/catch inside.
The problem is that this pass ( that obviously should fail ) passes even with the assert 1==2. For some reason I cant even use pdb/ipdb.set_trace() after the repository call.
Pytest show me this warning:
purchase_system/tests/test_domain/test_services/test_delete_sale.py::TestDeleteSale::test_delete_nonexistent_sale
/home/tamer.cuba/Documents/purchase-system/purchase_system/tests/test_domain/test_services/test_delete_sale.py:102: RuntimeWarning: coroutine 'DeleteSaleService.handle' was never awaited
self.service.handle(
-- Docs: https://docs.pytest.org/en/stable/warnings.html
How can I propagate de exceptions in tests using pytest-asyncio ?
I would to combine pytest and trio (or curio, if that is any easier), i.e. write my test cases as coroutine functions. This is relatively easy to achieve by declaring a custom test runner in conftest.py:
#pytest.mark.tryfirst
def pytest_pyfunc_call(pyfuncitem):
'''If item is a coroutine function, run it under trio'''
if not inspect.iscoroutinefunction(pyfuncitem.obj):
return
kernel = trio.Kernel()
funcargs = pyfuncitem.funcargs
testargs = {arg: funcargs[arg]
for arg in pyfuncitem._fixtureinfo.argnames}
try:
kernel.run(functools.partial(pyfuncitem.obj, **testargs))
finally:
kernel.run(shutdown=True)
return True
This allows me to write test cases like this:
async def test_something():
server = MockServer()
server_task = await trio.run(server.serve)
try:
# test the server
finally:
server.please_terminate()
try:
with trio.fail_after(30):
server_task.join()
except TooSlowError:
server_task.cancel()
But this is a lot of boilerplate. In non-async code, I would factor this out into a fixture:
#pytest.yield_fixture()
def mock_server():
server = MockServer()
thread = threading.Thread(server.serve)
thread.start()
try:
yield server
finally:
server.please_terminate()
thread.join()
server.server_close()
def test_something(mock_server):
# do the test..
Is there a way to do the same in trio, i.e. implement async fixtures? Ideally, I would just write:
async def test_something(mock_server):
# do the test..
Edit: the answer below is mostly irrelevant now – instead use pytest-trio and follow the instructions in its manual.
Your example pytest_pyfunc_call code doesn't work becaues it's a mix of trio and curio :-). For trio, there's a decorator trio.testing.trio_test that can be used to mark individual tests (like if you were using classic unittest or something), so the simplest way to write a pytest plugin function is to just apply this to each async test:
from trio.testing import trio_test
#pytest.mark.tryfirst
def pytest_pyfunc_call(pyfuncitem):
if inspect.iscoroutine(pyfuncitem.obj):
# Apply the #trio_test decorator
pyfuncitem.obj = trio_test(pyfuncitem.obj)
In case you're curious, this is basically equivalent to:
import trio
from functools import wraps, partial
#pytest.mark.tryfirst
def pytest_pyfunc_call(pyfuncitem):
if inspect.iscoroutine(pyfuncitem.obj):
fn = pyfuncitem.obj
#wraps(fn)
def wrapper(**kwargs):
trio.run(partial(fn, **kwargs))
pyfuncitem.obj = wrapper
Anyway, that doesn't solve your problem with fixtures – for that you need something much more involved.
I have following code in my tests module
def teardown_module():
clean_database()
def test1(): pass
def test2(): assert 0
and I want teardown_module() (some cleanup code) to be called only if some test failed. Otherwise (if all passed) this code shouldn't have to be called.
Can I do such a trick with PyTest?
You can. But it is a little bit of a hack.
As written here: http://pytest.org/latest/example/simple.html#making-test-result-information-available-in-fixtures
you do the following, to set up an attribute for saving the status of each phase of the testcall:
# content of conftest.py
import pytest
#pytest.mark.tryfirst
def pytest_runtest_makereport(item, call, __multicall__):
rep = __multicall__.execute()
setattr(item, "rep_" + rep.when, rep)
return rep
and in the fixture you just examine the condition on those attributes like this:
import pytest
#pytest.yield_fixture(scope="module", autouse=True)
def myfixture(request):
print "SETUP"
yield
# probably should not use "_collected" to iterate over test functions
if any(call.rep_call.outcome != "passed" for call in request.node._collected):
print "TEARDOWN"
This way if any of the tests associated with that module fixture is not "passed" (so "failed" or "skipped") then the condition holds.
The answer posted here and link to documentation was helpful but not sufficient for my needs. I needed a module teardown function to execute for each module independently if any test in that module (.py) file failed.
A complete sample project is available on GitHub
To start with, we need a hook to attach the test function result to
the test node. This is taken directly from the pytest docs:
# in conftest.py
#pytest.hookimpl(tryfirst=True, hookwrapper=True)
def pytest_runtest_makereport(item, call):
# execute all other hooks to obtain the report object
outcome = yield
rep = outcome.get_result()
# set a report attribute for each phase of a call, which can
# be "setup", "call", "teardown"
var_name = "rep_" + rep.when
setattr(item, var_name, rep)
After that, we need another hook for the test case to find the module and
store itself there, so the module can easily find its test cases. Perhaps
there's a better way, but I was unable to find one.
# also in conftest.py
#pytest.fixture(scope="function", autouse=True)
def _testcase_exit(request):
yield
parent = request.node.parent
while not isinstance(parent, pytest.Module):
parent = parent.parent
try:
parent.test_nodes.append(request.node)
except AttributeError:
parent.test_nodes = [request.node]
Once we do that, it's nice to have a decorator function to have the module on
completion look through its test nodes, find if there are any failures, and
then if there were call the function associated with the decorator:
# also also in conftest.py
def module_error_teardown(f):
#wraps(f)
#pytest.fixture(scope="module", autouse=True)
def wrapped(request, *args, **kwargs):
yield
try:
test_nodes = request.node.test_nodes
except AttributeError:
test_nodes = []
something_failed = False
for x in test_nodes:
try:
something_failed |= x.rep_setup.failed
something_failed |= x.rep_call.failed
something_failed |= x.rep_teardown.failed
except AttributeError:
pass
if something_failed:
f(*args, **kwargs)
return wrapped
Now we have all the necessary framework to work with. Now, a test file with a failing test case is easy to write:
from conftest import module_error_teardown
def test_something_that_fails():
assert False, "Yes, it failed."
def test_something_else_that_fails():
assert False, "It failed again."
#module_error_teardown
def _this_gets_called_at_the_end_if_any_test_in_this_file_fails():
print('')
print("Here's where we would do module-level cleanup!")