I am trying to learn unittest patching. I have a single file that both defines a function, then later uses that function. When I try to patch this function, its return value is giving me the real return value, not the patched return value.
How do I patch a function that is both defined and used in the same file? Note: I did try to follow the advice given here, but it didn't seem to solve my problem.
walk_dir.py
from os.path import dirname, join
from os import walk
from json import load
def get_config():
current_path =dirname(__file__)
with open(join(current_path, 'config', 'json', 'folder.json')) as json_file:
json_data = load(json_file)
return json_data['parent_dir']
def get_all_folders():
dir_to_walk = get_config()
for root, dir, _ in walk(dir_to_walk):
return [join(root, name) for name in dir]
test_walk_dir.py
from hello_world.walk_dir import get_all_folders
from unittest.mock import patch
#patch('walk_dir.get_config')
def test_get_all_folders(mock_get_config):
mock_get_config.return_value = 'C:\\temp\\test\\'
result = get_all_folders()
assert set(result) == set('C:\\temp\\test\\test_walk_dir')
Try declaring the patch in such way:
#patch('hello_world.walk_dir.get_config')
As you can see this answer to the question you linked, it's recommended that your import statements match your patch statements. In your case from hello_world.walk_dir import get_all_folders and #patch('walk_dir.get_config') doesn't match.
Related
In the following code, the request has the type of <class '_pytest.fixtures.SubRequest'>. I want to add type hint to the parameter request.
#pytest.fixture
def dlv_service(request: SubRequest): # How to import SubRequest?
print(type(request), request)
filepath = pathlib.Path(request.node.fspath.strpath)
f = filepath.with_name("file.json")
The following import doesn't work.
from pytest.fixtures import SubRequest
I've found one on the internet, hope this will help.
from _pytest.fixtures import SubRequest
I think it's worth trying, but not sure whether it could work, sorry.
For some applications, such as accessing the node property, you can import FixtureRequest, which is part of the public API and a superclass of SubRequest. See yourself:
from _pytest.fixtures import SubRequest
from pytest import FixtureRequest
issubclass(SubRequest, FixtureRequest)
hasattr(FixtureRequest, "node")
Applying this to your example:
from pathlib import Path
import pytest
from pytest import FixtureRequest
#pytest.fixture
def dlv_service(request: FixtureRequest) -> Path:
print(type(request), request)
filepath = Path(request.node.fspath.strpath)
return filepath.with_name("file.json")
Imagine that we have the following Data Base structure with the data stored in python files ready to be imported:
data_base/
foo_data/
rev_1.py
rev_2.py
bar_data/
rev_1.py
rev_2.py
rev_3.py
In my main script, I would like to import the last revision of the data available in the folder. For example, instead of doing this:
from data_base.foo_data.rev_2 import foofoo
from data_base.bar_data.rev_3 import barbar
I want to call a method:
import_from_db(path='data_base.foo_data', attr='foofoo', rev='last')
import_from_db(path='data_base.bar_data', attr='barbar', rev='last')
I could take a relative path to the Data Base and use glob.glob to search the last revision, but for this, I should know the path to the data_base folder, which complicates things (imagine that the parent folder of the data_base is in sys.path so the from data_base.*** import will work)
Is there an efficient way to maybe retrieve a full path knowing only part of it (data_base.foo_data)? Other ideas?
I think it's better to install the last version.
but going on with your flow, you may use getattr on the module:
from data_base import foo_data
i = 0
while True:
try:
your_module = getattr(foo_data, f'rev_{i}')
except AttributeError:
break
i += 1
# Now your_module is the latest rev
#JohnDoriaN 's idea led me to a quite simple solution:
import os, glob
def import_from_db(import_path, attr, rev_id=None):
"""
"""
# Get all the modules/folders names
dir_list = import_path.split('.')
# Import the last module
exec(f"from {'.'.join(dir_list[:-1])} import {dir_list[-1]}")
db_parent = locals()[dir_list[-1]]
# Get an absolute path to corresponding to the db_parent folder
abs_path = db_parent.__path__._path[0]
rev_path = os.path.join(abs_path, 'rev_*.py')
rev_names = [os.path.basename(x) for x in glob.glob(rev_path)]
if rev_id is None:
revision = rev_names[-1]
else:
revision = rev_names[rev_id]
revision = revision.split('.')[0]
# import attribute
exec(f'from {import_path}.{revision} import {attr}', globals())
Some explanations:
Apparently (I didn't know this), we can import a folder as a module; this module has a __path__ attribute (found out using the built-in dir method).
glob.glob allows us to use regex expressions to search for a required pattern for files in the directory.
using exec without parameters will import only in the local namespace (namespace of the method) so without polluting the global namespace.
using exec with globals() allows us to import in the global namespace.
I want to mock a ZipFile. In particular, I need a mock
Which passes a zipfile.is_zipfile() test,
Returns a list of strings for zipfile.ZipFile().namelist(), and
Uses only the standard library.
The code I am testing looks for potential Python modules1 within a given zip archive (i.e. .py, .zip, and .whl files):
# utils.py
import zipfile
from pathlib import Path
def find_modules(archive=None):
"""Find modules within a given zip archive.
Inputs:
archive (str/Path): Zip archive
Returns:
list (str): List of module names as strings
"""
possible_ext = ['.py'. '.zip', '.whl']
modules = []
if zipfile.is_zipfile(archive):
paths = [Path(p) for p in zipfile.ZipFile(archive).namelist()]
modules = [p.stem for p in paths if p.stem != '__init__' and p.suffix in possible_ext]
return modules
Voodoo solution
I have cobbled together the following test:
# test_utils.py
from mypackage import utils
from unittest import mock
class TestFunctions():
MOCK_LISTING = ['single_file_module.py', 'dummy.txt',
'package_namespace.zip', 'wheel_namespace-0.1-py3-none-any.whl']
#mock.patch('zipfile.ZipFile')
#mock.patch('zipfile.is_zipfile')
def test_find_modules_return_value(self, mock_is_zipfile, mock_zipfile):
mock_is_zipfile.return_value = True
mock_zipfile.return_value.namelist.return_value = self.MOCK_LISTING
modules = utils.find_modules('dummy_archive.zip')
assert len(modules) == 3
def main():
"""Main function used to run tests manually.
Use PyTest to run tests in bulk.
"""
tc = TestFunctions()
tc.test_find_modules_return_value()
if __name__ == '__main__':
import time
start_time = time.time()
main()
print("\nThe chosen tests have all passed.")
print("--- %s seconds ---" % (time.time() - start_time))
Questions
I found that a #mock.path('zipfile.ZipFile') alone wouldn't meet my needs; it failed a zipfile.is_zipfile() test.
If I'm mocking a ZipFile object, shouldn't it automatically pass a zipfile.is_zipfile() test?
I found that I couldn't use the same approach to overriding is_zipfile as I did namelist. That is, an additional #mock.patch('zipfile.is_zipfile') was needed. My understanding is that because a ZipFile defines a context, the first return_value overrides the __enter__ of the context, and then the next namespace is the ZipFile method level. Why doesn't the same approach work for both is_zipfile and namelist?
# Test doesn't work
# Fails on: assert 0 == 3
# + where 0 = len([])
#mock.patch('zipfile.ZipFile')
def test_find_modules_return_value(self, mock_zipfile):
mock_zipfile.return_value.is_zipfile.return_value = True
mock_zipfile.return_value.namelist.return_value = self.MOCK_LISTING
modules = utils.find_modules('dummy_archive.zip')
assert len(modules) == 3
Maybe I'm getting too far off-base and there's a simpler way to mock a .zip archive?
EDIT
Based on #Don Kirby's answer, the pattern I found most intuitive was:
def test_find_modules_return_value(self):
# Create mock zipfile and override the is_zipfile function
with mock.patch('mypackage.utils.zipfile') as mock_zipfile:
mock_zipfile.is_zipfile.return_value = True
mock_zipfile.namelist.return_value = self.MOCK_LISTING
# Since a ZipFile is a separate object, which returns a zipfile (note
# that that's lowercase), we need to mock the ZipFile and have it return
# the zipfile mock previously created.
with mock_patch('mypackage.utils.zipfile.ZipFile') as mock_ZipFile:
mock_ZipFile.return_value = mock_zipfile
modules = utils.find_modules("/dummy/path/to/check.zip")
assert len(modules) == 3
1 It's assumed that .zip files may contain modules and that .zip and .whl will be handled in a different process. The file names are all we care about in this step.
You have to patch is_zipfile() separately from ZipFile, because is_zipfile() is a function, not a method of the ZipFile class. I suppose you might be able to patch the whole zipfile module by patching mypackage.utils.zipfile, but that seems way more confusing.
The zipfile source code might be useful.
I want to look at a file and get the names of classes and check if the "Runconfig" name is inherited. So if a file has
class some_function(RunConfig):
I want to return true.
My code looks like this right now:
for file in list_of_files:
if file in ['some_file.py']:
for name,obj in inspect.getmembers(file):
if inspect.isclass(obj):
print("NAME",name,"obj",obj)
This returns objects but I don't see anything that says 'RunConfig' on it.
What am I missing here?
Thank you so much in advance!
You can do something like:
import importlib
import inspect
def is_class_inherited_in_file(file_name, class_ref):
module = importlib.import_module(file_name.split('.')[0])
module_members = inspect.getmembers(module)
for member in module_members:
if type(member[1]) == type and issubclass(member[1], class_ref):
return True
return False
>>> is_class_inherited_in_file('some_file.py', RunConfig)
True
Assumption:
The filename is in the working directory. If you would like to import from any directory, then do something like: How to import a module given the full path?
I have a main file which is looking at files within a /modules/ folder, it needs to look at every .py file and find all functions that have a specific attribute.
An example module will be like this:
def Command1_1():
True
Command1_1.command = ['cmd1']
def Command1_2():
True
The code I am currently using to look through each file and function is this:
for module in glob.glob('modules/*.py'):
print(module)
tree = ast.parse(open(module, "rt").read(), filename=PyBot.msggrp + module)
for item in [x.name for x in ast.walk(tree) if isinstance(x, ast.FunctionDef)]:
if item is not None:
print(str(item))
Below is what the code produces but I cannot find a way to show if a function has a ".command" attribute:
modules/Placeholder001.py
Command1_1
Command1_2
modules/Placeholder002.py
Command2_1
Command2_2
Command2_3
The easiest way is to import each file and then look for functions in its global scope. Functions can be identified with the use of callable. Checking if a function has an attribute can be done with hasattr.
The code to import a module from a path is taken from this answer.
from pathlib import Path
import importlib.util
def import_from_path(path):
spec = importlib.util.spec_from_file_location(path.stem, str(path))
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module
for module_path in Path('modules').glob('*.py'):
module = import_from_path(module_path)
for name, value in vars(module).items():
if callable(value):
has_attribute = hasattr(value, 'command')
print(name, has_attribute)
Output:
Command1_1 True
Command1_2 False