We Keep Coding

pytest how to mock functions from another module

I have a directory structure:
├── src
│ └── chkfixt
│ ├── __init__.py
│ ├── config.py
│ ├── main.py
│ └── util.py
└── tests
└── test_chkfixt.py
Files have contents as:
config.py
APP_NAME = 'not_mocked'
util.py
from pathlib import Path
from chkfixt.config import APP_NAME
def get_app_dir() -> str:
return Path(APP_NAME)
def get_metadata_pickle_file_path() -> Path:
app_dir = get_app_dir()
config_path = Path(app_dir) / "metadata.pkl"
return config_path
def save_metadata_to_pickle_file():
pickle_file = get_metadata_pickle_file_path()
print(f"saved to {pickle_file}")
main.py
from chkfixt.util import (get_metadata_pickle_file_path,
save_metadata_to_pickle_file)
print(f"pkl_file = {get_metadata_pickle_file_path()}")
save_metadata_to_pickle_file()
If I run main.py output is like this:
(deleteme) user#server:~/tmp/chkfixt$ python src/chkfixt/main.py
pkl_file = not_mocked/metadata.pkl
saved to not_mocked/metadata.pkl
In pytest, I need a different pkl_file for each test, so I have to mock get_metadata_pickle_file_path() function.
My first attempt in test_chkfixt.py was (ignore monkeypatch repetition for now) like this:
import pytest
from rich import print
from chkfixt.main import (get_metadata_pickle_file_path,
save_metadata_to_pickle_file)
#pytest.fixture
def mocked_pkl_file(tmp_path):
return tmp_path / "metadata.pkl"
def test_mocked_get_pkl(mocked_pkl_file, monkeypatch):
monkeypatch.setattr(
"chkfixt.main",
"get_metadata_pickle_file_path",
lambda: str(mocked_pkl_file),
)
print(get_metadata_pickle_file_path())
def test_mocked_save_pkl(mocked_pkl_file, monkeypatch):
monkeypatch.setattr(
"chkfixt.main",
"get_metadata_pickle_file_path",
lambda: str(mocked_pkl_file),
)
print(save_metadata_to_pickle_file())
But that produced errors:
_______________________________________________________________________________________________________________________________ test_mocked_get_pkl ________________________________________________________________________________________________________________________________
mocked_pkl_file = PosixPath('/tmp/pytest-of-user/pytest-89/test_mocked_get_pkl0/metadata.pkl'), monkeypatch = <_pytest.monkeypatch.MonkeyPatch object at 0x7ff900732ec0>
def test_mocked_get_pkl(mocked_pkl_file, monkeypatch):
> monkeypatch.setattr(
"chkfixt.main",
"get_metadata_pickle_file_path",
lambda: str(mocked_pkl_file),
)
E AttributeError: 'chkfixt.main' has no attribute 'get_metadata_pickle_file_path'
tests/test_chkfixt.py:14: AttributeError
_______________________________________________________________________________________________________________________________ test_mocked_save_pkl _______________________________________________________________________________________________________________________________
mocked_pkl_file = PosixPath('/tmp/pytest-of-user/pytest-89/test_mocked_save_pkl0/metadata.pkl'), monkeypatch = <_pytest.monkeypatch.MonkeyPatch object at 0x7ff9007b5db0>
def test_mocked_save_pkl(mocked_pkl_file, monkeypatch):
> monkeypatch.setattr(
"chkfixt.main",
"get_metadata_pickle_file_path",
lambda: str(mocked_pkl_file),
)
E AttributeError: 'chkfixt.main' has no attribute 'get_metadata_pickle_file_path'
tests/test_chkfixt.py:23: AttributeError
Here, I am confused with error that is saying 'chkfixt.main' has no attribute 'get_metadata_pickle_file_path'.
If I change test functions like this (merged "chkfixt.main", "get_metadata_pickle_file_path" to "chkfixt.main.get_metadata_pickle_file_path"):
def test_mocked_get_pkl(mocked_pkl_file, monkeypatch):
monkeypatch.setattr(
"chkfixt.main.get_metadata_pickle_file_path",
lambda: str(mocked_pkl_file),
)
print(get_metadata_pickle_file_path())
def test_mocked_save_pkl(mocked_pkl_file, monkeypatch):
monkeypatch.setattr(
"chkfixt.main.get_metadata_pickle_file_path",
lambda: str(mocked_pkl_file),
)
print(save_metadata_to_pickle_file())
I am not getting errors anymore, but test output is still not_mocked/metadata.pkl:
platform linux -- Python 3.10.4, pytest-7.1.2, pluggy-1.0.0
rootdir: /home/user/tmp/chkfixt
plugins: forked-1.4.0, cov-3.0.0, black-0.3.12, mypy-0.9.1, xdist-2.5.0, flake8-1.1.1
collecting ... pkl_file = not_mocked/metadata.pkl
saved to not_mocked/metadata.pkl
collected 2 items
tests/test_chkfixt.py not_mocked/metadata.pkl
.saved to not_mocked/metadata.pkl
None
.
As output shows, both paths are not_mocked/metadata.pkl, but I expect them to be from mocked_pkl_file fixture.
How to mock main.py get_metadata_pickle_file_path and save_metadata_to_pickle_file functions to use path from mocked_pkl_file fixture.

You should be patching the imported objects, not strings. Also, import the module, don't use from .. import ... statements, so that pytest is able to access the module it is patching:
import chkfixt.main
def test_mocked_save_pkl(mocked_pkl_file, monkeypatch):
monkeypatch.setattr(
chkfixt.main,
"get_metadata_pickle_file_path",
lambda: str(mocked_pkl_file),
)
See the how to monkey patch docs for more examples.

importlib.reload() not realoading

I am trying to create a leetcode like online judge. I need to reload the submission module but import.reload() does not work.
The code:
class Test:
current_exercise = None
current_name = None
def _import(self, exercise):
exercise = 'exercise' # for testing
if exercise == self.current_name:
module = sys.modules[f'puzzles.{exercise}']
self.current_exercise = importlib.reload(module) # <---- not working
else:
self.current_name = exercise
self.current_exercise = __import__(f'puzzles.{exercise}').exercise
def _test(self, exercise):
solution = self._import(exercise)
print(self.current_exercise.main())
if __name__=='__main__':
import shutil
t= Test()
# first run
t._test('exercise')
# copy another solution.py for reload test
shutil.copy(f"./puzzles/other_exercise/solution.py", f"./puzzles/exercise/solution.py")
# second run
t._test('exercise')
My directory;
.
├── codetest.py
├── puzzles
│   ├── __init__.py
│   ├── exercise
│   │   ├── __init__.py
│   │   ├── solution.py
│ ├── other_exercise
│ │ ├── __init__.py
│ │ ├── solution.py
exercise/solution.py:
def main():
print('EXERCISE')
exercise/init.py
from .solution import main
from .test import cases
other_exercise/solution.py:
def main():
print('OTHER EXERCISE')
Output:
> EXERCISE
> EXERCISE # <--- not sucessfull, should be 'OTHER EXERCISE'

This works:
import sys
import time
import importlib
class Test:
current_exercise = None
current_name = None
def _import(self, exercise):
if exercise == self.current_name:
self.current_exercise.solution = importlib.reload(self.current_exercise.solution)
else:
self.current_name = exercise
self.current_exercise = importlib.import_module(f'puzzles.{exercise}')
print('mod',self.current_exercise)
print('nam',self.current_exercise.__name__)
print('fil',self.current_exercise.__file__)
print('pkg',self.current_exercise.__package__)
def _test(self, exercise):
solution = self._import(exercise)
print(self.current_exercise.solution.main())
if __name__=='__main__':
import shutil
shutil.copy(f"./puzzles/exercise/solution.0", f"./puzzles/exercise/solution.py")
t= Test()
# first run
t._test('exercise')
# copy another solution.py for reload test
shutil.copy("./puzzles/other_exercise/solution.py", "./puzzles/exercise/solution.py")
print(open("./puzzles/exercise/solution.py").read())
# second run
t._test('exercise')

I went with an alternative; load solution.py as a text and create a module from that string. The module is not registered in sys.modules and can be overwritten. However imp is deprecated.
import imp
class Test:
current_exercise = None
def _import(self, exercise):
# load module code
with open(f'./puzzles/{exercise}/solution.py') as f:
code = f.read()
# register/create the module
self.current_exercise = imp.new_module('mymodule')
# import/fill the module
exec(code, self.current_exercise.__dict__)
def _test(self, exercise):
self._import(exercise)
print(self.current_exercise.main())

Get imports required for re-creating an object

Given an object produced during importing the code, produce the set of imports that are needed to execute that object creation code.
Case 1:
some_obj = module.submodule.Class(42)
get_imports for_object(some_obj)
>>> "import module.submodule"
Case 2 (Sometimes the root module does not import submodules automatically (e.g. Airflow operators)):
some_obj = submodule.Class(42)
get_imports for_object(some_obj)
>>> "from module import submodule"
Case 3 (stretch goal):
some_obj = submodule.Class(sub2.Class2(42))
get_imports for_object(some_obj)
>>> ["from module import submodule", "from module2 import sub2"]
The goal is to produce import lines such that prepending them to object instantiation code will make the instantiation work.

This'll do:
def get_object_imports(obj, sub_obj_class_name=None):
sub_obj_modules = []
if sub_obj_class_name is not None:
for _, attribute_value in obj.__dict__.items():
value_str = str(getattr(attribute_value,'__class__',''))
if ('.' + sub_obj_class_name) in value_str:
sub_obj_modules.append(attribute_value.__module__)
if sub_obj_modules != []:
sub_module_imports = [('import ' + sub_obj_module) for sub_obj_module
in sub_obj_modules]
return ['import ' + obj.__module__] + sub_module_imports
else:
return 'import ' + obj.__module__
Cases (1) & (2) are equivalent, in that running either imports the same module. Note that with above, objects with same class names but different module sources will be included.
Demo:
from module import class1
from other_module import submodule
obj1 = class1()
obj2 = obj1(submodule.class2())
print(get_object_imports(obj2, 'class2'))
# ['import module', 'import other_module.submodule']

Include one python file B inside file A where B uses variables defined in A

I have a Django project where I have multiple settings file which has lot of redundant data. The structure is as follows
development.py
app_one = 'http://localhost:8000'
app_two = 'http://localhost:9999'
abc_url = '{0}/some-url/'.format(app_one)
xyz_url = '{0}/some-url/'.format(app_two)
staging.py
app_one = 'http://staging.xyz.abc.com'
app_two = 'http://staging.pqr.abc.com'
abc_url = '{0}/some-url/'.format(app_one)
xyz_url = '{0}/some-url/'.format(app_two)
production.py
app_one = 'http://production.xyz.abc.com'
app_two = 'http://production.pqr.abc.com'
abc_url = '{0}/some-url/'.format(app_one)
xyz_url = '{0}/some-url/'.format(app_two)
In all the files abc_url and xyz_url are basically same url. The only thing that changes is the domain.
What I am looking to is,
Put all urls in separate file called app_one_urls.py and app_two_urls.py
Find a way to include this app_one_urls.py and app_two_urls.py in my development, staging and productions file
The final outcome can be as follows:
app_one_urls.py
abc_url = '{0}/some-url/'.format(app_one)
app_two_urls.py
xyz_url = '{0}/some-url/'.format(app_two)
are two separate files
in development.py I intend to do following
app_one = 'http://localhost:8000'
app_two = 'http://localhost:9999'
somehow get urls from app_one_urls and app_two_urls
Is it possible and feasible? If yes, I need help in understanding how.

There is no need to maintain separate files, you can define the configuration in dictionary, with key based on the environment type.
Here I am demonstrating based on hostname, as hostnames of my server differs like: my-host-prod, my-host-staging, my-host-dev. You may use the condition which uniquely defines your server.
import socket
def get_url_conf(my_host=socket.gethostname()):
def get_conf_setting(env_type):
return {'prod': {'app1': 'app1_prod_url',
'app2': 'app2_prod_url'},
'staging': {'app1': 'app1_staging_url',
'app2': 'app2_staging_url'},
'dev': {'app1': 'app1_dev_url',
'app2': 'app2_dev_url'},
'local': {'app1': 'app1_local_url',
'app2': 'app2_local_url'}
}[env_type]
if my_host.endswith('-prod'):
server_key = 'prod'
elif my_host.endswith('-staging'):
server_key = 'prod'
elif my_host.endswith('-dev'):
server_key = 'dev'
else: # In case someone is running on local system
server_key = 'local'
return get_conf_setting(server_key)
Now in your settings file, you may call these as:
abc_url = '{0}/some-url/'.format(get_url_conf()[`app1`])
xyz_url = '{0}/some-url/'.format(get_url_conf()[`app2`])

Yes, this is feasible.
You will need to arrange your settings as a module:
settings/
/__init__.py
/development.py
/staging.py
/production.py
/base.py
Contents of base.py :
SOME_ENVIRONMENT_INDEPENDENT_SETTING = 1
Contents of development.py:
from base import *
SOME_DEVELOPMENT_SETTING = 1
app_one = 'http://localhost:8000'
app_two = 'http://localhost:9999'
Content of production.py:
from base import *
SOME_PRODUCTION_SETTING = 1
app_one = 'http://production.xyz.abc.com'
app_two = 'http://production.pqr.abc.com'
Content of __init__.py:
import os
#If you want to pull environment from an environment variable
#ENVIRONMENT = os.environ.get('CURR_ENV','PROD')
ENVIRONMENT = "DEVELOPMENT"
if ENVIRONMENT == "PRODUCTION" :
try:
from production import *
except:
pass
elif ENVIRONMENT == "DEVELOPMENT" :
try:
from development import *
except:
pass
elif ENVIRONMENT == "STAGING":
try:
from staging import *
except:
pass
elif ENVIRONMENT.lower() == "DEVEL_LOCAL".lower():
try:
from devel_local import *
except:
pass
elif ENVIRONMENT.lower() == "PROD_PP".lower():
try:
from prod_pp import *
except:
pass
#common variables which change based on environment
abc_url = '{0}/some-url/'.format(app_one)
xyz_url = '{0}/some-url/'.format(app_two)

mocking a variable inside a file

I have a file named
a.py
file = "/home/test/abc.txt"
I am working on creating a unittest for another file which takes value of this file variable from a.py
How can I mock this variable name to any dummy file for example?
file = "/tmp/a.txt"

Use mock.patch:
import mock
#mock.patch('a.file', '/tmp/a.txt')
def test_a():
assert thing_that_uses_a_file == '/tmp/a.txt'

#tbm 's answer is work for me, on python 2.7
config.py
SERVICE_REQUIRED = [
("lrc:/etc/rc2_d/S47pppd", "legacy_run"),
("lrc:/etc/rc2_d/S89PRESERVE", "legacy_run"),
("lrc:/etc/rc2_d/S99sysinfo", "legacy_run"),
("lrc:/etc/rc2_d/S99tcpwindow", "legacy_run"),
("lrc:/etc/rc3_d/S99cpupool", "legacy_run"),
("lrc:/etc/rc3_d/S99logparse", "legacy_run"),
("lrc:/etc/rc3_d/S99nicmon", "legacy_run"),
("lrc:/etc/rc3_d/S99nwmond", "legacy_run")
]
file_a.py
from config import SERVICE_REQUIRED
def demo_func():
for name, state in SERVICE_REQUIRED:
print name, state
...
test_file_a.py
...
class TestFileA(unittest.TestCase):
#mock.patch('file_a.SERVICE_REQUIRED', [
('svc:/system/console-login:vt2', 'online'),
('svc:/system/system-log:rsyslog', 'online'),
('svc:/network/socket-config:default', 'disabled'),
('dump', 'legacy_run'),
('firewall', 'disabled'),
("/'; echo hello;'", 'online')
])
def test_demo_func(self):
print SERVICE_REQUIRED

In your specific case, why not just import a and then a.file = "/tmp/a.txt" ?
Which version of Python are you on? Python 3.x has unittest.mock which is backported to 2.x on pypi.
Anyway, if you are trying to create a context specific mock:
>>> from mock import Mock
>>>
>>> # applies to module imports but doing it as a class here
... class A(object):
... file = 'xyz'
...
>>> some_a = Mock(file='abc')
>>> some_a.file
'abc'
>>>
>>> actual_a = A()
>>> actual_a.file
'xyz'
>>>
>>>
>>> def some_test():
... A = Mock(file='abc')
... assert A.file == 'abc'
... assert A.file != 'xyz'
...
>>> some_test()
>>> # no assertion error
Are you trying to mock it at import time? Based on another SO answer:
>>> import sys
>>> sys.modules['A'] = Mock(file='abc')
>>> import A
>>>
>>> A.file
'abc'
>>> mocked_a = A
>>> mocked_a.file
'abc'
>>>