We Keep Coding

importlib.reload() not realoading

I am trying to create a leetcode like online judge. I need to reload the submission module but import.reload() does not work.
The code:
class Test:
current_exercise = None
current_name = None
def _import(self, exercise):
exercise = 'exercise' # for testing
if exercise == self.current_name:
module = sys.modules[f'puzzles.{exercise}']
self.current_exercise = importlib.reload(module) # <---- not working
else:
self.current_name = exercise
self.current_exercise = __import__(f'puzzles.{exercise}').exercise
def _test(self, exercise):
solution = self._import(exercise)
print(self.current_exercise.main())
if __name__=='__main__':
import shutil
t= Test()
# first run
t._test('exercise')
# copy another solution.py for reload test
shutil.copy(f"./puzzles/other_exercise/solution.py", f"./puzzles/exercise/solution.py")
# second run
t._test('exercise')
My directory;
.
├── codetest.py
├── puzzles
│   ├── __init__.py
│   ├── exercise
│   │   ├── __init__.py
│   │   ├── solution.py
│ ├── other_exercise
│ │ ├── __init__.py
│ │ ├── solution.py
exercise/solution.py:
def main():
print('EXERCISE')
exercise/init.py
from .solution import main
from .test import cases
other_exercise/solution.py:
def main():
print('OTHER EXERCISE')
Output:
> EXERCISE
> EXERCISE # <--- not sucessfull, should be 'OTHER EXERCISE'

This works:
import sys
import time
import importlib
class Test:
current_exercise = None
current_name = None
def _import(self, exercise):
if exercise == self.current_name:
self.current_exercise.solution = importlib.reload(self.current_exercise.solution)
else:
self.current_name = exercise
self.current_exercise = importlib.import_module(f'puzzles.{exercise}')
print('mod',self.current_exercise)
print('nam',self.current_exercise.__name__)
print('fil',self.current_exercise.__file__)
print('pkg',self.current_exercise.__package__)
def _test(self, exercise):
solution = self._import(exercise)
print(self.current_exercise.solution.main())
if __name__=='__main__':
import shutil
shutil.copy(f"./puzzles/exercise/solution.0", f"./puzzles/exercise/solution.py")
t= Test()
# first run
t._test('exercise')
# copy another solution.py for reload test
shutil.copy("./puzzles/other_exercise/solution.py", "./puzzles/exercise/solution.py")
print(open("./puzzles/exercise/solution.py").read())
# second run
t._test('exercise')

I went with an alternative; load solution.py as a text and create a module from that string. The module is not registered in sys.modules and can be overwritten. However imp is deprecated.
import imp
class Test:
current_exercise = None
def _import(self, exercise):
# load module code
with open(f'./puzzles/{exercise}/solution.py') as f:
code = f.read()
# register/create the module
self.current_exercise = imp.new_module('mymodule')
# import/fill the module
exec(code, self.current_exercise.__dict__)
def _test(self, exercise):
self._import(exercise)
print(self.current_exercise.main())

python AttributeError not finding a class within a module I am importing

I have a package containing a module called rigfuncs3.py
SDNpackage/
__init__.py
rigfuncs3.py
Code:
class rig():
def __init__(self, rigNumber, leftNozPlate, rightNozPlate, bottomNozPlate, row1_centre_focus):
self.rigNumber = rigNumber
self.leftNozPlate = leftNozPlate
self.rightNozPlate = rightNozPlate
self.bottomNozPlate = bottomNozPlate
self.row1_centre_focus = row1_centre_focus
def getRigName(self):
return self.rigNumber
A script called play_04.py imports the module rigfuncs from the package.
Code:
from SDNpack2 import rigfuncs3
instantiation = rigfuncs3.rig(1,1000.0, 2000.0, 3000.0, 4000.0)
print(dir(instantiation))
rig_num = instantiation.getRigName()
When run play_04 I get the error:
AttributeError: 'rig' object has no attribute 'getRigName'
I have no idea why, any ideas?

I still cannot replicate the problem you're having.
I've tried to copy your stated folder-structure like this:
C:.
│ play_04.py
│
└───SDNpack2
rigfuncs3.py
__init__.py <--- empty file
rigfuncs3.py:
class rig():
def __init__(self, rigNumber, leftNozPlate, rightNozPlate, bottomNozPlate, row1_centre_focus):
self.rigNumber = rigNumber
self.leftNozPlate = leftNozPlate
self.rightNozPlate = rightNozPlate
self.bottomNozPlate = bottomNozPlate
self.row1_centre_focus = row1_centre_focus
def getRigName(self):
return self.rigNumber
play_04.py:
from SDNpack2 import rigfuncs3
instantiation = rigfuncs3.rig(1, 1000.0, 2000.0, 3000.0, 4000.0)
print(instantiation)
print(instantiation.__dict__)
rig_num = instantiation.getRigName()
print(rig_num)
I change your call of dir(instantiation) into instantiation.__dict__ to just show the attributes of the object, not all built-in variables.
Output:
<SDNpack2.rigfuncs3.rig object at 0x02B1E628>
{'rigNumber': 1, 'leftNozPlate': 1000.0, 'rightNozPlate': 2000.0, 'bottomNozPlate': 3000.0, 'row1_centre_focus': 4000.0}
1
The only way I can replicate the problem in of itself is if I either change the indentation of getRigName or if I remove it all together from the class.
Your code works the way you have coded it, I see no actual problem here.

Populating DATA in the help() call

If I have the following directory structure:
handy/
- __init__.py
- utils.py
- dir1
- __init__.py
- script.py
I can populate DATA in help() by writing non-keywords into the __init__.py file, for example:
# __init__.py
hello = "xyz"
other = "z"
variables = 1
Now when I do help(handy), it shows:
DATA
hello = 'xyz'
other = 'z'
variables = 1
Are there any other ways to populate the help DATA from outside of the top-level __init__.py file, or is that the only way?

I'm not sure what you have in mind, but since handy/__init__.py is an executable script, you could do something like this:
__init__.py:
from .utils import *
hello = "xyz"
other = "z"
variables = 1
utils.py:
UTILS_CONSTANT = 42
def func():
pass
Which would result in:
>>> import handy
>>> help(handy)
Help on package handy:
NAME
handy
PACKAGE CONTENTS
utils
DATA
UTILS_CONSTANT = 42
hello = 'xyz'
other = 'z'
variables = 1
FILE
c:\stack overflow\handy\__init__.py
>>>
to what help(handy) displays.

Parametrize the test based on the list test-data from a json file

Is there a way to parametrize a test, when test has a list of different/multiple test-data?
example_test_data.json
{ "test_one" : [1,2,3], # this is the case, where the `test_one` test need to be parametrize.
"test_two" : "split",
"test_three" : {"three":3},
"test_four" : {"four":4},
"test_set_comparison" : "1234"
}
Directory structure:
main --
conftest.py # conftest file for my fixtures
testcases
project_1
(contains these files -- test_suite_1.py, config.json)
project_2
(contains these files -- test_suite_2.py, config.json)
workflows
libs
Using below code in conftest.py at top directory level, able to get/map the test data from json file for particular test case.
#pytest.yield_fixture(scope="class", autouse=True)
def test_config(request):
f = pathlib.Path(request.node.fspath.strpath)
print "File : %s" % f
config = f.with_name("config.json")
print "Config json file : %s" % config
with config.open() as fd:
testdata = json.loads(fd.read())
print "test data :", testdata
yield testdata
#pytest.yield_fixture(scope="function", autouse=True)
def config_data(request, test_config):
testdata = test_config
test = request.function.__name__
print "Class Name : %s" % request.cls.__name__
print "Testcase Name : %s" % test
if test in testdata:
test_args = testdata[test]
yield test_args
else:
yield {}
In my case:
#pytest.yield_fixture(scope="function", autouse=True)
def config_data(request, test_config):
testdata = test_config
test = request.function.__name__
print "Class Name : %s" % request.cls.__name__
print "Testcase Name : %s" % test
if test in testdata:
test_args = testdata[test]
if isinstance(test_args, list):
# How to parametrize the test
# yield test_args
else:
yield {}

I would handle the special parametrization case in pytest_generate_tests hook:
# conftest.py
import json
import pathlib
import pytest
#pytest.fixture(scope="class")
def test_config(request):
f = pathlib.Path(request.node.fspath.strpath)
config = f.with_name("config.json")
with config.open() as fd:
testdata = json.loads(fd.read())
yield testdata
#pytest.fixture(scope="function")
def config_data(request, test_config):
testdata = test_config
test = request.function.__name__
if test in testdata:
test_args = testdata[test]
yield test_args
else:
yield {}
def pytest_generate_tests(metafunc):
if 'config_data' not in metafunc.fixturenames:
return
config = pathlib.Path(metafunc.module.__file__).with_name('config.json')
testdata = json.loads(config.read_text())
param = testdata.get(metafunc.function.__name__, None)
if isinstance(param, list):
metafunc.parametrize('config_data', param)
Some notes: yield_fixture is deprecated so I replaced it with plain fixture. Also, you don't need autouse=True in fixtures that return values - you call them anyway.
Example tests and configs I used:
# testcases/project_1/config.json
{
"test_one": [1, 2, 3],
"test_two": "split"
}
# testcases/project_1/test_suite_1.py
def test_one(config_data):
assert config_data >= 0
def test_two(config_data):
assert config_data == 'split'
# testcases/project_2/config.json
{
"test_three": {"three": 3},
"test_four": {"four": 4}
}
# testcases/project_2/test_suite_2.py
def test_three(config_data):
assert config_data['three'] == 3
def test_four(config_data):
assert config_data['four'] == 4
Running the tests yields:
$ pytest -vs
============================== test session starts ================================
platform linux -- Python 3.6.5, pytest-3.4.1, py-1.5.3, pluggy-0.6.0 --
/data/gentoo64/usr/bin/python3.6
cachedir: .pytest_cache
rootdir: /data/gentoo64/home/u0_a82/projects/stackoverflow/so-50815777, inifile:
plugins: mock-1.6.3, cov-2.5.1
collected 6 items
testcases/project_1/test_suite_1.py::test_one[1] PASSED
testcases/project_1/test_suite_1.py::test_one[2] PASSED
testcases/project_1/test_suite_1.py::test_one[3] PASSED
testcases/project_1/test_suite_1.py::test_two PASSED
testcases/project_2/test_suite_2.py::test_three PASSED
testcases/project_2/test_suite_2.py::test_four PASSED
============================ 6 passed in 0.12 seconds =============================

Same tests over many similar data files

With python and unittest I have this structure of test directory:
tests/
__init__.py
test_001.py
data/
data_001_in.py
data_001_out.py
where
data_001_in.py : the input data to use in the functions to test
data_001_out.py : the output data expected from the function to test
I have the inputs and outputs in python dictionaries because it is easier for me than using json, sqlite, etc.
I try use a set of input/output data with the same format and apply the test over each pair of data:
tests/
__init__.py
test_001.py
data/
data_001_in.py
data_001_out.py
data_002_in.py
data_002_out.py
data_003_in.py
data_003_out.py
Is there any package/approach to make this task more easier?

inspirated in the question nose, unittest.TestCase and metaclass: auto-generated test_* methods not discovered, I solved with a metaclass. First, I change the directory data structure to
├── data
│   ├── __init__.py
│   ├── data001
│   │   ├── __init__.py
│   │   ├── datain.py
│   │   ├── dataout.py
│   └── data002
│   ├── __init__.py
│   ├── datain.py
│   ├── dataout.py
└── metatest.py
Second, I make a metaclass for create new test with the data in the subdirectories and base tests.
import unittest
import os
import copy
def data_dir():
return os.path.join(os.path.dirname(__file__), 'data')
def get_subdirs(dir_name):
""" retorna subdirectorios con path completo"""
subdirs = []
for f in os.listdir(dir_name):
f_path = os.path.join(dir_name, f)
if os.path.isdir(f_path):
subdirs.append(f)
return subdirs
def get_data_subdirs():
return get_subdirs(data_dir())
def data_py_load(file_name):
""" carga diccionario data desde archivo .py """
name = file_name.split('.py')[0]
path_name = 'data.' + name
exec_str = "from {} import *".format(path_name)
exec(exec_str)
return data
class TestDirectories(type):
def __new__(cls, name, bases, attrs):
subdirs = get_data_subdirs()
callables = dict([
(meth_name, meth) for (meth_name, meth) in attrs.items() if
meth_name.startswith('_test')
])
data = {}
for d in subdirs:
data[d] = {}
data[d]['name'] = d
out_path = "{}.dataout.py".format(d)
data[d]['out'] = data_py_load(out_path)
var_path = "{}.datain.py".format(d)
data[d]['in'] = data_py_load(var_path)
for meth_name, meth in callables.items():
for d in subdirs:
new_meth_name = meth_name[1:]
# name of test to add, _test to test
test_name = "{}_{}".format(new_meth_name, d)
# deep copy for dictionaries
testeable = lambda self, func=meth, args=copy.deepcopy(data[d]): func(self, args)
attrs[test_name] = testeable
return type.__new__(cls, name, bases, attrs)
class TestData(unittest.TestCase):
__metaclass__ = TestDirectories
def _test_name(self, data):
in_name = data['in']['name']
out_name = data['out']['name']
print in_name, out_name
self.assertEquals(in_name, out_name)
if __name__ == '__main__':
unittest.main(verbosity=2)
And, when I run
$ python metatest.py
test_name_data001 (__main__.TestData) ... Alice Alice
ok
test_name_data002 (__main__.TestData) ... Bob Bob
ok
----------------------------------------------------------------------
Ran 2 tests in 0.001s
OK