If you have seen my other question you will know I am having a very hard time at the moment with unit tests in Python. Two days of trying and I've made no progress.
In my method which is part of a Class there is several calls to a DAL.
car_registration = self.dal.cars.get_by_registration('121D121')
This DAL is configured in the base class. I want to completely overrid/mock these calls when running my unit tests and instead return predefined responses so I can continue with the method and ensure everything works as expected.
The method starts of with:
def change_registration(self):
body = json.loads(self.request.body)
registration = body['registration']
car = self.dal.cars.get_by_registration(registration)
My Python test file at the moment is:
class CarTestCase(unittest.TestCase):
def setUp(self):
self.car_controller = CarController()
def test_change_registrations(self):
self.car_controller.dal.cars.get_by_registration = MagicMock(return_value=3)
response = self.car_controller.change_registration()
I am expecting to get the response 3. However, an error is being thrown.
AttributeError: 'CarController' object has no attribute '_py_object'
It appears the mocking isn't working and it still trying to use the main DAL which isn't fully set up when using the unit tests. How do I prevent it for looking for the actual DAL but instead mocks?
I think you are not showing us the code that triggers the error because there is nothing wrong with your strategy. Using some imagination to mimic code we don't have I can write this and it runs with no problem:
import unittest
from unittest.mock import MagicMock
class CarList():
def get_by_registration(self, registration):
pass
class Dal:
def __init__(self):
self.cars = CarList()
pass
class CarController:
def __init__(self):
self.dal = Dal()
def change_registration(self):
registration = None
car = self.dal.cars.get_by_registration(registration)
return car
class CarTestCase(unittest.TestCase):
def setUp(self):
self.car_controller = CarController()
def test_change_registrations(self):
self.car_controller.dal.cars.get_by_registration =\
MagicMock(return_value=3)
result = self.car_controller.change_registration()
self.assertEqual(result, 3)
unittest.main()
Here my example:
# test_tool_file.py
import unittest
from unittest.mock import patch, Mock, call
import test_tools_file
class MyObject():
def __init__(self, data):
self.data = data
def get_data(self):
return self.data
def count(self):
return len(self.get_data())
class TestFile(unittest.TestCase):
""" Cas de tests.
"""
#patch("test_tools_file.MyObject.get_data")
def test_1(self, mock_get):
""" test_1
"""
mock_get.return_value = [1,2,3,4,5,6]
obj = MyObject(["12", "13"])
result = obj.count()
self.assertEqual(result, 6)
Personal opinion: I would suggest you to start simple. Use anything 'magic' once you understand what it provides you over the non-magic way. Using a non-magic solution tends to be easier to understand.
I think you have multiple simple solutions at hand. For what you tried to achieve, instead of:
self.car_controller.dal.cars.get_by_registration = MagicMock(return_value=3)
You could try:
self.car_controller.dal.cars.get_by_registration = lambda: 3
But you mentioned you want to replace all methods. In fact I would consider a 'simple' dependency injection. If you find it difficult to test, it might be a sign that the another design would be better (that's the idea of TDD - Test Driven Design). A simple dependency injection is one where you for example just pass dal to the constructor of CarController.
Here is a complete example with some variations of the test:
from unittest import TestCase
from unittest.mock import Mock
class SomeDal(object):
def get_something(self):
return 'something'
class SomeController(object):
def __init__(self, dal):
self.dal = dal
def upper_something(self):
return self.dal.get_something().upper()
class TestSomeController(TestCase):
def test_using_simple_patch(self):
controller = SomeController(SomeDal())
controller.dal.get_something = lambda: 'anything'
assert controller.upper_something() == 'ANYTHING'
def test_using_simple_patch_and_injection(self):
dal = SomeDal()
dal.get_something = lambda: 'anything'
controller = SomeController(dal)
assert controller.upper_something() == 'ANYTHING'
def test_using_simple_mock_class(self):
class MockDal(object):
def get_something(self):
return 'anything'
controller = SomeController(MockDal())
assert controller.upper_something() == 'ANYTHING'
def test_using_semi_magic_mock(self):
mock_dal = Mock(spec=SomeDal)
mock_dal.get_something.return_value = 'anything'
controller = SomeController(mock_dal)
assert controller.upper_something() == 'ANYTHING'
Related
Given this example code where we have a series of log processors, I can't help feeling there ought to be a more pythonic/efficient way of deciding which log processor to use to process some data:
class Component1ErrorLogProcessor:
def process(logToProcess):
# Do something with the logs
pass
class Component2ErrorLogProcessor:
def process(logToProcess):
# Do something with the logs
pass
class LogProcessor:
def __init__(self):
self.component1 = Component1ErrorLogProcessor()
self.component2 = Component2ErrorLogProcessor()
def process_line(self, line, component):
if component == "Component1Log-" or component == "[Component1]":
self.component1.process_errors(line)
elif component == "Component2Log-" or component == "[Component2]":
self.component2.process_errors(line)
I'd personally use the idea of registry, so you map each class to component names.
There are a bunch of different ways to go about this, here's a quick example by using a base class:
class ComponentLogProcessor(object):
_Mapping = {}
#classmethod
def register(cls, *component_names):
for name in component_names:
cls._Mapping[name] = cls
#classmethod
def cls_from_component(cls, component):
return cls._Mapping[component]
class Component1ErrorLogProcessor(ComponentLogProcessor):
def process(logToProcess):
# Do something with the logs
pass
Component1ErrorLogProcessor.register('Component1Log-', '[Component1]')
class Component2ErrorLogProcessor(ComponentLogProcessor):
def process(logToProcess):
# Do something with the logs
pass
Component2ErrorLogProcessor.register('Component2Log-', '[Component2]')
class LogProcessor:
def process_line(self, line, component):
ComponentLogProcessor.cls_from_component(component).process_errors(line)
I recently started to work with Python's classes, since I need to work with it through the use of OTree, a Python framework used for online experiment.
In one file, I define the pages that I want to be created, using classes. So essentially, in the OTree system, each class corresponds to a new page. The thing is, all pages (so classes) are basically the same, at the exception to some two parameters, as shown in the following code:
class Task1(Page):
form_model = 'player'
form_fields = ['Envie_WordsList_Toy']
def is_displayed(self):
return self.round_number == self.participant.vars['task_rounds'][1]
def vars_for_template(player):
WordsList_Toy= Constants.WordsList_Toy.copy()
random.shuffle(WordsList_Toy)
return dict(
WordsList_Toy=WordsList_Toy
)
#staticmethod
def live_method(player, data):
player.WTP_WordsList_Toy = int(data)
def before_next_page(self):
self.participant.vars['Envie_WordsList_Toy'] = self.player.Envie_WordsList_Toy
self.participant.vars['WTP_WordsList_Toy'] = self.player.WTP_WordsList_Toy
So here, the only thing that would change would be the name of the class, as well as the suffix of the variable WordsList_ used throughout this code, which is Toy.
Naively, what I tried to do is to define a function that would take those two parameters, such as this:
def page_creation(Task_Number,name_type):
class Task+str(Task_Number)(Page):
form_model = 'player'
form_fields = ['Envie_WordsList_'+str(name_type)]
def is_displayed(self):
return self.round_number == self.participant.vars['task_rounds'][1]
def vars_for_template(player):
WordsList_+str(name_type) = Constants.WordsList+str(name_type).copy()
random.shuffle(WordsList_+str(name_type))
return dict(
WordsList_+str(name_type)=WordsList_+str(name_type)
)
#staticmethod
def live_method(player, data):
player.WTP_WordsList_+str(name_type) = int(data)
def before_next_page(self):
self.participant.vars['Envie_WordsList_+str(name_type)'] = self.player.Envie_WordsList_+str(name_type)
self.participant.vars['WTP_WordsList_+str(name_type)'] = self.player.WTP_WordsList_+str(name_type)
Obviously, it does not work since I have the feeling that it is not possible to construct variables (or classes identifier) this way. I just started to really work on Python some weeks ago, so some of its aspects might escape me still. Could you help me on this issue? Thank you.
You can generate dynamic classes using the type constructor:
MyClass = type("MyClass", (BaseClass1, BaseClass2), {"attr1": "value1", ...})
Thus, according to your case, that would be:
cls = type(f"Task{TaskNumber}", (Page, ), {"form_fields": [f"Envive_WordList_{name_type}"], ...})
Note that you still have to construct your common methods like __init__, is_displayed and so on, as inner functions of the class factory:
def class_factory(*args, **kwargs):
...
def is_displayed(self):
return self.round_number == self.participant.vars['task_rounds']
def vars_for_template(player):
...
# Classmethod wrapping is done below
def live_method(player, data):
...
cls = type(..., {
"is_displayed": is_displayed,
"vars_for_template": vars_for_template,
"live_method": classmethod(live_method),
...,
}
#classmethod could be used as a function - {"live_method": classmethod(my_method)}
I have this main class
def main(args):
if type == train_pipeline_type:
strategy = TrainPipelineStrategy()
else:
strategy = TestPipelineStrategy()
for table in fetch_table_information_by_region(region):
split_required = DataUtils.load_from_dict(table, "split_required")
if split_required:
strategy.split(spark=spark, table_name=table_name,
data_loc=filtered_data_location, partition_column=partition_column,
split_output_dir= split_output_dir)
logger.info("Data Split for table : {} completed".format(table_name))
My TrainPipelineStrategy, and TestPipelineStrategy looks like this -
class PipelineTypeStrategy(object):
def partition_data(self, x):
# Something
def prepare_split_data(self, y):
# Something
def write_split_data(self, z):
# Something
def split(self, p):
# Something
class TrainPipelineStrategy(PipelineTypeStrategy):
""""""
class TestPipelineStrategy(PipelineTypeStrategy):
def write_split_data(self, y):
# Something else
My test case -
I need to test how many times split is called by mocking split functionality in main method.
Here is what i have tried -
#patch('module.PipelineTypeStrategy.TrainPipelineStrategy')
def test_split_data_main_split_data_call_count(self, fake_train):
fake_train_functions = mock.Mock()
fake_train_functions.split.return_value = None
fake_train.return_value = fake_train_functions
test_args = ["", "--x=6"]
SplitData.main(args=test_args)
assert fake_train_functions.split.call_count == 10
When i try to run my test, it creates the mock but ultimately ends up calling the actual split function. What am i doing wrong ?
The main issue with this code is that the way you set up the patch would be if TrainPipelineStrategy were a nested class of PipelineTypeStrategy, but TrainPipelineStrategy is a subclass of PipelineTypeStrategy.
Since TrainPipelineStrategy inherits from PipelineTypeStrategy it has access to split directly, so you can patch split without any reference to PipelineTypeStrategy (unless you specifically want to patch the version of split defined in PipelineTypeStrategy).
However, if you just want to mock the split method of the PipelineTypeStrategy class, you should use the patch.object decorator to mock just split instead of mocking the whole class as it's a bit more clean. Here's an example:
class TestClass(unittest.TestCase):
#patch.object(TrainPipelineStrategy, 'split', return_value=None)
def test_split_data_main_split_data_call_count(self, mock_split):
test_args = ["", "--x=6"]
SplitData.main(args=test_args)
self.assertEqual(mock_split.call_count, 10)
Assuming the following setup:
class A:
def __init__(self, nodes):
self.nodes=nodes
def update(self, bool_a=True):
if bool_a:
for n in self.nodes:
if hasattr(self.nodes[n], 'update'):
self.nodes[n].update()
class B:
def __init__(self, int_attr=5):
self.int_attr=int_attr
def update(self):
self.int_attr = 0
Let us assume that the list of nodes in class A is in fact a list of instances of class B.
How do I write a unit test for the update method of class A to check whether the update method of each class B node contained in self.nodes of class A was called?
In a more general setup, let us assume that there are multiple classes implementing update method and can be nodes within self.nodes of class A. How do I check that all update methods of self.nodes members were called?
I have tried the following, unsuccessfully:
mock_obj = MagicMock()
#patch('module.A.update', return_value=mock_obj)
def test_update(self, mock_obj):
nodes = {}
nodes['first'] = B(int_attr=1)
nodes['second'] = B(int_attr=2)
test_A = module.A(nodes=nodes)
test_A.update(bool_A=True)
self.assertTrue(mock_obj.called)
as suggested in mocking a function within a class method.
Edit: If we assume this particular case:
import unittest
import mock
from unittest import TestCase
class A:
def __init__(self, nodes):
self.nodes=nodes
def update(self, bool_a=True):
if bool_a:
to_update = [n for n in self.nodes]
while len(to_update) > 0:
if hasattr(self.nodes[to_update[-1]], 'update'):
self.nodes[to_update[-1]].update()
print('Update called.')
if self.nodes[to_update[-1]].is_updated:
to_update.pop()
class B:
def __init__(self, int_attr=5):
self.int_attr=int_attr
self.is_updated = False
def update(self):
self.int_attr = 0
self.is_updated = True
class TestEnsemble(TestCase):
def setUp(self):
self.b1 = B(1)
self.b2 = B(2)
self.b3 = B(3)
self.nodes = {}
self.nodes['1'] = self.b1
self.nodes['2'] = self.b2
self.nodes['3'] = self.b3
self.a = A(self.nodes)
#mock.patch('module.B.update')
def test_update(self, mock_update):
mock_update.return_value = None
self.a.update()
with self.subTest():
self.assertEqual(mock_update.call_count, 3)
Running unittest for this case results in an endless loop, since is_updated attribute never gets set to True because the update method of class B is mocked. How to measure the amount of time B.update was called within A.update in this case?
Update:
Tried this:
#mock.patch('dummy_script.B')
def test_update(self, mock_B):
self.a.update()
with self.subTest():
self.assertEqual(mock_B.update.call_count, 3)
The update function indeed runs 3 times now (I see it in the console output, as "Update called." is printed out three times), but the call_count of update method stays zero. Am I inspecting the wrong attribute / object?
How do I write a unit test for TestA.test_update() to see if B.update() was called?
This is just to give some ideas.
import mock
import unittest
import A
import B
class TestB(unittest.TestCase):
# only mock away update method of class B, this is python2 syntax
#mock.patch.object(B, 'update')
def test_update(self, mockb_update):
# B.update() does not return anything
mockb_update.return_value = None
nodes = {}
nodes['first'] = B(int_attr=1)
nodes['second'] = B(int_attr=2)
test_A = A(nodes)
test_A.update(bool_A=True)
self.assertTrue(mockb_update.called)
How do I check all B.update() were called for all A.nodes?
# same everthing except this
self.assertEqual(mockb_update.call_count, 2)
Running into an endless loop when B.is_udpated is not mocked after OP updated the code
mock a B.is_updated inside __init__ or mock class __init__ is a topic more complicated that the original post
Here is a few thoughts, B.is_updated cannot be just mock.patch, it is only available after the class B is initiated. so the option is to
a) mock B.__init__, or a class constructor
b) mock the whole class B, which is easier in your case, set is_updated to be True, will end the endless loop.
Combining answers from #Gang and #jonrsharpe, the following code snippets solve the questions I have asked above:
How do I write a unit test for TestA.test_update() to see if B.update() was called?
See #Gangs' answer.
How do I check all B.update() were called for all A.nodes?
See #Gangs' answer.
Running into an endless loop when B.is_udpated is not mocked after OP updated the code
As #jonrsharpe suggested, here the solution is to create one mock object per B instance in nodes and check for function calls separately:
class TestA(TestCase):
#mock.patch('module.B')
#mock.patch('module.B')
#mock.patch('module.B')
def test_update(self, mock_B1, mock_B2, mock_B3):
nodes = {}
nodes['1'] = mock_B1
nodes['2'] = mock_B2
nodes['3'] = mock_B3
a = A(nodes)
a.update()
with self.subTest():
self.assertEqual(mock_B1.update.call_count, 1)
with self.subTest():
self.assertEqual(mock_B2.update.call_count, 1)
with self.subTest():
self.assertEqual(mock_B3.update.call_count, 1)
Additionally, if you want for some reason to execute mocked functions (in case they set some flags or variables which affect runtime), one can write a test like this:
def test_fit_skip_ancestors_all(self):
nodes = {}
nodes['1'] = mock_B1
nodes['2'] = mock_B2
nodes['3'] = mock_B3
a = A(nodes)
with mock.patch.object(A.nodes['1'],'update',wraps=A.nodes['1'].update) as mock_B1, \
mock.patch.object(A.nodes['2'], 'update', wraps=A.nodes['2'].update) as mock_B2, \
mock.patch.object(A.nodes['3'], 'update', wraps=A.nodes['3'].update) as mock_B3:
a.update()
with self.subTest():
self.assertEqual(mock_B1.call_count, 1)
with self.subTest():
self.assertEqual(mock_B2.call_count, 1)
with self.subTest():
self.assertEqual(mock_B3.call_count, 1)
Here is an example of failure from a shell.
>>> from traits.api import Dict
>>> d=Dict()
>>> d['Foo']='BAR'
Traceback (most recent call last):
File "<stdin>", line 1, in <module>
TypeError: 'Dict' object does not support item assignment
I have been searching all over the web, and there is no indication of how to use Dict.
I am trying to write a simple app that displays the contents of a python dictionary. This link (Defining view elements from dictionary elements in TraitsUI) was moderately helpful except for the fact that the dictionary gets updated on some poll_interval and if I use the solution there (wrapping a normal python dict in a class derived from HasTraits) the display does not update when the underlying dictionary gets updated.
Here are the relevant parts of what I have right now. The last class can pretty much be ignored, the only reason I included it is to help understand how I intend to use the Dict.
pyNetObjDisplay.run_ext() gets called once per loop from the base classes run() method
class DictContainer(HasTraits):
_dict = {}
def __getattr__(self, key):
return self._dict[key]
def __getitem__(self, key):
return self._dict[key]
def __setitem__(self, key, value):
self._dict[key] = value
def __delitem__(self, key, value):
del self._dict[key]
def __str__(self):
return self._dict.__str__()
def __repr__(self):
return self._dict.__repr__()
def has_key(self, key):
return self._dict.has_key(key)
class displayWindow(HasTraits):
_remote_data = Instance(DictContainer)
_messages = Str('', desc='Field to display messages to the user.', label='Messages', multi_line=True)
def __remote_data_default(self):
tempDict = DictContainer()
tempDict._dict = Dict
#tempDict['FOO'] = 'BAR'
sys.stderr.write('SETTING DEFAULT DICTIONARY:\t%s\n' % tempDict)
return tempDict
def __messages_default(self):
tempStr = Str()
tempStr = ''
return tempStr
def traits_view(self):
return View(
Item('object._remote_data', editor=ValueEditor()),
Item('object._messages'),
resizable=True
)
class pyNetObjDisplay(pyNetObject.pyNetObjPubClient):
'''A derived pyNetObjPubClient that stores remote data in a dictionary and displays it using traitsui.'''
def __init__(self, hostname='localhost', port=54322, service='pyNetObject', poll_int=10.0):
self._display = displayWindow()
self.poll_int = poll_int
super(pyNetObjDisplay, self).__init__(hostname, port, service)
self._ui_running = False
self._ui_pid = 0
### For Testing Only, REMOVE THESE LINES ###
self.connect()
self.ns_subscribe(service, 'FOO', poll_int)
self.ns_subscribe(service, 'BAR', poll_int)
self.ns_subscribe(service, 'BAZ', poll_int)
############################################
def run_ext(self):
if not self._ui_running:
self._ui_running = True
self._ui_pid = os.fork()
if not self._ui_pid:
time.sleep(1.25*self.poll_int)
self._display.configure_traits()
for ((service, namespace, key), value) in self._object_buffer:
sys.stderr.write('TEST:\t' + str(self._display._remote_data) + '\n')
if not self._display._remote_data.has_key(service):
self._display._remote_data[service] = {}
if not self._display._remote_data[service].has_key(namespace):
#self._remote_data[service][namespace] = {}
self._display._remote_data[service][namespace] = {}
self._display._remote_data[service][namespace][key] = value
msg = 'Got Published ((service, namespace, key), value) pair:\t((%s, %s, %s), %s)\n' % (service, namespace, key, value)
sys.stderr.write(msg)
self._display._messages += msg
sys.stderr.write('REMOTE DATA:\n' + str(self._display._remote_data)
self._object_buffer = []
I think your basic problem has to do with notification issues for traits that live outside the model object, and not with "how to access those objects" per se [edit: actually no this is not your problem at all! But it is what I thought you were trying to do when I read your question with my biased mentality towards problems I have seen before and in any case my suggested solution will still work]. I have run into this sort of problem recently because of how I decided to design my program (with code describing a GUI separated modularly from the very complex sets of data that it can contain). You may have found my other questions, as you found the first one.
Having lots of data live in a complex data hierarchy away from the GUI is not the design that traitsui has in mind for your application and it causes all kinds of problems with notifications. Having a flatter design where GUI information is integrated into the different parts of your program more directly is the design solution.
I think that various workarounds might be possible for this in general (I have used some for instance in enabled_when listening outside model object) that don't involve dictionaries. I'm not sure what the most design friendly solution to your problem with dictionaries is, but one thing that works and doesn't interfere a lot with your design (but it is still a "somewhat annoying" solution) is to make everything in a dictionary be a HasTraits and thus tag it as listenable. Like so:
from traits.api import *
from traitsui.api import *
from traitsui.ui_editors.array_view_editor import ArrayViewEditor
import numpy as np
class DContainer(HasTraits):
_dict=Dict
def __getattr__(self, k):
if k in self._dict:
return self._dict[k]
class DItem(HasTraits):
_item=Any
def __init__(self,item):
super(DItem,self).__init__()
self._item=item
def setitem(self,val):
self._item=val
def getitem(self):
return self._item
def traits_view(self):
return View(Item('_item',editor=ArrayViewEditor()))
class LargeApplication(HasTraits):
d=Instance(DContainer)
stupid_listener=Any
bn=Button('CLICKME')
def _d_default(self):
d=DContainer()
d._dict={'a_stat':DItem(np.random.random((10,1))),
'b_stat':DItem(np.random.random((10,10)))}
return d
def traits_view(self):
v=View(
Item('object.d.a_stat',editor=InstanceEditor(),style='custom'),
Item('bn'),
height=500,width=500)
return v
def _bn_fired(self):
self.d.a_stat.setitem(np.random.random((10,1)))
LargeApplication().configure_traits()
Okay, I found the answer (kindof) in this question: Traits List not reporting items added or removed
when including Dict or List objects as attributes in a class one should NOT do it this way:
class Foo(HasTraits):
def __init__(self):
### This will not work as expected!
self.bar = Dict(desc='Description.', label='Name', value={})
Instead do this:
class Foo(HasTraits):
def __init__(self):
self.add_trait('bar', Dict(desc='Description.', label='Name', value={}) )
Now the following will work:
>>> f = Foo()
>>> f.bar['baz']='boo'
>>> f.bar['baz']
'boo'
Unfortunately for some reason the GUI generated with configure_traits() does not update it's view when the underlying data changes. Here is some test code that demonstrates the problem:
import os
import time
import sys
from traits.api import HasTraits, Str, Dict
from traitsui.api import View, Item, ValueEditor
class displayWindow(HasTraits):
def __init__(self, **traits):
super(displayWindow, self).__init__(**traits)
self.add_trait('_remote_data', Dict(desc='Dictionary to store remote data in.', label='Data', value={}) )
self.add_trait('_messages', Str(desc='Field to display messages to the user.', label='Messages', multi_line=True, value='') )
def traits_view(self):
return View(
Item('object._remote_data', editor=ValueEditor()),
Item('object._messages'),
resizable=True
)
class testObj(object):
def __init__(self):
super(testObj, self).__init__()
self._display = displayWindow()
self._ui_pid = 0
def run(self):
### Run the GUI in the background
self._ui_pid = os.fork()
if not self._ui_pid:
self._display.configure_traits()
i = 0
while True:
self._display._remote_data[str(i)] = i
msg = 'Added (key,value):\t("%s", %s)\n' % (str(i), i, )
self._display._messages += msg
sys.stderr.write(msg)
time.sleep(5.0)
i+=1
if __name__ == '__main__':
f = testObj()
f.run()