I am trying to understand how to mock in python external dependencies while doing mock methods argument matching and argument capture.
1) Argument matching:
class ExternalDep(object):
def do_heavy_calc(self, anInput):
return 3
class Client(object):
def __init__(self, aDep):
self._dep = aDep
def invokeMe(self, aStrVal):
sum = self._dep.do_heavy_calc(aStrVal)
aNewStrVal = 'new_' + aStrVal
sum += self._dep.do_heavy_calc(aNewStrVal)
class ClientTest(unittest.TestCase):
self.mockDep = MagicMock(name='mockExternalDep', spec_set=ExternalDep)
###
self.mockDep.do_heavy_calc.return_value = 5
### this will be called twice regardless of what parameters are used
### in mockito-python, it is possible to create two diff mocks (by param),like
###
### when(self.mockDep).do_heavy_calc('A').thenReturn(7)
### when(self.mockDep).do_heavy_calc('new_A').thenReturn(11)
###
### QUESTION: how could I archive the same result in MagicMock?
def setUp(self):
self.cut = Client(self.mockDep)
def test_invokeMe(self):
capturedResult = self.cut.invokeMe('A')
self.assertEqual(capturedResult, 10, 'Unexpected sum')
# self.assertEqual(capturedResult, 18, 'Two Stubs did not execute')
2) Argument Capturing
I cannot find good docs or examples on neither MagicMock or mockito-python able to accommodate the following mocking scenario:
class ExternalDep(object):
def save_out(self, anInput):
return 17
class Client(object):
def __init__(self, aDep):
self._dep = aDep
def create(self, aStrVal):
aNewStrVal = 'new_' + aStrVal if aStrVal.startswith('a')
self._dep.save_out(aNewStrVal)
class ClientTest(unittest.TestCase):
self.mockDep = MagicMock(name='mockExternalDep', spec_set=ExternalDep)
###
self.mockDep.save_out.return_value = 5
### this will be called with SOME value BUT how can I capture it?
### mockito-python does not seem to provide an answer to this situation either
### (unline its Java counterpart with ArgumentCaptor capability)
###
### Looking for something conceptually like this (using MagicMock):
### self.mockDep.save_out.argCapture(basestring).return_value = 11
###
### QUESTION: how could I capture value of parameters with which
### 'save_out' is invoked in MagicMock?
def setUp(self):
self.cut = Client(self.mockDep)
def test_create(self):
capturedResult = self.cut.create('Z')
self.assertEqual(capturedResult, 5, 'Unexpected sum')
### now argument will be of different value but we cannot assert on what it is
capturedResult = self.cut.create('a')
self.assertEqual(capturedResult, 5, 'Unexpected sum')
If anyone could show me how to accomplish these two mocking scenarios (using MagicMock), I would be very grateful! (Please ask if something is unclear.)
Something that might help you is to use assert_called_with with a Matcher.
This will allow you to have a finer grain access to the arguments on your calls. i.e.:
>>> def compare(self, other):
... if not type(self) == type(other):
... return False
... if self.a != other.a:
... return False
... if self.b != other.b:
... return False
... return True
>>> class Matcher(object):
def __init__(self, compare, some_obj):
self.compare = compare
self.some_obj = some_obj
def __eq__(self, other):
return self.compare(self.some_obj, other)
>>> match_foo = Matcher(compare, Foo(1, 2))
>>> mock.assert_called_with(match_foo)
Related
I had created a simple example to illustrate my issue. First is the setup say mydummy.py:
class TstObj:
def __init__(self, name):
self.name = name
def search(self):
return self.name
MyData = {}
MyData["object1"] = TstObj("object1")
MyData["object2"] = TstObj("object2")
MyData["object3"] = TstObj("object3")
def getObject1Data():
return MyData["object1"].search()
def getObject2Data():
return MyData["object2"].search()
def getObject3Data():
return MyData["object3"].search()
def getExample():
res = f"{getObject1Data()}{getObject2Data()}{getObject3Data()}"
return res
Here is the test that failed.
def test_get_dummy1():
dummy.MyData = MagicMock()
mydummy.MyData["object1"].search.side_effect = ["obj1"]
mydummy.MyData["object2"].search.side_effect = ["obj2"]
mydummy.MyData["object3"].search.side_effect = ["obj3"]
assert mydummy.getExample() == "obj1obj2obj3"
The above failed with run time error:
/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/unittest/mock.py:1078: StopIteration
Here is the test that passed:
def test_get_dummy2():
dummy.MyData = MagicMock()
mydummy.MyData["object1"].search.side_effect = ["obj1", "obj2", "obj3"]
assert mydummy.getExample() == "obj1obj2obj3"
Am I missing something? I would have expected test_get_dummy1() to work and test_get_dummy2() to fail and not vice versa. Where and how can I find/learn more information about mocking to explain what is going on...
MyData["object1"] is converted to this function call: MyData.__getitem__("object1"). When you call your getExample method, the __getitem__ method is called 3 times with 3 parameters ("object1", "object2", "object3").
To mock the behavior you could have written your test like so:
def test_get_dummy_alternative():
mydummy.MyData = MagicMock()
mydummy.MyData.__getitem__.return_value.search.side_effect = ["obj1", "obj2", "obj3"]
assert mydummy.getExample() == "obj1obj2obj3"
Note the small change from your version: mydummy.MyData["object1"]... became: mydummy.MyData.__getitem__.return_value.... This is the regular MagicMock syntax - we want to to change the return value of the __getitem__ method.
BONUS:
I often struggle with mock syntax and understanding what's happening under the hood. This is why I wrote a helper library: the pytest-mock-generator. It can show you the actual calls made to the mock object.
To use it in your case you could have added this "exploration test":
def test_get_dummy_explore(mg):
mydummy.MyData = MagicMock()
mydummy.getExample()
mg.generate_asserts(mydummy.MyData, name='mydummy.MyData')
When you execute this test, the following output is printed to the console, which contains all the asserts to the actual calls to the mock:
from mock import call
mydummy.MyData.__getitem__.assert_has_calls(calls=[call('object1'),call('object2'),call('object3'),])
mydummy.MyData.__getitem__.return_value.search.assert_has_calls(calls=[call(),call(),call(),])
mydummy.MyData.__getitem__.return_value.search.return_value.__str__.assert_has_calls(calls=[call(),call(),call(),])
You can easily derive from here what has to be mocked.
For example if I have a custom Python object like this;
#!/usr/bin/env python3
import os
base_dir = os.path.abspath(".")
class MyFile(dict):
def __init__(self, name, size = None, dir = base_dir):
self.name = name
self.path = os.path.join(dir, name)
self.bytes = size
and somewhere in my program, I initialize my object class;
a = MyFile(name = "foo", size = 10)
I want to be able to return the code used to create the object in the first place. For example;
print(a)
# <__main__.MyFile object at 0x102b84470>
# should instead print:
# MyFile(name = "foo", size = 10)
But since my object has some default attribute values, I only want those to show up in the output if they were explicitly included when the object was initialized;
b = MyFile(name = "bar", dir = "/home")
print(b)
# <__main__.MyFile object at 0x102b845c0>
# should instead print:
# MyFile(name = "bar", dir = "/home")
And to be clear, I am not trying to pull this from the source code, because a lot of my objects will be created dynamically, and I want to be able to return the same thing for them as well;
l = [ ("baz", 4), ("buzz", 12) ]
f = [ MyFile(name = n, size = s) for n, s in l ]
print(f)
# [<__main__.MyFile object at 0x1023844a8>, <__main__.MyFile object at 0x102384828>]
# should instead print:
# [ MyFile(name = "baz", size = 4), MyFile(name = "buzz", size = 12) ]
I saw the inspect library (https://docs.python.org/3/library/inspect.html) but it does not seem to have anything that does this. What am I missing? This functionality would be pretty analogous to R's dput function.
At a very basic level you can do this:
class MyClass:
def __init__(self, a, b):
self.a = a
self.b = b
def __repr__(self):
return f'{self.__class__.__name__}({self.a}, {self.b})'
class MyOtherClass(MyClass):
def method(self):
pass
c = MyClass(1, 2)
oc = MyOtherClass(3, 4)
print(c, oc)
Result:
MyClass(1, 2) MyOtherClass(3, 4)
This does what you ask, as well as taking subclassing into account to provide the correct class name. But of course things can get complicated for several reasons:
class MyClass:
def __init__(self, a, b):
self.a = a + 1
self.b = b if b < 10 else a
self.c = 0
def inc_c(self):
self.c += 1
def __repr__(self):
return f'{self.__class__.__name__}({self.a - 1}, {self.b})'
The value of c isn't covered by the constructor, so the proposed call would set it to 0. And Although you could compensate for the + 1 for a, the value of b will be more complicated - even more so if you realise someone could have changed the value later.
And then you need to consider that subclasses can override behaviour, etc. So, doing something like this only makes sense in very limited use cases.
As simple as replacing your code snippet with the following:
import os
base_dir = os.path.abspath(".")
class MyFile(object):
def __init__(self, name, size = None, dir = base_dir):
self.name = name
self.path = os.path.join(dir, name)
self.bytes = size
self.remember(name,size, dir)
def remember(self, name,size, dir):
self.s= '{}(name = \'{}\'{}{})'.format(self.__class__.__name__,name, ", size="+str(size) if size!=None else "", ', dir="'+dir+'"' if dir!=base_dir else "")
def __repr__(self):
return self.s
a) for a it returns:
MyFile(name = 'foo', size=10)
b) for b it returns:
MyFile(name = 'bar', dir="/home")
c) for f it returns:
[MyFile(name = 'baz', size=4), MyFile(name = 'buzz', size=12)]
Thanks to everyone who commented and answered. Ultimately, I incorporated their ideas and feedback into the following method, which allowed me to preserve the object's native __repr__ while still getting the behaviors I wanted.
#!/usr/bin/env python3
import os
base_dir = os.path.abspath(".")
class MyFile(dict):
"""
A custom dict class that auto-populates some keys based on simple input args
compatible with unittest.TestCase.assertDictEqual
"""
def __init__(self, name, size = None, dir = base_dir):
"""
standard init methods
"""
self.name = name
self.path = os.path.join(dir, name)
self.bytes = size
# auto-populate this key
self['somekey'] = self.path + ' ' + str(self.bytes)
# more logic for more complex keys goes here...
# use these later with `init` and `repr`
self.args = None
self.kwargs = None
#classmethod
def init(cls, *args, **kwargs):
"""
alternative method to initialize the object while retaining the args passed
"""
obj = cls(*args, **kwargs)
obj.args = args
obj.kwargs = kwargs
return(obj)
def repr(self):
"""
returns a text representation of the object that can be used to
create a new copy of an identical object, displaying only the
args that were originally used to create the current object instance
(do not show args that were not passed e.g. default value args)
"""
n = 'MyFile('
if self.args:
for i, arg in enumerate(self.args):
n += arg.__repr__()
if i < len(self.args) - 1 or self.kwargs:
n += ', '
if self.kwargs:
for i, (k, v) in enumerate(self.kwargs.items()):
n += str(k) + '=' + v.__repr__()
if i < len(self.kwargs.items()) - 1:
n += ', '
n += ')'
return(n)
Usage:
# normal object initialization
obj1 = MyFile('foo', size=10)
print(obj1) # {'somekey': '/Users/me/test/foo 10'}
# initialize with classmethod instead to preserve args
obj2 = MyFile.init("foo", size = 10)
print(obj2) # {'somekey': '/Users/me/test/foo 10'}
# view the text representation
repr = obj2.repr()
print(repr) # MyFile('foo', size=10)
# re-load a copy of the object from the text representation
obj3 = eval(repr)
print(obj3) # {'somekey': '/Users/me/test/foo 10'}
The use case for this being where I need to represent large simple data structures (dicts) in my Python code (integration tests), where the data values are dynamically generated from a smaller set of variables. But when I have many hundreds of such data structures that I need to include in the test case, it becomes infeasible to write the code for e.g. MyFile(...) out hundreds of times. This method allows me to use a script to ingest the data, and then print out compact Python code needed to recreate the data using my custom object class. Which I can then just copy/paste into my test cases.
I have a number of classes where there are functions inside are almost the same.
Say function x:
class A():
def x_A (self):
...
...do the same thing
...
run a function that is unique in class A itself, say u_A
...
...do the same thing
...
class B():
def x_B (self):
...
...do the same thing
..
run a function that is unique in class B itself, say u_B
...
...do the same thing
...
So I came up with an idea to re-write function x in a new class(say x_C in class C) to replace x_A and x_B. And I just have to import that new class when I need it. something like:
import C
class A():
def x_A (self):
C.x_C(u_A)
class B():
def x_B (self):
C.x_C(u_A)
but I am confused of how to pass in the unique function (u_A and u_B) as a variable and make python to run it properly.
class C():
def x_C (self,unique_function):
...
...do the same thing
..
run unique_function here
...
...do the same thing
...
Thx in advance
blow is newly edited:
hi trying to specify my question:
I have a number of crawlers, at the end of each I got "run_before_insert" to check if they can run properly.
Currently I just copy and paste this function at end of every finished crawler with some edits.
But now I would like to simplify my code by importing "run_before_insert" from other files, and then comes my questions.
def run_before_insert(self):
try:
#store_list = []
comp_name = 'HangTen'
start = time.time()
print('{} runBeforeInsert START'.format(comp_name), '\n')
###Here is the part where small edits in the function:
store_list = self.get_stores_2()
###the rest is the same
script_info = {}
running_time = round(time.time() - start,2)
total = str(len(store_list))
script_info['running_time'] = running_time
script_info['total_stores'] = total
print('\n{} total stores : {}'.format(comp_name,script_info['total_stores']), '\n')
print('{} running time : {}'.format(comp_name,script_info['running_time']), '\n')
print('{} runBeforeInsert Done'.format(comp_name), '\n')
print('\n')
return script_info
except Exception as e:
traceback.print_exc()
script_info = {}
script_info['running_time'] = '--'
script_info['total_stores'] = 'error'
return script_info
print(e)
Here is my code with reference to #juanpa.arrivillaga:
class run_pkg_class():
def __init__(self):
pass
def run_before_insert(self, store_function, company_name):
try:
comp_name = company_name
start = time.time()
print('{} runBeforeInsert START'.format(comp_name), '\n')
###
store_list = store_function()
###
script_info = {}
running_time = round(time.time() - start,2)
total = str(len(store_list))
script_info['running_time'] = running_time
script_info['total_stores'] = total
print('\n{} total stores : {}'.format(comp_name,script_info['total_stores']), '\n')
print('{} running time : {}'.format(comp_name,script_info['running_time']), '\n')
print('{} runBeforeInsert Done'.format(comp_name), '\n')
print('\n')
return script_info
except Exception as e:
traceback.print_exc()
script_info = {}
script_info['running_time'] = '--'
script_info['total_stores'] = 'error'
return script_info
print(e)
and import above into hangten crawler class:
def run_before_insert2(self):
rp = run_pkg_class()
rp.run_before_insert(self.get_id())
In this hangTen case, self.get_stores_2() will return a list.
"TypeError: 'list' object is not callable" occur while running.
Not sure for the reason
Python functions are first-class objects. They are like any other attribute. Just pass them directly:
import C
class A:
def x_A (self):
C.x_C(self.u_A)
class B:
def x_B (self):
C.x_C(self.u_B)
And in C, you just call it like so:
unique_function()
Given that C apparently doesn't care about the state in A and B though, I suspect these things shouldn't be classes to begin with.
If I understand correctly, you don't even need to import a module every time. Instead, you can create a basic class from which other classes will inherit the function. For example, classes B and C inherit function "power" from class A.
class A:
""" Example of class A """
def __init__(self):
self.num1 = num1
def power(self, num):
return num**3
class B (A):
""" Example of class B"""
def __init__(self, num1, num2):
super().__init__(num1)
self.num2 = num2
self.power_num1 = self.power(num1)
class C(A):
""" Example of class C"""
def __init__(self, num1, num2, num3):
super().__init__(num1)
self.num2 = num2
self.num3 = num3
def power_total(self):
print(self.power(self.num1) + self.power(self.num2)
+ self.power(self.num3))
Examples of use:
>>> c = C(1, 2, 3)
>>> c.power_total()
36
>>> b = B(2, 4)
>>> b.power_num1
8
TypeError: _slow_trap_ramp() takes 1 positional argument but 2 were given
def demag_chip(self):
coil_probe_constant = float(514.5)
field_sweep = [50 * i * (-1)**(i + 1) for i in range(20, 0, -1)] #print as list
for j in field_sweep:
ramp = self._slow_trap_ramp(j)
def _set_trap_ramp(self):
set_trap_ramp = InstrumentsClass.KeysightB2962A.set_trap_ramp
return set_trap_ramp
def _slow_trap_ramp(self):
slow_trap_ramp = ExperimentsSubClasses.FraunhoferAveraging.slow_trap_ramp
return slow_trap_ramp
The error is straightforward.
ramp = self._slow_trap_ramp(j)
You are calling this method with an argument j, but the method doesn't take an argument (other than self, which is used to pass the object).
Re-define your method to accept an argument if you want to pass it one:
def _slow_trap_ramp(self, j):
It looks like your code extract contains methods of some class, whose full definition is not shown, and you are calling one method from another method (self._slow_trap_ramp(j)). When you call a method, Python automatically passes self before any other arguments. So you need to change def _slow_trap_ramp(self) to def _slow_trap_ramp(self, j).
Update in response to comment
To really help, we would need to see more of the class you are writing, and also some info on the other objects you are calling. But I am going to go out on a limb and guess that your code looks something like this:
InstrumentsClass.py
class KeysightB2962A
def __init__(self):
...
def set_trap_ramp(self):
...
ExperimentsSubClasses.py
class FraunhoferAveraging
def __init__(self):
...
def slow_trap_ramp(self, j):
...
Current version of main.py
import InstrumentsClass, ExperimentsSubClasses
class MyClass
def __init__(self)
...
def demag_chip(self):
coil_probe_constant = float(514.5)
field_sweep = [50 * i * (-1)**(i + 1) for i in range(20, 0, -1)] #print as list
for j in field_sweep:
ramp = self._slow_trap_ramp(j)
def _set_trap_ramp(self):
set_trap_ramp = InstrumentsClass.KeysightB2962A.set_trap_ramp
return set_trap_ramp
def _slow_trap_ramp(self):
slow_trap_ramp = ExperimentsSubClasses.FraunhoferAveraging.slow_trap_ramp
return slow_trap_ramp
if __name__ == "__main__":
my_obj = MyClass()
my_obj.demag_chip()
If this is the case, then these are the main problems:
Python passes self and j to MyClass._slow_trap_ramp, but you've only defined it to accept self (noted above),
you are using class methods from KeysightB2962A and FraunhoferAveraging directly instead of instantiating the class and using the instance's methods, and
you are returning references to the methods instead of calling the methods.
You can fix all of these by changing the code to look like this (see embedded comments):
New version of main.py
import InstrumentsClass, ExperimentsSubClasses
class MyClass
def __init__(self)
# create instances of the relevant classes (note parentheses at end)
self.keysight = InstrumentsClass.KeysightB2962A()
self.fraun_averaging = ExperimentsSubClasses.FraunhoferAveraging()
def demag_chip(self):
coil_probe_constant = float(514.5)
field_sweep = [50 * i * (-1)**(i + 1) for i in range(20, 0, -1)] #print as list
for j in field_sweep:
ramp = self._slow_trap_ramp(j)
def _set_trap_ramp(self):
# call instance method (note parentheses at end)
return self.keysight.set_trap_ramp()
def _slow_trap_ramp(self, j): # accept both self and j
# call instance method (note parentheses at end)
return self.fraun_averaging.slow_trap_ramp(j)
if __name__ == "__main__":
my_obj = MyClass()
my_obj.demag_chip()
I am working on a django site and want to define some instance methods like below.
Class Auth(models.Model):
def wrap_has_perm(self, perm):
def wrap():
if self.is_staff and self.has_perm(perm):
return True
else:
return False
can_add_order = wrap_has_perm('finance.normal')
can_review_order = wrap_has_perm('finance.review')
is_leader = wrap_has_perm('finance.leader')
is_finance = wrap_has_perm('finance.finance')
I want to use can_add_order, can_review_order, is_leader, is_finance as django admin site's list_display element. But now these instance methods is illegal.(TypeError: wrap_has_perm() takes exactly 2 arguments (1 given))
How can I achieve these methods?
If I use partial:
def staff_has_perm(self, perm):
return self.is_staff and self.has_perm(perm)
can_add_order = partial(staff_has_perm, perm='finance.normal')
can_review_order = partial(staff_has_perm, perm='finance.review')
is_leader = partial(staff_has_perm, perm='finance.leader')
is_finance = partial(staff_has_perm, perm='finance.finance')
It raises (* TypeError: staff_has_perm() takes exactly 2 arguments (1 given));
Should I pass self to and how to ?
Move the self to wrap()'s definition:
def wrap_has_perm(perm):
def wrap(self):
However, a more Pythonic way to do this might be to use functools.partial:
from functools import partial
class Auth(models.Model):
def has_perm(self, perm):
# ...
can_add_order = partial(has_perm, perm='finance.normal')
can_review_order = partial(has_perm, perm='finance.review')
is_leader = partial(has_perm, perm='finance.leader')
is_finance = partial(has_perm, perm='finance.finance')