We Keep Coding

Not sure why MyMock.env["key1"].search.side_effect=["a", "b"] works but MyMock.env["key1"] = ["a"] with MyMock.env["key2"] = ["b"] does not work

I had created a simple example to illustrate my issue. First is the setup say mydummy.py:
class TstObj:
def __init__(self, name):
self.name = name
def search(self):
return self.name
MyData = {}
MyData["object1"] = TstObj("object1")
MyData["object2"] = TstObj("object2")
MyData["object3"] = TstObj("object3")
def getObject1Data():
return MyData["object1"].search()
def getObject2Data():
return MyData["object2"].search()
def getObject3Data():
return MyData["object3"].search()
def getExample():
res = f"{getObject1Data()}{getObject2Data()}{getObject3Data()}"
return res
Here is the test that failed.
def test_get_dummy1():
dummy.MyData = MagicMock()
mydummy.MyData["object1"].search.side_effect = ["obj1"]
mydummy.MyData["object2"].search.side_effect = ["obj2"]
mydummy.MyData["object3"].search.side_effect = ["obj3"]
assert mydummy.getExample() == "obj1obj2obj3"
The above failed with run time error:
/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/unittest/mock.py:1078: StopIteration
Here is the test that passed:
def test_get_dummy2():
dummy.MyData = MagicMock()
mydummy.MyData["object1"].search.side_effect = ["obj1", "obj2", "obj3"]
assert mydummy.getExample() == "obj1obj2obj3"
Am I missing something? I would have expected test_get_dummy1() to work and test_get_dummy2() to fail and not vice versa. Where and how can I find/learn more information about mocking to explain what is going on...

MyData["object1"] is converted to this function call: MyData.__getitem__("object1"). When you call your getExample method, the __getitem__ method is called 3 times with 3 parameters ("object1", "object2", "object3").
To mock the behavior you could have written your test like so:
def test_get_dummy_alternative():
mydummy.MyData = MagicMock()
mydummy.MyData.__getitem__.return_value.search.side_effect = ["obj1", "obj2", "obj3"]
assert mydummy.getExample() == "obj1obj2obj3"
Note the small change from your version: mydummy.MyData["object1"]... became: mydummy.MyData.__getitem__.return_value.... This is the regular MagicMock syntax - we want to to change the return value of the __getitem__ method.
BONUS:
I often struggle with mock syntax and understanding what's happening under the hood. This is why I wrote a helper library: the pytest-mock-generator. It can show you the actual calls made to the mock object.
To use it in your case you could have added this "exploration test":
def test_get_dummy_explore(mg):
mydummy.MyData = MagicMock()
mydummy.getExample()
mg.generate_asserts(mydummy.MyData, name='mydummy.MyData')
When you execute this test, the following output is printed to the console, which contains all the asserts to the actual calls to the mock:
from mock import call
mydummy.MyData.__getitem__.assert_has_calls(calls=[call('object1'),call('object2'),call('object3'),])
mydummy.MyData.__getitem__.return_value.search.assert_has_calls(calls=[call(),call(),call(),])
mydummy.MyData.__getitem__.return_value.search.return_value.__str__.assert_has_calls(calls=[call(),call(),call(),])
You can easily derive from here what has to be mocked.

How to pass a variable from a function to a class?

How can I print path outside function:
class FirstClas:
path = ''
def num(self):
path = "C:\\Users\\JOHN\\Desktop\\test.txt"
return path
print(path)
This method don't print anything.
This result:
C:\Python\python.exe C:/Users/JOHN/Desktop/test/tt.py
Process finished with exit code 0

You need to create an instance from the class that you created.
I would suggest doing this:
test = FirstClas()
print(test.num())
Hope this helps

Your method never gets called, and the class variable path is pointless here. Do:
class FirstClas:
def num(self):
path = "C:\\Users\\JOHN\\Desktop\\test.txt"
return path
print(FirstClas().num()) # note that this is outside the class!

I don't think you quite understand the purpose of classes, but here's how to make what you have "work" (in the sense that there are no fatal errors):
File global_variable.py
def init_global_variable():
"""initialize variable"""
global GLOBALS_DICT
GLOBALS_DICT = {}
def set_variable(name, value):
"""set variable"""
try:
GLOBALS_DICT[name] = value
return True
except KeyError:
return False
def get_variable(name):
"""get variable"""
try:
return GLOBALS_DICT[name]
except KeyError:
return "Not Found"
init_global_variable() # ADDED.
File tt.py
import os
#import lib.global_variable as glv
import global_variable as glv # Since I don't have your whole package.
class FirstClas:
def num(self):
path = "C:\\Users\\JOHN\\Desktop\\test.txt"
return path
def imag(self):
icon_file = os.path.join(
glv.get_variable("APP_PATH"),
glv.get_variable("DATA_DIR"),
"paths",
"PathExcel",
)
return icon_file
class Second:
# Put statements in a method so they don't run when the class is defined.
def run(self):
test = FirstClas()
print('first: ' + test.num())
print('second: ' + test.imag())
second = Second()
second.run()
Output:
first: C:\Users\JOHN\Desktop\test.txt
second: Not Found\Not Found\paths\PathExcel

the path does not changed(path = ' ') because you don't run the function num

Decorator factory parameter is a static value and not a variable's value

So I have
# my decorator factory
def execute_in(directory): # <-- I want this to be a variable's value which can change
def decorator(function):
def wrapper(*args, **kwargs):
os.chdir(directory)
print(directory) # currently is printing None which is my problem
value = function(*args, **kwargs)
os.chdir(home_dir)
return value
return wrapper
return decorator
and
# a function that runs after assigning General.archive_dir a value
#execute_in(General.archive_dir)
def get_data():
print(General.archive_dir) # will print the correct directory name
with open('data.csv', 'r') as f:
rows = [row for row in csv.reader(f, delimiter=',')]
return rows
My problem is that the decorator factory is using the value of the variable General.archive_dir instantiated at program start when its value is None. I want it to use the value of General.archive_dir at the time the decorated function is called. How can I do this?
I apologize if this question is unclear. If you can, please let me know how I can clarify it if needed.

One solution is calling #execute_in with a lambda.
directory inside wrapper would become a function that, when called, returns the current value.
archive_dir = None
# decorator factory
def execute_in(directory_path_getter):
def decorator(function):
def wrapper(*args, **kwargs):
print('from wrapper:', directory_path_getter()) # Notice the function call
value = function(*args, **kwargs)
return value
return wrapper
return decorator
#execute_in(lambda: archive_dir)
def get_data():
...
archive_dir = 'some directory'
print(get_data())
Prints:
from wrapper: some directory
from get_data: some directory
['some data']

If a decorator isn't strictly required, a context manager can also fulfill the task of temporarily changing directories.
import os
from contextlib import contextmanager
#contextmanager
def execute_in(directory):
orig_dir = os.getcwd()
os.chdir(directory)
try:
yield
finally:
os.chdir(orig_dir)
Using a context manager would allow for changing directories many times in one method, and can be nested.
settings = {
'archive_dir': './dir'
}
def get_data():
print(os.getcwd())
with execute_in(settings['archive_dir']):
print(' ' + os.getcwd())
with execute_in('bin'):
print(' ' + os.getcwd())
print(' ' + os.getcwd())
print(os.getcwd())
And when we run it
>>> get_data()
/home/they4kman/.PyCharm2019.2/config/scratches
/home/they4kman/.PyCharm2019.2/config/scratches/dir
/home/they4kman/.PyCharm2019.2/config/scratches/dir/bin
/home/they4kman/.PyCharm2019.2/config/scratches/dir
/home/they4kman/.PyCharm2019.2/config/scratches

Python mocking open and check for close

I am trying to mock open and want to check if close gets called at least once
class MyObject():
def __init__(self,path):
fp = open(path)
self.file_list = []
for line in fp:
self.file_list.append(line.strip())
fp.close()
def testsimpleFile():
fake_file = io.StringIO("data.csv\ndata2.csv")
with patch("builtins.open",return_value=fake_file,create=True) as mock_file:
f = MyObject("path/to/open/test.f")
mock_file.assert_called_once_with("/path/to/open/test.f")
golden_list = ["data.csv","data2.csv"]
assert f.file_list == golden_list
This is my working testcode until now and now i want to additionally check if the the close method was called i tried to add
mock_file.close.assert_called_once()
and
mock_file.fake_file.close.assert_called_once()
but both will not catch the method call.

The short of it is: You can't track that the function is being called with assert_called_once if the return value of open isn't a mock object. So, instead of making the return value a StringIO we can make it a MagicMock that will act like a file handle.
import io
from unittest.mock import patch, MagicMock
class MyObject():
def __init__(self,path):
fp = open(path)
self.file_list = []
for line in fp:
self.file_list.append(line.strip())
fp.close()
def testsimpleFile():
fake_file = MagicMock()
fake_file.__iter__.return_value = ["data.csv", "data2.csv"]
with patch("builtins.open", return_value=fake_file, create=True) as mock_file:
f = MyObject("/path/to/open/test.f")
mock_file.assert_called_once_with("/path/to/open/test.f")
golden_list = ["data.csv", "data2.csv"]
assert f.file_list == golden_list
fake_file.close.assert_called_once()

Generate graph of the imports

I'm getting close to my final goal, which is to generate a nice graph between modules and other imported modules.
For example if x imports from y and z, and y imports from t and v I would like to have:
x -> y, z
y -> t, v
Now I already have my import hook defined as below, but running it on a simple file I don't get what I would expect:
python study_imports.py CollectImports simple.py
('study_imports.py', 'study_imports')
Where simple.py actually imports from study_imports.
The problem is that I want to see "simple.py" instead of "study_imports.py", is there a way to get the path of the file actually importing the other module?
class CollectImports(object):
"""
Import hook, adds each import request to the loaded set and dumps
them to file
"""
def __init__(self, output_file):
self.loaded = set()
self.output_file = output_file
def __str__(self):
return str(self.loaded)
def cleanup(self):
"""Dump the loaded set to file
"""
dumped_str = '\n'.join(x for x in self.loaded)
open(self.output_file, 'w').write(dumped_str)
def find_module(self, module_name, package=None):
#TODO: try to find the name of the package which is actually
#importing something else, and how it's doing it
#use a defualtdict with empty sets as the storage for this job
entry = (__file__, module_name)
self.loaded.add(str(entry))

Maybe with the inspect module.
Module a.py
import inspect
print inspect.stack()
Module b.py
import a
when running b.py, I got :
[
(<frame object at 0x28a9b70>, '/path/a.py', 5, '<module>', ['print inspect.stack()\n'], 0),
(<frame object at 0x28a9660>, 'b.py', 2, '<module>', ['import to_import\n'], 0)
]
Looks like the second frame contains what you need.

So I looked in snakefood a bit better and I ended up rewriting my code using the AST.
Snakefood still uses the compiler, which is deprecated and much slower than using the ast.
The result is great, for example this is a visitor:
from ast import parse, NodeVisitor
class ImportVisitor(NodeVisitor):
def __init__(self):
self.imported = set()
super(ImportVisitor, self).__init__()
def __str__(self):
return '\n'.join(x for x in self.imported)
def visit_Import(self, node):
for n in node.names:
self.imported.add(n.name)
#that we are using
def visit_ImportFrom(self, node):
self.imported.add(node.module)
Which can be usef for example as:
def gen_module_imports(mod):
try:
at = parse(open(mod).read())
except SyntaxError:
print("file %s has a syntax error, please fix it" % mod)
return []
else:
v = ImportVisitor()
v.visit(at)
return v.imported

The inspect trick seems to work fine :)
I get something like simple.py: set(['study_imports']) in the imports.log.
Class CollectImports(object):
"""
Import hook, adds each import request to the loaded set and dumps
them to file
"""
def __init__(self, output_file):
self.loaded = defaultdict(lambda: set())
self.output_file = output_file
def __str__(self):
return str(self.loaded)
def cleanup(self):
"""Dump the loaded set to file
"""
dumped_str = '\n'.join(('%s: %s' % (k, v)) for k, v in self.loaded.items())
open(self.output_file, 'w').write(dumped_str)
def find_module(self, module_name, package=None):
st = inspect.stack()
self.loaded[st[1][1]].add(module_name)