How can I re-instantiate a variable for each of my PyTests?
Specifically, I want to create a new StringIO() object each time.
My current code is this:
output = StringIO()
def info_to_string(text):
output.write(text)
def write_to_file(some_text):
for row in some_text:
info_to_string(row)
I need output to be set up each time there is a new test fixture.
Copy and pastable code to test:
from io import StringIO
import pytest
output = StringIO()
def info_to_string(text):
output.write(text)
def write_to_file(some_text):
for row in some_text:
info_to_string(row)
def test_1():
write_to_file(['hello', 'there', 'what', 'is', 'up'])
print(output)
assert output.getvalue() == "hellotherewhatisup"
def test_2():
write_to_file(['nothing', 'much'])
assert output.getvalue() == "nothingmuch"
#This will error as the output is "hellotherewhatisupnothingmuch" if run after test_1
So I would need a new output = stringIO() for each test.
Incase anyone sees this in the future, the way I did this was by creating a class, re-initialising it each fixture
class WriteToString:
def __init__(self):
self.output = StringIO()
def info_to_string(self, text):
self.output.write(text)
#pytest.fixture
def write_to_string():
return WriteToString()
and changing the tests to:
def test_2(write_to_string):
write_to_file(['nothing', 'much'])
assert write_to_string.output.getvalue() == "nothingmuch"
I'm not sure I quite understand your questions, but you can do something like below, which will create a new StringIO instance every time you pass it into your test function. If you want to return a different string every time, I don't think you're looking for a fixture, but just a generic function call that does the work for you.
import pytest
from StringIO import StringIO
#pytest.fixture(scope='function')
def info_to_string():
output = StringIO()
output.write('blah')
return output
def test_something(info_to_string):
assert isinstance(info_to_string, StringIO)
assert info_to_string.getvalue() == 'blah'
I think the easiest way to test file writing operation is to use tempfile.
But once StringIO is mentioned as a part of testing strategy my suggestion would be to split file writing into two parts. This gives a clean entry point for StringIO buffer.
from pathlib import Path
from io import StringIO
import pytest
# Separate your data export to 2 functions, one operating
# with a filename, other with a file object.
def to_file(filename, content):
path = Path(filename)
if not path.exists():
path.touch()
with path.open('a') as f_obj:
write_to_file(f_obj, content)
# This is code under test. It must not be connected to your
# test code in any way, even if you have some version of "monkey-patching"
# in mind.
def write_to_file(file_object, content):
"""Dump *content* list of strings to *filename* with *writer*."""
for x in content:
file_object.write(str(x))
# This is test with StringIO buffer
def test_write_to_file_with_buffer():
buffer = StringIO()
# note you make several calls to write_to_file in one test, not across tests
# this helps keep tests isolated/independent
write_to_file(buffer, ['hello', 'there', 'what', 'is', 'up'])
write_to_file(buffer, ['nothing', 'much'])
assert buffer.getvalue() == 'hellotherewhatisupnothingmuch'
Related
I have a class that is doing a lot of stuff. In the end, it saves everything into a pickle. When I rerun this class I want to read the pickle instead of doing everything again. Unfortunately it the variable is always empty if I unpickle. Why is that so?
import pandas as pd
class Test:
def __init__(path, value):
# path points to a .txt file but its in the same folder as the pickle
data_path, data = os.path.split(path)
pickle_path = os.path.join(data_path, name.split('.')[1] + '.pickle'
if os.path.isfile(pickle_path):
self = pd.read_pickle(path)
else:
# do a ton of stuff and safe it as pickle afterwards
variable = Test(path, value)
In this case variable is empty if I read from pickle but correct if I do all the stuff...
If I want to cache some calculation results I will load/dump the object outside the class, something like,
pickle_path = os.path.join(data_path, name.split('.')[1] + '.pickle'
if os.path.isfile(pickle_path):
with open(pickle_path, 'rb') as f:
variable = pickle.load(f) # use cached results
else:
variable = Test() # do all the calculations
I need to mock pathlib.Path.open using pytest-mock.
The real open_func opens a yaml-file. The return value is a regular dict. How can I mock Path.open to just load another yaml-file called test-config.yaml?
My code is not working properly as conf will simply become a str ("test_config.yaml"). It should be a dict.
from pathlib import Path
import yaml
def open_func():
with Path.open(Path("./config.yaml")) as f:
return yaml.load(f, Loader=yaml.FullLoader)
def test_open_func(mocker):
mocker.patch("pathlib.Path.open", mocker.mock_open(read_data="test_config.yaml"))
conf = open_func()
assert isinstance(conf, dict)
EDIT:
To get closer to my real world problem, I am providing the following code. I have a class TryToMock, that basically takes two files as inputs. The method load_files simply loads these files (which are actually .yaml files) and returns the output. These .yaml files are really some configuration files.
In my unit tests, I will be calling TryToMocknumerous times through pytest's parametrize. Therefore, I would like to load the original configuration files via a fixture. Then I am able to monkeypatch some entries in my various tests before running load_files.
In order not to load the original files again, I need to mock the Path.open function in TryToMock. I would like to pass the monkeypatched yaml files instead (i.e. in the form of a dict). The difficulty is that I must discriminate between the two files. That is I can't simply mock the Path.open function with the same file content.
# TryToMock.py
from pathlib import Path
import yaml
# In my current working folder, I have to .yaml files containing the following
# content for illustrative purpose:
#
# file1.yaml = {'name': 'test1', 'file_type': 'yaml'}
# file2.yaml = {'schema': 'test2', 'currencies': ['EUR', 'USD', 'JPY']}
class TryToMock:
def __init__(self, file_to_mock_1, file_to_mock_2):
self._file_to_mock_1 = file_to_mock_1
self._file_to_mock_2 = file_to_mock_2
def load_files(self):
with Path.open(self._file_to_mock_1) as f:
file1 = yaml.load(f, Loader=yaml.FullLoader)
with Path.open(self._file_to_mock_2) as f:
file2 = yaml.load(f, Loader=yaml.FullLoader)
return file1, file2
# test_TryToMock.py
import os
from pathlib import Path
import pytest
import yaml
from tests import TryToMock
def yaml_files_for_test(yaml_content):
names = {"file1.yaml": file1_content, "file2.yaml": file2_content}
return os.path.join("./", names[os.path.basename(yaml_content)])
#pytest.fixture(scope="module")
def file1_content():
with Path.open(Path("./file1.yaml")) as f:
return yaml.load(f, Loader=yaml.FullLoader)
#pytest.fixture(scope="module")
def file2_content():
with Path.open(Path("./file2.yaml")) as f:
return yaml.load(f, Loader=yaml.FullLoader)
def test_try_to_mock(file1_content, file2_content, monkeypatch, mocker):
file_1 = Path("./file1.yaml")
file_2 = Path("./file2.yaml")
m = TryToMock.TryToMock(file_to_mock_1=file_1, file_to_mock_2=file_2)
# Change some items
monkeypatch.setitem(file1_content, "file_type", "json")
# Mocking - How does it work when I would like to use mock_open???
# How should the lambda function look like?
mocker.patch(
"pathlib.Path.open",
lambda x: mocker.mock_open(read_data=yaml_files_for_test(x)),
)
files = m.load_files()
assert files[0]["file_type"] == "json"
You have to provide the actual file contents to the read_data argument of mock_open. You can just create the data in your test:
test_yaml = """
foo:
bar:
- VAR: "MyVar"
"""
def test_open_func(mocker):
mocker.patch("pathlib.Path.open", mocker.mock_open(read_data=test_yaml))
conf = open_func()
assert conf == {'foo': {'bar': [{'VAR': 'MyVar'}]}}
Or you can read the data from your test file:
def test_open_func(mocker):
with open("my_fixture_path/test.yaml") as f:
contents = f.read()
mocker.patch("pathlib.Path.open", mocker.mock_open(read_data=contents))
conf = open_func()
assert isinstance(conf, dict)
The last case can be also re-written to replace the path argument in the open call by your test path:
def test_open_func(mocker):
mocker.patch("pathlib.Path.open", lambda path: open("test.yaml"))
conf = open_func()
assert isinstance(conf, dict)
or, if you have different test files for different configs, something like:
def yaml_path_for_test(yaml_path):
names = {
"config.yaml": "test.yaml",
...
}
return os.path.join(my_fixture_path, names[os.path.basename(yaml_path)])
def test_open_func3(mocker):
mocker.patch("pathlib.Path.open", lambda path: open(yaml_path_for_test(path)))
conf = open_func()
assert isinstance(conf, dict)
This is probably what you wanted to achieve in your test code.
UPDATE:
This is related to the second part of the question (after the edit). If you have the module-scoped fixtures that preload the fixture files as in the question, you can do something like this:
def test_open_func(mocker, file1_content, file2_content):
def yaml_files_for_test(path):
contents = {"file1.yaml": file1_content,
"file2.yaml": file2_content}
data = contents[os.path.basename(path)]
mock = mocker.mock_open(read_data=yaml.dump(data))
return mock.return_value
mocker.patch("pathlib.Path.open", yaml_files_for_test)
conf = open_func()
assert isinstance(conf, dict)
or, if you prefer not to use nested functions:
def yaml_files_for_test(path, mocker, content1, content2):
contents = {"file1.yaml": content1,
"file2.yaml": content2}
data = contents[os.path.basename(path)]
mock = mocker.mock_open(read_data=yaml.dump(data))
return mock.return_value
def test_open_func5(mocker, file1_content, file2_content):
mocker.patch("pathlib.Path.open",
lambda path: yaml_files_for_test(path, mocker,
file2_content, file2_content))
conf = open_func()
assert isinstance(conf, dict)
Can I configure python to have matlab like print, so that when I just have a function
returnObject()
that it simply prints that object without me having to type print around it? I assume this is not easy, but something like if an object does not get bound by some other var it should get printed, so that this would work.
a = 5 #prints nothing
b = getObject() #prints nothing
a #prints 5
b #prints getObject()
getObject() #prints the object
If you use an ipython notebook individual cells work like this. But you can only view one object per cell by typing the objects name. To see multiple objects you'd need to call print, or use lots of cells.
You could write a script to modify the original script based on a set of rules that define what to print, then run the modified script.
A basic script to do this would be:
f = open('main.py', 'r')
p = open('modified.py', 'w')
p.write('def main(): \n')
for line in f:
temp = line
if len(temp) == 1:
temp = 'print(' + line + ')'
p.write('\t' + temp)
p.close()
from modified import main
main()
The script main.py would then look like this:
x = 236
x
output:
236
Idea is as follows: parse AST of Python code, replace every expression with call to print and content of expression as argument and then run the modified version. I'm not sure whether it works with every code, but you might try. Save it as matlab.py and run your code as python3 -m matlab file.py.
#!/usr/bin/env python3
import ast
import os
import sys
class PrintAdder(ast.NodeTransformer):
def add_print(self, node):
print_func = ast.Name("print", ast.Load())
print_call = ast.Call(print_func, [node.value], [])
print_statement = ast.Expr(print_call)
return print_statement
def visit_Expr(self, node):
if isinstance(node.value, ast.Call) and node.value.func.id == 'print':
return node
return self.add_print(node)
def main():
import argparse
parser = argparse.ArgumentParser()
parser.add_argument('infile', type=argparse.FileType(), nargs='?', default='-')
args = parser.parse_args()
with args.infile as infile:
code = infile.read()
file_name = args.infile.name
tree = ast.parse(code, file_name, 'exec')
tree = PrintAdder().visit(tree)
tree = ast.fix_missing_locations(tree)
bytecode = compile(tree, file_name, 'exec')
exec(bytecode)
if __name__ == '__main__':
main()
Using mock_open, I can capture the data from writes using the with [...] as construct. However, testing that what I have is correct is a little tricky. For example, I can do this:
>>> from mock import mock_open
>>> m = mock_open()
>>> with patch('__main__.open', m, create=True):
... with open('foo', 'w') as h:
... h.write('some stuff')
...
>>> m.mock_calls
[call('foo', 'w'),
call().__enter__(),
call().write('some stuff'),
call().__exit__(None, None, None)]
>>> m.assert_called_once_with('foo', 'w')
>>> handle = m()
>>> handle.write.assert_called_once_with('some stuff')
But I want to do compare what I think should have been written to what was. In effect something like this:
>>> expected = 'some stuff'
>>> assert(expected == m.all_that_was_written)
The problem I am facing with call is that different versions of json (2.0.9 vs 1.9) seem to print things differently. No, I cannot just update to the latest json.
The actual error I am getting is this:
E AssertionError: [call('Tool_000.json', 'w'),
call().__enter__(),
call().write('['),
call().write('\n '),
call().write('"1.0.0"'),
call().write(', \n '),
call().write('"2014-02-27 08:58:02"'),
call().write(', \n '),
call().write('"ook"'),
call().write('\n'),
call().write(']'),
call().__exit__(None, None, None)]
!=
[call('Tool_000.json', 'w'),
call().__enter__(),
call().write('[\n "1.0.0"'),
call().write(', \n "2014-02-27 08:58:02"'),
call().write(', \n "ook"'),
call().write('\n'),
call().write(']'),
call().__exit__(None, None, None)]
In effects, the calls are different but the end result is the same.
The code I am testing is fairly simple:
with open(get_new_file_name(), 'w') as fp:
json.dump(lst, fp)
So, creating another method that passes the file pointer seems overkill.
You can patch open() to return StringIO object and then check the contents.
with mock.patch('module_under_test.open', create=True) as mock_open:
stream = io.StringIO()
# patching to make getvalue() work after close() or __exit__()
stream.close = mock.Mock(return_value=None)
mock_open.return_value = stream
module_under_test.do_something() # this calls open()
contents = stream.getvalue()
assert(contents == expected)
Edit: added patch for stream.close to avoid exception on stream.getvalue().
mock_open is not fully featured yet. It works well if you are mocking files to be read but it does not yet have enough features for testing written files. The question clearly shows this deficiency.
My solution is to not use mock_open if you are testing the written content. Here is the alternative:
import six
import mock
import unittest
class GenTest(unittest.TestCase):
def test_open_mock(self):
io = six.BytesIO()
io_mock = mock.MagicMock(wraps=io)
io_mock.__enter__.return_value = io_mock
io_mock.close = mock.Mock() # optional
with mock.patch.object(six.moves.builtins, 'open', create=True, return_value=io_mock):
# test using with
with open('foo', 'w') as h:
expected = 'some stuff'
h.write(expected)
self.assertEquals(expected, io.getvalue())
# test using file handle directly
io.seek(0); io.truncate() # reset io
expected = 'other stuff'
open('bar', 'w').write(expected)
self.assertEquals(expected, io.getvalue())
# test getvalue after close
io.seek(0); io.truncate() # reset io
expected = 'closing stuff'
f = open('baz', 'w')
f.write(expected)
f.close()
self.assertEquals(expected, io.getvalue())
if __name__ == '__main__':
unittest.main()
Here's what I will do, write a method that returns the complete string from all calls of the write method.
class FileIOTestCase(unittest.TestCase):
""" For testing code involving file io operations """
def setUp(self):
""" patches the open function with a mock, to be undone after test. """
self.mo = mock_open()
patcher = patch("builtins.open", self.mo)
patcher.start()
self.addCleanup(patcher.stop)
def get_written_string(self):
return ''.join(c[0][0] for c in self.mo.return_value.write.call_args_list)
An example of how to use it
class TestWriteFile(FileIOTestCase):
def test_write_file__csv(self):
save.write_file("a,b\n1,2", "directory", "C6L")
self.mo.assert_called_once_with(os.path.join("directory", "C6L.csv"), 'w')
self.assertEqual(self.get_written_string(), "a,b\n1,2")
This question already has answers here:
How do I mock the filesystem in Python unit tests?
(5 answers)
Closed 1 year ago.
I have a source code that opens a csv file and sets up a header to
value association. The source code is given below:
def ParseCsvFile(source):
"""Parse the csv file.
Args:
source: file to be parsed
Returns: the list of dictionary entities; each dictionary contains
attribute to value mapping or its equivalent.
"""
global rack_file
rack_type_file = None
try:
rack_file = source
rack_type_file = open(rack_file) # Need to mock this line.
headers = rack_type_file.readline().split(',')
length = len(headers)
reader = csv.reader(rack_type_file, delimiter=',')
attributes_list=[] # list of dictionaries.
for line in reader:
# More process to happeng. Converting the rack name to sequence.
attributes_list.append(dict((headers[i],
line[i]) for i in range(length)))
return attributes_list
except IOError, (errno, strerror):
logging.error("I/O error(%s): %s" % (errno, strerror))
except IndexError, (errno, strerror):
logging.error('Index Error(%s), %s' %(errno, strerror))
finally:
rack_type_file.close()
I am trying to mock the following statement
rack_type_file = open(rack_file)
How do I mock open(...) function?
This is admittedly an old question, hence some of the answers are outdated.
In the current version of the mock library there is a convenience function designed for precisely this purpose. Here's how it works:
>>> from mock import mock_open
>>> m = mock_open()
>>> with patch('__main__.open', m, create=True):
... with open('foo', 'w') as h:
... h.write('some stuff')
...
>>> m.mock_calls
[call('foo', 'w'),
call().__enter__(),
call().write('some stuff'),
call().__exit__(None, None, None)]
>>> m.assert_called_once_with('foo', 'w')
>>> handle = m()
>>> handle.write.assert_called_once_with('some stuff')
Documentation is here.
To mock built-in function open with mox use __builtin__ module:
import __builtin__ # unlike __builtins__ this must be imported
m = mox.Mox()
m.StubOutWithMock(__builtin__, 'open')
open('ftphelp.yml', 'rb').AndReturn(StringIO("fake file content"))
m.ReplayAll()
# call the code you want to test that calls `open`
m.VerifyAll()
m.UnsetStubs()
Note that __builtins__ is not always a module, it can be of type dict, please use __builtin__ (with no "s") module to refer to system built-in methods.
More about __builtin__ module: http://docs.python.org/library/builtin.html
There are two ways that I like to do this, depending on the situation.
If your unit test is going to call ParseCsvFile directly I would add a new kwarg to ParseCsvFile:
def ParseCsvFile(source, open=open):
# ...
rack_type_file = open(rack_file) # Need to mock this line.
Then your unit test can pass a different open_func in order to accomplish the mocking.
If your unit test calls some other function that in turn calls ParseCsvFile then passing around open_func just for tests is ugly. In that case I would use the mock module. This lets you alter a function by name and replace it with a Mock object.
# code.py
def open_func(name):
return open(name)
def ParseCsvFile(source):
# ...
rack_type_file = open_func(rack_file) # Need to mock this line.
# test.py
import unittest
import mock
from StringIO import StringIO
#mock.patch('code.open_func')
class ParseCsvTest(unittest.TestCase):
def test_parse(self, open_mock):
open_mock.return_value = StringIO("my,example,input")
# ...
Is simple with decorator (Python3):
def my_method():
with open(file="/1.txt", mode='r', encoding='utf-8') as file:
return file.read().strip()
#mock.patch("builtins.open", create=True)
def test_my_method(mock_open):
mock_open.side_effect = [
mock.mock_open(read_data="A").return_value
]
resA = my_method()
assert resA == "A"
mock_open.mock_calls == [mock.call(file="/1.txt", mode='r', encoding='utf-8')]
I took the liberty of re-writing your sample function:
Assume your function is located in a file named code.py
# code.py
import csv
import logging
def ParseCsvFile(source):
"""Parse the csv file.
Args:
source: file to be parsed
Returns: the list of dictionary entities; each dictionary contains
attribute to value mapping or its equivalent.
"""
global rack_file
rack_file = source
attributes_list = []
try:
rack_type_file = open(rack_file)
except IOError, (errno, strerror):
logging.error("I/O error(%s): %s", errno, strerror)
else:
reader = csv.DictReader(rack_type_file, delimiter=',')
attributes_list = [line for line in reader] # list of dictionaries
rack_type_file.close()
return attributes_list
A simple test case would be:
# your test file
import __builtin__
import unittest
import contextlib
from StringIO import StringIO
import mox
import code
#contextlib.contextmanager
def mox_replayer(mox_instance):
mox_instance.ReplayAll()
yield
mox_instance.VerifyAll()
class TestParseCSVFile(unittest.TestCase):
def setUp(self):
self.mox = mox.Mox()
def tearDown(self):
self.mox.UnsetStubs()
def test_parse_csv_file_returns_list_of_dicts(self):
TEST_FILE_NAME = 'foo.csv'
self.mox.StubOutWithMock(__builtin__, 'open')
open(TEST_FILE_NAME).AndReturn(StringIO("name,age\nfoo,13"))
with mox_replayer(self.mox):
result = code.ParseCsvFile(TEST_FILE_NAME)
self.assertEqual(result, [{'age': '13', 'name': 'foo'}]) # works!
if __name__ == '__main__':
unittest.main()
EDIT:
% /usr/bin/python2.6
Python 2.6.1 (r261:67515, Jun 24 2010, 21:47:49)
[GCC 4.2.1 (Apple Inc. build 5646)] on darwin
Type "help", "copyright", "credits" or "license" for more information.
>>> import __builtin__
>>> import mox
>>> mock = mox.Mox()
>>> mock.StubOutWithMock(__builtin__, 'open')
>>> mock.UnsetStubs()
Works fine on 2.6 using mox 0.53
Hi I was having a similar problem, and was tearing my hair out flipping between different mocking libraries. I finally found a solution that I am happy with, and maybe it might help you? In the end I went with the Mocker library http://labix.org/mocker and here is the code for mocking open:
from mocker import Mocker
from StringIO import StringIO
import __builtin__
mocker = Mocker()
sourceFile = 'myTestFile.txt'
__builtin__.open = mocker.mock()
__builtin__.open(sourceFile)
mocker.result(StringIO('this,is,a,test,file'))
<the rest of your test setup goes here>
mocker.replay()
ParseCsvFile(sourceFile)
mocker.restore()
mocker.verify()
Incidentaly the reason I went with Mocker is because I was testing a function which used open to read a file, and then used open again to overwrite the same file with new data. What I needed to be able to do was test the case where the initial file didn't exist, so set up a mock, that threw an IOError the first time, and then worked the second time. The setup for which looked like this:
from mocker import Mocker
import __builtin__
mocker = Mocker()
mockFileObject = mocker.mock()
__builtin__.open = mocker.mock()
__builtin__.open('previousState.pkl', 'r')
mocker.throw(IOError('Boom'))
__builtin__.open('previousState.pkl','w')
mocker.result(mockFileObject)
<rest of test setup >
mocker.replay()
<test>
mocker.restore() #required to restore the open method
mocker.verify()
Hope this helps!
#mock.patch decorator (2.7 example)
This is now much easier:
import your_script.py
import __builtin__
import mock
#mock.patch("__builtin__.open")
def test_example(self, mock_open):
your_script.your_method()
self.assertEqual(mock_open.call_count, 1)
>>> class A(object):
... def __init__(self):
... self.x = open('test.py')
...
>>> old_open = open
>>> def open(s):
... return "test\n"
...
>>> a = A()
>>> a.x
'test\n'
>>> open = old_open
>>> a = A()
>>> a.x
<open file 'test.py', mode 'r' at 0xb7736230>