In my python package I have a configuration module that reads a yaml file (when creating the instance) at an explicit location, i.e. something like
class YamlConfig(object):
def __init__(self):
filename = os.path.join(os.path.expanduser('~'), '.hanzo\\config.yml')
with open(filename) as fs:
self.cfg = yaml.load(fs.read())
Now what should I do when writing my unit test if I don't want to use the explicitly specified file? Instead I want to create a temporary config.yml to be used for testing.
I could simply allow for a specified filename in __init__(), but I strongly prefer forcing the filename location. I.e. like this
class YamlConfig(object):
def __init__(self, filename=os.path.join(os.path.expanduser('~'), '.hanzo\\config.yml')):
with open(filename) as fs:
self.cfg = yaml.load(fs.read())
Is there other ways to solve my issue? I guess it might be possible using mock right way? Also feel free to give any comments about upside and downside.
You need to run the test inside the mock context:
import unittest.mock as umock
with umock.patch('__main__.open', umock.mock_open(read_data='yaml data')):
# Run your test here and open the file
with open('filename') as f:
You might need to use library yaml to generate yaml output for read_data
I did this for one of my projects. Here's what my function looked like (simplified):
def parse_template(cve: str) -> Dict:
"""Parse yaml templates and return relevant information
Args:
cve: CVE id to identify respective template
Returns:
Dict: return CVE name
"""
year = cve.split("-")[1]
file_path = f"/cves/{year}/{cve}.yaml"
with open(file_path, "r") as f:
data = yaml.load(f, Loader=yaml.SafeLoader)
return {
"name": data.get("name", "None")
}
To write tests for this, we first create a python fixture which patches the "open" function which is responsible for opening my yaml file.
#pytest.fixture
def mock_template(mocker: MockFixture):
"""Mock yaml file"""
sample_dict={
"name":"Example cve name"
}
mocked_yaml = mocker.mock_open(read_data=yaml.dump(sample_dict))
mocker.patch("builtins.open", mocked_yaml)
Then we just use this fixture to validate our function with something like:
def test_parse_template(mock_template:pytest.fixture) -> None:
"""Tests for opening yaml template and extract data"""
result=parse_template(cve="CVE-2022-1337")
assert "name" in result.keys() # or any other validation
Related
I want to add value on my ini file means I want to write on my ini file, but I am getting configparser.NoSectionError: No section: 'locators', though I have gone through all solutions as well, they are not informative at all.
My Logic is that:
conf.ini file:
[locators]
#Home Page Locators
DATE_PICKER_HEADER_CLASS_NAME = XCUIElementTypeStaticText
DATE_WHEEL_PICKER_ACCESSIBILITY_ID =
Now I want to add value on DATE_WHEEL_PICKER_ACCESSIBILITY_ID, so here I am using this:
def write_config(key, value):
config = ConfigParser()
config.set('locators', key, value)
with open(file_name, 'w') as configfile:
config.write(configfile)
and I am calling method like that:
write_config("DATE_WHEEL_PICKER_ACCESSIBILITY_ID", "Feb 8, 2023")
now file_name variable returns mein absolute path, which I have created with some other method, which shows the result like that:
/Users/tp/Documents/Pract/iOSUICatalogAutomation/ConfigurationData/conf.ini
Also I have tried with passing absolute path directly as well, but its not working at all.
I had to write a custom function to load a yaml file from the current working directory. The function itself works and my intention was to write it in a pure fashion but my senior colleague told me that the way I wrote this function is utterly bad and I have to rewrite it.
Which commandment in Python did I violate? Can anyone tell me what I did wrong here and how a "professional" solution would look like?
from typing import Dict
import yaml
from yaml import SafeLoader
from pathlib import Path
import os
def read_yaml_from_cwd(file: str) -> Dict:
"""[reads a yaml file from current working directory]
Parameters
----------
file : str
[.yaml or .yml file]
Returns
-------
Dict
[Dictionary]
"""
path = os.path.join(Path.cwd().resolve(), file)
if os.path.isfile(path):
with open(path) as f:
content = yaml.load(f, Loader=SafeLoader)
return content
else:
return None
content = read_yaml_from_cwd("test.yaml")
print(content)
The significant parts of your function can be reduced to this:
import yaml
from yaml import SafeLoader
def read_yaml_from_cwd(file):
try:
with open(file) as f:
return yaml.load(f, Loader=SafeLoader)
except Exception:
pass
In this way, the function will either return a dict object or None if either the file cannot be opened or parsed by the yaml loader
I want to test this method:
class Data(object):
def save(self, filename=''):
if filename:
self.filename = filename
if not self.filename:
raise ValueError('Please provide a path to save to')
with open(self.filename, 'w') as f:
pickle.dump(self, f)
I can set up the test to make sure pickle.dump gets called, and that the first argument is the object:
#patch('pickle.dump')
def test_pickle_called(self, dump):
self.data.save('foo.pkl')
self.assertTrue(dump.called)
self.assertEquals(self.data, dump.call_args[0][0])
I'm not sure what to do for the second argument, though. If I open a new file for the test, it's not going to be the same as what gets called for the execution. I'd at least like to be sure I'm opening the right file. Would I just mock open and make sure that gets called with the right name at some point?
Patch open() and return an instance of writeable StringIO from it. Load pickled data from that StringIO and test its structure and values (test that it's equivalent to self.data). Something like this:
import builtins # or __builtin__ for Python 2
builtins.open = open = Mock()
open.return_value = sio = StringIO()
self.data.save('foo.pkl')
new_data = pickle.load(sio.getvalue())
self.assertEqual(self.data, new_data)
I am using ConfigParser to read the runtime configuration of a script.
I would like to have the flexibility of not providing a section name (there are scripts which are simple enough; they don't need a 'section'). ConfigParser will throw a NoSectionError exception, and will not accept the file.
How can I make ConfigParser simply retrieve the (key, value) tuples of a config file without section names?
For instance:
key1=val1
key2:val2
I would rather not write to the config file.
You can do this in a single line of code.
In python 3, prepend a fake section header to your config file data, and pass it to read_string().
from configparser import ConfigParser
parser = ConfigParser()
with open("foo.conf") as stream:
parser.read_string("[top]\n" + stream.read()) # This line does the trick.
You could also use itertools.chain() to simulate a section header for read_file(). This might be more memory-efficient than the above approach, which might be helpful if you have large config files in a constrained runtime environment.
from configparser import ConfigParser
from itertools import chain
parser = ConfigParser()
with open("foo.conf") as lines:
lines = chain(("[top]",), lines) # This line does the trick.
parser.read_file(lines)
In python 2, prepend a fake section header to your config file data, wrap the result in a StringIO object, and pass it to readfp().
from ConfigParser import ConfigParser
from StringIO import StringIO
parser = ConfigParser()
with open("foo.conf") as stream:
stream = StringIO("[top]\n" + stream.read()) # This line does the trick.
parser.readfp(stream)
With any of these approaches, your config settings will be available in parser.items('top').
You could use StringIO in python 3 as well, perhaps for compatibility with both old and new python interpreters, but note that it now lives in the io package and readfp() is now deprecated.
Alternatively, you might consider using a TOML parser instead of ConfigParser.
Alex Martelli provided a solution for using ConfigParser to parse .properties files (which are apparently section-less config files).
His solution is a file-like wrapper that will automagically insert a dummy section heading to satisfy ConfigParser's requirements.
Enlightened by this answer by jterrace, I come up with this solution:
Read entire file into a string
Prefix with a default section name
Use StringIO to mimic a file-like object
ini_str = '[root]\n' + open(ini_path, 'r').read()
ini_fp = StringIO.StringIO(ini_str)
config = ConfigParser.RawConfigParser()
config.readfp(ini_fp)
EDIT for future googlers: As of Python 3.4+ readfp is deprecated, and StringIO is not needed anymore. Instead we can use read_string directly:
with open('config_file') as f:
file_content = '[dummy_section]\n' + f.read()
config_parser = ConfigParser.RawConfigParser()
config_parser.read_string(file_content)
You can use the ConfigObj library to do that simply : http://www.voidspace.org.uk/python/configobj.html
Updated: Find latest code here.
If you are under Debian/Ubuntu, you can install this module using your package manager :
apt-get install python-configobj
An example of use:
from configobj import ConfigObj
config = ConfigObj('myConfigFile.ini')
config.get('key1') # You will get val1
config.get('key2') # You will get val2
The easiest way to do this is to use python's CSV parser, in my opinion. Here's a read/write function demonstrating this approach as well as a test driver. This should work provided the values are not allowed to be multi-line. :)
import csv
import operator
def read_properties(filename):
""" Reads a given properties file with each line of the format key=value. Returns a dictionary containing the pairs.
Keyword arguments:
filename -- the name of the file to be read
"""
result={ }
with open(filename, "rb") as csvfile:
reader = csv.reader(csvfile, delimiter='=', escapechar='\\', quoting=csv.QUOTE_NONE)
for row in reader:
if len(row) != 2:
raise csv.Error("Too many fields on row with contents: "+str(row))
result[row[0]] = row[1]
return result
def write_properties(filename,dictionary):
""" Writes the provided dictionary in key-sorted order to a properties file with each line of the format key=value
Keyword arguments:
filename -- the name of the file to be written
dictionary -- a dictionary containing the key/value pairs.
"""
with open(filename, "wb") as csvfile:
writer = csv.writer(csvfile, delimiter='=', escapechar='\\', quoting=csv.QUOTE_NONE)
for key, value in sorted(dictionary.items(), key=operator.itemgetter(0)):
writer.writerow([ key, value])
def main():
data={
"Hello": "5+5=10",
"World": "Snausage",
"Awesome": "Possum"
}
filename="test.properties"
write_properties(filename,data)
newdata=read_properties(filename)
print "Read in: "
print newdata
print
contents=""
with open(filename, 'rb') as propfile:
contents=propfile.read()
print "File contents:"
print contents
print ["Failure!", "Success!"][data == newdata]
return
if __name__ == '__main__':
main()
Having ran into this problem myself, I wrote a complete wrapper to ConfigParser (the version in Python 2) that can read and write files without sections transparently, based on Alex Martelli's approach linked on the accepted answer. It should be a drop-in replacement to any usage of ConfigParser. Posting it in case anyone in need of that finds this page.
import ConfigParser
import StringIO
class SectionlessConfigParser(ConfigParser.RawConfigParser):
"""
Extends ConfigParser to allow files without sections.
This is done by wrapping read files and prepending them with a placeholder
section, which defaults to '__config__'
"""
def __init__(self, *args, **kwargs):
default_section = kwargs.pop('default_section', None)
ConfigParser.RawConfigParser.__init__(self, *args, **kwargs)
self._default_section = None
self.set_default_section(default_section or '__config__')
def get_default_section(self):
return self._default_section
def set_default_section(self, section):
self.add_section(section)
# move all values from the previous default section to the new one
try:
default_section_items = self.items(self._default_section)
self.remove_section(self._default_section)
except ConfigParser.NoSectionError:
pass
else:
for (key, value) in default_section_items:
self.set(section, key, value)
self._default_section = section
def read(self, filenames):
if isinstance(filenames, basestring):
filenames = [filenames]
read_ok = []
for filename in filenames:
try:
with open(filename) as fp:
self.readfp(fp)
except IOError:
continue
else:
read_ok.append(filename)
return read_ok
def readfp(self, fp, *args, **kwargs):
stream = StringIO()
try:
stream.name = fp.name
except AttributeError:
pass
stream.write('[' + self._default_section + ']\n')
stream.write(fp.read())
stream.seek(0, 0)
return ConfigParser.RawConfigParser.readfp(self, stream, *args,
**kwargs)
def write(self, fp):
# Write the items from the default section manually and then remove them
# from the data. They'll be re-added later.
try:
default_section_items = self.items(self._default_section)
self.remove_section(self._default_section)
for (key, value) in default_section_items:
fp.write("{0} = {1}\n".format(key, value))
fp.write("\n")
except ConfigParser.NoSectionError:
pass
ConfigParser.RawConfigParser.write(self, fp)
self.add_section(self._default_section)
for (key, value) in default_section_items:
self.set(self._default_section, key, value)
Blueicefield's answer mentioned configobj, but the original lib only supports Python 2. It now has a Python 3+ compatible port:
https://github.com/DiffSK/configobj
APIs haven't changed, see it's doc.
I'm trying to build tests for some models that have a FileField. The model looks like this:
class SolutionFile(models.Model):
'''
A file from a solution.
'''
solution = models.ForeignKey(Solution)
file = models.FileField(upload_to=make_solution_file_path)
I have encountered two problems:
When saving data to a fixture using ./manage.py dumpdata, the file contents are not saved, only the file name is saved into the fixture. While I find this to be the expected behavior as the file contents are not saved into the database, I'd like to somehow include this information in the fixture for tests.
I have a test case for uploading a file that looks like this:
def test_post_solution_file(self):
import tempfile
import os
filename = tempfile.mkstemp()[1]
f = open(filename, 'w')
f.write('These are the file contents')
f.close()
f = open(filename, 'r')
post_data = {'file': f}
response = self.client.post(self.solution.get_absolute_url()+'add_solution_file/', post_data,
follow=True)
f.close()
os.remove(filename)
self.assertTemplateUsed(response, 'tests/solution_detail.html')
self.assertContains(response, os.path.basename(filename))
While this test works just fine, it leaves the uploaded file in the media directory after finishing. Of course, the deletion could be taken care of in tearDown(), but I was wondering if Django had another way of dealing with this.
One solution I was thinking of was using a different media folder for tests which must be kept synced with the test fixtures. Is there any way to specify another media directory in settings.py when tests are being run? And can I include some sort of hook to dumpdata so that it syncs the files in the media folders?
So, is there a more Pythonic or Django-specific way of dealing with unit tests involving files?
Django provides a great way to write tests on FileFields without mucking about in the real filesystem - use a SimpleUploadedFile.
from django.core.files.uploadedfile import SimpleUploadedFile
my_model.file_field = SimpleUploadedFile('best_file_eva.txt', b'these are the contents of the txt file')
It's one of django's magical features-that-don't-show-up-in-the-docs :). However it is referred to here.
You can override the MEDIA_ROOT setting for your tests using the #override_settings() decorator as documented:
from django.test import override_settings
#override_settings(MEDIA_ROOT='/tmp/django_test')
def test_post_solution_file(self):
# your code here
I've written unit tests for an entire gallery app before, and what worked well for me was using the python tempfile and shutil modules to create copies of the test files in temporary directories and then delete them all afterwards.
The following example is not working/complete, but should get you on the right path:
import os, shutil, tempfile
PATH_TEMP = tempfile.mkdtemp(dir=os.path.join(MY_PATH, 'temp'))
def make_objects():
filenames = os.listdir(TEST_FILES_DIR)
if not os.access(PATH_TEMP, os.F_OK):
os.makedirs(PATH_TEMP)
for filename in filenames:
name, extension = os.path.splitext(filename)
new = os.path.join(PATH_TEMP, filename)
shutil.copyfile(os.path.join(TEST_FILES_DIR, filename), new)
#Do something with the files/FileField here
def remove_objects():
shutil.rmtree(PATH_TEMP)
I run those methods in the setUp() and tearDown() methods of my unit tests and it works great! You've got a clean copy of your files to test your filefield that are reusable and predictable.
with pytest and pytest-django, I use this in conftest.py file:
import tempfile
import shutil
from pytest_django.lazy_django import skip_if_no_django
from pytest_django.fixtures import SettingsWrapper
#pytest.fixture(scope='session')
##pytest.yield_fixture()
def settings():
"""A Django settings object which restores changes after the testrun"""
skip_if_no_django()
wrapper = SettingsWrapper()
yield wrapper
wrapper.finalize()
#pytest.fixture(autouse=True, scope='session')
def media_root(settings):
tmp_dir = tempfile.mkdtemp()
settings.MEDIA_ROOT = tmp_dir
yield settings.MEDIA_ROOT
shutil.rmtree(tmp_dir)
#pytest.fixture(scope='session')
def django_db_setup(media_root, django_db_setup):
print('inject_after')
might be helpful:
https://dev.funkwhale.audio/funkwhale/funkwhale/blob/de777764da0c0e9fe66d0bb76317679be964588b/api/tests/conftest.py
https://framagit.org/ideascube/ideascube/blob/master/conftest.py
https://stackoverflow.com/a/56177770/5305401
This is what I did for my test. After uploading the file it should end up in the photo property of my organization model object:
import tempfile
filename = tempfile.mkstemp()[1]
f = open(filename, 'w')
f.write('These are the file contents')
f.close()
f = open(filename, 'r')
post_data = {'file': f}
response = self.client.post("/org/%d/photo" % new_org_data["id"], post_data)
f.close()
self.assertEqual(response.status_code, 200)
## Check the file
## org is where the file should end up
org = models.Organization.objects.get(pk=new_org_data["id"])
self.assertEqual("These are the file contents", org.photo.file.read())
## Remove the file
import os
os.remove(org.photo.path)