We Keep Coding

Understanding class variable behavior

We came across the need to have a dynamic class variable in the following code in python 2.
from datetime import datetime
from retrying import retry
class TestClass(object):
SOME_VARIABLE = None
def __init__(self, some_arg=None):
self.some_arg = some_arg
#retry(retry_on_exception=lambda e: isinstance(e, EnvironmentError), wait_fixed=3000 if SOME_VARIABLE == "NEEDED" else 1000, stop_max_attempt_number=3)
def some_func(self):
print("Running {} at {}".format(self.some_arg, datetime.now()))
if self.some_arg != "something needed":
raise EnvironmentError("Unexpected value")
TestClass.SOME_VARIABLE = "NEEDED"
x = TestClass()
x.some_func()
Output:
Running None at 2021-07-26 19:40:22.374736
Running None at 2021-07-26 19:40:23.376027
Running None at 2021-07-26 19:40:24.377523
Traceback (most recent call last):
File "/home/raj/tmp/test_test.py", line 19, in <module>
x.some_func()
File "/home/raj/.local/share/virtualenvs/test-DzpjW1fZ/lib/python2.7/site-packages/retrying.py", line 49, in wrapped_f
return Retrying(*dargs, **dkw).call(f, *args, **kw)
File "/home/raj/.local/share/virtualenvs/test-DzpjW1fZ/lib/python2.7/site-packages/retrying.py", line 212, in call
raise attempt.get()
File "/home/raj/.local/share/virtualenvs/test-DzpjW1fZ/lib/python2.7/site-packages/retrying.py", line 247, in get
six.reraise(self.value[0], self.value[1], self.value[2])
File "/home/raj/.local/share/virtualenvs/test-DzpjW1fZ/lib/python2.7/site-packages/retrying.py", line 200, in call
attempt = Attempt(fn(*args, **kwargs), attempt_number, False)
File "/home/raj/tmp/test_test.py", line 14, in some_func
raise EnvironmentError("Unexpected value")
EnvironmentError: Unexpected value
We can see that the value of SOME_VARIABLE is not being updated.
Trying to understand if there is way in which we can update SOME_VARIABLE dynamically. The use case is to have dynamic timings in the retry function based on SOME_VARIABLE value at runtime.

Your class definition is equivalent, based on the definition of decorator syntax, to
class TestClass(object):
SOME_VARIABLE = None
def __init__(self, some_arg=None):
self.some_arg = some_arg
decorator = retry(retry_on_exception=lambda e: isinstance(e, EnvironmentError),
wait_fixed=3000 if SOME_VARIABLE == "NEEDED" else 1000,
stop_max_attempt_number=3)
def some_func(self):
...
some_func = decorator(some_func)
Note that retry is called long before you change the value of TestClass.SOME_VARIABLE (indeed, before the class object that will be bound to TestClass even exists), so the comparison SOME_VARIABLE == "NEEDED" is evaluated when SOME_VARIABLE still equals None.
To have the retry behavior configured at run-time, try something like
class TestClass(object):
SOME_VARIABLE = None
def __init__(self, some_arg=None):
self.some_arg = some_arg
def _some_func_implemenation(self):
print("Running {} at {}".format(self.some_arg, datetime.now()))
if self.some_arg != "something needed":
raise EnvironmentError("Unexpected value")
def some_func(self):
wait = 3000 if self.SOME_VARIABLE == "NEEDED" else 1000
impl = retry(retry_on_exception=lambda e: isinstance(e, EnvironmentError),
wait_fixed=wait,
stop_max_attempt_number=3)(self._some_func)
return impl()
some_func becomes a function that, at runtime, creates a function (based on the private _some_func) with the appropriate retry behavior, then calls it.
(Not tested; I may have gotten the interaction between the bound method self._some_func and retry wrong.)

Set attribue in custom class derived from uuid.UUID: AttributeError

I'm porting some code from IronPython to CPython (3.8) + Python.NET and I have a custom class that's broken in an odd way: it gives me an AttributeError even though the member is present in __dict__. This class derives from uuid.UUID to add support to BLE short UUIDs:
import uuid
class UUID(uuid.UUID):
"""Represents a more general version of the uuid.UUID, so we can have both
regular and short UUIDs.
"""
def __init__(self, hex=None, bytes=None, is_short=False, **kwargs):
try:
super(UUID, self).__init__(hex=hex, bytes=bytes, **kwargs)
self.__dict__['is_short'] = False
except ValueError as val_ex:
if hex is None or len(hex) != 4:
raise val_ex
# Remove braces GUIDs
hex_digits = hex.strip('{}').replace('-', '')
# remove RFC 4122 URN's 'urn:uuid:deadbeef-1234-fedc-5678-deadbeefaaaa'
hex_digits = hex_digits.replace('uuid:', '').replace('urn:', '')
if len(hex_digits) != 4:
raise ValueError('badly formed hexadecimal UUID string')
self.__dict__['int'] = int(hex, 16)
self.__dict__['is_short'] = True
if is_short is True:
self.__dict__['is_short'] = True
def __str__(self):
if self.is_short:
hex = '%04x' % self.int
return '%s' % (hex[:4])
else:
return super(UUID, self).__str__()
And I'm using it like so:
class Test_UUID(unittest.TestCase):
def test_create(self):
x = pybletestlibrary.UUID("1800")
pprint(x.__dict__)
print(x)
The above code yields:
{'int': 6144, 'is_short': True}
E
======================================================================
ERROR: test_create (__main__.Test_UUID)
----------------------------------------------------------------------
Traceback (most recent call last):
File "test\test_pybletestlibrary.py", line 27, in test_create
print(x)
File "my_uuid.py", line 66, in __str__
hex = '%04x' % self.int
AttributeError: int
----------------------------------------------------------------------
Ran 1 test in 0.003s
FAILED (errors=1)
I can't use setattr() because uuid.UUID is immutable. I don't understand why the AttributeError as int is present in __dict__. Any suggestions?
Thanks!!

Here's what I mean:
import uuid
class UUID(uuid.UUID):
"""Represents a more general version of the uuid.UUID, so we can have both
regular and short UUIDs.
"""
def __init__(self, hex=None, bytes=None, is_short=False, **kwargs):
self.is_short = False
if len(hex) == 4:
self.is_short = True
hex = "0000"+hex+"-0000-1000-8000-00805f9b34fb"
super(UUID, self).__init__(hex=hex, bytes=bytes, **kwargs)
def __str__(self):
val = super(UUID, self).__str__()
if self.is_short:
return val[4:8]
else:
return val
You might have to override some of the other attributes, too.

How to mock nested function?

I would like to mock some nested functions from a specific function:
# tools.py
def cpu_count():
def get_cpu_quota():
return int(load("/sys/fs/cgroup/cpu/cpu.cfs_quota_us"))
def get_cpu_period():
return int(load("/sys/fs/cgroup/cpu/cpu.cfs_period_us"))
def get_cpus():
try:
cfs_quota_us = get_cpu_quota()
cfs_period_us = get_cpu_period()
if cfs_quota_us > 0 and cfs_period_us > 0:
return int(math.ceil(cfs_quota_us / cfs_period_us))
except:
pass
return multiprocessing.cpu_count()
return get_cpus()
# test_tools.py
class TestTools(unittest.TestCase):
#patch("tools.cpu_count")
def test_cpu_count_in_container(self, cpu_count_mock):
cpu_count_mock.return_value.get_cpu_quota.return_value = 5000
cpu_count_mock.return_value.get_cpu_period.return_value = 1000
cpus = tools.cpu_count()
self.assertEqual(5, cpus)
However, when running this test I have the follow error:
FAIL: test_cpu_count_in_container (tests.util.tools_test.ToolsTest)
----------------------------------------------------------------------
Traceback (most recent call last):
File "/home/uilian/Development/foo/venv/lib/python3.7/site-packages/mock/mock.py", line 1305, in patched
return func(*args, **keywargs)
File "/home/uilian/Development/foo/tests/util/tools_test.py", line 154, in test_cpu_count_in_container
self.assertIsInstance(cpus, int)
AssertionError: <MagicMock name='cpu_count()' id='140490818322384'> is not an instance of <class 'int'>
Is it possible mocking nested functions?
If not, what's the best approach for this case?

python: return exception from function

suppose I have the following function:
def test():
...
if x['error']:
raise
This would raise an exception regardless if x['error'] is defined or not.
Instead if I try this, it doesn't throw any exception:
def test():
...
try:
if x['error']:
raise
except:
return
How can I test for a specific value and return an exception if it is defined, and to return successfully if it is not defined?

If you want to return error as string:
>>> def test():
try:
if x['error']:raise
except Exception as err:
return err
>>> test()
NameError("name 'x' is not defined",)
If you want to an error to occur:
>>> def test():
try:
if x['error']:raise
except:
raise
>>> test()
Traceback (most recent call last):
File "<pyshell#20>", line 1, in <module>
test()
File "<pyshell#19>", line 3, in test
if x['error']:raise
NameError: name 'x' is not defined

def test():
...
if x.get(‘error’):
raise
You can avoid unintentionally raising an error using the dictionary's built in get function. Get will return None if the value at the specified key does not exist instead of throwing an exception.

try this one
def check_not_exist(d,k):
#if keys exists in dict,raise it
if k in d:
raise
else:
return True

Problems mocking nested classes in DNSResolver

I have the following code:
import unittest, mock
class MockedRRData(object):
def to_text(self):
return '0 example.com.'
class MockedResponse(object):
answer = [[MockedRRData()]]
class MockedReturnValue(object):
response = MockedResponse()
class MockedDNSResolver(object):
def query(self, domain_name, query_type):
return MockedReturnValue()
class DNSQueryTest(unittest.TestCase):
def setUp(self):
# MockedRRData = mock.Mock(to_text=lambda: '0 example.com.')
# MockedResponse = mock.Mock(answer=[[MockedRRData()]])
# MockedReturnValue = mock.Mock()
# MockedReturnValue.attach_mock(MockedResponse, 'response')
# MockedReturnValue = mock.Mock(response=MockedResponse())
self.fake_dns_resolver = mock.Mock(query=lambda *args, **kwargs: MockedReturnValue())
def test_mock(self):
for rrset in self.fake_dns_resolver.query('a', 'b').response.answer:
for rrdata in rrset:
print(rrdata.to_text())
unittest.main()
It works, but I'd like to transition to fully using mock classes - as shown in the comments. The problem is that when I uncomment even just the last line of my comments, I get the following:
E
======================================================================
ERROR: test_mock (__main__.DNSQueryTest)
----------------------------------------------------------------------
Traceback (most recent call last):
File "/tmp/a.py", line 33, in test_mock
for rrset in self.fake_dns_resolver.query('a', 'b').response.answer:
TypeError: 'Mock' object is not iterable
----------------------------------------------------------------------
Ran 1 test in 0.000s
FAILED (errors=1)
How do I fix that?

Apparently, I made the mistake of trying to instantiate mocks twice by adding extra () (mock.Mock returns an instance, not a class template). Here's the working code - compare the previously commented part.
import unittest, mock
class MockedRRData(object):
def to_text(self):
return '0 profound.mail.pairserver.com.'
class MockedResponse(object):
answer = [[MockedRRData()]]
class MockedReturnValue(object):
response = MockedResponse()
class MockedDNSResolver(object):
def query(self, domain_name, query_type):
return MockedReturnValue()
class DNSQueryTest(unittest.TestCase):
def setUp(self):
MockedRRData = mock.Mock(to_text=lambda: '0 profound.mail.pairserver.com.')
MockedResponse = mock.Mock(answer=[[MockedRRData]])
MockedReturnValue = mock.Mock(response=MockedResponse)
self.fake_dns_resolver = mock.Mock(query=lambda *args, **kwargs: MockedReturnValue)
def test_mock(self):
for rrset in self.fake_dns_resolver.query('a', 'b').response.answer:
for rrdata in rrset:
print(rrdata.to_text())
unittest.main()