We Keep Coding

How can I have json.dumps treat my class as a dict?

I would like to create a custom Python class that JSON-serializes like a dict. Taking Python's duck-typing at its name, I thought I could create a class that looks and quacks exactly like a dict. However, the class shown below is apparently not dict-like enough for json.dumps -- the code below produces the error TypeError: Object of type TotallyADict is not JSON serializable. What can I change about TotallyADict so that the default encoder for json.dumps will output {"a": 1, "b": 2, "c": 3}?
I know this immediate issue can be resolved by creating a custom encoder, but that is not an acceptable solution in the larger issue this specific problem has been distilled from.
Another attempted solution is to have TotallyADict inherit from dict rather than MutableMapping. This does not throw any exceptions, but in that case json.dumps(x) yields {}; apparently the data source the default encoder for json.dumps uses for dicts is not any of the overridden methods below.
What I want here is to able to use attribute semantics (x.c = x.a + x.b) but still serialize into a JSON object. So, a possible suggestion that does not seem to work is TypedDict (would have to be x['c'] = x['a'] + x['b']). Intercepting attribute assignment and retrievals via __setattr__ and __getattribute__ and redirecting to entries self which inherits from dict seems to work well enough, so that's my default solution. But I'm surprised that the one time I actually want to use duck-typing rather than strict(ish) typing, it doesn't seem to work.
from collections.abc import MutableMapping
import json
class TotallyADict(MutableMapping):
def __init__(self, a, b, c):
self.a = a
self.b = b
self.c = c
self._fields = {'a', 'b', 'c'}
def __getitem__(self, key):
if key in self._fields:
return getattr(self, key)
else:
raise KeyError('"{}" is not a field in {}'.format(key, type(self).__name__))
def __setitem__(self, key, value):
if key in self._fields:
setattr(self, key, value)
else:
raise KeyError('"{}" is not a field in {}'.format(key, type(self).__name__))
def __delitem__(self, key):
raise RuntimeError('Cannot delete fields from {}'.format(type(self).__name__))
def __iter__(self):
return iter(self._fields)
def __len__(self):
return len(self._fields)
def __contains__(self, k):
return k in self._fields
def copy(self):
return type(self)(**{k: getattr(self, k) for k in self._fields})
def __repr__(self):
return '{' + ', '.join('"{}": {}'.format(k, repr(getattr(self, k))) for k in self._fields) + '}'
def get(self, key, default=None):
if key in self._fields:
value = getattr(self, key)
if value is None:
value = default
return value
else:
raise KeyError('"{}" is not a field in {}'.format(key, type(self).__name__))
def setdefault(self, key, default=None):
if key in self._fields:
value = getattr(self, key)
if value is None:
value = default
setattr(self, key, value)
return value
else:
raise KeyError('"{}" is not a field in {}'.format(key, type(self).__name__))
def pop(self, key, value=None):
raise RuntimeError('Cannot delete fields from {}'.format(type(self).__name__))
def keys(self):
return self._fields
def items(self):
return [(k, getattr(self, k)) for k in self._fields]
def values(self):
return [getattr(self, k) for k in self._fields]
def __eq__(self, other):
if type(self) is type(other):
for k in self._fields:
if getattr(self, k) != getattr(other, k):
return False
return True
else:
return False
def __ne__(self, other):
return not self.__eq__(other)
x = TotallyADict(1, 2, 3)
print(json.dumps(x))

The issue here is your _fields variable. This wont serialize to a JSON object as {'c', 'b', 'a'} is not valid json. If you look at the x.__dict__ property you can see what this object will be represented as.
{'a': 1, 'b': 2, 'c': 3, '_fields': {'c', 'b', 'a'}}
If you change _fields to a list you could also use the default parameter in JSON.dumps
These are the changes I made to get what you are looking for to work
self._fields = ['a', 'b', 'c']
print(json.dumps(x, default=vars))
Here is the full code with my canges.
from collections.abc import MutableMapping
import json
class TotallyADict(MutableMapping):
def __init__(self, a, b, c):
self.a = a
self.b = b
self.c = c
self._fields = ['a', 'b', 'c']
def __getitem__(self, key):
if key in self._fields:
return getattr(self, key)
else:
raise KeyError('"{}" is not a field in {}'.format(key, type(self).__name__))
def __setitem__(self, key, value):
if key in self._fields:
setattr(self, key, value)
else:
raise KeyError('"{}" is not a field in {}'.format(key, type(self).__name__))
def __delitem__(self, key):
raise RuntimeError('Cannot delete fields from {}'.format(type(self).__name__))
def __iter__(self):
return iter(self._fields)
def __len__(self):
return len(self._fields)
def __contains__(self, k):
return k in self._fields
def copy(self):
return type(self)(**{k: getattr(self, k) for k in self._fields})
def __repr__(self):
return '{' + ', '.join('"{}": {}'.format(k, repr(getattr(self, k))) for k in self._fields) + '}'
def get(self, key, default=None):
if key in self._fields:
value = getattr(self, key)
if value is None:
value = default
return value
else:
raise KeyError('"{}" is not a field in {}'.format(key, type(self).__name__))
def setdefault(self, key, default=None):
if key in self._fields:
value = getattr(self, key)
if value is None:
value = default
setattr(self, key, value)
return value
else:
raise KeyError('"{}" is not a field in {}'.format(key, type(self).__name__))
def pop(self, key, value=None):
raise RuntimeError('Cannot delete fields from {}'.format(type(self).__name__))
def keys(self):
return self._fields
def items(self):
return [(k, getattr(self, k)) for k in self._fields]
def values(self):
return [getattr(self, k) for k in self._fields]
def __eq__(self, other):
if type(self) is type(other):
for k in self._fields:
if getattr(self, k) != getattr(other, k):
return False
return True
else:
return False
def __ne__(self, other):
return not self.__eq__(other)
x = TotallyADict(1, 2, 3)
print(json.dumps(x, default=vars))
You could also try using a UserDict
https://docs.python.org/3/library/collections.html#collections.UserDict

In some instances, the easiest solution is the best one. In this case, create a to_dict() function that returns the data inside your custom class as a Python dictionary before json dumping it.
This way, you can manipulate the data within your class at your leisure, and convert it to a dictionary when other libraries expect a dictionary. Then if you need the opposite, just write another function that parses dict into your custom class.
Since this class is intended to hold data, I recommend using DataClasses.
Then you can just add this function to your class to get its attributes as a dict:
from dataclasses import dataclass, asdict
def get_as_dict(self):
return {k: v for k, v in asdict(self).items() if self._dataclass_fields_[k].repr}

How to force calling __getitem__ using items() and values() in dict inheritance in python

I use python3.7 and for some application create a class inherited from a dict, but have a problem to implement items() and values() methods to made it work correctly. This class overrides many methods, but here I placed a very simplified example just to illustrate exact problem:
class MyFunction:
def __call__(self):
return 5
class MyDict(dict):
def __getitem__(self, key):
item = super().__getitem__(key)
if isinstance(item, MyFunction):
return item()
else:
return item
def get(self, key, default=None):
if self.__contains__(key):
return self.__getitem__(key)
if isinstance(default, MyFunction):
return default()
return default
# def __copy__(self):
# return type(self)(self)
#
# def copy(self):
# return self.__copy__()
# def __iter__(self):
# return super().__iter__()
d = MyDict(a=MyFunction(), b=3)
I want than I get a value by key instances of MyFunction be called. This works well:
for k in d:
print(k, d[k])
and prints the expected output:
a 5
b 3
But these two do not:
for v in d.values():
print(v)
for k, v in d.items():
print(k, v)
They print the function's repr.
How can I achive them to call __getitem__?
Remark: It can be some kind of dict built-in class optimization (I would like to not inherit form UserDict or Mapping). For example if I uncomment:
def __iter__(self):
return super().__iter__()
The calls:
new_d = d.copy()
new_d = dict(d)
new_d = dict(**d)
will call the __getitem__

According to the Python documentation, accessing a dict-like object with the d[k] syntax is just syntactic sugar for d.__getitem__(k).
However, as you discovered, the default implementation of values() and items() do not call __getitem__() at all. If you want them to, you'll have to implement them yourself.
Here is an implementation that hopefully does what you want:
class MyFunction:
def __call__(self):
return 5
class MyDict(dict):
def __getitem__(self, key):
item = super().__getitem__(key)
if isinstance(item, MyFunction):
return item()
else:
return item
def values(self):
for k in self:
yield(self[k])
def items(self):
for k in self:
yield(k, self[k])
def get(self, key, default=None):
try:
return self[key]
except KeyError:
return default
d = MyDict(a=MyFunction(), b=3)
for k in d:
print(d.get(k))
for v in d.values():
print(v)
for k, v in d.items():
print(k, v)
This is the output:
5
3
5
3
a 5
b 3

Why does defaultdict register keys without assignments

In python 3, if you define a defaultdict, trigger the default values to be returned with a non-existent key, then that key will be automatically put into the dictionary?? This:
foo = defaultdict(int)
1 in foo # False
foo[1] # 0
1 in foo # True????
This seems erroneous to me. I feel like the purpose of defaultdict is to allow the user to get a default value without putting that key in the dictionary. Why did the language developers choose this and how can I avoid it?

It's not about language design. You can choose not to use collection.defaultdict.
You can define your own dictionary that act as you want by defining __missing__ method:
>>> class MyDefaultDict(dict):
... def __init__(self, default_factory):
... self.default_factory = default_factory
... def __missing__(self, key):
... # called by dict.__getitem__ when key is not in the dictionary.
... return self.default_factory()
...
>>>
>>> foo = MyDefaultDict(int)
>>> 1 in foo
False
>>> foo[1]
0
>>> 1 in foo
False
SIDE NOTE: defaultdict is implemented using __missing__.

You cannot avoid this using defaultdict as it's in the core implementation of it:
class defaultdict:
#staticmethod
def __new__(cls, default_factory=None, **kwargs):
# Some code (e.g. urllib.urlparse) expects that basic defaultdict
# functionality will be available to subclasses without them
# calling __init__().
self = super(defaultdict, cls).__new__(cls)
self.d = {}
return self
def __init__(self, default_factory=None, **kwargs):
self.d = kwargs
self.default_factory = default_factory
def __getitem__(self, key):
try:
return self.d[key]
except KeyError:
v = self.__missing__(key)
self.d[key] = v
return v
def __setitem__(self, key, v):
self.d[key] = v
def __delitem__(self, key):
del self.d[key]
def __contains__(self, key):
return key in self.d
def __missing__(self, key):
if self.default_factory is None:
raise KeyError(key)
return self.default_factory()
You can see clearly here
def __getitem__(self, key):
try:
return self.d[key]
except KeyError:
v = self.__missing__(key)
self.d[key] = v
return v
That after getting a default value for missing key, it registers the key you tried in the dict.
That is really important in a case like having a list map for example:
from collections import defaultdict
list_map = defaultdict(list)
list_map['some_key'].append(5)
print(list_map['some_key'])
will output:
[5]
here you can see that I'm trying to get a new key and at the same I want to add a new element to the list referenced by that key, in your proposal that won't be valid since you will return me a default empty list every time that is linked to Nothing.
in your implementation it will output instead
[]
You could change the implementation if you like, it's a small change but you have to count for the consequences of your change.

Count reads from python dictionary with unpacking

I am interested in counting the number of accesses to a dictionary's values. I am unsure how to include dictionary unpacking in the counter. Any tips?
from collections import defaultdict
class LDict(dict):
def __init__(self, *args, **kwargs):
'''
This is a read-counting dictionary
'''
super().__init__(*args, **kwargs)
self._lookup = defaultdict(lambda : 0)
def __getitem__(self, key):
retval = super().__getitem__(key)
self._lookup[key] += 1
return retval
def __setitem__(self, key, value):
super().__setitem__(key, value)
self._lookup[key] = self._lookup.default_factory()
def __delitem__(self, key):
super().__delitem__(self, key)
_ = self._lookup[key]
del self._lookup[key]
def list_unused(self):
return [key for key in self if self._lookup[key] == 0]
l = LDict(a='apple', b='bugger')
print({**l, **l})
print(l.list_unused())
_ = l['a']
print(l.list_unused())

You need to override more methods. Access is not centralized through __getitem__(): other methods like copy(), items(), etc. access the keys without going through __getitem()__. I would assume the ** operator uses items(), but you will need to handle ALL of the methods to keep track of EVERY access. In many cases you will have to make a judgement call. For example, does __repr__() count as an access? The returned string contains every key and value formatted, so I think it does.
I would recommend overriding all of these methods, because you have to do bookkeeping on assignment too.
def __repr__(self):
def __len__(self):
def __iter__(self):
def clear(self):
def copy(self):
def has_key(self, k):
def update(self, *args, **kwargs):
def keys(self):
def values(self):
def items(self):
EDIT: So apparently there's an important caveat here that directly relates to your implementation. if LDict extends dict, then none of these methods are invoked during the dictionary unpacking { **l, **l}.
Apparently you can follow the advice here though, and implement LDict without extending dict. This worked for me:
from collections import MutableMapping
class LDict(MutableMapping):
def __init__(self, *args, **kwargs):
'''
This is a read-counting dictionary
'''
self._lookup = defaultdict(lambda : 0)
self.data = {}
if kwargs:
self.data.update(kwargs)
def __getitem__(self, key):
retval = self.data[key]
self._lookup[key] += 1
return retval
def __setitem__(self, key, value):
self.data[key] = value
self._lookup[key] = self._lookup.default_factory()
def __delitem__(self, key):
del self.data[key]
_ = self._lookup[key]
del self._lookup[key]
def items(self):
print('items is being called!')
yield from self.data.items()
def __iter__(self):
print('__iter__ is being called!')
yield from self.data
def __len__(self):
return len(self.data)
def list_unused(self):
return [key for key in self if self._lookup[key] == 0]
l = LDict(a='apple', b='bugger')
print({**l, **l})
print(l.list_unused())
_ = l['a']
print(l.list_unused())
which produces the output:
__iter__ is being called!
__iter__ is being called!
{'b': 'bugger', 'a': 'apple'}
__iter__ is being called!
[]
__iter__ is being called!
[]
(I only implemented the bare minimum to get example to work, I still recommend implementing the set of methods I listed about if you want your counts to be correct!)
So I guess the answer to your question is you have to
Implement the __iter__(self) method
DO NOT inherit from dict().

A dict-like class that uses transformed keys

I'd like a dict-like class that transparently uses transformed keys on lookup, so that I can write
k in d # instead of f(k) in d
d[k] # instead of d[f(k)]
d.get(k, v) # instead of d.get(f(k), v)
etc. (Imagine for example that f does some kind of canonicalization, e.g. f(k) returns k.lower().)
It seems that I can inherit from dict and override individual operations, but not that there is a centralized spot for such transformation that all keys go through. That means I have to override all of __contains__, __getitem__, get, and possibly __missing__, etc. This gets too tedious and error-prone, and not very attractive unless this overhead outweighs that of manually substituting f(k) for every call on a plain dict.

Well, the idiomatic way to do it is probably using dimo414's answer. For the case where the transform is not pure (do not always evaluates the same result value given the same argument):
class Foo(dict):
def __init__(self, transform, *args, **kwargs):
super(Foo, self).__init__(self, *args, **kwargs)
assert isfunction(transform), u'Transform argument must be a function.'
self._transform = transform
def get(self, k, d=None):
return super(Foo, self).get(self._transform(k), d)
def __getitem__(self, item):
return super(Foo, self).__getitem__(self._transform(item))
def __contains__(self, item):
return super(Foo, self).__contains__(self._transform(item))
def __repr__(self):
return '<Foo instance {}>'.format(id(self))
Testing:
>>> import datetime
>>> # {0: '0', 1: '1', 2: '2' ... 99: '99'}
>>> x = Foo(lambda x: (datetime.datetime.now() - x).seconds, ((i, str(i)) for i in range(10)))
>>> t = datetime.datetime.now()
>>> x.get(t)
'5'
>>> x[t]
'12'
Not that tedious but I don't like how it smells (in terms of design).

I'm not sure why your question is being downvoted, it's a reasonable thing to want. In Java, Guava provides several map transformation utilities which provide views into the backing map like you're describing. However they don't provide a Maps.transformKeys() method because it's actually not a very useful function. See How to convert Map<String, String> to Map<Long, String> using guava and Why Guava does not provide a way to transform map keys for details as to why.
In short, it's not possible to efficiently provide key transformations in the general case. Rather than creating the complex and possibly inconsistent data structure you're envisioning, the best thing to do is likely to just create a new dict applying your key transformation, e.g.:
{ f(k): v for k, v in d.iteritems() }

Since you want to maintain the exact same signature as dict(), I
propose creating a factory function to wrap a TransformDict to provide
the same signature.
def transform_dict(transform_key):
def _transform_dict(*args, **kwargs):
return TransformDict(transform_key, *args, **kwargs)
return _transform_dict
Which can be used as:
>>> LowerDict = transform_dict(lambda k: k.lower())
>>> lower_dict = LowerDict({'FOO': 1}, BaR=2)
TransformDict(<function <lambda> at 0x12345678>, {'foo': 1, 'bar': 2})
The TransformDict should implement the MutableMapping abstract
base class so that any potentially missed dict method will not pass
silently. All methods dealing with transforming the key can be
implemented in terms of __contains__(), __getitem__(),
__setitem__(), and __delitem__().
import collections
import sys
class TransformDict(collections.MutableMapping):
def __init__(self, __transform_key, *args, **kwargs):
self.data = dict(*args, **kwargs)
self.transform_key = __transform_key
# Key methods.
def __contains__(self, key):
key = self.transform_key(key)
return key in self.data
def __getitem__(self, key):
key = self.transform_key(key)
return self.data[key]
def __setitem__(self, key, value):
key = self.transform_key(key)
self.data[key] = value
def __delitem__(self, key):
key = self.transform_key(key)
del self.data[key]
# Operator methods.
def __iter__(self):
return iter(self.data)
def __len__(self):
return len(self.data)
def __eq__(self, other):
if isinstance(other, TransformDict):
other = other.data
return self.data == other
def __ne__(self, other):
return not (self == other)
def __repr__(self):
return "{}({!r}, {!r})".format(self.__class__.__name__, self.transform_key, self.data)
# Accessor methods.
def get(self, key, default=None):
if key in self:
return self[key]
return default
def keys(self):
return self.data.keys()
def items(self):
return self.data.items()
def values(self):
return self.data.values()
if sys.version_info[0] == 2:
def iterkeys(self):
return self.data.iterkeys()
def itervalues(self):
return self.data.itervalues()
def iteritems(self):
return self.data.iteritems()
def viewkeys(self):
return self.data.viewkeys()
def viewvalues(self):
return self.data.viewvalues()
def viewitems(self):
return self.data.viewitems()
# Mutable methods.
def clear(self):
self.data.clear()
def pop(self, key, default=KeyError):
if key in self or default is KeyError:
value = self[key]
del self[key]
return value
return default
def popitem(self):
return self.data.popitem()
def setdefault(self, key, default=None):
if key not in self:
self[key] = default
return default
return self[key]
def update(self, other):
for key for other:
self[key] = other[key]
# Miscellaneous methods.
def copy(self):
return self.__class__(self.transform_key, self.data)