I haven't been able to find out how to do this in the PyYAML documentation. I want to represent python classes I've defined in YAML, and have a default value given to a parameter in the constructor if it's not specified in the YAML. For example:
>>> class Test(yaml.YAMLObject):
... yaml_tag = u"!Test"
... def __init__(self, foo, bar=3):
... self.foo = foo
... self.bar = bar
... def __repr__(self):
... return "%s(foo=%r, bar=%r)" % (self.__class__.__name__, self.foo, self.bar)
...
>>> yaml.load("""
... --- !Test
... foo: 5
... """)
Traceback (most recent call last):
File "<stdin>", line 4, in <module>
File "<stdin>", line 7, in __repr__
AttributeError: 'Test' object has no attribute 'bar'
I expected that it would create a Test object with bar=3, but I guess it bypasses my constructor when it creates the object. If I include a mapping for bar in the YAML, everything works as expected:
>>> yaml.load("""
... --- !Test
... foo: 5
... bar: 42
... """)
Test(foo=5, bar=42)
Does anyone know how I can have it use a default value?
I encountered the same problem: yaml_tag doesn't work for some reason. So I used alternative approach:
import yaml
def constructor(loader, node) :
fields = loader.construct_mapping(node)
return Test(**fields)
yaml.add_constructor('!Test', constructor)
class Test(object) :
def __init__(self, foo, bar=3) :
self.foo = foo
self.bar = bar
def __repr__(self):
return "%s(foo=%r, bar=%r)" % (self.__class__.__name__, self.foo, self.bar)
print yaml.load("""
- !Test { foo: 1 }
- !Test { foo: 10, bar: 20 }""")
Output:
[Test(foo=1, bar=3), Test(foo=10, bar=20)]
Based on alexanderlukanin13's answer. Here's my cut.
import yaml
YAMLObjectTypeRegistry = {}
def register_type(target):
if target.__name__ in YAMLObjectTypeRegistry:
print "{0} already in registry.".format(target.__name__)
elif 'yaml_tag' not in target.__dict__.keys():
print target.__dict__
raise TypeError("{0} must have yaml_tag attribute".format(
target.__name__))
elif target.__dict__['yaml_tag'] is None:
pass
else:
YAMLObjectTypeRegistry[target.__name__] = target
yaml.add_constructor(
target.__dict__['yaml_tag'],
lambda loader, node: target(**loader.construct_mapping(node)))
print "{0} added to registry.".format(target.__name__)
class RegisteredYAMLObjectType(type):
def __new__(meta, name, bases, class_dict):
cls = type.__new__(meta, name, bases, class_dict)
register_type(cls)
return cls
class RegisteredYAMLObject(object):
__metaclass__=RegisteredYAMLObjectType
yaml_tag = None
You can then use it like this:
class MyType(registry.RegisteredYAMLObject):
yaml_tag = u'!mytype'
def __init__(self, name, attr1='default1', attr2='default2'):
super(MyType, self).__init__()
self.name = name
self.attr1 = attr1
self.attr2 = attr2
The answers above work well, but here is a way to make initialisation work fully with a class based approach:
UNSPECIFIED = object()
class SomeYAMLObject(yaml.YAMLObject):
#classmethod
def from_yaml(cls, loader, node):
arg_spec = inspect.getfullargspec(cls.__init__)
arg_spec.args.remove("self")
arg_defaults = reversed(list(
zip_longest(
reversed(arg_spec.args),
reversed(arg_spec.defaults or []),
fillvalue=UNSPECIFIED)))
kwarg_defaults = reversed(list(
zip_longest(
reversed(arg_spec.kwonlyargs),
reversed(arg_spec.kwonlydefaults or []),
fillvalue=UNSPECIFIED)))
node_mapping = loader.construct_mapping(node)
used_nodes = set()
# fill args first
args = []
for a,d in arg_defaults:
if a in node_mapping:
args.append(node_mapping[a])
used_nodes.add(a)
elif d is not UNSPECIFIED:
args.append(d)
else:
raise Exception(f"Tag {cls.yaml_tag} is missing '{a}' argument")
# then kwargs
kwargs = {}
for a,d in kwarg_defaults:
if a in node_mapping:
kwargs[a] = node_mapping[a]
used_nodes.add(a)
elif d is not UNSPECIFIED:
args[a] = d
# if it accepts additional kwargs, fill with leftover kwargs
if arg_spec.varkw and len(used_nodes) != len(node_mapping):
for k,v in node_mapping:
if k not in used_nodes:
kwargs[k] = v
return cls(*args,**kwargs)
It's a bit lengthy, but gives a nice error if required positional arguments without a default value are missing.
Related
I need to intercept setattr and getattr after init completion, i.e. if main class doesn't have required attribute, it would look for it in subclass Extra, or when setting attribute, if it's not in main class then setting went to subclass Extra, how to understand that init was executed and intercept it only after completion? Here's the code I tried to do it with but it didn't work
class Test:
def __init__(self):
self.default_name = "Michael"
def __setattr__(self, key, value):
if not hasattr(self, key):
self.Extra.__dict__[key] = value;
self.__dict__[key] = v
def __getattr__(self, item):
if not hasattr(self, item):
return self.Extra.__dict__[item]
class Extra:
pass
user = Test()
user.default_name = "Tomas"
user.some_data = "test"
print(user.default_name)
print(user.some_data)
Direct operation attribute dictionary:
class Test:
def __init__(self):
vars(self)['default_name'] = "Michael"
vars(self)['extra'] = Test.Extra()
def __setattr__(self, key, value):
if key not in vars(self):
setattr(self.extra, key, value)
else:
vars(self)[key] = value
def __getattr__(self, item):
return getattr(self.extra, item)
class Extra:
pass
Test:
>>> user = Test()
>>> user.default_name
'Michael'
>>> user.default_name = 'Tomas'
>>> user.default_name
'Tomas'
>>> user.some_data = 'test'
>>> user.some_data
'test'
>>> vars(user)
{'default_name': 'Tomas', 'extra': <__main__.Test.Extra object at 0x000001D5151D6380>}
I create a metaclass that manage the inheritance of __init__ function parameters through parent class.
Let me show you what I mean :
class A(metaclass= MagicMeta):
def __init__(self, a, taunt = None):
print(locals())
self.a = a
self.taunt = taunt
class B(A, metaclass=MagicMeta):
def __init__(self, b):
self.b = b
class Foo(B,metaclass=MagicMeta):
def __init__(self,yolo, name ='empty', surname = None):
print(yolo,a,b)
self.name = name
self.surname= surname
o =Foo(1,2,3,taunt='taunted')
o.taunt
>>> 'taunted'
o.b
>>> 2
My metaclass is working well when it's writing in the same file than my class
But when I import it ,I get this error:
TypeError: super(type, obj): obj must be an instance or subtype of type when my metaclass is imported
my metaclass:
import re
from inspect import Parameter
def get_args(f):
args = list()
kwargs = dict()
for param in inspect.signature(f).parameters.values():
if (param.kind == param.POSITIONAL_OR_KEYWORD):
if param.default ==Parameter.empty:
args.append(param.name)
else:
kwargs[param.name]= param.default
return args, kwargs
def compileKwargs(dct):
string =""
poke = False
for k, o in dct.items():
if type(o) == str:
string+= k+"='"+o+"', "
else:
string+= k+"="+str(o)+", "
return string
def compileKwargs2(dct):
string =""
poke = False
for k, o in dct.items():
if type(o) == str:
string+= k+"='"+k+"', "
else:
string+= k+"="+k+", "
return string
def stringArgs(liste):
return " ".join([e+"," for e in liste])
def compileArgs(liste1,liste2):
liste1.extend([e for e in liste2 if e not in liste1])
return liste1
def editFuncName(actual: str, replace:str):
#print('EDITFUNCNAME')
#print(actual)
string = re.sub('(?<=def ).*?(?=\()',replace, actual)
#print('string', string)
return string
import inspect
from textwrap import dedent, indent
def processCode(code : list):
string=""
#print('processcode')
for i,e in enumerate(code):
#print('row', e)
#print('dedent', e)
if i != 0:
string+=indent(dedent(e),'\t')
else :
string+=dedent(e)
return string
import types
class MagicMeta(type):
def __init__(cls, name, bases, dct):
#print(bases,dct)
setattr(cls,'_CODE_', dict())
#GET THE __init__ code function and its arg and kwargs
# for the class and the inherited class
func = cls.__init__
cls._CODE_[func.__name__]= inspect.getsourcelines(func)
args2 =get_args(cls.__bases__[0].__init__)
setattr(cls,'_ARGS_', dict())
cls._ARGS_[func.__name__]=[get_args(func), args2]
lines = cls._CODE_['__init__']
string= lines[0][0]
arg, kwarg = cls._ARGS_['__init__'][0]
arg2, kwarg2 = cls._ARGS_['__init__'][1]
comparg = stringArgs(compileArgs(arg, arg2))
#------------------------------------------------------
#PROCESS arg and kwargs to manage it as string
dct = {**kwarg ,**kwarg2}
#print(dct)
newargs = comparg + compileKwargs(dct)
string = re.sub('(?<=\().*?(?=\))',newargs, string)
print(type(arg2))
print(arg2)
superarg =stringArgs([a for a in arg2 if a != 'self']) + compileKwargs2(kwarg2)
arg =stringArgs([a for a in arg2 if a != 'self'])
printt = "print({})\n".format(arg)
printtt = "print(locals())\n"
print(superarg)
#--------------------------------------------------------
#ADD the super().__init__ in the __init__ function
superx = "super({},self).{}({})\n".format(cls.__name__, func.__name__, superarg)
#superx = "super().{}({})\n".format( func.__name__, superarg)
print(superx)
code = lines[0]
#print('LINE DEF', code[0])
#--------------------------------------------------------
#BUILD the code of the new __init__ function
code[0]= editFuncName(string, 'tempo')
code.insert(1, printt)
code.insert(2, "print(self, type(self))\n")
if len(bases)>0:
code.insert(3, superx)
print('code:',code)
codestr = processCode(code)
#print('précompile', codestr)
#--------------------------------------------------------
#REPLACE the __init__ function code
comp = compile(codestr, '<string>','exec')
#print(comp)
exec(comp)
cls.__init__ = types.MethodType(eval('tempo'), cls)
#print(eval('tempo.__code__'))
#--------------------------------------------------------
I would avoid to set the code of my metaclass each time I need it
furthermore, I think , for deeply understanding python, it is a good opportunity to learn why import change the class error behaviour, when its inner code is modify dynamically
So I think that with some tweaks, you can solve this problem without using a metaclass here. It looks like you want to use inheritance to have your Foo instance contain b, a, and taunt. But you are passing them in using positional args which is problematic. One solution would be to ingest *args and **kwargs, and pass them to ancestor classes in super calls. Then we could access and remove args[0] to set b and a. That is concerning because if the inheritance order is changed, what is b and a change.
For example:
class A:
def __init__(self, *args):
args = list(args)
a = args.pop(0)
self.a = a
try:
super().__init__(*args)
except TypeError:
pass
class B:
def __init__(self, *args):
args = list(args)
b = args.pop(0)
self.b = b
try:
super().__init__(*args)
except TypeError:
pass
class C(B,A):
pass
class D(A, B):
pass
c = C('a', 'b')
d = D('a', 'b')
>>> c.__dict__
{'b': 'a', 'a': 'b'}
>>> d.__dict__
{'a': 'a', 'b': 'b'}
We cannot depend upon this method to reliably set what a and b are.
So instead we should use keyword args.
This code uses keyword arguments to set those ancestor parameters:
class NewA:
def __init__(self, *, a, taunt = None):
self.a = a
self.taunt = taunt
class NewB(NewA):
def __init__(self, *, b, **kwargs):
super().__init__(**kwargs)
self.b = b
class NewFoo(NewB):
def __init__(self, yolo, name ='empty', surname = None, **kwargs):
super().__init__(**kwargs)
self.name = name
self.surname= surname
f = NewFoo(1, b=2, a=3,taunt='taunted')
>>> print(f.__dict__)
{'a': 3, 'taunt': 'taunted', 'b': 2, 'name': 'empty', 'surname': None}
In it we use a new ability in python3 to require named arguments.
When we try to omit a, we get:
>>> NewFoo(1, b=2, taunt='taunted')
Traceback (most recent call last):
File "<stdin>", line 1, in <module>
File "<stdin>", line 3, in __init__
File "<stdin>", line 3, in __init__
TypeError: __init__() missing 1 required keyword-only argument: 'a'
We can require these named arguments by:
using a * as an argument which tells python that all arguments after it are named
for those named arguments do not set a default value
Per the python docs on function signatures Parameters after “*” or “*identifier” are keyword-only parameters and may only be passed used keyword arguments.
Let's say I have a class like this:
class C:
def __init__(self, stuff: int):
self._stuff = stuff
#property
def stuff(self) -> int:
return self._stuff
then stuff is read-only:
c = C(stuff=10)
print(c.stuff) # prints 10
and
c.stuff = 2
fails as expected
AttributeError: can't set attribute
How can I get the identical behavior using a dataclass? If I wanted to also have a setter, I could do:
#dataclass
class DC:
stuff: int
_stuff: int = field(init=False, repr=False)
#property
def stuff(self) -> int:
return self._stuff
#stuff.setter
def stuff(self, stuff: int):
self._stuff = stuff
But how could I do it without the #stuff.setter part?
This answer extends directly from my other post on using descriptor classes, which is a convenient and handy way to define properties, more or less.
Since dataclasses does not offer a field(frozen=True) approach, I think this one can instead work for you.
Here is a straightforward example of usage below:
from dataclasses import dataclass, MISSING
from typing import Generic, TypeVar
_T = TypeVar('_T')
class Frozen(Generic[_T]):
__slots__ = (
'_default',
'_private_name',
)
def __init__(self, default: _T = MISSING):
self._default = default
def __set_name__(self, owner, name):
self._private_name = '_' + name
def __get__(self, obj, objtype=None):
value = getattr(obj, self._private_name, self._default)
return value
def __set__(self, obj, value):
if hasattr(obj, self._private_name):
msg = f'Attribute `{self._private_name[1:]}` is immutable!'
raise TypeError(msg) from None
setattr(obj, self._private_name, value)
#dataclass
class DC:
stuff: int = Frozen()
other_stuff: str = Frozen(default='test')
dc = DC(stuff=10)
# raises a TypeError: Attribute `stuff` is immutable!
# dc.stuff = 2
# raises a TypeError: Attribute `other_stuff` is immutable!
# dc.other_stuff = 'hello'
print(dc)
# raises a TypeError: __init__() missing 1 required positional argument: 'stuff'
# dc = DC()
Another option, is to use a metaclass which automatically applies the #dataclass decorator. This has a few advantages, such as being able to use dataclasses.field(...) for example to set a default value if desired, or to set repr=False for instance.
Note that once #dataclass_transform comes out in PY 3.11, this could potentially be a good use case to apply it here, so that it plays more nicely with IDEs in general.
In any case, here's a working example of this that I was able to put together:
from dataclasses import dataclass, field, fields
class Frozen:
__slots__ = ('private_name', )
def __init__(self, name):
self.private_name = '_' + name
def __get__(self, obj, objtype=None):
value = getattr(obj, self.private_name)
return value
def __set__(self, obj, value):
if hasattr(obj, self.private_name):
msg = f'Attribute `{self.private_name[1:]}` is immutable!'
raise TypeError(msg) from None
setattr(obj, self.private_name, value)
def frozen_field(**kwargs):
return field(**kwargs, metadata={'frozen': True})
def my_meta(name, bases, cls_dict):
cls = dataclass(type(name, bases, cls_dict))
for f in fields(cls):
# if a dataclass field is supposed to be frozen, then set
# the value to a descriptor object accordingly.
if 'frozen' in f.metadata:
setattr(cls, f.name, Frozen(f.name))
return cls
class DC(metaclass=my_meta):
other_stuff: str
stuff: int = frozen_field(default=2)
# DC.stuff = property(lambda self: self._stuff)
dc = DC(other_stuff='test')
print(dc)
# raises TypeError: Attribute `stuff` is immutable!
# dc.stuff = 41
dc.other_stuff = 'hello'
print(dc)
To get the boilerplate reduction that dataclass provides I found the only way to do this is with a descriptor.
In [236]: from dataclasses import dataclass, field
In [237]: class SetOnce:
...: def __init__(self):
...: self.block_set = False
...: def __set_name__(self, owner, attr):
...: self.owner = owner.__name__
...: self.attr = attr
...: def __get__(self, instance, owner):
...: return getattr(instance, f"_{self.attr}")
...: def __set__(self, instance, value):
...: if not self.block_set:
...: self.block_set = True
...: setattr(instance, f"_{self.attr}", value)
...: else:
...: raise AttributeError(f"{self.owner}.{self.attr} cannot be set.")
In [239]: #dataclass
...: class Foo:
...: bar:str = field(default=SetOnce())
In [240]: test = Foo("bar")
In [241]: test.bar
Out[241]: 'bar'
In [242]: test.bar = 1
---------------------------------------------------------------------------
AttributeError Traceback (most recent call last)
<ipython-input-242-9cc7975cd08b> in <module>
----> 1 test.bar = 1
<ipython-input-237-bddce9441c9a> in __set__(self, instance, value)
12 self.value = value
13 else:
---> 14 raise AttributeError(f"{self.owner}.{self.attr} cannot be set.")
15
AttributeError: Foo.bar cannot be set.
In [243]: test
Out[247]: Foo(bar='bar')
from dataclasses import dataclass
#dataclass(frozen=True)
class YourClass:
"""class definition"""
https://docs.python.org/3/library/dataclasses.html#frozen-instances
After instantiation of the class, when trying to change any of its properties, the exception is raised.
Because using the decorator in the class definition essentially triggers the #dataclass decorator to use the property object as a default field, it doesn't play nice. You can set the property outside like:
>>> from dataclasses import dataclass, field
>>> #dataclass
... class DC:
... _stuff: int = field(repr=False)
... stuff: int = field(init=False)
...
>>> DC.stuff = property(lambda self: self._stuff) # dataclass decorator cant see this
>>> dc = DC(42)
>>> dc
DC(stuff=42)
>>> dc.stuff
42
>>> dc.stuff = 99
Traceback (most recent call last):
File "<stdin>", line 1, in <module>
AttributeError: can't set attribute
import operator
#dataclass
class Enum:
name: str = property(operator.attrgetter("_name"))
def __init__(self, name):
self._name = name
You can do it by combining three things:
Set frozen to False (the default);
Use __post_init__, which is called after the auto-generated __init__ finishes, to mark when the initial setting of values is set and the read-only behavior has to start;
Create your own version of __setattr__ to enforce the read-only behavior after the initial assignment.
Example Person class with a read-only ID field and a read-write name field:
from dataclasses import dataclass
#dataclass
class Person(object):
id : str
name : str
def __post_init__(self):
self._initialized = True
def __setattr__(self, key, value):
if "_initialized" not in self.__dict__:
# we are still inside __init__, assign all values
super().__setattr__(key, value)
elif key == 'id':
# __init__ has finished, enforce read-only attributes
raise AttributeError(f'Attribute id is read-only')
else:
# set read-write attributes normally
super().__setattr__(key, value)
p = Person(id="1234", name="John Doe")
p.name = "John Wick" # succeeds
p.id = "3456" # fails
I haven't implemented __delattr__ in this example, but it could follow the same logic we used on __setattr__.
Using a decorator so you don't need to write this much code for each class:
from typing import Optional, Iterable, Callable, Union
from dataclasses import dataclass
def readonlyattr(attrs : Optional[Union[str, Iterable[str]]] = None):
# ensure attrs is a set of strings
if isinstance(attrs, str):
attrs = set([attrs])
elif not isinstance(attrs, set):
attrs = set(attrs)
# return decorator
def wrap_readonly_attributes(cls: type):
# update post_init method
def make_post_init(cls: type, method: Callable):
def post_init(self, *args, **kwargs):
self._initialized = True
if method:
method(self, *args, **kwargs)
else:
for base in cls.__bases__:
try:
getattr(base, "__post_init__")(self, *args, **kwargs)
except AttributeError:
pass
return post_init
setattr(cls, "__post_init__", make_post_init(cls, getattr(cls, "__post_init__", None)))
# update setattr method
def make_setattr(cls: type, method: Callable):
def new_setattr(self, key, value):
if "_initialized" not in self.__dict__:
if method:
method(self, key, value)
else:
super().__setattr__(key, value)
elif key in attrs:
raise AttributeError(f'Attribute {key} is read-only')
else:
if method:
method(self, key, value)
else:
super().__setattr__(key, value)
return new_setattr
setattr(cls, "__setattr__", make_setattr(cls, getattr(cls, "__setattr__", None)))
return cls
return wrap_readonly_attributes
#dataclass
#readonlyattr(["id", "passport_no"])
class Person(object):
id : str
passport_no : str
name : str
p = Person(id="1234", passport_no="AB12345", name="John Doe")
print(p)
p.name = "John Wick" # succeeds
p.id = "3456" # fails
I'm working with classes that have a lot of instance variables, and I want to have classes that inherit every instance variables from them. something like this:
class foo(object):
def __init__(self,thing1,thing2,thing3,thing4,thing5,thingetc):
self.1 = thing1
self.2 = thing2
self.3 = thing3
self.4 = thing4
self.5 = thing5
self.etc = thingetc
class bar(foo):
self.6 = []
a = bar
print a.3
obviously this won't work, but all the documentation that I can find on line is confusing. How do you inherit variables in cases like this?
Currently, your code is invalid syntax as a digit cannot be at the very front of a variable name. However, you can use *args with __dict__:
class foo:
def __init__(self, *args):
self.__dict__ = dict(zip(['var{}'.format(i) for i in range(1, len(args)+1)], args))
f = foo(*range(15))
print(f.var1)
print(f.var14)
Output:
0
13
Use this as a template for your inheritance, emphasis on the super() method:
class Foo:
def __init__(self):
self.name = 'Foo'
class Bar(Foo):
def __init__(self):
super().__init__()
b = Bar()
b.name
# outputs 'Foo'
For your specific type of class (that takes an unknown number of initialization arguments, i.e. *args):
class Foo:
def __init__(self, *args):
self.name = 'Foo'
for i, arg in enumerate(args):
setattr(self, 'thing_' + str(i), arg)
class Bar(Foo):
def __init__(self, *args):
super().__init__(*args)
b = Bar('hello', 'world')
b.name
# outputs 'Foo'
b.thing_0
# outputs 'hello'
b.thing_1
# outputs 'world'
Now I would personally use the **kwargs over *args for specifying unique instance attributes:
class Foo:
def __init__(self, **kwargs):
self.name = 'Foo'
for att in kwargs:
setattr(self, att, kwargs[att])
class Bar(Foo):
def __init__(self, **kwargs):
super().__init__(**kwargs)
b = Bar(value = 4, area = 3.14)
b.name
# outputs 'Foo'
b.value
# outputs 4
b.area
# outputs 3.14
Ok, lets say I have a really simple class i.e.:
class Test(object):
pass
What I would like to do is to define some default setter and getter methods
which are automatically applied to a new object member at creation time. In the example below a.x should always be uppercase, i.e.:
a = Test()
a.x = "foo"
print a.x
>>> FOO
If I create x within the class I would get this behavior like this:
class Test(object):
def __init__(self):
self._x = ""
#property
def x(self):
return self._x
#x.setter(self, string):
self._x = string.upper()
So is there any possibility to do this without defining setter and getter methods for each member ?? Thank a lot.
EDIT: With creation time I meant the creation time of a.x not of the class instance.
The simplest way is probably to override __setattr__, and change any string values to uppercase:
>>> class Test(object):
def __setattr__(self, attr, val):
if isinstance(val, basestring):
val = val.upper()
super(Test, self).__setattr__(attr, val)
>>> t = Test()
>>> t.x = 'foo'
>>> t.x
'FOO'
Subclass a dict;
In [1]: %cpaste
Pasting code; enter '--' alone on the line to stop or use Ctrl-D.
:class Struct(dict):
: """A dict subclass where you can simply use a dot to access attributes."""
:
: def __getattr__(self, name):
: return self[name]
:
: def __setattr__(self, name, value):
: self[name] = value
:--
In [2]: a = Struct()
In [3]: a.x = "foo"
In [4]: a.x
Out[4]: 'foo'
In [5]: a.length = 14
In [6]: a
Out[6]: {'length': 14, 'x': 'foo'}
That sounds like a use-case for pythons Descriptor Proctocol.
class WithDescriptors:
x = UpperCaseDescriptor()
y = UpperCaseDescriptor()
z = UpperCaseDescriptor()
class UperCaseDescriptor(object):
def __init__(self):
self.val = ''
def __get__(self, obj, objtype):
return self.val.upper()
def __set__(self, obj, val):
self.val = val
Thats just an outline and i didnt test the code to work!
If you want to extend such behaviour to every attribute of an instance,
even which are not existent, you should consider metaclasses.