I'm trying to write a tracker class where the instances of the tracker class track the sub-classes of another class that are in the scope of the tracker instance.
More concretely, the following is an example of what I am trying to achieve:
class Foo(object): pass
class FooTracker(object):
def __init__(self):
# use Foo.__subclasses__() or a metaclass to track subclasses
# - but how do I filter this to only get the ones in scope?
self.inscope = <something magic goes here>
ft1 = FooTracker()
assert ft1.inscope == []
class Bar(Foo): pass
ft2 = FooTracker()
assert ft2.inscope == [<class '__main__.Bar'>]
def afunction():
class Baz(Foo): pass # the global definition of Bar is now hidden
class Bar(Foo): pass
ft3 = FooTracker()
assert (set(ft3.inscope) == set([<class '__main__.afunction.<locals>.Baz'>,
<class '__main__.afunction.<locals>.Bar'>])
ft4 = FooTracker() # afunction.Baz and afunction.Bar are no longer in scope
assert ft4.inscope == [<class '__main__.Bar'>]
So I want the instances of FooTracker to track the sub-classes of Foo that were in scope at the time the FooTracker object was created.
I've tried a few different things, such as parsing the qualified names of the Foo sub-classes and using exec() to do the name resolution but the fundamental problem is that it always works out the sub-classes relative to the scope within FooTracker.__init__() and not where it was called.
My only other thought was to try something with inspect.currentframe() but even if this were possible it would probably be too much of a hack and would make the code too brittle (e.g., there is a comment in the docs that not all Python implementations will have frame support in the interpreter").
There's no easy way to do exactly what you're asking for. But you might be able to use some Python features to get something with a roughly similar API, without as much hassle.
One option would be to require each subclass to be decorated with a method of your Tracker class. This would make it really easy to keep track of them, since you'd just append each caller of the method to a list:
class Tracker:
def __init__(self):
self.subclasses = []
def register(self, cls):
self.subclasses.append(cls)
return cls
class Foo(): pass
foo_tracker = Tracker()
#foo_tracker.register
class FooSubclass1(Foo): pass
#foo_tracker.register
class FooSubclass2(Foo): pass
print(foo_tracker.subclasses)
This doesn't actually require that the classes being tracked are subclasses of Foo, all classes (and even non-class objects) can be tracked if you pass them to the register method. Decorator syntax makes it a little nicer than just appending each class to a list after you define it, but not by a whole lot (you still repeat yourself a fair amount, which may be annoying unless you make the tracker and method names very short).
A slightly trickier version might get passed the base class, so that it would detect subclasses automatically (via Foo.__subclasses__). To limit the subclasses it detects (rather than getting all subclasses of the base that have ever existed), you could make it behave as a context manager, and only track new subclasses defined within a with block:
class Tracker:
def __init__(self, base):
self.base = base
self._exclude = set()
self.subclasses = set()
def __enter__(self):
self._exclude = set(self.base.__subclasses__())
return self
def __exit__(self, *args):
self.subclasses = set(self.base.__subclasses__()) - self._exclude
return False
class Foo(): pass
class UntrackedSubclass1(Foo): pass
with Tracker(Foo) as foo_tracker:
class TrackedSubclass1(Foo): pass
class TrackedSubclass2(Foo): pass
class UntrackedSubclass2(Foo): pass
print(foo_tracker.subclasses)
If you're using Python 3.6 or later, you can do the tracking a different way by injecting an __init_subclass__ class method into the tracked base class, rather than relying upon __subclasses__. If you don't need to support class hierarchies that are already using __init_subclass__ for their own purposes (and you don't need to support nested trackers), it can be quite elegant:
class Tracker:
def __init__(self, base):
self.base = base
self.subclasses = []
def __enter__(self):
#classmethod
def __init_subclass__(cls, **kwargs):
self.subclasses.append(cls)
self.base.__init_subclass__ = __init_subclass__
return self
def __exit__(self, *args):
del self.base.__init_subclass__
return False
class Foo(): pass
class UntrackedSubclass1(Foo): pass
with Tracker(Foo) as foo_tracker:
class TrackedSubclass1(Foo): pass
class TrackedSubclass2(Foo): pass
class UntrackedSubclass2(Foo): pass
print(foo_tracker.subclasses)
One nice feature of this version is that it automatically tracks deeper inheritance hierarchies. If a subclass of a subclass is created within the with block, that "grandchild" class will still be tracked. We could make the previous __subclasses__ based version work this way too, if you wanted, by adding another function to recursively expand out the subclasses of each class we find.
If you do want to play nice with existing __init_subclass__ methods, or want to be able to nest trackers, you need to make the code a bit more complicated. Injecting a well behaved classmethod in a reversible way is tricky since you need handle both the case where the base class has its own method, and the case where it's inheriting a version from its parents.
class Tracker:
def __init__(self, base):
self.base = base
self.subclasses = []
def __enter__(self):
if '__init_subclass__' in self.base.__dict__:
self.old_init_subclass = self.base.__dict__['__init_subclass__']
else:
self.old_init_subclass = None
#classmethod
def __init_subclass__(cls, **kwargs):
if self.old_init_subclass is not None:
self.old_init_subclass.__get__(None, cls)(**kwargs)
else:
super(self.base, cls).__init_subclass__(**kwargs)
self.subclasses.append(cls)
self.base.__init_subclass__ = __init_subclass__
return self
def __exit__(self, *args):
if self.old_init_subclass is not None:
self.base.__init_subclass__ = self.old_init_subclass
else:
del self.base.__init_subclass__
return False
class Foo:
def __init_subclass__(cls, **kwargs):
super().__init_subclass__(**kwargs)
print("Foo!")
class Bar(Foo): pass # every class definition from here on prints "Foo!" when it runs
with Tracker(Bar) as tracker1:
class Baz(Bar): pass
with Tracker(Foo) as tracker2:
class Quux(Foo): pass
with Tracker(Bar) as tracker3:
class Plop(Bar): pass
# four Foo! lines will have be printed by now by Foo.__init_subclass__
print(tracker1.subclasses) # will describe Baz and Plop, but not Quux
print(tracker2.subclasses) # will describe Quux and Plop
print(tracker3.subclasses) # will describe only Plop
Related
Currently I am writing a Python program with a plugin system. To develop a new plugin a new class must be created and inherit from a base plugin class. Now it should be possible to add optional functions via mixins. Some mixins provide new functions others access builtin types of the base class and can act with them or change them.
In the following a simplified structure:
import abc
import threading
class Base:
def __init__(self):
self.config = dict()
if hasattr(self, "edit_config"):
self.edit_config()
def start(self):
"""Starts the Plugin"""
if hasattr(self, "loop"):
self._loop()
class AMixin:
def edit_config(self):
self.config["foo"] = 123
class BMixin(abc.ABC):
def _loop(self):
thread = threading.Thread(target=self.loop, daemon=True)
thread.start()
#abc.abstractmethod
def loop(self):
"""Override this method with a while true loop to establish a ongoing loop
"""
pass
class NewPlugin(Base, AMixin, BMixin):
def loop(self):
while True:
print("Hello")
plugin = NewPlugin()
plugin.start()
What is the best way to tackle this problem?
EDIT: I need to make my question more specific. The question is whether the above is the Pythonic way and is it possible to ensure that the mixin are inherited exclusively in conjunction with the Base class. Additionally it would be good in an IDE like VSCode to get support for e.g. autocomplete when accessing builtin types of the Base class, like in AMixin, without inheriting from it of course.
If you want to allow but not require subclasses to define some behaviour in a method called by the base class, the simplest way is to declare the method in the base class, have an empty implementation, and just call the method unconditionally. This way you don't have to check whether the method exists before calling it.
class Base:
def __init__(self):
self.config = dict()
self.edit_config()
def start(self):
self.loop()
def edit_config(self):
pass
def loop(self):
pass
class AMixin:
def edit_config(self):
self.config["foo"] = 123
class NewPlugin(AMixin, Base):
def loop(self):
for i in range(10):
print("Hello")
Note that you have to write AMixin before Base in the list of superclasses, so that its edit_config method overrides the one from Base, and not the other way around. You can avoid this by writing class AMixin(Base): so that AMixin.edit_config always overrides Base.edit_config in the method resolution order.
If you want to require subclasses to implement one of the methods, then you can raise TypeError() instead of pass in the base class's method.
I would move the calls to the methods provided by the mix-ins to __init__ methods defined by those classes.
import abc
import threading
class Base:
def __init__(self, **kwargs):
super.__init__(**kwargs)
self.config = dict()
class AMixin:
def __init__(self, **kwargs):
super().__init__(**kwargs)
self.edit_config()
def edit_config(self):
self.config["foo"] = 123
class BMixin(abc.ABC):
def __init__(self, **kwargs):
super().__init__(**kwargs):
self.loop()
def _loop(self):
thread = threading.Thread(target=self.loop, daemon=True)
thread.start()
#abc.abstractmethod
def loop(self):
"""Override this method with a while true loop to establish a ongoing loop
"""
pass
class NewPlugin(Base, AMixin, BMixin):
pass
When you instantiate a concrete subclass of NewPlugin, Base.__init__, AMixin.__init__, and BMixin.__init__ will be called in that order.
I read that it is considered bad practice to create a variable in the class namespace and then change its value in the class constructor.
(One of my sources: SoftwareEngineering SE: Is it a good practice to declare instance variables as None in a class in Python.)
Consider the following code:
# lib.py
class mixin:
def __init_subclass__(cls, **kwargs):
cls.check_mixin_subclass_validity(cls)
super().__init_subclass__(**kwargs)
def check_mixin_subclass_validity(subclass):
assert hasattr(subclass, 'necessary_var'), \
'Missing necessary_var'
def method_used_by_subclass(self):
return self.necessary_var * 3.14
# app.py
class my_subclass(mixin):
necessary_var = None
def __init__(self, some_value):
self.necessary_var = some_value
def run(self):
# DO SOME STUFF
self.necessary_var = self.method_used_by_subclass()
# DO OTHER STUFF
To force its subclass to declare the variable necessary_var, the class mixin uses the metaclass subclass_validator.
And the only way I know to makes it work on app.py side, is to initialized necessary_var as a class variable.
I am missing something or is it the only way to do so?
Short answer
You should check that attributes and methods exist at instantiation of a class, not before. This is what the abc module does and it has good reasons to work like this.
Long answer
First, I would like to point out that it seems what you want to check is that an instance attribute exists.
Due to Python dynamic nature, it is not possible to do so before an instance is created, that is after the call to __init__. We could define Mixin.__init__, but we would then have to rely on the users of your API to have perfect hygiene and to always call super().__init__.
One option is thus to create a metaclass and add a check in its __call__ method.
class MetaMixin(type):
def __call__(self, *args, **kwargs):
instance = super().__call__(*args, **kwargs)
assert hasattr(instance, 'necessary_var')
class Mixin(metaclass=MetaMixin):
pass
class Foo(Mixin):
def __init__(self):
self.necessary_var = ...
Foo() # Works fine
class Bar(Mixin):
pass
Bar() # AssertionError
To convince yourself that it is good practice to do this at instantiation, we can look toward the abc module which uses this behaviour.
from abc import abstractmethod, ABC
class AbstractMixin(ABC):
#abstractmethod
def foo(self):
...
class Foo(AbstractMixin):
pass
# Right now, everything is still all good
Foo() # TypeError: Can't instantiate abstract class Foo with abstract methods foo
As you can see the TypeError was raise at instantiation of Foo() and not at class creation.
But why does it behave like this?
The reason for that is that not every class will be instantiated, consider the example where we want to inherit from Mixin to create a new mixin which checks for some more attributes.
class Mixin:
def __init_subclass__(cls, **kwargs):
assert hasattr(cls, 'necessary_var')
super().__init_subclass__(**kwargs)
class MoreMixin(Mixin):
def __init_subclass__(cls, **kwargs):
assert hasattr(cls, 'other_necessary_var')
super().__init_subclass__(**kwargs)
# AssertionError was raised at that point
class Foo(MoreMixin):
necessary_var = ...
other_necessary_var = ...
As you see, the AssertionError was raised at the creation of the MoreMixin class. This is clearly not the desired behaviour since the Foo class is actually correctly built and that is what our mixin was supposed to check.
In conclusion, the existence of some attribute or method should be done at instantiation, Otherwise, you are preventing a whole lot of helpful inheritance techniques. This is why the abc module does it like that and this is why we should.
I have the following simplified scheme:
class NetworkAnalyzer(object):
def __init__(self):
print('is _score_funct implemented?')
#staticmethod
def _score_funct(network):
raise NotImplementedError
class LS(NetworkAnalyzer):
#staticmethod
def _score_funct(network):
return network
and I am looking for what I should use instead of print('is _score_funct implemented?') in order to figure out if a subclass has already implemented _score_funct(network) or not.
Note: If there is a more pythonic/conventional way of structuring the code, I would also appreciate its mention. The reason I defined it this way is, some NetworkAnalyzer subclasses have _score_funct in their definition, and the ones that dont have it will have different initialization of variables although they will have the same structure
Use an abstract base class and you won't be able to instantiate the class unless it implements all of the abstract methods:
import abc
class NetworkAnalyzerInterface(abc.ABC):
#staticmethod
#abc.abstractmethod
def _score_funct(network):
pass
class NetworkAnalyzer(NetworkAnalyzerInterface):
def __init__(self):
pass
class LS(NetworkAnalyzer):
#staticmethod
def _score_funct(network):
return network
class Bad(NetworkAnalyzer):
pass
ls = LS() # Ok
b = Bad() # raises TypeError: Can't instantiate abstract class Bad with abstract methods _score_funct
I'm not a metaclass/class specialist but here's a method that works in your simple case (not sure it works as-is in a complex/nested class namespace):
To check if the method was overridden, you could try a getattr on the function name, then check the qualified name (class part is enough using string partitionning):
class NetworkAnalyzer(object):
def __init__(self):
funcname = "_score_funct"
d = getattr(self,funcname)
print(d.__qualname__.partition(".")[0] == self.__class__.__name__)
if _score_funct is defined in LS, d.__qualname__ is LS._score_funct, else it's NetworkAnalyzer._score_funct.
That works if the method is implemented at LS class level. Else you could replace by:
d.__qualname__.partition(".")[0] != "NetworkAnalyzer"
Of course if the method is overridden with some code which raises an NotImplementedError, that won't work... This method doesn't inspect methods code (which is hazardous anyway)
class Email():
def __init__(self, store_number):
self.store_number = store_number
def amethod(self):
pass
What is the correct way to pass variables from a sub-class to a parent-class?
should I do:
class MoreSpecificEmail():
def __init__(self, store_number):
Email.__init__(self, store_number=store_number)
def another_method(self):
pass
or:
class MoreSpecificEmail():
def __init__(self, store_number):
self.store_number = store_number
Email.__init__(self, store_number=self.store_number)
I have just been using different abbreviations of store_number in each sub-class to help clarify what's going on in my head. I am sure that is the wrong way, though.
What you currently have isn't inheritance; neither of your classes actually inherits from anything! Firstly, Email should be a "new-style class", inheriting from object:
class Email(object):
# ^ note inheritance from object
def __init__(self, store_number):
self.store_number = store_number
def amethod(self):
pass
Then MoreSpecificEmail should inherit from Email - as it doesn't have any additional instantiation parameters, it can just use the inherited __init__ and doesn't need to define its own:
class MoreSpecificEmail(Email):
# ^ note inheritance from Email
# note no need to define __init__
def another_method(self):
pass
For an example where there are additional __init__ parameters, note that you should use super and rely on the superclass's __init__ to assign the parameters it takes - you only need to assign the attributes that don't get handled by the superclass:
class MoreSpecificEmail(Email):
def __init__(self, store_number, something_else):
super(MoreSpecificEmail, self).__init__(store_number)
# ^ pass it straight on
self.something_else = something_else
def another_method(self):
pass
For more information, see the Python class tutorial.
This doesn't work:
def register_method(name=None):
def decorator(method):
# The next line assumes the decorated method is bound (which of course it isn't at this point)
cls = method.im_class
cls.my_attr = 'FOO BAR'
def wrapper(*args, **kwargs):
method(*args, **kwargs)
return wrapper
return decorator
Decorators are like the movie Inception; the more levels in you go, the more confusing they are. I'm trying to access the class that defines a method (at definition time) so that I can set an attribute (or alter an attribute) of the class.
Version 2 also doesn't work:
def register_method(name=None):
def decorator(method):
# The next line assumes the decorated method is bound (of course it isn't bound at this point).
cls = method.__class__ # I don't really understand this.
cls.my_attr = 'FOO BAR'
def wrapper(*args, **kwargs):
method(*args, **kwargs)
return wrapper
return decorator
The point of putting my broken code above when I already know why it's broken is that it conveys what I'm trying to do.
I don't think you can do what you want to do with a decorator (quick edit: with a decorator of the method, anyway). The decorator gets called when the method gets constructed, which is before the class is constructed. The reason your code isn't working is because the class doesn't exist when the decorator is called.
jldupont's comment is the way to go: if you want to set an attribute of the class, you should either decorate the class or use a metaclass.
EDIT: okay, having seen your comment, I can think of a two-part solution that might work for you. Use a decorator of the method to set an attribute of the method, and then use a metaclass to search for methods with that attribute and set the appropriate attribute of the class:
def TaggingDecorator(method):
"Decorate the method with an attribute to let the metaclass know it's there."
method.my_attr = 'FOO BAR'
return method # No need for a wrapper, we haven't changed
# what method actually does; your mileage may vary
class TaggingMetaclass(type):
"Metaclass to check for tags from TaggingDecorator and add them to the class."
def __new__(cls, name, bases, dct):
# Check for tagged members
has_tag = False
for member in dct.itervalues():
if hasattr(member, 'my_attr'):
has_tag = True
break
if has_tag:
# Set the class attribute
dct['my_attr'] = 'FOO BAR'
# Now let 'type' actually allocate the class object and go on with life
return type.__new__(cls, name, bases, dct)
That's it. Use as follows:
class Foo(object):
__metaclass__ = TaggingMetaclass
pass
class Baz(Foo):
"It's enough for a base class to have the right metaclass"
#TaggingDecorator
def Bar(self):
pass
>> Baz.my_attr
'FOO BAR'
Honestly, though? Use the supported_methods = [...] approach. Metaclasses are cool, but people who have to maintain your code after you will probably hate you.
Rather than use a metaclass, in python 2.6+ you should use a class decorator. You can wrap the function and class decorators up as methods of a class, like this real-world example.
I use this example with djcelery; the important aspects for this problem are the "task" method and the line "args, kw = self.marked[klass.dict[attr]]" which implicitly checks for "klass.dict[attr] in self.marked". If you want to use #methodtasks.task instead of #methodtasks.task() as a decorator, you could remove the nested def and use a set instead of a dict for self.marked. The use of self.marked, instead of setting a marking attribute on the function as the other answer did, allows this to work for classmethods and staticmethods which, because they use slots, won't allow setting arbitrary attributes. The downside of doing it this way is that the function decorator MUST go above other decorators, and the class decorator MUST go below, so that the functions are not modified / re=wrapped between one and the other.
class DummyClass(object):
"""Just a holder for attributes."""
pass
class MethodTasksHolder(object):
"""Register tasks with class AND method decorators, then use as a dispatcher, like so:
methodtasks = MethodTasksHolder()
#methodtasks.serve_tasks
class C:
#methodtasks.task()
##other_decorators_come_below
def some_task(self, *args):
pass
#methodtasks.task()
#classmethod
def classmethod_task(self, *args):
pass
def not_a_task(self):
pass
#..later
methodtasks.C.some_task.delay(c_instance,*args) #always treat as unbound
#analagous to c_instance.some_task(*args) (or C.some_task(c_instance,*args))
#...
methodtasks.C.classmethod_task.delay(C,*args) #treat as unbound classmethod!
#analagous to C.classmethod_task(*args)
"""
def __init__(self):
self.marked = {}
def task(self, *args, **kw):
def mark(fun):
self.marked[fun] = (args,kw)
return fun
return mark
def serve_tasks(self, klass):
setattr(self, klass.__name__, DummyClass())
for attr in klass.__dict__:
try:
args, kw = self.marked[klass.__dict__[attr]]
setattr(getattr(self, klass.__name__), attr, task(*args,**kw)(getattr(klass, attr)))
except KeyError:
pass
#reset for next class
self.marked = {}
return klass