I found this useful decorator that allows you to pass in some optional arguments
def mlflow_experiment(
_func=None,
*,
experiment_name=None
):
def experiment_decorator(func):
#functools.wraps(func)
def experiment_wrapper(self, *args, **kwargs):
nonlocal experiment_name
experiment_id = (
mlflow.set_experiment(experiment_name)
if experiment_name is not None
else None
)
...
value = func(self, *args, **kwargs)
return value
return experiment_wrapper
if _func is None:
return experiment_decorator
else:
return experiment_decorator(_func)
So in a use case like this where I just pass in a string to experiment_name, the code works flawlessly.
#mlflow_experiment(autolog=True, experiment_name = 'blarg')
def train_mlflow(self, maxevals=50, model_id=0):
...
I've always had a hard time figuring out scope in decorators but I wasn't surprised that using passing an instance variable defined in __init__ does NOT work.
class LGBOptimizerMlfow:
def __init__(self, arg):
self.arg = arg
#mlflow_experiment(autolog=True, experiment_name = self.arg)
def train_mlflow(self, maxevals=50, model_id=0):
...
>>> `NameError: name 'self' is not defined`
Just to see if scoping was an issue, I declared the variable outside the class and it worked.
And just for the heck of it I decided to declare a global variable inside the class which also works but its less than ideal, especially if I want to pass it into the class or a method as a optional argument.
class LGBOptimizerMlfow:
global arg
arg = 'hi'
#mlflow_experiment(autolog=True, experiment_name = arg)
def train_mlflow(self, maxevals=50, model_id=0):
...
Any help to revise the code so that the decorator accepts an instance variable would be lovely.
Thank you!
Decorators are called while the class is being defined, and self is simply a parameter used for each instance method, not something the class itself provides. So self is not defined at the time you need it to be for use as an argument to your decorator.
You need to modify experiment_wrapper to take a name directly from its self argument, rather than from an argument to mflow_experiment. Something like
def mlflow_experiment(
_func=None,
*,
experiment_name=None,
tracking_uri=None,
autolog=False,
run_name=None,
tags=None,
):
def experiment_decorator(func):
#functools.wraps(func)
def experiment_wrapper(self, *args, **kwargs):
nonlocal tracking_uri
experiment_name = getattr(self, 'experiment_name', None)
experiment_id = (
mlflow.set_experiment(experiment_name)
if experiment_name is not None
else None
)
...
with mlflow.start_run(experiment_id=experiment_id
, run_name=run_name
, tags=tags):
value = func(self, *args, **kwargs)
return value
return experiment_wrapper
if _func is None:
return experiment_decorator
else:
return experiment_decorator(_func)
Then you need to make sure that each instance has an experiment name (or None) associated with it.
class LGBOptimizerMlfow:
def __init__(self, arg, experiment_name=None):
self.arg = arg
self.experiment_name = experiment_name
#mlflow_experiment(autolog=True, experiment_name = self.arg)
def train_mlflow(self, maxevals=50, model_id=0):
...
Another alternative is to make experiment_name an argument to train_mflow, making it easier to create different names with the same method. (This may be closer to what you were intending.)
class LGBOptimizerMlfow:
def __init__(self, arg):
self.arg = arg
#mlflow_experiment(autolog=True)
def train_mlflow(self, maxevals=50, model_id=0, experiment_name=None):
if experiment_name is None:
self.experiment_name = self.arg
...
The definition of the decorator remains the same as shown above.
Related
Lets say my class has many function, and I want to apply my decorator on each one of them. I have researched for a while, and find https://stackoverflow.com/a/6307917/18859252. By using metaclass, I can decorate all function in one line.
Here is my code (framework)
class myMetaClass(type):
def __new__(cls, name, bases, local):
for attr in local:
value = local[attr]
if callable(value) and attr != '__init__':
local[attr] = log_decorator(local['Variable'], value)
return super().__new__(cls, name, bases, local)
class log_decorator():
def __init__(self, Variable, func):
self.Variable = Variable
self.func = func
def __call__(self, *args, **kargs):
start_time = time.time()
self.func(*args, **kargs)
end_time = time.time()
class Test(metaclass = myMetaClass):
Variable = Some_Class
check_test = Some_Class
def __init__(self, **args):
self.connect = Some_Class(**args)
def A(self, a, b):
self.connect.abc
pass
then use like this
def Flow():
test = Test(**args)
test.A(a, b)
But here is problem, it show exception like:
TypeError:A() missing 1 required positional argument: 'self'
I have no idea about this problem. I'd be very grateful if anyone has an answer or if there is a better way.
The piece you are missing (and the bit I don't fully understand, but has to do with functions or methods as descriptors and how python will attach an instance as the self parameter) is that log_decorator() is an instance of that class and not a function or method (even though you have defined a __call__() method which makes it callable.)
Here is some code which just slightly changes the syntax needed, but gives you the results you want:
import functools
class log_decorator:
def __init__(self, Variable): # Note that the only parameter is Variable
self.Variable = Variable
def __call__(self, func):
#functools.wraps(func)
def decorated(*args, **kwargs):
start_time = time.time()
func(*args, **kwargs)
end_time = time.time()
return decorated
class myMetaClass(type):
def __new__(cls, name, bases, local):
for attr in local:
value = local[attr]
if callable(value) and attr != '__init__':
# Note the change in syntax vvv
local[attr] = log_decorator(local['Variable'])(value)
return super().__new__(cls, name, bases, local)
How do class decorators for methods in classes work? Here is a sample of what I've done through some experimenting:
from functools import wraps
class PrintLog(object):
def __call__(self, func):
#wraps(func)
def wrapped(*args):
print('I am a log')
return func(*args)
return wrapped
class foo(object):
def __init__(self, rs: str) -> None:
self.ter = rs
#PrintLog()
def baz(self) -> None:
print('inside baz')
bar = foo('2')
print('running bar.baz()')
bar.baz()
And this works perfectly fine. However, I was under the impression that decorators do not need to be called with (), but when I remove the brackets from #PrintLog(), I get this error:
def baz(self) -> None:
TypeError: PrintLog() takes no arguments
Is there something I am missing/do not understand? I've also tried passing in a throwaway arg with __init__(), and it works.
class PrintLog(object):
def __init__(self, useless):
print(useless)
def __call__(self, func):
#wraps(func)
def wrapped(*args):
print('I am a log')
return func(*args)
return wrapped
class foo(object):
def __init__(self, rs: str) -> None:
self.ter = rs
#PrintLog("useless arg that I'm passing to __init__")
def baz(self) -> None:
print('inside baz')
Again, this works, but I don't want to pass any argument to the decorator.
tl;dr: This question in python 3.x.
Help appreciated!
Class decorators accept the function as a subject within the __init__ method (hence the log message), so your decorator code should look like:
class PrintLog(object):
def __init__(self, function):
self.function = function
def __call__(self):
#wraps(self.function)
def wrapped(*args):
print('I am a log')
return self.function(*args)
return wrapped
Sorry if this doesn’t work, I’m answering on my mobile device.
EDIT:
Okay so this is probably not what you want, but this is the way to do it:
from functools import update_wrapper, partial, wraps
class PrintLog(object):
def __init__(self, func):
update_wrapper(self, func)
self.func = func
def __get__(self, obj, objtype):
"""Support instance methods."""
return partial(self.__call__, obj)
def __call__(self, obj, *args, **kwargs):
#wraps(self.func)
def wrapped(*args):
print('I am a log')
return self.func(*args)
return wrapped(obj, *args)
class foo(object):
def __init__(self, rs: str) -> None:
self.ter = rs
#PrintLog
def baz(self) -> None:
print('inside baz')
bar = foo('2')
print('running bar.baz()')
bar.baz()
The decorator has to have the __get__ method defined because you're applying the decorator to an instance method. How would a descriptor have the context of the foo instance?
Ref: Decorating Python class methods - how do I pass the instance to the decorator?
There is a big picture you're missing.
#decorator
def foo(...):
function_definition
is almost identical (except for some internal mangling) to
temp = foo
foo = decorator(temp)
It doesn't matter what the decorator is, as long as it can act like a function.
Your example is equivalent to:
baz = PrintLog("useless thing")(<saved defn of baz>)
Since PrintLog is a class, PrintLog(...) creates an instance of PrintLog. That instance has a __call__ method, so it can act like a function.
Some decorators are designed to take arguments. Some decorators are designed not to take arguments. Some, like #lru_cache, are pieces of Python magic which look to see if the "argument" is a function (so the decorator is being used directly) or a number/None, so that it returns a function that then becomes the decorator.
I have a following decorator.
def allow_disable_in_tests(func):
#functools.wraps(func)
def wrapper(*args, **kwargs):
version = ??? # name of the func of method’s class name
need_to_switch_off_in_tests = cache.get('switch_off_in_tests', version=version)
if settings.IM_IN_TEST_MODE and need_to_switch_off_in_tests:
return None
value = func(*args, **kwargs)
return value
return wrapper
There are 2 types of objects this decorator might accept as a func:
Standalone function.
Method of the class(bound method, static method and class method all possible)
Question is how to get from inside decorator name of the :
function in case of func is standalone function.
method’s class name in case of func is method of the class
This will be used in version .
Decorator should be able to work with both types of objects.
Thanks
You can use __qualname__ and __module__ to derive this information. __qualname__ will describe where the class is defined within a module according to the class or function it was defined in.
However, you are putting test logic in production code, which is a bit of a code smell. You'd be better off using the monkey patching features of your testing framework to patch these functions when running your test suite. For example, with pytest:
import pytest
from functools import wraps
from inspect import signature
class FuncPatch:
def __init__(self, parent, name, retval=None):
self.parent = parent
self.name = name
self.retval = retval
def get_things_to_patch():
import mymodule
return (
FuncPatch(mymodule, 'my_func'),
FuncPatch(mymodule.MyClass, 'method'),
FuncPatch(mymodule.MyClass, 'static'),
FuncPatch(mymodule.MyClass, 'class_', retval='special'),
)
def create_test_function(func, retval, decorator=None):
func = getattr(func, '__func__', func) # unwrap if classmethod or normal method
sig = signature(func)
#wraps(func)
def f(*args, **kwargs):
# check func was called with correct params raises TypeError if wrong
sig.bind(*args, **kwargs)
return retval
if decorator:
f = decorator(f)
return f
#pytest.fixture
def patch_all_the_things(monkeypatch):
for patch in get_things_to_patch():
decorator = None
if (isinstance(patch.parent, type)
and not callable(patch.parent.__dict__[patch.name])
):
# quick hack to detect staticmethod or classmethod
decorator = type(patch.parent.__dict__[patch.name])
to_patch = getattr(patch.parent, patch.name)
func = create_test_function(to_patch, patch.retval, decorator)
monkeypatch.setattr(patch.parent, patch.name, func)
# things to test
def my_func():
return 'my_func'
class MyClass:
#staticmethod
def static():
return 'static'
#classmethod
def class_(cls):
return 'class'
def method(self):
return 'method'
# actual tests
def test_my_func(patch_all_the_things):
assert my_func() is None
def test_my_class(patch_all_the_things):
assert MyClass().method() is None
assert MyClass.method(MyClass()) is None
assert MyClass.static() is None
assert MyClass.class_() == 'special'
I have classes like this:
class Tool(object):
def do_async(*args):
pass
for which I want to automatically generate non-async methods that make use of the async methods:
class Tool(object):
def do_async(*args):
pass
def do(*args):
result = self.do_async(*args)
return magical_parser(result)
This gets to be particularly tricky because each method needs to be accessible as both an object and class method, which is normally achieved with this magical decorator:
class class_or_instance(object):
def __init__(self, fn):
self.fn = fn
def __get__(self, obj, cls):
if obj is not None:
f = lambda *args, **kwds: self.fn(obj, *args, **kwds)
else:
f = lambda *args, **kwds: self.fn(cls, *args, **kwds)
functools.update_wrapper(f, self.fn)
return f
How can I make these methods, and make sure they're accessible as both class and object methods? This seems like something that could be done with decorators, but I am not sure how.
(Note that I don't know any of the method names in advance, but I know that all of the methods that need new buddies have _async at the end of their names.)
I think I've gotten fairly close, but this approach does not appropriately set the functions as class/object methods:
def process_asyncs(cls):
methods = cls.__dict__.keys()
for k in methods:
methodname = k.replace("_async","")
if 'async' in k and methodname not in methods:
#class_or_instance
def method(self, verbose=False, *args, **kwargs):
response = self.__dict__[k](*args,**kwargs)
result = self._parse_result(response, verbose=verbose)
return result
method.__docstr__ = ("Returns a table object.\n" +
cls.__dict__[k].__docstr__)
setattr(cls,methodname,MethodType(method, None, cls))
Do not get the other method from the __dict__; use getattr() instead so the descriptor protocol can kick in.
And don't wrap the method function in a MethodType() object as that'd neutralize the descriptor you put on method.
You need to bind k to the function you generate; a closured k would change with the loop:
#class_or_instance
def method(self, verbose=False, _async_method_name=k, *args, **kwargs):
response = getattr(self, _async_method_name)(*args,**kwargs)
result = self._parse_result(response, verbose=verbose)
return result
cls.__dict__[methodname] = method
Don't forget to return cls at the end; I've changed this to use a separate function to create a new scope to provide a new local name _async_method_name instead of a keyword parameter; this avoids difficulties with *args and explicit keyword arguments:
def process_asyncs(cls):
def create_method(async_method):
#class_or_instance
def newmethod(self, *args, **kwargs):
if 'verbose' in kwargs:
verbose = kwargs.pop('verbose')
else:
verbose = False
response = async_method(*args,**kwargs)
result = self._parse_result(response, verbose=verbose)
return result
return newmethod
methods = cls.__dict__.keys()
for k in methods:
methodname = k.replace("_async","")
if 'async' in k and methodname not in methods:
async_method = getattr(cls, k)
setattr(cls, methodname, create_method(async_method))
return cls
I want to construct classes for use as decorators with the following principles intact:
It should be possible to stack multiple such class decorators on top off 1 function.
The resulting function name pointer should be indistinguishable from the same function without a decorator, save maybe for just which type/class it is.
Ordering off the decorators should not be relevant unless actually mandated by the decorators. Ie. independent decorators could be applied in any order.
This is for a Django project, and the specific case I am working on now the method needs 2 decorators, and to appear as a normal python function:
#AccessCheck
#AutoTemplate
def view(request, item_id) {}
#AutoTemplate changes the function so that instead of returning a HttpResponse, it just returns a dictionary for use in the context. A RequestContext is used, and the template name is inferred from the method name and module.
#AccessCheck adds additional checks on the user based on the item_id.
I am guessing it's just to get the constructor right and copy the appropriate attributes, but which attributes are these?
The following decorator won't work as I describe:
class NullDecl (object):
def __init__ (self, func):
self.func = func
def __call__ (self, * args):
return self.func (*args)
As demonstrated by the following code:
#NullDecl
#NullDecl
def decorated():
pass
def pure():
pass
# results in set(['func_closure', 'func_dict', '__get__', 'func_name',
# 'func_defaults', '__name__', 'func_code', 'func_doc', 'func_globals'])
print set(dir(pure)) - set(dir(decorated));
Additionally, try and add "print func.name" in the NullDecl constructor, and it will work for the first decorator, but not the second - as name will be missing.
Refined eduffy's answer a bit, and it seems to work pretty well:
class NullDecl (object):
def __init__ (self, func):
self.func = func
for n in set(dir(func)) - set(dir(self)):
setattr(self, n, getattr(func, n))
def __call__ (self, * args):
return self.func (*args)
def __repr__(self):
return self.func
A do-nothing decorator class would look like this:
class NullDecl (object):
def __init__ (self, func):
self.func = func
for name in set(dir(func)) - set(dir(self)):
setattr(self, name, getattr(func, name))
def __call__ (self, *args):
return self.func (*args)
And then you can apply it normally:
#NullDecl
def myFunc (x,y,z):
return (x+y)/z
The decorator module helps you writing signature-preserving decorators.
And the PythonDecoratorLibrary might provide useful examples for decorators.
To create a decorator that wraps functions in a matter that make them indistinguishable from the original function, use functools.wraps.
Example:
def mydecorator(func):
#functools.wraps(func):
def _mydecorator(*args, **kwargs):
do_something()
try:
return func(*args, **kwargs)
finally:
clean_up()
return _mydecorator
# ... and with parameters
def mydecorator(param1, param2):
def _mydecorator(func):
#functools.wraps(func)
def __mydecorator(*args, **kwargs):
do_something(param1, param2)
try:
return func(*args, **kwargs)
finally:
clean_up()
return __mydecorator
return _mydecorator
(my personal preference is to create decorators using functions, not classes)
The ordering of decorators is as follows:
#d1
#d2
def func():
pass
# is equivalent to
def func():
pass
func = d1(d2(func))