Is there a way to disallow overriding given keyword arguments in a partial? Say I want to create function bar which always has a set to 1. In the following code:
from functools import partial
def foo(a, b):
print(a)
print(b)
bar = partial(foo, a=1)
bar(b=3) # This is fine and prints 1, 3
bar(a=3, b=3) # This prints 3, 3
You can happily call bar and set a to 3. Is it possible to create bar out of foo and make sure that calling bar(a=3, b=3) either raises an error or silently ignores a=3 and keeps using a=1 as in the partial?
Do you really need to use partial?
You can just define a new function bar normally with fewer arguments.
Like:
def bar(a):
b = 1
foo(a, b)
This will give error.
Or like:
def bar(a, b=1):
b = 1 #ignored provided b
foo(a, b)
This will ignore b.
EDIT: use lambda if you want to oneline these:
like: bar = lambda a:foo(a,b=1) or bar = lambda a,b=1:foo(a,b=1)
This is by design. The documentation for partial says (emphasize mine):
functools.partial(func, /, *args, **keywords)
Return a new partial object which when called will behave like func called with the positional arguments args and keyword arguments keywords. If more arguments are supplied to the call, they are appended to args. If additional keyword arguments are supplied, they extend and override keywords.
If you do not want that, you can manually reject frozen keyword arguments:
def freeze(f, **kwargs):
frozen = kwargs
def wrapper(*args, **kwargs):
kwargs.update(frozen)
return f(*args, **kwargs)
return wrapper
You can now do:
>>> baz = freeze(foo, a=1)
>>> baz(b=3, a=2)
1
3
If you want to raise an error when overriding occurs, you can create a custom implementation of partial:
class _partial:
def __init__(self, f, **kwargs):
self.f, self.kwargs = f, kwargs
def __call__(self, **kwargs):
if (p:=next((i for i in kwargs if i in self.kwargs), None)) is not None:
raise ValueError(f"parameter '{p}' already specified")
return self.f(**self.kwargs, **kwargs)
def foo(a, b):
print(a)
print(b)
>>> bar = _partial(foo, a=1)
>>> bar(b=3)
1
3
>>> bar(a=3, b=3)
Traceback (most recent call last):
File "<stdin>", line 1, in <module>
File "<stdin>", line 6, in __call__
ValueError: parameter 'a' already specified
Related
I have multiple classes, many of them having the same initialisation code for one parameter. Therefore I wanted to add the argument with the wrapper.
Since the code is already in production and this parameter is last in all calls, but the signatures have different lengths and the parameter can be position only, it is not trivial to "catch" this argument from the args and kwargs.
The following "works", as long as step is a kwarg, but if not, it is in *args and will be passed to the function, which correctly throws because it got too many arguments:
def stepable(func):
#functools.wraps(func)
def wrapper(self, *args, step=1, **kwargs):
func(self, *args, **kwargs)
self.step = step # and other stuff, depending on step
return wrapper
But even if I would catch it with len(args)>len(inspect.signature(func).parameters) (there are no *args in the functions parameters) the signature shown to the Users is wrong (because I used #wraps).
How can I add the parameter(/default) so that inspect will get it? Or basically "do the inverse of functools.partial"?
Your problem is that functools.wraps copy the original signature. Here, you will have to manually process and change it. If could be simple enough if you could be sure that none of the wrapped method could have:
a step parameter
a *args (VAR_POSITIONAL) parameter
a **kwargs (VAR_KEYWORD) parameter
And if the step parameter had no default value
But anyway, the inspect module provides everything to deal with signature.
I would define step to be the last POSITIONAL_OR_KEYWORD parameter in the wrapped function
Possible code:
def stepable(func):
oldsig = inspect.signature(func)
# search if a VAR_POSITIONAL or VAR_KEYWORD is present
# if yes insert step parameter before it, else insert it in last position
params = list(oldsig.parameters.values())
for i, param in enumerate(params):
if param.kind == inspect.Parameter.VAR_POSITIONAL:
break
if param.kind == inspect.Parameter.VAR_KEYWORD:
break
else:
i = len(params)
# new parameter name is step or step_[_...] if step if already present
name = "step"
while name in oldsig.parameters:
name += '_'
newparam = inspect.Parameter(name,
inspect.Parameter.POSITIONAL_OR_KEYWORD,
default = 1)
params.insert(i, newparam)
# we can now build the signature for the wrapper function
sig = oldsig.replace(parameters = params)
#functools.wraps(func)
def wrapper(self, *args, **kwargs):
bound = sig.bind(self, *args, **kwargs) # compute the bound parameter list
bound.apply_defaults()
step = bound.arguments[name] # extract and remove step
del bound.arguments[name]
cr = func(*bound.args, **bound.kwargs) # call original function
self.step = step
return cr
wrapper.__signature__ = sig
return wrapper
Demo:
>>> class A:
#stepable
def func(self, a, b=1):
"""This is a test"""
print(a,b)
>>> a = A()
>>> a.func(5)
5 1
>>> a.step
1
>>> a.func(5,6)
5 6
>>> a.step
1
>>> a.func(5,6,7)
5 6
>>> a.step
7
>>> help(a.func)
Help on method func in module __main__:
func(a, b=1, step=1) method of __main__.A instance
This is a test
Hope this helps
def stepable(func):
#functools.wraps(func)
def wrapper(self, *args, step=1, **kwargs):
return func(self, *args, **kwargs, step=step)
return wrapper
class A:
#stepable
def __init__(self,a, b, **kwargs):
self.a = a
self.b = b
for k in kwargs:
setattr(self, k, kwargs[k])
It will pass provided step or 1 to the function
Though you have to pass step as named argument only on instantiation and it will make your code more readable.
In Python 3.4+, functools.wraps preserves the signature of the function it wraps. Unfortunately, if you create decorators that are meant to be stacked on top of each other, the second (or later) decorator in the sequence will be seeing the generic *args and **kwargs signature of the wrapper and not preserving the signature of the original function all the way at the bottom of the sequence of decorators. Here's an example.
from functools import wraps
def validate_x(func):
#wraps(func)
def wrapper(*args, **kwargs):
assert kwargs['x'] <= 2
return func(*args, **kwargs)
return wrapper
def validate_y(func):
#wraps(func)
def wrapper(*args, **kwargs):
assert kwargs['y'] >= 2
return func(*args, **kwargs)
return wrapper
#validate_x
#validate_y
def foo(x=1, y=3):
print(x + y)
# call the double wrapped function.
foo()
This gives
-------------------------------------------------------------------------
KeyError Traceback (most recent call last)
<ipython-input-5-69c17467332d> in <module>
22
23
---> 24 foo()
<ipython-input-5-69c17467332d> in wrapper(*args, **kwargs)
4 #wraps(func)
5 def wrapper(*args, **kwargs):
----> 6 assert kwargs['x'] <= 2
7 return func(*args, **kwargs)
8 return wrapper
KeyError: 'x'
and if you switch the order of the decorators, you get the same key error for 'y'.
I tried replacing wraps(func) with wraps(func.__wrapped__) in the second decorator, but this still doesn't work (not to mention it requires the programmer to explicitly know where in the stack of decorators they are working for given wrapper functionality).
I also took a look at inspect.signature(foo) and this seems to give the right thing, but I found that this is because inspect.signature has a follow_wrapped parameter that defaults to True so it somehow knows to follow the sequence of wrapped functions, but apparently the regular method call framework for invoking foo() will not follow this same protocol for resolve args and kwargs of the outer decorated wrapper.
How can I just have wraps faithfully passthrough the signature so that wraps(wraps(wraps(wraps(f)))) (so to speak) always faithfully replicated the signature of f?
You are not actually passing any arguments to you function foo so *args and **kwargs are empty for both decorators. If you pass arguments the decorators will work just fine
foo(x=2, y = 3) # prints 5
You can try to get default function arguments using inspect
You can't really get the default values without using inspect and you also need to account for positional args (*args) vs keyword args (**kwargs). So normalize the data if it's there if it's missing then inspect the function
import inspect
from functools import wraps
def get_default_args(func):
signature = inspect.signature(func)
return {
k: v.default
for k, v in signature.parameters.items()
if v.default is not inspect.Parameter.empty
}
def validate_x(func):
#wraps(func)
def wrapper(*args, **kwargs):
if args and not kwargs and len(args) == 2:
kwargs['x'] = args[0]
kwargs['y'] = args[1]
args = []
if not args and not kwargs:
kwargs = get_default_args(func)
assert kwargs['x'] <= 2
return func(*args, **kwargs)
return wrapper
def validate_y(func):
#wraps(func)
def wrapper(*args, **kwargs):
if args and not kwargs and len(args) == 2:
kwargs['x'] = args[0]
kwargs['y'] = args[1]
args = []
if not args and not kwargs:
kwargs = get_default_args(func)
assert kwargs['y'] >= 2
return func(*args, **kwargs)
return wrapper
#validate_x
#validate_y
def foo(x=1, y=3):
print(x + y)
# call the double wrapped function.
foo()
# call with positional args
foo(1, 4)
# call with keyword args
foo(x=2, y=10)
This prints
4
5
12
Your diagnosis is incorrect; in fact, functools.wraps preserves the signature of the double-decorated function:
>>> import inspect
>>> inspect.signature(foo)
<Signature (x=1, y=3)>
We can also observe that it is not a problem with calling the function with the wrong signature, since that would raise a TypeError, not a KeyError.
You seem to be under the impression that when just one decorator is used, kwargs will be populated with the argument default values. This doesn't happen at all:
def test_decorator(func):
#wraps(func)
def wrapper(*args, **kwargs):
print('args:', args)
print('kwargs:', kwargs)
return func(*args, **kwargs)
return wrapper
#test_decorator
def foo(x=1):
print('x:', x)
The output is:
>>> foo()
args: ()
kwargs: {}
x: 1
So as you can see, neither args nor kwargs receives the argument's default value, even when just one decorator is used. They are both empty, because foo() calls the wrapper function with no positional arguments and no keyword arguments.
The actual problem is that your code has a logical error. The decorators validate_x and validate_y expect the arguments to be passed as keyword arguments, but in fact they might be passed as positional arguments or not at all (so the default values would apply), in which case 'x' and/or 'y' won't be present in kwargs.
There is no easy way to make your decorators work with an argument which could be passed as either keyword or positional; if you make the arguments keyword-only, then you can test whether 'x' or 'y' are in kwargs before validating them.
def validate_x(func):
#wraps(func)
def wrapper(*args, **kwargs):
if 'x' in kwargs and kwargs['x'] > 2:
raise ValueError('Invalid x, should be <= 2, was ' + str(x))
return func(*args, **kwargs)
return wrapper
#validate_x
def bar(*, x=1): # keyword-only arg, prevent passing as positional arg
...
It's usually better to explicitly raise an error, instead of using assert, because your program can be run with assert disabled.
Beware also that it's possible to declare a function like #validate_x def baz(*, x=5): ... where the default x is invalid. This won't raise any error because the default argument value isn't checked by the decorator.
I need to write a method that takes in 3 arguments:
a string with the name of a function
an ordered list of arguments to that function. This includes arguments with default values and *varargs, but does not include **kwargs
a dict representing any additional keyword arguments, or None if there are none
And I need to use this input to retrieve a function and call it. For example:
def dispatch(name, args, kwargs=None):
do_magic_here(name, args, kwargs)
def meth1():
print "meth1"
def meth2(a, b):
print "meth2: %s %s" % (a, b)
def meth3(a, **kwargs):
print "meth3: " + a
for k,v in kwargs.iteritems():
print "%s: %s" % (k,v)
And I need to be able to call things like this:
>>> dispatch("meth1", [])
meth1
>>> dispatch("meth2", [1, 3])
meth2: 1 3
>>> dispatch("meth3", [1], {"hello":2, "there":3})
meth3: 1
hello: 2
there: 3
I could do this:
def do_magic_here(name, args, kwargs=None):
if name=="meth1":
meth1()
if name=="meth2":
meth2(args[0], args[1])
if name=="meth3":
meth3(args[0], **kwargs)
But I'm trying to dispatch like 40 methods, and that number may expand, so I'm hoping there's a more programmatic way to do it. I'm looking at something with getattr, but I can't quite figure it out.
I would just use
def dispatch(name, *args, **kwargs):
func_name_dict[name](*args, **kwargs)
with
func_name_dict = {'meth1':meth1,
'meth2':meth2,
...}
Allowing you to pass args and kwargs through more naturally and transparently:
>>> dispatch("meth2", 1, 3)
meth2: 1 3
You can of course use globals() or locals() in place of the dict, but you might need to be careful about which functions in each namespace you do or don't want to expose to the caller
Indeed, getattr will get you there.
class X:
def a(self):
print('a called')
def b(self, arg):
print('b called with ' + arg)
x = X()
getattr(x, 'a')()
# a called
getattr(x, 'b')('foo')
# b called with foo
Just like getattr handles methods and fields the same way, you can handle
functions and variables not associated with a class by referencing locals() or globals().
If you want to refer to a function in the global scope:
globals()['meth'](args)
For example:
def dispatch(name, *args, **kwargs):
globals()[name](*args, **kwargs)
dispatch('meth3', 'hello', foo='bar')
# meth3: hello
# foo: bar
Remember in Python you can always pass a list of arguments or dict of keyword arguments using the **:
dispatch('meth3', *['hello'], **{'foo':'bar'})
If you truly prefer to pass arguments as list/dict to dispatch:
def dispatch(name, args, kwargs):
globals()[name](*args, **kwargs)
dispatch('meth3', ['hello'], {'foo': 'bar'})
I have a code that has the pattern "function(a)(**kwargs)". I know only the pattern "function (a, **kwargs)". What does it mean if there is a second set of arguments in separate paranthesis?
Shortened to what I think is relevant the code looks like this:
myprog1.py
def factory(cid, classes=CLASS_CACHE):
some code ...
myprog2.py
from myprog1 import factory
...
class Client(object):
def __init__(self, operations, factory):
self.factory = factory
def some_function()
chk = self.factory(test)(**kwargs)
factory is a function, test is a string (naming an object).
function(a)(**kwargs) calls the returned value of function(a) with keyword arguments unpacked from **kwargs. E.g the below code
def f():
def inner(**ka):
print(ka) # print received keyword arguments
return inner # return a callable function object
f()(argument='here')
outputs
{'argument': 'here'}
**kwargs is a syntax construction, which makes function arguments from dictionary. For example:
def a(b, c):
print b + c
args = {'b': 1, 'c': 2}
a(**args) # will print 3
In you code, search for definition of kwargs. I bet you some_function has **kwargs in arguments list like so:
def some_function(**kwargs):
So, your code chk = self.factory(test)(**kwargs) will do this thing:
Call self.factory method with test argument.
self.factory returns function
Returned function will be called with arguments, which is passed in some_function as arguments.
I'm dreaming of a Python method with explicit keyword args:
def func(a=None, b=None, c=None):
for arg, val in magic_arg_dict.items(): # Where do I get the magic?
print '%s: %s' % (arg, val)
I want to get a dictionary of only those arguments the caller actually passed into the method, just like **kwargs, but I don't want the caller to be able to pass any old random args, unlike **kwargs.
>>> func(b=2)
b: 2
>>> func(a=3, c=5)
a: 3
c: 5
So: is there such an incantation? In my case, I happen to be able to compare each argument against its default to find the ones that are different, but this is kind of inelegant and gets tedious when you have nine arguments. For bonus points, provide an incantation that can tell me even when the caller passes in a keyword argument assigned its default value:
>>> func(a=None)
a: None
Tricksy!
Edit: The (lexical) function signature has to remain intact. It's part of a public API, and the primary worth of the explicit keyword args lies in their documentary value. Just to make things interesting. :)
I was inspired by lost-theory's decorator goodness, and after playing about with it for a bit came up with this:
def actual_kwargs():
"""
Decorator that provides the wrapped function with an attribute 'actual_kwargs'
containing just those keyword arguments actually passed in to the function.
"""
def decorator(function):
def inner(*args, **kwargs):
inner.actual_kwargs = kwargs
return function(*args, **kwargs)
return inner
return decorator
if __name__ == "__main__":
#actual_kwargs()
def func(msg, a=None, b=False, c='', d=0):
print msg
for arg, val in sorted(func.actual_kwargs.iteritems()):
print ' %s: %s' % (arg, val)
func("I'm only passing a", a='a')
func("Here's b and c", b=True, c='c')
func("All defaults", a=None, b=False, c='', d=0)
func("Nothin'")
try:
func("Invalid kwarg", e="bogon")
except TypeError, err:
print 'Invalid kwarg\n %s' % err
Which prints this:
I'm only passing a
a: a
Here's b and c
b: True
c: c
All defaults
a: None
b: False
c:
d: 0
Nothin'
Invalid kwarg
func() got an unexpected keyword argument 'e'
I'm happy with this. A more flexible approach is to pass the name of the attribute you want to use to the decorator, instead of hard-coding it to 'actual_kwargs', but this is the simplest approach that illustrates the solution.
Mmm, Python is tasty.
Here is the easiest and simplest way:
def func(a=None, b=None, c=None):
args = locals().copy()
print args
func(2, "egg")
This give the output: {'a': 2, 'c': None, 'b': 'egg'}.
The reason args should be a copy of the locals dictionary is that dictionaries are mutable, so if you created any local variables in this function args would contain all of the local variables and their values, not just the arguments.
More documentation on the built-in locals function here.
One possibility:
def f(**kw):
acceptable_names = set('a', 'b', 'c')
if not (set(kw) <= acceptable_names):
raise WhateverYouWantException(whatever)
...proceed...
IOW, it's very easy to check that the passed-in names are within the acceptable set and otherwise raise whatever you'd want Python to raise (TypeError, I guess;-). Pretty easy to turn into a decorator, btw.
Another possibility:
_sentinel = object():
def f(a=_sentinel, b=_sentinel, c=_sentinel):
...proceed with checks `is _sentinel`...
by making a unique object _sentinel you remove the risk that the caller might be accidentally passing None (or other non-unique default values the caller could possibly pass). This is all object() is good for, btw: an extremely-lightweight, unique sentinel that cannot possibly be accidentally confused with any other object (when you check with the is operator).
Either solution is preferable for slightly different problems.
How about using a decorator to validate the incoming kwargs?
def validate_kwargs(*keys):
def entangle(f):
def inner(*args, **kwargs):
for key in kwargs:
if not key in keys:
raise ValueError("Received bad kwarg: '%s', expected: %s" % (key, keys))
return f(*args, **kwargs)
return inner
return entangle
###
#validate_kwargs('a', 'b', 'c')
def func(**kwargs):
for arg,val in kwargs.items():
print arg, "->", val
func(b=2)
print '----'
func(a=3, c=5)
print '----'
func(d='not gonna work')
Gives this output:
b -> 2
----
a -> 3
c -> 5
----
Traceback (most recent call last):
File "kwargs.py", line 20, in <module>
func(d='not gonna work')
File "kwargs.py", line 6, in inner
raise ValueError("Received bad kwarg: '%s', expected: %s" % (key, keys))
ValueError: Received bad kwarg: 'd', expected: ('a', 'b', 'c')
This is easiest accomplished with a single instance of a sentry object:
# Top of module, does not need to be exposed in __all__
missing = {}
# Function prototype
def myFunc(a = missing, b = missing, c = missing):
if a is not missing:
# User specified argument a
if b is missing:
# User did not specify argument b
The nice thing about this approach is that, since we're using the "is" operator, the caller can pass an empty dict as the argument value, and we'll still pick up that they did not mean to pass it. We also avoid nasty decorators this way, and keep our code a little cleaner.
There's probably better ways to do this, but here's my take:
def CompareArgs(argdict, **kwargs):
if not set(argdict.keys()) <= set(kwargs.keys()):
# not <= may seem weird, but comparing sets sometimes gives weird results.
# set1 <= set2 means that all items in set 1 are present in set 2
raise ValueError("invalid args")
def foo(**kwargs):
# we declare foo's "standard" args to be a, b, c
CompareArgs(kwargs, a=None, b=None, c=None)
print "Inside foo"
if __name__ == "__main__":
foo(a=1)
foo(a=1, b=3)
foo(a=1, b=3, c=5)
foo(c=10)
foo(bar=6)
and its output:
Inside foo
Inside foo
Inside foo
Inside foo
Traceback (most recent call last):
File "a.py", line 18, in
foo(bar=6)
File "a.py", line 9, in foo
CompareArgs(kwargs, a=None, b=None, c=None)
File "a.py", line 5, in CompareArgs
raise ValueError("invalid args")
ValueError: invalid args
This could probably be converted to a decorator, but my decorators need work. Left as an exercise to the reader :P
Perhaps raise an error if they pass any *args?
def func(*args, **kwargs):
if args:
raise TypeError("no positional args allowed")
arg1 = kwargs.pop("arg1", "default")
if kwargs:
raise TypeError("unknown args " + str(kwargs.keys()))
It'd be simple to factor it into taking a list of varnames or a generic parsing function to use. It wouldn't be too hard to make this into a decorator (python 3.1), too:
def OnlyKwargs(func):
allowed = func.__code__.co_varnames
def wrap(*args, **kwargs):
assert not args
# or whatever logic you need wrt required args
assert sorted(allowed) == sorted(kwargs)
return func(**kwargs)
Note: i'm not sure how well this would work around already wrapped functions or functions that have *args or **kwargs already.
Magic is not the answer:
def funky(a=None, b=None, c=None):
for name, value in [('a', a), ('b', b), ('c', c)]:
print name, value