Here is my problem, i'm working with an API,
precisely with a high-order function that only accepts functions with N arguments. (I cannot monkey-patch this API).
#this is an example of a high order function i may encounter
#there are many more of such functions in the API that require N ammount of arguments
#this example fct required 3 arg, but a valid solution should adapt to any required args count
def high_order_function(f):
"""high order function expecting a function with 3 arguments!"""
print(f"\nprocessing function {f.__name__}")
if f.__code__.co_argcount!=3:
raise Exception(f"Error Expecting a function with 3 arguments, the passed function got {f.__code__.co_argcount}")
print("Function is Ok")
#...
return None
And my problem is that I simply cannot use any wrapper because of this check.
what am I supposed to do ?
def my_wrapper(func):
import functools
#functools.wraps(func)
def inner(*args, **kwargs):
print("wrapped1!")
r = func(*args,**kwargs)
print("wrapped2!")
return r
return inner
def original(a, b, c):
return None
wrapped = my_wrapper(original)
high_order_function(original)
#ok!
high_order_function(wrapped)
#will cause error
#because wrapped.__code__.co_argcount == 0 and is readonly!
After a lot of tinkering, I found a pretty procedural way that might work for you.
The trick was to use __code__.replace(). There are some caveats, probably more than I know.
def high_order_function(f):
"""high order function expecting a function with 3 arguments!"""
print(f"\nprocessing function {f.__name__}")
if f.__code__.co_argcount!=3:
raise Exception(f"Error Expecting a function with 3 arguments, the passed function got {f.__code__.co_argcount}")
print("Function is Ok")
#...
return None
def my_wrapper(func):
import functools
#functools.wraps(func)
def inner(a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p, q, r, s, t, u, v, w, x, y, z):
kwargs = locals().copy()
del kwargs["func"]
print("wrapped1!")
r = func(**kwargs) # func(*kwargs.values()) would work too
print("wrapped2!")
return r
func_args = func.__code__.co_varnames
inner.__code__ = inner.__code__.replace(co_varnames=func_args, co_argcount=len(func_args))
return inner
def original(a, b, c):
return None
wrapped = my_wrapper(original)
high_order_function(original)
high_order_function(wrapped)
Result
processing function original
Function is Ok
processing function original
Function is Ok
functools.wraps changes the name of inner to original
Caveats
__code__.replace() raised ValueError: code: varnames is too small when inner's parameters were *args or **kwargs
If inner instead had no parameters then locals() inside it would not get the supplied values, therefore you got the whole alphabet instead
Inside inner you can access the parameters by the letter like normal if you're sure it´s supplied otherwise you'll get IndexError: tuple index out of range
I recommend to use e.g. kwargs.get("d") instead
__code__.replace may only be for 3.8+, it has sys.version_info >= (3, 8) in the source code
Why not define 2 version of the inner function in your wrapper based on whether you need to pass the wrapped function to higher_order_function or not.
Something like this:
def high_order_function(f):
"""high order function expecting a function with 3 arguments!"""
print(f"\nprocessing function {f.__name__}")
if f.__code__.co_argcount != 3:
raise Exception(
f"Error Expecting a function with 3 arguments, the passed function got {f.__code__.co_argcount}")
print("Function is Ok")
# ...
return None
def my_wrapper(func, higher_order_compatible=True): # switch to control whether should be compatible with the higher_order_function or not
import functools
if higher_order_compatible:
#functools.wraps(func)
def inner(a, b, c, *args, **kwargs): # extra args to satisfy the condition
print("wrapped1!")
r = func(a, b, c, *args, **kwargs)
print("wrapped2!")
return r
else:
#functools.wraps(func)
def inner(*args, **kwargs): # normal wrapper
print("wrapped1!")
r = func(*args, **kwargs)
print("wrapped2!")
return r
return inner
def original(a, b, c):
return None
wrapped_compat = my_wrapper(original, higher_order_compatible=True)
wrapped_nocompat = my_wrapper(original, higher_order_compatible=False)
print("Original")
high_order_function(original)
print("Compatible")
high_order_function(wrapped_compat)
print("Not compatible")
try:
high_order_function(wrapped_nocompat)
except:
print("no not working")
Results in:
Original
processing function original
Function is Ok
Compatible
processing function original
Function is Ok
Not compatible
processing function original
no not working
co_argcount: number of arguments (not including keyword only arguments, * or ** args)
Hence the goal is to bypass such definition. Make a fake signature with 3 fake parameters, these are taken into consideration by the code attribute co_argcount. Then the parameters of the original function must by passed as keys.
def wrapper(f):
def extended_signature(fake1=None, fake2=None, fake3=None, **kwargs):
return f(**kwargs)
return extended_signature
def a(q, w): print(q, w)
a_wrapped = wrapper(a)
high_order_function(a_wrapped)(q=1, w=2)
#processing function true_signature
#Function is Ok
This is an attempt to solve the problem
i feel like it's almost a potential solution
However it is not working as expected, strange. f()for a function object class is not as f.__call__() hmm
def my_wrapper(func):
import copy
def inner(*args, **kwargs):
print("wrapped front")
r = func(*args, **kwargs)
print("wrapped end")
return r
newfunc = copy.deepcopy(func)
newfunc.__name__ = func.__name__ + "_wrapped"
newfunc.__call__ = inner
return newfunc
def original(a, b, c=6):
print("original",a,b,c)
return None
###testing if the original function work
high_order_function(original)
#will pass requirement
###testing if the wrap works?
high_order_function(my_wrapper(original))
#will pass requirement, however the wrap did not work
well, here's a solution, couldn't find a procedural way to generate the functions... it scales up to 5 forced arguments
def my_wrapper(func):
"""see https://stackoverflow.com/questions/73601340/how-to-wrap-a-function-with-accurate-code-argcount?noredirect=1#comment129973896_73601340
yes this is a shit show, did not found a procedural way to generate functions. tried exec() code generation & was also a mess"""
#find back the expected arguments so func.__code__.co_argcount will be accurate
if (func.__defaults__ is not None):
force_arg = func.__code__.co_argcount - len(func.__defaults__)
else: force_arg = func.__code__.co_argcount
import functools
if (force_arg==0):
#functools.wraps(func)
def inner(**kwargs):
print("wrapped1!")
r = func(**kwargs)
print("wrapped2!")
return r
elif (force_arg==1):
#functools.wraps(func)
def inner(a,**kwargs):
print("wrapped1!")
r = func(a,**kwargs)
print("wrapped2!")
return r
elif (force_arg==2):
#functools.wraps(func)
def inner(a,b,**kwargs):
print("wrapped1!")
r = func(a,b,**kwargs)
print("wrapped2!")
return r
elif (force_arg==3):
#functools.wraps(func)
def inner(a,b,c,**kwargs):
print("wrapped1!")
r = func(a,b,c,**kwargs)
print("wrapped2!")
return r
elif (force_arg==4):
#functools.wraps(func)
def inner(a,b,c,d,**kwargs):
print("wrapped1!")
r = func(a,b,c,d,**kwargs)
print("wrapped2!")
return r
elif (force_arg==5):
#functools.wraps(func)
def inner(a,b,c,d,e,**kwargs):
print("wrapped1!")
r = func(a,b,c,d,e,**kwargs)
print("wrapped2!")
return r
else: raise Exception("my_wrapper() do not support more than 5 forced argument")
return inner
Related
Is there any way to check inside function f1 in my example if calling a function (here decorated or not_decorated) has a specific decorator (in code #out)? Is such information passed to a function?
def out(fun):
def inner(*args, **kwargs):
fun(*args, **kwargs)
return inner
#out
def decorated():
f1()
def not_decorated():
f1()
def f1():
if is_decorated_by_out: # here I want to check it
print('I am')
else:
print('I am not')
decorated()
not_decorated()
Expected output:
I am
I am not
To be clear, this is egregious hackery, so I don't recommend it, but since you've ruled out additional parameters, and f1 will be the same whether wrapped or not, you've left hacks as your only option. The solution is to add a local variable to the wrapper function for the sole purpose of being found by means of stack inspection:
import inspect
def out(fun):
def inner(*args, **kwargs):
__wrapped_by__ = out
fun(*args, **kwargs)
return inner
def is_wrapped_by(func):
try:
return inspect.currentframe().f_back.f_back.f_back.f_locals.get('__wrapped_by__') is func
except AttributeError:
return False
#out
def decorated():
f1()
def not_decorated():
f1()
def f1():
if is_wrapped_by(out):
print('I am')
else:
print('I am not')
decorated()
not_decorated()
Try it online!
This assumes a specific degree of nesting (the manual back-tracking via f_back to account for is_wrapped_by itself, f1, decorated and finally to inner (from out). If you want to determine if out was involved anywhere in the call stack, make is_wrapped_by loop until the stack is exhausted:
def is_wrapped_by(func):
frame = None
try:
# Skip is_wrapped_by and caller
frame = inspect.currentframe().f_back.f_back
while True:
if frame.f_locals.get('__wrapped_by__') is func:
return True
frame = frame.f_back
except AttributeError:
pass
finally:
# Leaving frame on the call stack can cause cycle involving locals
# which delays cleanup until cycle collector runs;
# explicitly break cycle to save yourself the headache
del frame
return False
If you are open to creating an additional parameter in f1 (you could also use a default parameter), you can use functools.wraps and check for the existence of the __wrapped__ attribute. To do so, pass the wrapper function to f:
import functools
def out(fun):
#functools.wraps(fun)
def inner(*args, **kwargs):
fun(*args, **kwargs)
return inner
#out
def decorated():
f1(decorated)
def not_decorated():
f1(not_decorated)
def f1(_func):
if getattr(_func, '__wrapped__', False):
print('I am')
else:
print('I am not')
decorated()
not_decorated()
Output:
I am
I am not
Suppose you have a function decoration like this one
def double_arg(fun):
def inner(x):
return fun(x*2)
return inner
however you can't access it (it's inside a 3rd party lib or something). In this case you can wrap it into another function that adds the name of the decoration to the resulting function
def keep_decoration(decoration):
def f(g):
h = decoration(g)
h.decorated_by = decoration.__name__
return h
return f
and replace the old decoration by the wrapper.
double_arg = keep_decoration(double_arg)
You can even write a helper function that checks whether a function is decorated or not.
def is_decorated_by(f, decoration_name):
try:
return f.decorated_by == decoration_name
except AttributeError:
return False
Example of use...
#double_arg
def inc_v1(x):
return x + 1
def inc_v2(x):
return x + 1
print(inc_v1(5))
print(inc_v2(5))
print(is_decorated_by(inc_v1, 'double_arg'))
print(is_decorated_by(inc_v2, 'double_arg'))
Output
11
6
True
False
I have tree types of sub-functions:
one without any parameters (arguments),
second with one parameter
third with multiple parameters (tuple)
I am trying to pass that functions and its arguments to another function which sum results of all sub-functions and return the sum value.
Parameters in that function should be: names of each sub-function as position arguments (*args) and arguments of each subfunction as key-value arguments (*kvargs).
Example:
def no_arg()
def one_arg(a)
def multiple_args(a, b, c, e, f)
# execution of function_results_sum:
function_results_sum(
no_arg, one_arg, multiple_args,
one_arg=23,
multiple_args=(1, 2, 3, 4, 5))
What i have done so far:
def no_arg():
return 5
def ident(x):
return x
def mult(x, y):
return x * y
def function_results_sum(*args, **kwargs):
return no_arg() + ident(kwargs[ident.__name__]) + mult(*kwargs[mult.__name__])
The code above is passing arguments to each sub-function, but sub-function names are hardcoded. I would like to modify the current code to be able to get function names from *args. Below I wrote a pseudocode expressing more less what i am trying to achieve:
def function_results_sum(*args, **kwargs):
for functionName in args:
result = sum(funcionName(kwargs))
return result
I have already spent all day struggling with that problem, so please don't write me that "using google doesn't hurt" ;)
Something like this would work:
def no_arg():
return 5
def one_arg(x):
return x
def multiple_args(x, y):
return x * y
def function_results_sum(*args, **kwargs):
result = 0
for func in args:
result += func(*kwargs[func.__name__])
return result
Output:
function_results_sum(
no_arg, one_arg, multiple_args,
no_arg=(),
one_arg=(23, ),
multiple_args=(1,5))
33
The only difference between what you are asking is that you have to put args in a tuple to then unpack as args to pass in later.
If you dont want to have to supply anything for no argument functions, you can double check if the func name is in kwargs:
def function_results_sum(*args, **kwargs):
result = 0
for func in args:
if func.__name__ i kwargs:
result += func(*kwargs[func.__name__])
else:
result += func()
return result
Post of R Nar is exactly what I tried to achieve. I added additional if statement to verify if kwarg is an integer or a tuple. Thanks that it is not neccessary to put all **kwargs in a tuple. Thank you guys for help!
def function_results_sum(*args, **kwargs):
result = 0
for func in args:
if func.__name__ in kwargs:
if type(kwargs[func.__name__]) == int:
result += func(kwargs[func.__name__])
elif type(kwargs[func.__name__]) == tuple:
result += func(*kwargs[func.__name__])
else:
result += func()
return result
result = function_results_sum(no_arg, ident, mult, ident=2, mult=(3, 4))
print(result)
by goolging 'python determine number of args for passed function' I found How can I find the number of arguments of a Python function?
I'm pretty sure you don't want the **kwars key, value syntax so I use a func_list regular arg and *args
from inspect import signature
def function_results_sum(func_list, *args):
arg_gen = (e for e in args)
return sum([func(*(next(arg_gen)
for _ in range(len(signature(func).parameters))))
for func in func_list])
function_results_sum([no_arg, ident, mult], 7,8,9)
84
the input can be made flatter by parsing *args for Functions and (presumed) arguments (anything not Type Function)
from inspect import signature
import types
def function_results_sum(*args):
func_gen = (e for e in args if isinstance(e, types.FunctionType))
arg_gen = (e for e in args if not isinstance(e, types.FunctionType))
return sum(func(*(next(arg_gen)
for _ in range(len(signature(func).parameters))))
for func in func_gen)
function_results_sum(no_arg, ident, mult, 10,6,90)
555
order of functions and order of args are important, but separately, can be interleaved:
function_results_sum(no_arg, 10, ident, 6, 90, mult)
Out[399]: 555
Why does this :
def fn(proc, *args, **kwargs):
cache = proc.cache = {}
def cached_execution(cache, *args, **kwargs):
if proc in cache:
if args in cache[proc]:
return cache[proc][args]
res = proc(args)
cache[proc] = {args: res}
return res
return cached_execution(cache, proc, *args, **kwargs)
#fn
def cached_fibo(n):
if n == 1 or n == 0:
return n
else:
return cached_fibo(n-1) + cached_fibo(n-2)
print cached_fibo(100)
throw an exception like this:
NameError: global name 'cached_fibo' is not defined
What fundamental concept am I missing?
(Conceptually, **kwargs is for decoration only. Not utilizing in retrieving the cached result, but don't worry about it).
A decorator should return a function, not the result of calling a function.
But this leads us to the next mistake: when you're passing cache and proc to cached_execution function they land in *args which in turn gets passed to proc. This doesn't make sense. Just let cache and proc be captured within the inner method:
def fn(proc, *args, **kwargs):
cache = proc.cache = {}
def cached_execution(*args, **kwargs):
if proc in cache:
if args in cache[proc]:
return cache[proc][args]
res = proc(*args)
cache[proc] = {args: res}
return res
return cached_execution
Another problem: you were not unpacking args. You should call proc(*args) instead of proc(args) (already fixed above).
The wrapper seems a little malformed. Here is an updated version:
def fn(proc):
cache = proc.cache = {}
def cached_execution(*args, **kwargs):
if proc in cache:
if args in cache[proc]:
return cache[proc][args]
res = proc(args[0])
cache[proc] = {args: res}
return res
return cached_execution
You were trying to run the wrapper function inside the wrapper instead of returning it to be run as the function, causing issues.
The next issue is that the argument you supply is a list of tuples *args at proc(args) when you only want the first one, so needs to turn into proc(args[0])
I am writing a function which gets users from database and returns a list of user objects.
Function signature is as given below:
def select_users(self,userid,firstname,lastname,emailid,tenants,groups):
result = self.authservice.select_users(userid,firstname,lastname,emailid,tenants,groups)
In this function, I call select_users method of authservice object which will return a list of custom user objects. But if any of input parameters has '' value then it must be converted to None because self.authservice.select_users cannot handle empty strings. I can check each element value and convert it to None if it is empty, but I want it to be generic and reusable. If I could write a different function which can give me updated list of input parameters it would be very helpful. Please let me know how do I do that?
I would write a generic decorator, like this
def convert_empty_to_none(func):
def inner_function(*args, **kwargs):
args = (None if item == "" else item for item in args)
kwargs = {k:(None if v == "" else v) for k, v in kwargs.items()}
return func(*args, **kwargs)
return inner_function
#convert_empty_to_none
def test_function(a, b, c):
print a, b, c
test_function("", "", "")
Output
None None None
Evil way:
def select_users(self, *args):
new_args = [(None if arg == '' else arg) for arg in args]
result = self.authservice.select_users(*new_args)
Decorator solution is also pretty evil: changing function arguments to spare writing a couple of function calls doesn't seem such a great idea.
In the real life I would go with the explicit:
def never_blank(s):
return None if s == '' else s
def select_users(self, userid,firstname,lastname,emailid,tenants,groups):
result = self.authservice.select_users(userid,never_blank(firstname),never_blank(lastname),emailid,
never_blank(tenants),groups)
Tedious? Sure. Clean? Yep. Will bite you in the ass in the future? Nope.
Create a function and use it like a function type decorator
def sanitize(func):
def handler(*args, **kwargs):
args = (e if e != '' else None for e in args)
kwargs = {k:(v if v != '' else None) for k, v in kwargs.items()}
return func(*args, **kwargs)
return handler
#sanitize
def select_users(self,userid,firstname,lastname,emailid,tenants,groups):
result = self.authservice.select_users(userid,firstname,lastname,emailid,tenants,groups)
Benifits
You do not need to modify the signature
The caller would still have clear idea, what parameters the function expects
Generic and can be used for any function call
Is a decorator, so can easily be used in a non-intrusive fashion
You could use a decorator to create a generic wrapper that will replace every empty string with None.
def none_for_empty_string(func):
def wrapper(*args, **kwargs):
args = tuple(arg if arg != '' else None for arg in args)
kwargs = {k : v if v != '' else None for k, v in kwargs.iteritems()}
return func(*args, **kwargs)
return wrapper
#none_for_empty_string
def select_users(self,userid,firstname,lastname,emailid,tenants,groups):
...
I am new to Django and come from Java/Spring background.
I am wondering if there are decorators something like following that can be done in Django or Python?
Want
def addToList(#not_none a, #not_none b):
# so that I do not check for nullity explicitly
do_things_with_a(a)
do_things_with_b(b)
Since this is something which is pretty easy to get in Java, just looking if Python/Django has it
One doesn't typically constraint data-types in Python. Also, decorators can only be applied to classes and to methods/functions.
Although, you shouldn't really be doing this, this is how you would.
(You could amend this to accept argument names to enforce constraints on with a little work).
def not_none(f):
def func(*args, **kwargs):
if any(arg is None for arg in args):
raise ValueError('function {}: does not take arguments of None'.format(f.__name__))
return f(*args, **kwargs)
return func
#not_none
def test(a, b):
print a, b
You can write a decorator rejectNone as follows:
def rejectNone(f):
def myF(*args, **kwargs):
if None in args or None in kwargs.values():
raise Exception('One of the arguments passed to {0} is None.'.format(f.__name__)
return f(*args, **kwargs)
return myF
#rejectNone
def f(a, b, k=3):
print a * b
You will now get an Exception if you try to call f with a None argument. Note that decorators can be applied to functions or class methods but you can't put them in front of function parameters.
I know this is late, but to those who it may be helpful.
I have a simple repo based off of Jon's answer that accepts arguments for nullable fields here.
def not_none(nullable_parameters=None):
def the_actual_test(f, args, filter_array):
has_none = False
bad_parameters = []
if type(filter_array) is str:
filter_array = [filter_array]
if not filter_array:
if any(arg[1] is None for arg in args):
raise ValueError('function {}: Parameters cannot be None. '.format(f.__name__))
elif type(filter_array) is list:
for a in args:
for ff in filter_array:
if a[0] != ff:
if a[1] is None:
has_none = True
bad_parameters.append(a[0])
break
if has_none:
raise ValueError('function {}: Parameters {} cannot be None. '.format(f.__name__, bad_parameters))
def real_decorator(f):
v_names = f.__code__.co_varnames
def wrapper(*args, **kwargs):
n_args = []
for a in range(0, len(args)):
n_args.append((v_names[a], args[a]))
the_actual_test(f, n_args, nullable_parameters)
result = f(*args, **kwargs)
return result
return wrapper
return real_decorator
Usage
from not_none import not_none
#not_none()
def no_none(a,b):
return (a,b)
#not_none(nullable_parameters=["b"])
def allow_b_as_none(a,b):
return (a,b)
#passes
no_none(1,1)
#fails
no_none(None,1)
#passes
allow_b_as_none(1,None)
#fails
allow_b_as_none(None,1)
After my first answer got deleted. Here is an updated version:
I tried to use the very nice answer from Bigbob556677, but for me it didn't work with **kwargs, so I edited it and put it in a Gist, here: https://gist.github.com/devTechi/6e633ded72cc83637f34b1a3f4a96984 (code also below)
I didn't test it with just *args, but with (I posted more or less just the gist-link) **kwargs it works nicely.
def not_none(nullable_parameters=None):
# values given by real_decorator (see below)
def the_actual_test(f, expected_args_with_given, allowed_nullable_args):
has_none = False
bad_parameters = []
for key, value in expected_args_with_given.items():
if (value is None and nullable_parameters is None) or \
(value is None and key not in nullable_parameters):
bad_parameters.append(key)
has_none = True
if has_none:
raise ValueError("[Function '{}' of '{}'] - IMPORTANT: Parameters '{}' cannot be None. ".format(f.__name__, f.__module__, bad_parameters))
# here the code REALLY begins
def not_null_decorator(original_func):
import inspect
has_self = False
# f.__code__.co_varnames --> local variables (not only parameters), see: https://python-reference.readthedocs.io/en/latest/docs/code/varnames.html
# get declared arguments from ogirinal function
argspec = inspect.getargspec(original_func)
if 'self' in argspec.args:
argnames = argspec.args[1:] # no self
has_self = True
else:
argnames = argspec.args
args_dict = dict.fromkeys(argnames)
def get_args(*args, **kwargs):
for arg in args:
if arg in args_dict.keys():
args_dict[arg] = arg
for key, value in kwargs.items():
if key in args_dict.keys():
args_dict[key] = value
return args_dict
def wrapper_with_self(self, *args, **kwargs):
the_actual_test(original_func, get_args(*args, **kwargs), nullable_parameters)
return original_func(self, *args, **kwargs)
def wrapper(*args, **kwargs):
the_actual_test(original_func, get_args(*args, **kwargs), nullable_parameters)
return original_func(*args, **kwargs)
if has_self:
return wrapper_with_self
else:
return wrapper
return not_null_decorator
Usage:
from .nullable_decorator import not_none
#not_none(nullable_parameters=["nullable_arg1", "nullable_arg2"])
def some_function(self, nullable_arg1=None, nullable_arg2=None, non_nullable_arg1=None):
pass
#not_none()
def some_other_function(self, non_nullable_arg1=None, non_nullable_arg2=None):
pass