I have a class that I want to share in a read-only fashion with children processes in a pool, so I prepared a proxy of a class but it didn't work. The following is a simplified example of my problem.
from multiprocessing.managers import BaseManager
class TestClass:
def __init__(self, a):
self.a = a
def b(self):
print self.a
class MyManager(BaseManager): pass
MyManager.register('test', TestClass)
if __name__ == '__main__':
manager = MyManager()
manager.start()
t = TestClass(1)
print t.a
mt = manager.test(2)
mt.b()
mt.a
When I run this code I get:
1
2
Traceback (most recent call last):
File "multiprocess_example_stackexchange.py", line 20, in <module>
mt.a
AttributeError: 'AutoProxy[test]' object has no attribute 'a'
It seems that I cannot access the attribute of a shared object directly via a proxy. Is the only way using a method that gets the attribute, or am I doing something wrong?
The Proxy objects used by multiprocessing.BaseManager and its sub-classes normally only expose methods from the objects they're referring to, not attributes. Now, there is multiprocessing.Manager().Namespace, which provides a Proxy sub-class that does provide access to attributes, rather than methods. We can create our own Proxy type which inherits from that, which enables access to all our attributes, as well as access to our b function:
from multiprocessing.managers import BaseManager, NamespaceProxy
class TestClass(object):
def __init__(self, a):
self.a = a
def b(self):
print self.a
class MyManager(BaseManager): pass
class TestProxy(NamespaceProxy):
# We need to expose the same __dunder__ methods as NamespaceProxy,
# in addition to the b method.
_exposed_ = ('__getattribute__', '__setattr__', '__delattr__', 'b')
def b(self):
callmethod = object.__getattribute__(self, '_callmethod')
return callmethod('b')
MyManager.register('test', TestClass, TestProxy)
if __name__ == '__main__':
manager = MyManager()
manager.start()
t = TestClass(1)
print t.a
mt = manager.test(2)
print mt.a
mt.a = 5
mt.b()
Output:
1
2
5
Edit:
If you want to be able to dynamically add methods from your original class to a Proxy class, you can do something like this:
from multiprocessing.managers import BaseManager, NamespaceProxy
import inspect
class TestClass(object):
def __init__(self, a):
self.a = a
def b(self):
print self.a
class AnotherClass(object):
def __init__(self, a):
self.a = a
def c(self):
print self.a
class MyManager(BaseManager): pass
class ProxyBase(NamespaceProxy):
_exposed_ = ('__getattribute__', '__setattr__', '__delattr__')
class TestProxy(ProxyBase): pass
class AnotherProxy(ProxyBase): pass
def register_proxy(name, cls, proxy):
for attr in dir(cls):
if inspect.ismethod(getattr(cls, attr)) and not attr.startswith("__"):
proxy._exposed_ += (attr,)
setattr(proxy, attr,
lambda s: object.__getattribute__(s, '_callmethod')(attr))
MyManager.register(name, cls, proxy)
register_proxy('test', TestClass, TestProxy)
register_proxy('another', AnotherClass, AnotherProxy)
if __name__ == '__main__':
manager = MyManager()
manager.start()
mt = manager.test(2)
ma = manager.another(3)
mt.b()
ma.c()
mt.a = 5
ma.a = 6
mt.b()
ma.c()
After spending few hours to reading the source codes, here is the simplest ways to implement the proxy class to expose all attributes and methods:
class TestProxy(NamespaceProxy):
_exposed_ = tuple(dir(Test))
def __getattr__(self, name):
result = super().__getattr__(name)
if isinstance(result, types.MethodType):
def wrapper(*args, **kwargs):
self._callmethod(name, args)
return wrapper
return result
BaseManager.register('Test', Test, TestProxy)
manager = BaseManager()
test = manager.Test()
Also, here is an auto proxy method:
def Proxy(target):
dic = {'types': types}
exec('''def __getattr__(self, key):
result = self._callmethod('__getattribute__', (key,))
if isinstance(result, types.MethodType):
def wrapper(*args, **kwargs):
self._callmethod(key, args)
return wrapper
return result''', dic)
proxyName = target.__name__ + "Proxy"
ProxyType = type(proxyName, (NamespaceProxy,), dic)
ProxyType._exposed_ = tuple(dir(target))
return ProxyType
TestProxy = Proxy(Test)
BaseManager.register('Test', Test, TestProxy)
manager = BaseManager()
test = manager.Test()
Since I do not have enough reputation to comment, I am posting an answer. The otherwise excellent answer by #shtse8 has a bug. I wanted to point this out since this page is one of the top hits when you search for queries regarding NamespaceProxy and the said answer has been used by others as well.
The problem lies in the below code:
class TestProxy(NamespaceProxy):
_exposed_ = tuple(dir(Test))
def __getattr__(self, name):
result = super().__getattr__(name)
if isinstance(result, types.MethodType):
def wrapper(*args, **kwargs):
self._callmethod(name, args) # Result not returned
return wrapper
return result
If you use this class (or the equivalent 'auto proxy' method mentioned in the answer) to create proxy objects, then all functions which are programmed to return a value will always return a NoneType instead (if you access the functions from the proxy). Moreover, they will not pass on any keyword arguments either. This is because in the wrapper the result of the method call use kwargs and neither is the call itself being returned. Therefore, we need to pass kwargs and put a return in the line self._callmethod(name, args).
The TestProxy class should then become:
class TestProxy(NamespaceProxy):
_exposed_ = tuple(dir(Test))
def __getattr__(self, name):
result = super().__getattr__(name)
if isinstance(result, types.MethodType):
def wrapper(*args, **kwargs):
return self._callmethod(name, args, kwargs) # Note the return here
return wrapper
return result
The 'auto proxy' function would then become:
def Proxy(target):
dic = {'types': types}
exec('''def __getattr__(self, key):
result = self._callmethod('__getattribute__', (key,))
if isinstance(result, types.MethodType):
def wrapper(*args, **kwargs):
return self._callmethod(key, args, kwargs)
return wrapper
return result''', dic)
proxyName = target.__name__ + "Proxy"
ProxyType = type(proxyName, (NamespaceProxy,), dic)
ProxyType._exposed_ = tuple(dir(target))
return ProxyType
Update: Edited code and explanation to pass kwargs as well. Check Viktor's answer below
Here's a less verbose alternative that I found to work well in practice. Not sure if there are any disadvantages.
class TestClass:
def __init__(self, a):
self.a = a
def b(self):
print self.a
def wrap_test_class(*args, **kwargs):
obj = TestClass(*args, **kwargs)
obj.get_a = lambda: obj.a
return obj
class MyManager(BaseManager): pass
MyManager.register('test', wrap_test_class)
This allows you to access a by calling proxy_object.get_a()
This is an example of passing parameters (example: __getitem__) or not (example: __len__):
class TestProxy(NamespaceProxy):
_exposed_ = ('__getattribute__', '__setattr__', '__delattr__','__len__','__getitem__')
def __len__(self):
callmethod = object.__getattribute__(self, '_callmethod')
return callmethod('__len__')
def __getitem__(self,index):
callmethod = object.__getattribute__(self, '_callmethod')
return callmethod('__getitem__',(index,))
Charcit's solution was working for me except I made a small completion/bugfixing. There kwargs cannot be passed to the called methods. So the fixed version:
class TestProxy(NamespaceProxy):
_exposed_ = tuple(dir(Test))
def __getattr__(self, name):
result = super().__getattr__(name)
if isinstance(result, types.MethodType):
def wrapper(*args, **kwargs):
return self._callmethod(name, args, kwargs) # args and kwargs!
return wrapper
return result
Didn't test the 'autoproxy' method but this fix should apply there too:
def Proxy(target):
dic = {'types': types}
exec('''def __getattr__(self, key):
result = self._callmethod('__getattribute__', (key,))
if isinstance(result, types.MethodType):
def wrapper(*args, **kwargs):
return self._callmethod(key, args, kwargs)
return wrapper
return result''', dic)
proxyName = target.__name__ + "Proxy"
ProxyType = type(proxyName, (NamespaceProxy,), dic)
ProxyType._exposed_ = tuple(dir(target))
return ProxyType
I was originally using the answer by #shtse8 (as modified by #Charchit-Agarwal and #Viktor), which mostly worked for me, but it was trying to pickle the methods returned by the __getattr__ function, which didn't work in some use cases.
So here is another solution which is closer to the implementation of NamespaceProxy, but adds in the public methods the same way AutoProxy does.
def MakeProxyWithAttrs(target):
""" Create a derived NamespaceProxy class for `target`. """
# This bit follows what multiprocessing.managers.MakeProxy normally does.
dic = {}
public_methods = [m for m in dir(target) if m[0] != '_']
for meth in public_methods:
exec('''def %s(self, *args, **kwds):
return self._callmethod(%r, args, kwds)
'''%(meth,meth), dic)
# NamespaceProxy starts with __getattribute__ defined, so subclass from that
# rather than BaseProxy, as MakeProxy normally does.
proxy_name = target.__name__ + "_Proxy"
ProxyType = type(proxy_name, (NamespaceProxy,), dic)
# Expose all the public methods and also __getattribute__ and __setattr__.
ProxyType._exposed_ = tuple(public_methods + ['__getattribute__', '__setattr__'])
return ProxyType
class Manager(multiprocessing.managers.BaseManager): pass
test_proxy = MakeProxyWithAttrs(test_func)
Manager.register('test', test_func, test_proxy)
manager = Manager()
manager.start()
test = manager.test()
Related
Lets say my class has many function, and I want to apply my decorator on each one of them. I have researched for a while, and find https://stackoverflow.com/a/6307917/18859252. By using metaclass, I can decorate all function in one line.
Here is my code (framework)
class myMetaClass(type):
def __new__(cls, name, bases, local):
for attr in local:
value = local[attr]
if callable(value) and attr != '__init__':
local[attr] = log_decorator(local['Variable'], value)
return super().__new__(cls, name, bases, local)
class log_decorator():
def __init__(self, Variable, func):
self.Variable = Variable
self.func = func
def __call__(self, *args, **kargs):
start_time = time.time()
self.func(*args, **kargs)
end_time = time.time()
class Test(metaclass = myMetaClass):
Variable = Some_Class
check_test = Some_Class
def __init__(self, **args):
self.connect = Some_Class(**args)
def A(self, a, b):
self.connect.abc
pass
then use like this
def Flow():
test = Test(**args)
test.A(a, b)
But here is problem, it show exception like:
TypeError:A() missing 1 required positional argument: 'self'
I have no idea about this problem. I'd be very grateful if anyone has an answer or if there is a better way.
The piece you are missing (and the bit I don't fully understand, but has to do with functions or methods as descriptors and how python will attach an instance as the self parameter) is that log_decorator() is an instance of that class and not a function or method (even though you have defined a __call__() method which makes it callable.)
Here is some code which just slightly changes the syntax needed, but gives you the results you want:
import functools
class log_decorator:
def __init__(self, Variable): # Note that the only parameter is Variable
self.Variable = Variable
def __call__(self, func):
#functools.wraps(func)
def decorated(*args, **kwargs):
start_time = time.time()
func(*args, **kwargs)
end_time = time.time()
return decorated
class myMetaClass(type):
def __new__(cls, name, bases, local):
for attr in local:
value = local[attr]
if callable(value) and attr != '__init__':
# Note the change in syntax vvv
local[attr] = log_decorator(local['Variable'])(value)
return super().__new__(cls, name, bases, local)
I have a complex unpickable object that has properties (defined via getters and setters) that are of complex and unpickable type as well. I want to create a multiprocessing proxy for the object to execute some tasks in parallel.
The problem: While I have succeeded to make the getter methods available for the proxy object, I fail to make the getters return proxies for the unpickable return objects.
My setup resembles the following:
from multiprocessing.managers import BaseManager, NamespaceProxy
class A():
#property
def a(self):
return B()
#property
def b(self):
return 2
# unpickable class
class B():
def __init__(self, *args):
self.f = lambda: 1
class ProxyBase(NamespaceProxy):
_exposed_ = ('__getattribute__', '__setattr__', '__delattr__')
class AProxy(ProxyBase): pass
class BProxy(ProxyBase): pass
class MyManager(BaseManager):pass
MyManager.register('A', A, AProxy)
if __name__ == '__main__':
with MyManager() as manager:
myA = manager.A()
print(myA.b) # works great
print(myA.a) # raises error, because the object B is not pickable
I know that I can specify the result type of a method when registering it with the manager. That is, I can do
MyManager.register('A', A, AProxy, method_to_typeid={'__getattribute__':'B'})
MyManager.register('B', B, BProxy)
if __name__ == '__main__':
with MyManager() as manager:
myA = manager.A()
print(myA.a) # works great!
print(myA.b) # returns the same as myA.a ?!
It is clear to me that my solution does not work since the __getattr__ method applies to all properties, whereas I only want it to return a proxy for B when property a is accessed. How could I achieve this?
As a side question: if I remove the *args argument from the __init__ method of B, I get an error that it is called with the wrong number of arguments. Why? How could I resolve this?
I don't this is possible without some hacks, since the choice to return a value or proxy is made based on the method name alone, and not the type of the return value (from Server.serve_client):
try:
res = function(*args, **kwds)
except Exception as e:
msg = ('#ERROR', e)
else:
typeid = gettypeid and gettypeid.get(methodname, None)
if typeid:
rident, rexposed = self.create(conn, typeid, res)
token = Token(typeid, self.address, rident)
msg = ('#PROXY', (rexposed, token))
else:
msg = ('#RETURN', res)
Also keep in mind exposing __getattribute__ in an unpickable class's proxy basically breaks the proxy functionality when calling methods.
But if you're willing to hack it and just need attribute access, here is a working solution (note calling myA.a.f() still won't work, the lambda is an attribute and is not proxied, only methods are, but that's a different problem).
import os
from multiprocessing.managers import BaseManager, NamespaceProxy, Server
class A():
#property
def a(self):
return B()
#property
def b(self):
return 2
# unpickable class
class B():
def __init__(self, *args):
self.f = lambda: 1
self.pid = os.getpid()
class HackedObj:
def __init__(self, obj, gettypeid):
self.obj = obj
self.gettypeid = gettypeid
def __getattribute__(self, attr):
if attr == '__getattribute__':
return object.__getattribute__(self, attr)
obj = object.__getattribute__(self, 'obj')
result = object.__getattribute__(obj, attr)
if isinstance(result, B):
gettypeid = object.__getattribute__(self, 'gettypeid')
# This tells the server that the return value of this method is
# B, for which we've registered a proxy.
gettypeid['__getattribute__'] = 'B'
return result
class HackedDict:
def __init__(self, data):
self.data = data
def __setitem__(self, key, value):
self.data[key] = value
def __getitem__(self, key):
obj, exposed, gettypeid = self.data[key]
if isinstance(obj, A):
gettypeid = gettypeid.copy() if gettypeid else {}
# Now we need getattr to update gettypeid based on the result
# luckily BaseManager queries the typeid info after the function
# has been invoked
obj = HackedObj(obj, gettypeid)
return (obj, exposed, gettypeid)
class HackedServer(Server):
def __init__(self, registry, address, authkey, serializer):
super().__init__(registry, address, authkey, serializer)
self.id_to_obj = HackedDict(self.id_to_obj)
class MyManager(BaseManager):
_Server = HackedServer
class ProxyBase(NamespaceProxy):
_exposed_ = ('__getattribute__', '__setattr__', '__delattr__')
class AProxy(ProxyBase): pass
class BProxy(ProxyBase): pass
MyManager.register('A', callable=A, proxytype=AProxy)
MyManager.register('B', callable=B, proxytype=BProxy)
if __name__ == '__main__':
print("This process: ", os.getpid())
with MyManager() as manager:
myB = manager.B()
print("Proxy process, using B directly: ", myB.pid)
myA = manager.A()
print('myA.b', myA.b)
print("Proxy process, via A: ", myA.a.pid)
The key to the solution is to replace the _Server in our manager, and then wrap the id_to_obj dict with the one that performs the hack for the specific method we need.
The hack consists on populating the gettypeid dict for the method, but only after it has been evaluated and we know the return type to be one that we would need a proxy for. And we're lucky in the order of evaluations, gettypeid is accessed after the method has been called.
Also luckily gettypeid is used as a local in the serve_client method, so we can return a copy of it and modify it and we don't introduce any concurrency issues.
While this was a fun exercise, I have to say I really advise against this solution, if you're dealing with external code that you cannot modify, you should simply create your own wrapper class that has explicit methods instead of #property accessors, proxy your own class instead, and use method_to_typeid.
I have child classes which inherit some basic functionality from a parent class.
The child classes shall have a generic constructor prepare_and_connect_constructor() which does some magic around the object creation of the parent class.
For simplicity, the magic is done by a simple function based decorator (ultimately, it should be a part of the parent class).
def decorate_with_some_magic(func):
def prepare_and_connect(*args, **kwargs):
print("prepare something")
print("create an object")
obj = func(*args, **kwargs)
print("connect obj to something")
return obj
return prepare_and_connect
class Parent:
def __init__(self, a):
self.a = a
def __repr__(self):
return f"{self.a}"
class Child(Parent):
#classmethod
#decorate_with_some_magic
def prepare_and_connect_constructor(cls, a, b):
""" use the generic connection decorator right on object creation """
obj = super().__init__(a)
# put some more specific attributes (over the parents class)
obj.b = b
return obj
def __init__(self, a, b):
""" init without connecting """
super().__init__(a)
self.b = b
def __repr__(self):
return f"{self.a}, {self.b}"
if __name__ == '__main__':
print(Child.prepare_and_connect_constructor("special child", "needs some help"))
Using this code i finally get
obj = super().__init__(a)
TypeError: __init__() missing 1 required positional argument: 'a'
when running prepare_and_connect_constructor().
Actually I would expect that the super.__init__(a) call should be the same as in Child.__init__.
I guess the reason is related to the classmethod but I can't figure it out.
What's wrong with this call?
Update: In general what was wrong is that __init__ doesn't return an object.
Due to the hints and thoughts from the answers i modified my code to achieve what i need:
class Parent:
def __init__(self, a):
self.a = a
#staticmethod
def decorate_with_some_magic(func):
def prepare_and_connect(*args, **kwargs):
print("prepare something")
print("create an object")
obj = func(*args, **kwargs)
print("connect obj to something")
return obj
return prepare_and_connect
def __repr__(self):
return f"{self.a}"
class ChildWithOneName(Parent):
#classmethod
#Parent.decorate_with_some_magic
def prepare_and_connect_constructor(cls, a, b):
""" use the generic connection decorator right on object creation """
obj = super().__new__(cls)
obj.__init__(a, b)
print("Does the same as in it's __init__ method")
return obj
def __init__(self, a, b):
""" init without connecting """
super().__init__(a)
self.b = b
def __repr__(self):
return f"{self.a}, {self.b}"
class GodChild(Parent):
#classmethod
#Parent.decorate_with_some_magic
def prepare_and_connect_constructor(cls, a, names):
""" use the generic connection decorator right on object creation """
obj = super().__new__(cls)
obj.__init__(a, names)
# perform some more specific operations
obj.register_all_names(names)
print("And does some more stuff than in it's __init__ method")
return obj
def __init__(self, a, already_verified_names):
""" init without connecting """
super().__init__(a)
self.verified_names = already_verified_names
def register_all_names(self, names=[]):
self.verified_names = []
def verify(text):
return True
for name in names:
if verify(name):
self.verified_names.append(name)
def __repr__(self):
return f"{self.a}, {self.verified_names}"
if __name__ == '__main__':
print(ChildWithOneName.prepare_and_connect_constructor("special child", "needs some help"), end='\n\n')
print(GodChild.prepare_and_connect_constructor("unknown child", "needs some verification"), end='\n\n')
print(ChildWithOneName("my child", "is clean and doesn't need extra magic"))
decorate_with_some_magic is now a part of the Parent class (using a staticmethod) as it is a related generic functionality
Each child class (added one more for illustration) has it's own prepare_and_connect_constructor classmethod, which calls its own constructor and does optionally some additional work
You have a slight misunderstanding of the magic methods __init__ and __new__. __new__ creates a new object, e.g. returns a instance of the class. __init__ just modifies the object in place. So an easy fix for your problem would be de following:
#classmethod
#decorate_with_some_magic
def prepare_and_connect_constructor(cls, a, b):
""" use the generic connection decorator right on object creation """
obj = super().__new__(cls)
obj.__init__(a)
# put some more specific attributes (over the parents class)
obj.b = b
return obj
I however don't think you should use it like this. Instead, your probably should overwrite __new__
Decorators work on callables. Since there is no difference between calling a function and initiating a class, you can use your decorator directly on the class:
def decorate_with_some_magic(func):
def prepare_and_connect(*args, **kwargs):
print("prepare something")
print("create an object")
obj = func(*args, **kwargs)
print("connect obj to something")
return obj
return prepare_and_connect
class Parent:
#classmethod
def prepare_and_connect_constructor(cls, a, b):
return decorate_with_some_magic(cls)(a, b)
def __init__(self, a):
self.a = a
def __repr__(self):
return f"{self.a}"
class Child(Parent):
def __init__(self, a, b):
""" init without connecting """
super().__init__(a)
self.b = b
def __repr__(self):
return f"{self.a}, {self.b}"
if __name__ == '__main__':
normal_child = Child("normal child", "no help needed")
print(normal_child)
special_child = Child.prepare_and_connect_constructor("special child", "needs some help")
print(special_child)
Output:
normal child, no help needed
prepare something
create an object
connect obj to something
special child, needs some help
I'm trying to decorate all methods in class and i succeded with this code, but i'm also trying to log calls to operators like * + - / , is there any way to decorate them or something like getattr(self,"*") to log the calls ?
class Logger(object):
def __init__(self, bool):
self.bool = bool
def __call__(self, cls):
class DecoratedClass(cls):
def __init__(cls, *args, **kwargs):
super().__init__(*args, **kwargs)
if not(self.bool):
return
methods = [func for func in dir(cls)
if callable(getattr(cls, func))
and not func.startswith("__class")]
for func in methods:
old_func = getattr(cls, func)
def decorated_function(fname, fn):
def loggedFunction(*args, **kwargs):
print("Calling {0} from {3} with params {1} and kwargs {2}".format(fname.upper(), args, kwargs, cls))
return fn(*args, **kwargs)
return loggedFunction
setattr(cls, func, decorated_function(func, old_func))
return DecoratedClass
#Logger(True)
class DummyClass():
def __init__(self,foo):
self.foo = foo
def bar(self):
print(self.foo)
def __mul__(self,other):
print("Hello",other)
if __name__ == '__main__':
a = DummyClass('hola')
a.method()
a.__mul__(a) #this is logged
print(a*a) #this is not logged by decorator
Thanks to Łukasz, here is a working script.
A difficulty I encountered is to handle multiple instances and avoid to decorate multiple times the same class methods. To handle this problem, I keep track of the decorated class methods (cls.__logged).
Another difficulty is to deal with the magic methods like __setattr__, __getattribute__, __repr__, ... My solution is to ignore them, except for a list that you must define at start (loggable_magic_methods).
from functools import wraps
loggable_magic_methods = ['__mul__',]
def is_magic_method(method):
return method.startswith('__')
class Logger(object):
def __init__(self, bool):
self.bool = bool
def __call__(self, cls):
class LoggedClass(cls):
cls.__logged = []
def __init__(instance, *args, **kwargs):
super().__init__(*args, **kwargs)
if not(self.bool):
return
methods = [funcname for funcname in dir(instance)
if callable(getattr(instance, funcname))
and (funcname in loggable_magic_methods or not is_magic_method(funcname))]
def logged(method):
#wraps(method)
def wrapper(*args, **kwargs):
print (method.__name__, args, kwargs, cls)
return method(*args, **kwargs)
return wrapper
for funcname in methods:
if funcname in cls.__logged:
continue
if is_magic_method(funcname):
setattr(cls, funcname, logged(getattr(cls, funcname)))
cls.__logged.append(funcname)
else:
setattr(instance, funcname, logged(getattr(instance, funcname)))
return LoggedClass
#Logger(True)
class DummyClass():
def __init__(self, foo, coef):
self.foo = foo
self.coef = coef
def bar(self):
print(self.foo)
def __mul__(self, other):
print(self.foo)
print(other.foo)
return self.coef * other.coef
if __name__ == '__main__':
a = DummyClass('hola', 1)
a.bar()
print()
print(a.__mul__(a))
print()
print(a*a)
print()
b = DummyClass('gracias', 2)
b.bar()
print()
print(b.__mul__(a))
print()
print(b*a)
Currently you are patching values on instance. Your usage of cls in __init__ signature is false friend - actually it's old plain self in this case.
If you want to override magic methods, interpreter looks for them on class objects, not on instances.
Minimal example:
class DummyClass:
def __init__(self, foo):
self.foo = foo
def __mul__(self, other):
return self.foo * other.foo
def logged(method):
def wrapper(*args, **kwargs):
print (method.__name__, args, kwargs)
return method(*args, **kwargs)
return wrapper
DummyClass.__mul__ = logged(DummyClass.__mul__)
a = DummyClass(1)
b = DummyClass(2)
assert a * a == 1
assert a * b == 2
assert b * b == 4
Each call is logged.
>>> a = DummyClass(1)
>>> b = DummyClass(2)
>>> assert a * a == 1
__mul__ (<__main__.DummyClass object at 0x00000000011BFEB8>, <__main__.DummyClass object at 0x00000000011BFEB8>) {}
>>> assert a * b == 2
__mul__ (<__main__.DummyClass object at 0x00000000011BFEB8>, <__main__.DummyClass object at 0x00000000011BF080>) {}
>>> assert b * b == 4
__mul__ (<__main__.DummyClass object at 0x00000000011BF080>, <__main__.DummyClass object at 0x00000000011BF080>) {}
I'll leave a task of rewriting monkey-patching approach to you.
I am looking for a way to intercept instance method calls in class MyWrapper below:
class SomeClass1:
def a1(self):
self.internal_z()
return "a1"
def a2(self):
return "a2"
def internal_z(self):
return "z"
class SomeClass2(SomeClass1):
pass
class MyWrapper(SomeClass2):
# def INTERCEPT_ALL_FUNCTION_CALLS():
# result = Call_Original_Function()
# self.str += result
# return result
def __init__(self):
self.str = ''
def getFinalResult(self):
return self.str
x = MyWrapper()
x.a1()
x.a2()
I want to intercept all function calls make through my wrapper class. In my wrapper class I want to keep track of all the result strings.
result = x.getFinalResult()
print result == 'a1a2'
Some quick and dirty code:
class Wrapper:
def __init__(self, obj):
self.obj = obj
self.callable_results = []
def __getattr__(self, attr):
print("Getting {0}.{1}".format(type(self.obj).__name__, attr))
ret = getattr(self.obj, attr)
if hasattr(ret, "__call__"):
return self.FunctionWrapper(self, ret)
return ret
class FunctionWrapper:
def __init__(self, parent, callable):
self.parent = parent
self.callable = callable
def __call__(self, *args, **kwargs):
print("Calling {0}.{1}".format(
type(self.parent.obj).__name__, self.callable.__name__))
ret = self.callable(*args, **kwargs)
self.parent.callable_results.append(ret)
return ret
class A:
def __init__(self, val): self.val = val
def getval(self): return self.val
w = Wrapper(A(10))
print(w.val)
w.getval()
print(w.callable_results)
Might not be thorough, but could be a decent starting point, I guess.
You could wrap your methods with decorators a instanciation time:
#!/usr/bin/env python
import inspect
def log(func):
def _logged(*args, **kw):
print "[LD] func", func.__name__, "called with:", args, kw
result = func(*args, **kw)
print "[LD] func", func.__name__, "returned:", result
return result
return _logged
class A(object):
def __init__(self):
for x in inspect.getmembers(self, (inspect.ismethod)):
if not x[0].startswith('__'):
setattr(self, x[0], log(getattr(self, x[0])))
def hello(self):
print "Hello"
def bye(self):
print "Bye"
return 0
Now if you call hello or bye, the call goes through log first:
a = A()
a.hello()
a.bye()
# [LD] func hello called with: () {}
# Hello
# [LD] func hello returned: None
# [LD] func bye called with: () {}
# Bye
# [LD] func bye returned: 0
What you want to do is quite similar to this question.
You should take your example code in the reverse order, i mean creating a class to record return values of method calls, and make the classes you want to watch inherit from it.
Which would give something like this
class RetValWatcher(object):
def __init__(self):
self.retvals = []
def __getattribute__(self, name):
attr = super(RetValWatcher, self).__getattribute__(name)
if callable(attr):
def wrapped(*args, **kwargs):
retval = attr(*args, **kwargs)
self.retvals.append(retval)
return retval
return wrapped
else:
return attr
def getFinalResult(self):
return ''.join(self.retvals)
class MyClass(RetValWatcher):
def a(self):
self.internal_z()
return 'a1'
def b(self):
return 'b1'
def internal_z(self):
return 'z'
x = MyClass()
x.a()
x.b()
print x.getFinalResult()
#'za1b1'
With some minor changes, this method would also allow you to record return values across all RetValWatcher instances.
Edit: added changes suggested by singularity's comment
Edit2: forgot to handle the case where attr is not a method (thx singularity again)
Edit3: fixed typo