I am currently using this piece of code :
class FileSystem(metaclass=Singleton):
"""File System manager based on Spark"""
def __init__(self, spark):
self._path = spark._jvm.org.apache.hadoop.fs.Path
self._fs = spark._jvm.org.apache.hadoop.fs.FileSystem.get(
spark._jsc.hadoopConfiguration()
)
#classmethod
def without_spark(cls):
with Spark() as spark:
return cls(spark)
My object depends obviously on the Spark object (another object that I created - If you need to see its code, I can add it but I do not think it is required for my current issue).
It can be used in 2 differents ways resulting the same behavior :
fs = FileSystem.without_spark()
# OR
with Spark() as spark:
fs = FileSystem(spark)
My problem is that, even if FileSystem is a singleton, using the class method without_spark makes me enter (__enter__) the context manager of spark, which lead to a connection to spark cluster, which takes a lot of time. How can I make that the first execution of without_spark do the connection, but the next one only returns the already created instance?
The expected behavior would be something like this :
#classmethod
def without_spark(cls):
if not cls.exists: # I do not know how to persist this information in the class
with Spark() as spark:
return cls(spark)
else:
return cls()
I think you are looking for something like
import contextlib
class FileSystem(metaclass=Singleton):
"""File System manager based on Spark"""
spark = None
def __init__(self, spark):
self._path = spark._jvm.org.apache.hadoop.fs.Path
self._fs = spark._jvm.org.apache.hadoop.fs.FileSystem.get(
spark._jsc.hadoopConfiguration()
)
#classmethod
def without_spark(cls):
if cls.spark is None:
cm = cls.spark = Spark()
else:
cm = contextlib.nullcontext(cls.spark)
with cm as s:
return cls(s)
The first time without_spark is called, a new instance of Spark is created and used as a context manager. Subsequent calls reuse the same Spark instance and use a null context manager.
I believe your approach will work as well; you just need to initialize exists to be False, then set it to True the first (and every, really) time you call the class method.
class FileSystem(metaclass=Singleton):
"""File System manager based on Spark"""
exists = False
def __init__(self, spark):
self._path = spark._jvm.org.apache.hadoop.fs.Path
self._fs = spark._jvm.org.apache.hadoop.fs.FileSystem.get(
spark._jsc.hadoopConfiguration()
)
#classmethod
def without_spark(cls):
if not cls.exists:
cls.exists = True
with Spark() as spark:
return cls(spark)
else:
return cls()
Can't you make the constructor argument optional, and initiate the Spark lazily, e.g. in a property (or functools.cached_property):
from functools import cached_property
class FileSystem(metaclass=Singleton):
def __init__(self, spark=None):
self._spark = spark
#cached_property
def spark(self):
if self._spark:
return self._spark
return self._spark := Spark()
#cached_property
def path(self):
return self.spark._jvm.org.apache.hadoop.fs.Path
#cached_property
def fs(self):
with self.spark:
return self.spark._jvm.org.apache.hadoop.fs.FileSystem.get(
self.spark._jsc.hadoopConfiguration()
)
I'm trying to inject dependencies into my Django view (controller?). Here's some background.
Normally, the urls.py file is what handles the routing. It is usually something like this:
urlpatterns = [
path("", views.get_all_posts, name="get_all_posts"),
path("<int:post_id>", views.get_post, name="get_post"),
path("create", views.create_post, name="create_post"),
]
The problem with this, is that once you get to create_post for instance, you might have a dependency on a service that creates posts:
# views.py
...
def create_post(self):
svc = PostCreationService()
svc.create_post()
This kind of pattern is difficult to test. While I know python testing libraries have tools to mock this sort of thing, I'd rather inject the dependency into the view. Here's what I came up with.
A Controller class that has a static method, export(deps) that takes in a list of dependencies and returns a list of url pattern objects:
class ApiController(object):
#staticmethod
def export(**deps):
ctrl = ApiController(**deps)
return [
path("", ctrl.get_all_posts, name="get_all_posts"),
path("<int:post_id>", ctrl.get_post, name="get_post"),
path("create", ctrl.create_post, name="create_post"),
]
def __init__(self, **deps):
self.deps = deps
def get_all_posts():
pass
...
This looks janky, but I'm not aware of any other way to do what I'm trying to do. The controller needs to return a list of url patterns, and it also needs to take in a list of dependencies. Using the above technique, I can do this in urls.py:
urlpatterns = ApiController.export(foo_service=(lambda x: x))
I am now free to use foo_service in any of the methods of ApiController.
Note:
One alternative would be for the constructor to return the list of urls, but I don't see that as a huge improvement over this. In fact, it strikes me as being more confusing because the class constructor would return a list instead of an instance of the class.
Note 2:
I'm aware that python has mocking tools for mocking class members. Please don't suggest using them. I'd like to use DI as the way to control and manage dependencies.
Any ideas on what the best way to do this is?
Consider injecting using decorators:
from functools import wraps
class ServiceInjector:
def __init__(self):
self.deps = {}
def register(self, name=None):
name = name
def decorator(thing):
"""
thing here can be class or function or anything really
"""
if not name:
if not hasattr(thing, "__name__"):
raise Exception("no name")
thing_name = thing.__name__
else:
thing_name = name
self.deps[thing_name] = thing
return thing
return decorator
def inject(self, func):
#wraps(func)
def decorated(*args, **kwargs):
new_args = args + (self.deps, )
return func(*new_args, **kwargs)
return decorated
# usage:
si = ServiceInjector()
# use func.__name__, registering func
#si.register()
def foo(*args):
return sum(args)
# we can rename what it's been registered as, here, the class is registered
# with name `UpperCase` instead of the class name `UpperCaseRepresentation`
#si.register(name="UpperCase")
class UpperCaseRepresentation:
def __init__(self, value):
self.value = value
def __str__(self):
return self.value.upper()
#register float
si.register(name="PI")(3.141592653)
# inject into functions
#si.inject
def bar(a, b, c, _deps): # the last one in *args would be receiving the dependencies
UpperCase, PI, foo = _deps['UpperCase'], _deps['PI'], _deps['foo']
print(UpperCase('abc')) # ABC
print(PI) # 3.141592653
print(foo(a, b, c, 4, 5)) # = 15
bar(1, 2, 3)
# inject into class methods
class Foo:
#si.inject
def my_method(self, a, b, _deps, kwarg1=30):
return _deps['foo'](a, b, kwarg1)
print(Foo().my_method(1, 2, kwarg1=50)) # = 53
You could take a look at https://github.com/ets-labs/python-dependency-injector, but that is a pretty big setup.
You could also create something small like a Service factory
# services.py
class ServiceFactory:
def __init__(self):
self.__services = {}
def register(self, name, service_class):
# Maybe add some validation
self.__services[name] = service_class
def create(self, name, *args, **kwargs):
# Maybe add some error handling or fallbacks
return self.__services[name](*args, **kwargs)
factory = ServiceFactory()
# In your settings.py for example
from services import factory
factory.register('post_creation', PostCreationService)
# Or maybe in apps.ready do auto_load that will loop all apps and get config from services.py
# In your views.py
from services import factory
def create_post(self):
svc = factory.create('post_creation')
svc.create_post()
# In your tests.py
from services import factory
def setUp(self):
factory.register('post_creation', FakePostCreationService)
While reading Dependency Injection Principles, Practices, and Patterns and trying to apply the examples to a django app I came up with the following:
# views.py
class IndexView(View):
# Must include this to bypass django's validation
product_service: IProductService = None
# Init method not necessary but more explicit
def __init__(self, product_service: IProductService):
self.product_service = product_service
def get(self, request):
self.product_service.do_stuff()
...
# urls.py
# Construct dependencies. I guess this is the closest to the entry-point we can get
# with Django.
repo = DjangoProductRepository()
product_service = ProductService(repo)
urlpatterns = [
path('admin/', admin.site.urls),
path("",
IndexView.as_view(product_service=product_service),
name="index"),
]
This is only an updated version of rabbit.aaron reply above. My idea is to be able to specify which dependencies to inject instead of getting a dictionary with all registered dependencies.
from functools import wraps
class ServiceInjector:
deps = {}
def register(self, name=None):
name = name
def decorator(thing):
"""
thing here can be class or function or anything really
"""
if not name:
if not hasattr(thing, '__name__'):
raise Exception('no name')
thing_name = thing.__name__
else:
thing_name = name
self.__class__.deps[thing_name] = thing
return thing
return decorator
class inject:
def __init__(self, *args):
self.selected_deps = args
def __call__(self, func):
#wraps(func)
def decorated(*args, **kwargs):
selected_deps = {k: v for k, v in ServiceInjector.deps.items() if k in self.selected_deps}
new_kwargs = {**kwargs, **selected_deps}
return func(*args, **new_kwargs)
return decorated
Usage:
si = ServiceInjector()
# use func.__name__, registering func
#si.register()
def foo(*args):
return sum(args)
Custom naming still works
#si.register(name='uppercase')
class UpperCaseRepresentation:
def __init__(self, value):
self.value = value
def __str__(self):
return self.value.upper()
Register float
si.register(name="PI")(3.141592653)
Inject into functions
#si.inject('foo', 'PI', 'uppercase')
def bar(a, b, c, uppercase: UpperCaseRepresentation, **kwargs):
"""
You can specify dependencies as keyword arguments and add typehint annotation.
"""
UpperCase, foo = kwargs['UpperCase'], kwargs['foo']
print(uppercase('abc')) # ABC
print(PI) # 3.141592653
print(foo(a, b, c, 4, 5)) # = 15
bar(1, 2, 3)
Inject into class methods
class Bar:
#si.inject('foo')
def my_method(self, a, b, foo, kwarg1=30):
return foo(a, b, kwarg1)
print(Bar().my_method(1, 2, kwarg1=50)) # = 53
You could go the flask route and export a class instance with a property that initializes and caches the service on first access. E.g:
def default_factory():
pass
# service.py
class ServiceProvider:
def __init__(self, create_instance=default_factory):
self.create_instance = create_instance
_instance = None
#property
def service(self):
if self._instance:
return self._instance
self._instance = self.create_instance()
return self._instance
service_provider = ServiceProvider()
from .service import service_provider
# views.py
def view(request):
service_provider.service.do_stuff()
# etc.
This has the advantages of being easy to mock and not having any magic.
The most boring solution I could come up with involves using class variables:
# Module services.post_service
def default_create_post():
return "foo"
class Provider:
create_post = default_create_post
Then you could import and use normally in a view or elsewhere:
from services import post_service
post_service.Provider.create_post()
# Should return "foo"
And when testing it could be swapped out before being called:
from django.test import TestCase
from services import post_service
from unittest.mock import patch
class MyTestCase(TestCase):
#patch('services.post_service.default_create_post')
def test_some_view(self, mock_create_post):
mock_create_post.return_value = "bar"
post_service.Provider.create_post = mock_create_post
# Now when calling post_service.Provider.create_post it should just return "bar"
I'm trying to do some advanced wizardry and I don't completely understand the problem which is no doubt why I'm having trouble solving it.
If I do something super hacky and let the child (DailyPic) know about it's parent (Site) then I can make everything work, but I can't figure out how to dynamically do this just from Site.
I'm trying to dynamically add the fantastic reify (slightly modified) to DailyPic but have the DailyPic instance call a method on an instance of Site.
hacky but it works
class reify():
def __init__(self, wrapped, name=None):
self.wrapped = wrapped
if name is None:
from functools import update_wrapper
update_wrapper(self, wrapped)
else:
self.wrapped.__name__ = name
def __get__(self, inst, objtype=None):
if inst is None:
return self
val = self.wrapped(inst)
setattr(inst, self.wrapped.__name__, val)
return val
class Site:
def __init__(self):
self.counter = 0
def mk_filename(self, pic):
self.counter += 1
return f"{self.__class__.__name__}-{self.counter}.{pic.ext()}"
def save(self):
pic = DailyPic()
from functools import partial
pic._mk_filename = partial(self.mk_filename, pic)
return pic
class DailyPic:
def ext(self):
return 'gif' # simplified for example
#reify
def filename(self):
return self._mk_filename() # YUCK!!!
# works but ugly...
class Site1(Site): pass
class Site2(Site): pass
pic1 = Site1().save()
pic2 = Site2().save()
print(pic1.filename) # Site1-1
print(pic2.filename) # Site2-1
I'm trying to get to something like this:
class Site:
def save(self):
pic = DailyPic()
DailyPic.filename = reify(
functools.partial(lambda pic: self.mk_filename(pic)),
'filename'
)
return pic
# sorta works...
print(pic1.filename) # Site2-1 , should be Site1-1
print(pic2.filename) # Site2-2 , should be Site2-1
If an advanced wizard could chime in that would be fantastic!
unless you really, really, need it to be a property, just ignore reify and properties and set a function on each instance that overwrites itself when run once.
class Site:
def save(self):
pic = DailyPic()
def filename(pic_self):
val = self.mk_filename(pic_self)
pic_self.filename = lambda _: val
return val
pic.filename = filename
return pic
then it's just:
print(pic1.filename())
if you really want it to be a property you could always set pic._filename = filename and have a property on DailyPic that returns self._filename(). which is basically what you were doing to begin with.
Question about objects in python. I have created the following object....
class http(object):
def __init__(self):
self._resource = None
self._response = None
#property
def resource(self):
return self._resource
#resource.setter
def resource(self, value):
self._resource = "http://127.0.0.1:8000/%s" % value
def get(self, resource=None):
self.resource = resource
self._response = requests.get(self.resource)
return self._response
Init does not need anything at this stage so I was hoping I could create the object like this....
content = http.get("users/")
but it won't let me do this, instead I have to pass use the syntax http() but pass nothing...
content = http().get("users/")
which seems silly if I don't pass anything to __init__. I'm wondering how the a Python package like requests achieves the following syntax....
requests.get('https://api.github.com/user')
without doing this...
requests().get('https://api.github.com/user')
why, what does requests package do different?
Requests defines some extra methods that create an instance of Requests.request behind the scenes. You can do the same thing for your http class.
class http(object):
def get(self, resource=None):
self.resource = resource
self._response = requests.get(self.resource)
return self._response
def get(resource=None):
temp_instance = http()
return temp_instance.get(resource)
I have a model where I want to use a class method to set the default of for a property:
class Organisation(db.Model):
name=db.StringProperty()
code=db.StringProperty(default=generate_code())
#classmethod
def generate_code(cls):
import random
codeChars='ABCDEF0123456789'
while True: # Make sure code is unique
code=random.choice(codeChars)+random.choice(codeChars)+\
random.choice(codeChars)+random.choice(codeChars)
if not cls.all().filter('code = ',code).get(keys_only=True):
return code
But I get a NameError:
NameError: name 'generate_code' is not defined
How can I access generate_code()?
As I said in a comment, I would use a classmethod to act as a factory and always create you entity through there. It keeps things simpler and no nasty hooks to get the behaviour you want.
Here is a quick example.
class Organisation(db.Model):
name=db.StringProperty()
code=db.StringProperty()
#classmethod
def generate_code(cls):
import random
codeChars='ABCDEF0123456789'
while True: # Make sure code is unique
code=random.choice(codeChars)+random.choice(codeChars)+\
random.choice(codeChars)+random.choice(codeChars)
if not cls.all().filter('code = ',code).get(keys_only=True):
return code
#classmethod
def make_organisation(cls,*args,**kwargs):
new_org = cls(*args,**kwargs)
new_org.code = cls.generate_code()
return new_org
import random
class Test(object):
def __new__(cls):
cls.my_attr = cls.get_code()
return super(Test, cls).__new__(cls)
#classmethod
def get_code(cls):
return random.randrange(10)
t = Test()
print t.my_attr
You need specify the class name: Organisation.generate_code()