I have different set of config files initialized under a Config class and would need them updated dynamically in the run time.
config.py
# Package import
import reusable.common as common
import reusable.JSON_utils as JSON
class Config:
def __init__(self):
# run_config
run_file = common.file_path('run_config.JSON')
self.run_dict = JSON.read(run_file)
self.env_list = list(self.run_dict.keys())
# connection_config
self.conn_file = common.file_path('connection_config.JSON')
self.conn_dict = JSON.read(self.conn_file)
# snapshot_config
self.snap_file = common.file_path('snapshot_config.JSON')
self.snap_dict = JSON.read(self.snap_file)
For example I have to iterate through different environments like (DEV, STAGE, QA, PROD) and want to update conn_dict('env') to 'QA' from 'DEV' after DEV tasks are completed. Currently I have the update dict/JSON code in main() but I want to have this as a method inside the Config class
# Package import
import reusable.config as config
import reusable.JSON_utils as JSON
config_obj = config.Config()
for env in config_obj.env_list:
config_obj.conn_dict['env'] = env
JSON.write(config_obj.conn_dict, config_obj.conn_file)
src_id_list = config_obj.run_dict.get(env).get('snapshot')
# do stuff in the current env
for src_id in src_id_list:
config_obj.snap_dict['source_id'] = str(src_id)
JSON.write(config_obj.snap_dict, config_obj.snap_file)
# do stuff for the current data source
Q1 Which method is the optimal and conventional way for this. Class or instant or static method? kindly explain as I'm not clear with those completely
Q2 Can we have a unified method inside the class that takes the dict variable and function as parameters to update the dict and JSON file. If yes how it can be achieved?
I've been struggling to achieve this for a while now and it seems that I can't find my way around this. I have the following main entry point for my FastAPI project:
from fastapi import FastAPI
from starlette.middleware.cors import CORSMiddleware
from starlette.responses import RedirectResponse
from app.core.config import get_api_settings
from app.api.api import api_router
def get_app() -> FastAPI:
api_settings = get_api_settings()
server = FastAPI(**api_settings.fastapi_kwargs)
server.add_middleware(
CORSMiddleware,
allow_origins=["*"],
allow_credentials=True,
allow_methods=["*"],
allow_headers=["*"],
)
server.include_router(api_router, prefix="/api")
#server.get("/", include_in_schema=False)
def redirect_to_docs() -> RedirectResponse:
return RedirectResponse(api_settings.docs_url)
return server
app = get_app()
Nothing too fancy so far. As you can see, I'm importing get_api_settings which holds my entire service config and it looks like this:
from functools import lru_cache
from typing import Any, Dict
from pydantic import BaseSettings
class APISettings(BaseSettings):
"""This class enables the configuration of your FastAPI instance
through the use of environment variables.
Any of the instance attributes can be overridden upon instantiation by
either passing the desired value to the initializer, or by setting the
corresponding environment variable.
Attribute `xxx_yyy` corresponds to environment variable `API_XXX_YYY`.
So, for example, to override `api_prefix`, you would set the environment
variable `API_PREFIX`.
Note that assignments to variables are also validated, ensuring that
even if you make runtime-modifications to the config, they should have
the correct types.
"""
# fastapi.applications.FastAPI initializer kwargs
debug: bool = False
docs_url: str = "/docs"
openapi_prefix: str = ""
openapi_url: str = "/openapi.json"
redoc_url: str = "/redoc"
title: str = "Api Backend"
version: str = "0.1.0"
# Custom settings
disable_docs: bool = False
environment: str
#property
def fastapi_kwargs(self) -> Dict[str, Any]:
"""This returns a dictionary of the most commonly used keyword
arguments when initializing a FastAPI instance.
If `self.disable_docs` is True, the various docs-related arguments
are disabled, preventing spec from being published.
"""
fastapi_kwargs: Dict[str, Any] = {
"debug": self.debug,
"docs_url": self.docs_url,
"openapi_prefix": self.openapi_prefix,
"openapi_url": self.openapi_url,
"redoc_url": self.redoc_url,
"title": self.title,
"version": self.version
}
if self.disable_docs:
fastapi_kwargs.update({
"docs_url": None,
"openapi_url": None,
"redoc_url": None
})
return fastapi_kwargs
class Config:
case_sensitive = True
# env_file should be dynamic depending on the
# `environment` env variable
env_file = ""
env_prefix = ""
validate_assignment = True
#lru_cache()
def get_api_settings() -> APISettings:
"""This function returns a cached instance of the APISettings object.
Caching is used to prevent re-reading the environment every time the API
settings are used in an endpoint.
If you want to change an environment variable and reset the cache
(e.g., during testing), this can be done using the `lru_cache` instance
method `get_api_settings.cache_clear()`.
"""
return APISettings()
I'm trying to prepare this service for multiple environments:
dev
stage
prod
For each of the above, I have three different .env files as follow:
core/configs/dev.env
core/configs/stage.env
core/configs/prod.env
As an example, here is how a .env file looks like:
environment=dev
frontend_service_url=http://localhost:3000
What I can't get my head around is how to dynamically set the env_file = "" in my Config class based on the environment attribute in my APISettings BaseSettings class.
Reading through Pydantic's docs I thought I can use the customise_sources classmethod to do something like this:
def load_envpath_settings(settings: BaseSettings):
environment = # not sure how to access it here
for env in ("dev", "stage", "prod"):
if environment == env:
return f"app/configs/{environment}.env"
class APISettings(BaseSettings):
# ...
class Config:
case_sensitive = True
# env_file = "app/configs/dev.env"
env_prefix = ""
validate_assignment = True
#classmethod
def customise_sources(cls, init_settings, env_settings, file_secret_settings):
return (
init_settings,
load_envpath_settings,
env_settings,
file_secret_settings,
)
but I couldn't find a way to access the environment in my load_envpath_settings. Any idea how to solve this? Or if there's another way to do it? I've also tried creating another #property in my APISettings class which which would basically be the same as the load_envpath_settings but I couldn't refer it back in the Config class.
First; usually you'd copy the file you want to have active into the .env file, and then just load that. If you however want to let that .env file control which of the configurations that are active:
You can have two sets of configuration - one that loads the initial configuration (i.e. which environment is the active one) from .env, and one that loads the actual application settings from the core/configs/<environment>.env file.
class AppSettings(BaseSettings):
environment:str = 'development'
This would be affected by the configuration given in .env (which is the default file name). You'd then use this value to load the API configuration by using the _env_file parameter, which is support on all BaseSettings instances.
def get_app_settings() -> AppSettings:
return AppSettings()
def get_api_settings() -> APISettings:
app_settings = get_app_settings()
return APISettings(_env_file=f'core/configs/{app_settings.environment}.env') # or os.path.join() and friends
I would like to import a constant from an external file. I have two files in one directory.
constants.py file:
SOME_CONSTANT = 'something'
And import this into settings.py
import constants
someVariable = constants.SOME_CONSTANT
But pylint write that Module 'constants' has no 'SOME_CONSTANT' member
Can't really tell how you made your constants, but ideally you'd want to store them in your class.
#Constants.Py
class Province:
CITY = 'Toronto'
#Settings.Py
from Constants import Province
someVariable = Province.CITY
>>> 'Toronto'
I have to import my config.py file in my code and then basically pass the REPOS_TO_CHECK values to a function which should be in the form of a list such that when I pass the list to my build_url function which generates a url when the REPOS_TO_CHECK variable is passed to it from the config.py file
I have to generate the url like GET /repos/:owner/:repo
GITHUB_URL = 'https://testurl.com'
how do I pass the REPOS_TO_CHECK parameter from the config.py file as a list so that when I pass
def build_url(['owner','repo']):
the url generated will be https://testurl.com/:owner/:repo
:owner, :repo are in the REPOS_TO_CHECK in the config.py file.
I can access the config.py file by importing config.py in my code and then accessing the values by using the config. for example: config.GITHUB_URL ,gives me 'https://testurl.com'
This is my config file:
GITHUB_URL = 'https://testurl.com'
GITHUB_ACCESS_TOKEN = 'access token'
REPOS_TO_CHECK = [
('owner1', 'repo1'),
('owner2', 'repo2'),]
You can import your config.py as any other file given it's in the same folder: (otherwise navigate to it)
import config
Then you can call the variable like config.REPOS_TO_CHECK
To generate the url you can simply use the variables given in the function call:
If you're using python 3.6 use f strings:
def generate_url(list_here):
return f'http://test.com/{list_here[0]}/{list_here[1]}'
Else use .format()
def generate_url(list_here):
return 'http://test.com/{0}/{1}'.format(list_here[0], list_here[1])
I'm trying to use Python's PyYAML to create a custom tag that will allow me to retrieve environment variables with my YAML.
import os
import yaml
class EnvTag(yaml.YAMLObject):
yaml_tag = u'!Env'
def __init__(self, env_var):
self.env_var = env_var
def __repr__(self):
return os.environ.get(self.env_var)
settings_file = open('conf/defaults.yaml', 'r')
settings = yaml.load(settings_file)
And inside of defaults.yaml is simply:
example: !ENV foo
The error I keep getting:
yaml.constructor.ConstructorError:
could not determine a constructor for the tag '!ENV' in
"defaults.yaml", line 1, column 10
I plan to have more than one custom tag as well (assuming I can get this one working)
Your PyYAML class had a few problems:
yaml_tag is case sensitive, so !Env and !ENV are different tags.
So, as per the documentation, yaml.YAMLObject uses meta-classes to define itself, and has default to_yaml and from_yaml functions for those cases. By default, however, those functions require that your argument to your custom tag (in this case !ENV) be a mapping. So, to work with the default functions, your defaults.yaml file must look like this (just for example) instead:
example: !ENV {env_var: "PWD", test: "test"}
Your code will then work unchanged, in my case print(settings) now results in {'example': /home/Fred} But you're using load instead of safe_load -- in their answer below, Anthon pointed out that this is dangerous because the parsed YAML can overwrite/read data anywhere on the disk.
You can still easily use your YAML file format, example: !ENV foo—you just have to define an appropriate to_yaml and from_yaml in class EnvTag, ones that can parse and emit scalar variables like the string "foo".
So:
import os
import yaml
class EnvTag(yaml.YAMLObject):
yaml_tag = u'!ENV'
def __init__(self, env_var):
self.env_var = env_var
def __repr__(self):
v = os.environ.get(self.env_var) or ''
return 'EnvTag({}, contains={})'.format(self.env_var, v)
#classmethod
def from_yaml(cls, loader, node):
return EnvTag(node.value)
#classmethod
def to_yaml(cls, dumper, data):
return dumper.represent_scalar(cls.yaml_tag, data.env_var)
# Required for safe_load
yaml.SafeLoader.add_constructor('!ENV', EnvTag.from_yaml)
# Required for safe_dump
yaml.SafeDumper.add_multi_representer(EnvTag, EnvTag.to_yaml)
settings_file = open('defaults.yaml', 'r')
settings = yaml.safe_load(settings_file)
print(settings)
s = yaml.safe_dump(settings)
print(s)
When this program is run, it outputs:
{'example': EnvTag(foo, contains=)}
{example: !ENV 'foo'}
This code has the benefit of (1) using the original pyyaml, so nothing extra to install and (2) adding a representer. :)
I'd like to share how I resolved this as an addendum to the great answers above provided by Anthon and Fredrick Brennan. Thank you for your help.
In my opinion, the PyYAML document isn't real clear as to when you might want to add a constructor via a class (or "metaclass magic" as described in the doc), which may involve re-defining from_yaml and to_yaml, or simply adding a constructor using yaml.add_constructor.
In fact, the doc states:
You may define your own application-specific tags. The easiest way to do it is to define a subclass of yaml.YAMLObject
I would argue the opposite is true for simpler use-cases. Here's how I managed to implement my custom tag.
config/__init__.py
import yaml
import os
environment = os.environ.get('PYTHON_ENV', 'development')
def __env_constructor(loader, node):
value = loader.construct_scalar(node)
return os.environ.get(value)
yaml.add_constructor(u'!ENV', __env_constructor)
# Load and Parse Config
__defaults = open('config/defaults.yaml', 'r').read()
__env_config = open('config/%s.yaml' % environment, 'r').read()
__yaml_contents = ''.join([__defaults, __env_config])
__parsed_yaml = yaml.safe_load(__yaml_contents)
settings = __parsed_yaml[environment]
With this, I can now have a seperate yaml for each environment using an env PTYHON_ENV (default.yaml, development.yaml, test.yaml, production.yaml). And each can now reference ENV variables.
Example default.yaml:
defaults: &default
app:
host: '0.0.0.0'
port: 500
Example production.yaml:
production:
<<: *defaults
app:
host: !ENV APP_HOST
port: !ENV APP_PORT
To use:
from config import settings
"""
If PYTHON_ENV == 'production', prints value of APP_PORT
If PYTHON_ENV != 'production', prints default 5000
"""
print(settings['app']['port'])
If your goal is to find and replace environment variables (as strings) defined in your yaml file, you can use the following approach:
example.yaml:
foo: !ENV "Some string with ${VAR1} and ${VAR2}"
example.py:
import yaml
# Define the function that replaces your env vars
def env_var_replacement(loader, node):
replacements = {
'${VAR1}': 'foo',
'${VAR2}': 'bar',
}
s = node.value
for k, v in replacements.items():
s = s.replace(k, v)
return s
# Define a loader class that will contain your custom logic
class EnvLoader(yaml.SafeLoader):
pass
# Add the tag to your loader
EnvLoader.add_constructor('!ENV', env_var_replacement)
# Now, use your custom loader to load the file:
with open('example.yaml') as yaml_file:
loaded_dict = yaml.load(yaml_file, Loader=EnvLoader)
# Prints: "Some string with foo and bar"
print(loaded_dict['foo'])
It's worth noting, you don't necessarily need to create a custom EnvLoader class. You can call add_constructor directly on the SafeLoader class or the yaml module itself. However, this can have an unintended side-effect of adding your loader globally to all other modules that rely on those loaders, which could potentially cuase problems if those other modules have their own custom logic for loading that !ENV tag.
There are several problems with your code:
!Env in your YAML file is not the same as !ENV in your code.
You are missing the classmethod from_yaml that has to be provided for EnvTag.
Your YAML document specifies a scalar for !Env, but the subclassing mechanism for yaml.YAMLObject calls construct_yaml_object which in turn calls construct_mapping so a scalar is not allowed.
You are using .load(). This is unsafe, unless you have complete control over the YAML input, now and in the future. Unsafe in the sense that uncontrolled YAML can e.g. wipe or upload any information from your disc. PyYAML doesn't warn you for that possible loss.
PyYAML only supports most of YAML 1.1, the latest YAML specification is 1.2 (from 2009).
You should consistently indent your code at 4 spaces at every level (or 3 spaces, but not 4 at the first and 3 a the next level).
your __repr__ doesn't return a string if the environment variable is not set, which will throw an error.
So change your code to:
import sys
import os
from ruamel import yaml
yaml_str = """\
example: !Env foo
"""
class EnvTag:
yaml_tag = u'!Env'
def __init__(self, env_var):
self.env_var = env_var
def __repr__(self):
return os.environ.get(self.env_var, '')
#staticmethod
def yaml_constructor(loader, node):
return EnvTag(loader.construct_scalar(node))
yaml.add_constructor(EnvTag.yaml_tag, EnvTag.yaml_constructor,
constructor=yaml.SafeConstructor)
data = yaml.safe_load(yaml_str)
print(data)
os.environ['foo'] = 'Hello world!'
print(data)
which gives:
{'example': }
{'example': Hello world!}
Please note that I am using ruamel.yaml (disclaimer: I am the author of that package), so you can use YAML 1.2 (or 1.1) in your YAML file. With minor changes you can do the above with the old PyYAML as well.
You can do this by subclassing of YAMLObject as well, and in a safe way:
import sys
import os
from ruamel import yaml
yaml_str = """\
example: !Env foo
"""
yaml.YAMLObject.yaml_constructor = yaml.SafeConstructor
class EnvTag(yaml.YAMLObject):
yaml_tag = u'!Env'
def __init__(self, env_var):
self.env_var = env_var
def __repr__(self):
return os.environ.get(self.env_var, '')
#classmethod
def from_yaml(cls, loader, node):
return EnvTag(loader.construct_scalar(node))
data = yaml.safe_load(yaml_str)
print(data)
os.environ['foo'] = 'Hello world!'
print(data)
This will give you the same results as above.