I am trying to add a CDK Aspect to this AWS workshop
https://github.com/aws-samples/aws-cdk-intro-workshop/tree/master/code/python/pipelines-workshop
My Aspect would prefix "my-custom-prefix-" to all role and policy names (org requirement don't ask why). And also adds a permission boundary to all roles.
#jsii.implements(IAspect)
class PermissionBoundaryAspect:
def __init__(self, permission_boundary: Union[_iam.ManagedPolicy, str]) -> None:
self.permission_boundary = permission_boundary
def visit(self, construct_ref: IConstruct) -> None:
if isinstance(scope, iam.Role):
iam_role = scope.node.find_child('Resource')
iam_role.add_property_override(property_path='PermissionsBoundary', value=self.permission_boundary)
if iam_role.role_name.startswith("my-custom-prefix-"):
pass
else:
newrolename = f"{iam_prefix}{iam_role.role_name}"
iam_role.add_property_override(property_path='RoleName', value=newrolename)
and I use it in my app.py
app = cdk.App()
WorkshopPipelineStack(app, "WorkshopPipelineStack")
cftdeveloperboundary = f"arn:aws:iam::{t_account_id}:policy/my-boundary-policy"
cdk.Aspects.of(app).add(PermissionBoundaryAspect(cftdeveloperboundary))
app.synth()
Next I do cdk synth and in the CloudFormation template see the boundary is attached to roles and policy names are prefixed but not role names. The roles names are all RoleName: my-custom-prefix- whereas policy names are properly prefixed e.g. PolicyName: my-custom-prefix-PipelineRoleDefaultPolicy7BDC1ABB
Related
New to AWS CDK and I'm trying to create a load balanced fargate service with the construct ApplicationLoadBalancedFargateService.
I have an existing image on ECR that I would like to reference and use. I've found the ecs.ContainerImage.from_ecr_repository function, which I believe is what I should use in this case. However, this function takes an IRepository as a parameter and I cannot find anything under aws_ecr.IRepository or aws_ecr.Repository to reference a pre-existing image. These constructs all seem to be for making a new repository.
Anyone know what I should be using to get the IRepository object for an existing repo? Is this just not typically done this way?
Code is below. Thanks in Advance.
from aws_cdk import (
# Duration,
Stack,
# aws_sqs as sqs,
)
from constructs import Construct
from aws_cdk import (aws_ec2 as ec2, aws_ecs as ecs,
aws_ecs_patterns as ecs_patterns,
aws_route53,aws_certificatemanager,
aws_ecr)
class NewStack(Stack):
def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None:
super().__init__(scope, construct_id, **kwargs)
_repo = aws_ecr.Repository(self, 'id1', repository_uri = repo_uri)
vpc = ec2.Vpc(self, "applications", max_azs=3) # default is all AZs in region
cluster = ecs.Cluster(self, "id2", vpc=vpc)
hosted_zone = aws_route53.HostedZone.from_lookup(self,
'id3',
domain_name = 'domain'
)
certificate = aws_certificatemanager.Certificate.from_certificate_arn(self,
id4,
'cert_arn'
)
image = ecs.ContainerImage.from_ecr_repository(self, _repo)
ecs_patterns.ApplicationLoadBalancedFargateService(self, "id5",
cluster=cluster, # Required
cpu=512, # Default is 256
desired_count=2, # Default is 1
task_image_options=ecs_patterns.ApplicationLoadBalancedTaskImageOptions(
image = image,
container_port=8000),
memory_limit_mib=2048, # Default is 512
public_load_balancer=True,
domain_name = 'domain_name',
domain_zone = hosted_zone,
certificate = certificate,
redirect_http = True)
You are looking for from_repository_attributes() to create an instance of IRepository from an existing ECR repository.
I've been struggling to achieve this for a while now and it seems that I can't find my way around this. I have the following main entry point for my FastAPI project:
from fastapi import FastAPI
from starlette.middleware.cors import CORSMiddleware
from starlette.responses import RedirectResponse
from app.core.config import get_api_settings
from app.api.api import api_router
def get_app() -> FastAPI:
api_settings = get_api_settings()
server = FastAPI(**api_settings.fastapi_kwargs)
server.add_middleware(
CORSMiddleware,
allow_origins=["*"],
allow_credentials=True,
allow_methods=["*"],
allow_headers=["*"],
)
server.include_router(api_router, prefix="/api")
#server.get("/", include_in_schema=False)
def redirect_to_docs() -> RedirectResponse:
return RedirectResponse(api_settings.docs_url)
return server
app = get_app()
Nothing too fancy so far. As you can see, I'm importing get_api_settings which holds my entire service config and it looks like this:
from functools import lru_cache
from typing import Any, Dict
from pydantic import BaseSettings
class APISettings(BaseSettings):
"""This class enables the configuration of your FastAPI instance
through the use of environment variables.
Any of the instance attributes can be overridden upon instantiation by
either passing the desired value to the initializer, or by setting the
corresponding environment variable.
Attribute `xxx_yyy` corresponds to environment variable `API_XXX_YYY`.
So, for example, to override `api_prefix`, you would set the environment
variable `API_PREFIX`.
Note that assignments to variables are also validated, ensuring that
even if you make runtime-modifications to the config, they should have
the correct types.
"""
# fastapi.applications.FastAPI initializer kwargs
debug: bool = False
docs_url: str = "/docs"
openapi_prefix: str = ""
openapi_url: str = "/openapi.json"
redoc_url: str = "/redoc"
title: str = "Api Backend"
version: str = "0.1.0"
# Custom settings
disable_docs: bool = False
environment: str
#property
def fastapi_kwargs(self) -> Dict[str, Any]:
"""This returns a dictionary of the most commonly used keyword
arguments when initializing a FastAPI instance.
If `self.disable_docs` is True, the various docs-related arguments
are disabled, preventing spec from being published.
"""
fastapi_kwargs: Dict[str, Any] = {
"debug": self.debug,
"docs_url": self.docs_url,
"openapi_prefix": self.openapi_prefix,
"openapi_url": self.openapi_url,
"redoc_url": self.redoc_url,
"title": self.title,
"version": self.version
}
if self.disable_docs:
fastapi_kwargs.update({
"docs_url": None,
"openapi_url": None,
"redoc_url": None
})
return fastapi_kwargs
class Config:
case_sensitive = True
# env_file should be dynamic depending on the
# `environment` env variable
env_file = ""
env_prefix = ""
validate_assignment = True
#lru_cache()
def get_api_settings() -> APISettings:
"""This function returns a cached instance of the APISettings object.
Caching is used to prevent re-reading the environment every time the API
settings are used in an endpoint.
If you want to change an environment variable and reset the cache
(e.g., during testing), this can be done using the `lru_cache` instance
method `get_api_settings.cache_clear()`.
"""
return APISettings()
I'm trying to prepare this service for multiple environments:
dev
stage
prod
For each of the above, I have three different .env files as follow:
core/configs/dev.env
core/configs/stage.env
core/configs/prod.env
As an example, here is how a .env file looks like:
environment=dev
frontend_service_url=http://localhost:3000
What I can't get my head around is how to dynamically set the env_file = "" in my Config class based on the environment attribute in my APISettings BaseSettings class.
Reading through Pydantic's docs I thought I can use the customise_sources classmethod to do something like this:
def load_envpath_settings(settings: BaseSettings):
environment = # not sure how to access it here
for env in ("dev", "stage", "prod"):
if environment == env:
return f"app/configs/{environment}.env"
class APISettings(BaseSettings):
# ...
class Config:
case_sensitive = True
# env_file = "app/configs/dev.env"
env_prefix = ""
validate_assignment = True
#classmethod
def customise_sources(cls, init_settings, env_settings, file_secret_settings):
return (
init_settings,
load_envpath_settings,
env_settings,
file_secret_settings,
)
but I couldn't find a way to access the environment in my load_envpath_settings. Any idea how to solve this? Or if there's another way to do it? I've also tried creating another #property in my APISettings class which which would basically be the same as the load_envpath_settings but I couldn't refer it back in the Config class.
First; usually you'd copy the file you want to have active into the .env file, and then just load that. If you however want to let that .env file control which of the configurations that are active:
You can have two sets of configuration - one that loads the initial configuration (i.e. which environment is the active one) from .env, and one that loads the actual application settings from the core/configs/<environment>.env file.
class AppSettings(BaseSettings):
environment:str = 'development'
This would be affected by the configuration given in .env (which is the default file name). You'd then use this value to load the API configuration by using the _env_file parameter, which is support on all BaseSettings instances.
def get_app_settings() -> AppSettings:
return AppSettings()
def get_api_settings() -> APISettings:
app_settings = get_app_settings()
return APISettings(_env_file=f'core/configs/{app_settings.environment}.env') # or os.path.join() and friends
I am new to cdk and trying to create an instance profile with CDK+Python with the following code. I have already created the Role (gitLabRunner-glue) successfully thru CDK and wanting to use it with the intance profile. However, when i run the following code, i get an error gitLabRunner-glue already exists
Can somebody please explain what am i missing ?
from aws_cdk import core as cdk
from aws_cdk import aws_glue as glue
from aws_cdk import aws_ec2 as _ec2
from aws_cdk import aws_iam as _iam
class Ec2InstanceProfile(cdk.Stack):
def __init__(self, scope: cdk.Construct, construct_id: str, **kwargs) -> None:
super().__init__(scope, construct_id, **kwargs)
# role = _iam.Role(self, "instanceprofilerole", role_name="gitLabRunner-glue",
# assumed_by=_iam.ServicePrincipal(service='ec2.amazonaws.com'))
ec2gitLabRunnerinstanceprofile = _iam.CfnInstanceProfile(
self,
"ec2gitLabRunnerinstanceprofile",
instance_profile_name="ec2-gitLabRunner-glue",
roles=["gitLabRunner-glue"] # also tried with this[role.role_name]
)
Does your AWS account already have a role with that name in it?
the Cfn Functions in cdk represent constructs and services that have not been fully hooked into all that is CDK. As such, they often don't do things that others would - where as a CloudFormation Template for the instance profile may just hook into the existing role, the coding in the back of this cfn function may go ahead and create a role item in the template output.
if you do a cdk synth, look in your cdk.out directory, find your cloudformation template, then do a search for gitLabRunner-glue - you may find there is a AWS::IAM::ROLE being created, indicating when CloudFormation attempts to run based of the template created by cdk it tries to create a new resource and it cant.
You have a couple options to try:
As you tried, uncomment the role again and use role.role_name but name the role something else or, as CDK recommends, don't include a name and let it name it for you
search your aws account for the role and delete it
If you absolutely cannot delete the existing role or cannot create a new one with a new name, then import the role, using (based off your imports)
role = _iam.Role.from_role_arn(self, "ImportedGlueRole", role_arn="arn:aws:of:the:role", add_grants_to_resources=True)
be wary a bit of the add_grants_to_resources - if its not your role to mess with cdk can make changes if you make that true and that could cause issues elsewhere - but if its not true, then you have to update the Role itself in the aws console (or cli) to accept your resources as able to assume it.
I made it work like this, not the desired model though, but given the limitations of cdk, i couldn't find any other way.
from aws_cdk import core as cdk
from aws_cdk import aws_glue as glue
from aws_cdk import aws_ec2 as _ec2
from aws_cdk import aws_iam as _iam
class Ec2InstanceProfile(cdk.Stack):
def __init__(self, scope: cdk.Construct, construct_id: str, **kwargs) -> None:
super().__init__(scope, construct_id, **kwargs)
boundary = _iam.ManagedPolicy.from_managed_policy_arn(self, "Boundary",
"arn:aws:iam::${ACCOUNT_ID}:policy/app-perm-boundary")
# role = _iam.Role(self, "instanceprofilerole", role_name="gitLabRunner-glue",
# assumed_by=_iam.ServicePrincipal(service='ec2.amazonaws.com'))
ec2_gitlabrunner_glue = _iam.Role(
self, 'ec2-gitlabrunner-glue',
role_name='gitLabRunner-glue',
description='glue service role to be attached to the runner',
# inline_policies=[write_to_s3_policy],
assumed_by=_iam.ServicePrincipal('ec2.amazonaws.com'),
permissions_boundary=boundary
)
ec2gitLabRunnerinstanceprofile = _iam.CfnInstanceProfile(
self,
"gitLabRunnerinstanceprofile",
instance_profile_name="gitLabRunner-glue",
roles=["gitLabRunner-glue"]
)
I've seen two different methods of using depends in Fastapi authentication:
Method 1:
#app.get('/api/user/me')
async def user_me(user: dict = Depends(auth)):
return user
and method 2:
#app.get('/api/user/me', dependencies=[Depends(auth)])
async def user_me(user: dict):
return user
What is the difference between method 1 and method 2 and which is better for securing an API i.e. requiring authentication?
As #Omer Alkin correctly noted, a dependency needs to be specified in the path operation parameter list when we want to use its return value (user or token or smth.). Here's an example from the documentation:
async def get_current_user(token: str = Depends(oauth2_scheme)):
user = fake_decode_token(token)
return user
#app.get("/users/me")
async def read_users_me(current_user: User = Depends(get_current_user)):
return current_user
If the return value of dependency is not important to us or it is not returned, but only a side effect is important, for example, the dependency throws an exception, then we can specify the dependency in the path operation decorator.
In this case, we can also execute the dependency (do authentication) immediately for a group of operations, using APIRouter:
async def get_token_header(x_token: str = Header(...)):
if x_token != "fake-super-secret-token":
raise HTTPException(status_code=400, detail="X-Token header invalid")
router = APIRouter(
prefix="/items",
tags=["items"],
dependencies=[Depends(get_token_header)],
responses={404: {"description": "Not found"}},
)
It should also be noted that you can reuse the same dependency in the path operation or its sub dependencies, as FastAPI implements the cache policy by default:
If one of your dependencies is declared multiple times for the same path operation, for example, multiple dependencies have a common sub-dependency, FastAPI will know to call that sub-dependency only once per request.
In some cases you don't really need the return value of a dependency inside your path operation function. Or the dependency doesn't return a value. But you still need it to be executed/solved. For those cases, instead of declaring a path operation function parameter with Depends, you can add a list of dependencies to the path operation decorator.
More detail and tips can be found in here: https://fastapi.tiangolo.com/tutorial/dependencies/dependencies-in-path-operation-decorators/
I get this error message:
{logging_mixin.py:112} INFO - [2020-03-22 12:34:53,672] {local_task_job.py:103} INFO - Task exited with return code -6
when I use the list_keys or read_key methods of S3 hook. The get_credentials method works fine though. Have searched around and can't find why this occurs.
I'm using apache-airflow==1.10.9, boto3==1.12.21, botocore==1.15.21
Here's my code for my custom operator that makes use of the S3Hook:
class SASValueToRedshiftOperator(BaseOperator):
"""Custom Operator for extracting data from SAS source code.
Attributes:
ui_color (str): color code for task in Airflow UI.
"""
ui_color = '#358150'
#apply_defaults
def __init__(self,
aws_credentials_id="",
redshift_conn_id="",
table="",
s3_bucket="",
s3_key="",
sas_value="",
columns="",
*args, **kwargs):
"""Extracts label mappings from SAS source code and store as Redshift table
Args:
aws_credentials_id (str): Airflow connection ID for AWS key and secret.
redshift_conn_id (str): Airflow connection ID for redshift database.
table (str): Name of table to load data to.
s3_bucket (str): S3 Bucket Name Where SAS source code is store.
s3_key (str): S3 Key Name for SAS source code.
sas_value (str): value to search for in sas file for extraction of data.
columns (list): resulting data column names.
Returns:
None
"""
super(SASValueToRedshiftOperator, self).__init__(*args, **kwargs)
self.aws_credentials_id = aws_credentials_id
self.redshift_conn_id = redshift_conn_id
self.table = table
self.s3_bucket = s3_bucket
self.s3_key = s3_key
self.sas_value = sas_value
self.columns = columns
def execute(self, context):
"""Executes task for staging to redshift.
Args:
context (:obj:`dict`): Dict with values to apply on content.
Returns:
None
"""
s3 = S3Hook(self.aws_credentials_id)
redshift_conn = BaseHook.get_connection(self.redshift_conn_id)
self.log.info(s3)
self.log.info(s3.get_credentials())
self.log.info(s3.list_keys(self.s3_bucket))
s3 = S3Hook(self.aws_credentials_id)
s3.list_keys(bucket_name=s3_bucket, prefix= s3_path, delimiter=delimiter)