I'm trying to use the AssumeRole in such a way that i'm traversing multiple accounts and retrieving assets for those accounts. I've made it to this point:
import boto3
stsclient = boto3.client('sts')
assumedRoleObject = sts_client.assume_role(
RoleArn="arn:aws:iam::account-of-role-to-assume:role/name-of-role",
RoleSessionName="AssumeRoleSession1")
Great, i have the assumedRoleObject. But now i want to use that to list things like ELBs or something that isn't a built-in low level resource.
How does one go about doing that? If i may ask - please code out a full example, so that everyone can benefit.
Here's a code snippet from the official AWS documentation where an s3 resource is created for listing all s3 buckets. boto3 resources or clients for other services can be built in a similar fashion.
# create an STS client object that represents a live connection to the
# STS service
sts_client = boto3.client('sts')
# Call the assume_role method of the STSConnection object and pass the role
# ARN and a role session name.
assumed_role_object=sts_client.assume_role(
RoleArn="arn:aws:iam::account-of-role-to-assume:role/name-of-role",
RoleSessionName="AssumeRoleSession1"
)
# From the response that contains the assumed role, get the temporary
# credentials that can be used to make subsequent API calls
credentials=assumed_role_object['Credentials']
# Use the temporary credentials that AssumeRole returns to make a
# connection to Amazon S3
s3_resource=boto3.resource(
's3',
aws_access_key_id=credentials['AccessKeyId'],
aws_secret_access_key=credentials['SecretAccessKey'],
aws_session_token=credentials['SessionToken'],
)
# Use the Amazon S3 resource object that is now configured with the
# credentials to access your S3 buckets.
for bucket in s3_resource.buckets.all():
print(bucket.name)
To get a session with an assumed role:
import botocore
import boto3
import datetime
from dateutil.tz import tzlocal
assume_role_cache: dict = {}
def assumed_role_session(role_arn: str, base_session: botocore.session.Session = None):
base_session = base_session or boto3.session.Session()._session
fetcher = botocore.credentials.AssumeRoleCredentialFetcher(
client_creator = base_session.create_client,
source_credentials = base_session.get_credentials(),
role_arn = role_arn,
extra_args = {
# 'RoleSessionName': None # set this if you want something non-default
}
)
creds = botocore.credentials.DeferredRefreshableCredentials(
method = 'assume-role',
refresh_using = fetcher.fetch_credentials,
time_fetcher = lambda: datetime.datetime.now(tzlocal())
)
botocore_session = botocore.session.Session()
botocore_session._credentials = creds
return boto3.Session(botocore_session = botocore_session)
# usage:
session = assumed_role_session('arn:aws:iam::ACCOUNTID:role/ROLE_NAME')
ec2 = session.client('ec2') # ... etc.
The resulting session's credentials will be automatically refreshed when required which is quite nice.
Note: my previous answer was outright wrong but I can't delete it, so I've replaced it with a better and working answer.
You can assume role using STS token, like:
class Boto3STSService(object):
def __init__(self, arn):
sess = Session(aws_access_key_id=ARN_ACCESS_KEY,
aws_secret_access_key=ARN_SECRET_KEY)
sts_connection = sess.client('sts')
assume_role_object = sts_connection.assume_role(
RoleArn=arn, RoleSessionName=ARN_ROLE_SESSION_NAME,
DurationSeconds=3600)
self.credentials = assume_role_object['Credentials']
This will give you temporary access key and secret keys, with session token. With these temporary credentials, you can access any service. For Eg, if you want to access ELB, you can use the below code:
self.tmp_credentials = Boto3STSService(arn).credentials
def get_boto3_session(self):
tmp_access_key = self.tmp_credentials['AccessKeyId']
tmp_secret_key = self.tmp_credentials['SecretAccessKey']
security_token = self.tmp_credentials['SessionToken']
boto3_session = Session(
aws_access_key_id=tmp_access_key,
aws_secret_access_key=tmp_secret_key, aws_session_token=security_token
)
return boto3_session
def get_elb_boto3_connection(self, region):
sess = self.get_boto3_session()
elb_conn = sess.client(service_name='elb', region_name=region)
return elb_conn
with reference to the solution by #jarrad which is not working as of Feb 2021, and as a solution that does not use STS explicitly please see the following
import boto3
import botocore.session
from botocore.credentials import AssumeRoleCredentialFetcher, DeferredRefreshableCredentials
def get_boto3_session(assume_role_arn=None):
session = boto3.Session(aws_access_key_id="abc", aws_secret_access_key="def")
if not assume_role_arn:
return session
fetcher = AssumeRoleCredentialFetcher(
client_creator=_get_client_creator(session),
source_credentials=session.get_credentials(),
role_arn=assume_role_arn,
)
botocore_session = botocore.session.Session()
botocore_session._credentials = DeferredRefreshableCredentials(
method='assume-role',
refresh_using=fetcher.fetch_credentials
)
return boto3.Session(botocore_session=botocore_session)
def _get_client_creator(session):
def client_creator(service_name, **kwargs):
return session.client(service_name, **kwargs)
return client_creator
the function can be called as follows
ec2_client = get_boto3_session(role_arn='my_role_arn').client('ec2', region_name='us-east-1')
If you want a functional implementation, this is what I settled on:
def filter_none_values(kwargs: dict) -> dict:
"""Returns a new dictionary excluding items where value was None"""
return {k: v for k, v in kwargs.items() if v is not None}
def assume_session(
role_session_name: str,
role_arn: str,
duration_seconds: Optional[int] = None,
region_name: Optional[str] = None,
) -> boto3.Session:
"""
Returns a session with the given name and role.
If not specified, duration will be set by AWS, probably at 1 hour.
If not specified, region will be left unset.
Region can be overridden by each client or resource spawned from this session.
"""
assume_role_kwargs = filter_none_values(
{
"RoleSessionName": role_session_name,
"RoleArn": role_arn,
"DurationSeconds": duration_seconds,
}
)
credentials = boto3.client("sts").assume_role(**assume_role_kwargs)["Credentials"]
create_session_kwargs = filter_none_values(
{
"aws_access_key_id": credentials["AccessKeyId"],
"aws_secret_access_key": credentials["SecretAccessKey"],
"aws_session_token": credentials["SessionToken"],
"region_name": region_name,
}
)
return boto3.Session(**create_session_kwargs)
def main() -> None:
session = assume_session(
"MyCustomSessionName",
"arn:aws:iam::XXXXXXXXXXXX:role/TheRoleIWantToAssume",
region_name="us-east-1",
)
client = session.client(service_name="ec2")
print(client.describe_key_pairs())
import json
import boto3
roleARN = 'arn:aws:iam::account-of-role-to-assume:role/name-of-role'
client = boto3.client('sts')
response = client.assume_role(RoleArn=roleARN,
RoleSessionName='RoleSessionName',
DurationSeconds=900)
dynamodb_client = boto3.client('dynamodb', region_name='us-east-1',
aws_access_key_id=response['Credentials']['AccessKeyId'],
aws_secret_access_key=response['Credentials']['SecretAccessKey'],
aws_session_token = response['Credentials']['SessionToken'])
response = dynamodb_client.get_item(
Key={
'key1': {
'S': '1',
},
'key2': {
'S': '2',
},
},
TableName='TestTable')
print(response)
#!/usr/bin/env python3
import boto3
sts_client = boto3.client('sts')
assumed_role = sts_client.assume_role(RoleArn = "arn:aws:iam::123456789012:role/example_role",
RoleSessionName = "AssumeRoleSession1",
DurationSeconds = 1800)
session = boto3.Session(
aws_access_key_id = assumed_role['Credentials']['AccessKeyId'],
aws_secret_access_key = assumed_role['Credentials']['SecretAccessKey'],
aws_session_token = assumed_role['Credentials']['SessionToken'],
region_name = 'us-west-1'
)
# now we make use of the role to retrieve a parameter from SSM
client = session.client('ssm')
response = client.get_parameter(
Name = '/this/is/a/path/parameter',
WithDecryption = True
)
print(response)
Assuming that 1) the ~/.aws/config or ~/.aws/credentials file is populated with each of the roles that you wish to assume and that 2) the default role has AssumeRole defined in its IAM policy for each of those roles, then you can simply (in pseudo-code) do the following and not have to fuss with STS:
import boto3
# get all of the roles from the AWS config/credentials file using a config file parser
profiles = get_profiles()
for profile in profiles:
# this is only used to fetch the available regions
initial_session = boto3.Session(profile_name=profile)
# get the regions
regions = boto3.Session.get_available_regions('ec2')
# cycle through the regions, setting up session, resource and client objects
for region in regions:
boto3_session = boto3.Session(profile_name=profile, region_name=region)
boto3_resource = boto3_session.resource(service_name='s3', region_name=region)
boto3_client = boto3_session.client(service_name='s3', region_name=region)
[ do something interesting with your session/resource/client here ]
Credential Setup (boto3 - Shared Credentials File)
Assume Role Setup (AWS)
After a few days of searching, this is the simplest solution I have found. explained here but does not have a usage example.
import boto3
for profile in boto3.Session().available_profiles:
boto3.DEFAULT_SESSION = boto3.session.Session(profile_name=profile)
s3 = boto3.resource('s3')
for bucket in s3.buckets.all():
print(bucket)
This will switch the default role you will be using. To not make the profile the default, just do not assign it to boto3.DEFAULT_SESSION. but instead, do the following.
testing_profile = boto3.session.Session(profile_name='mainTesting')
s3 = testing_profile.resource('s3')
for bucket in s3.buckets.all():
print(bucket)
Important to note that the .aws credentials need to be set in a specific way.
[default]
aws_access_key_id = default_access_id
aws_secret_access_key = default_access_key
[main]
aws_access_key_id = main_profile_access_id
aws_secret_access_key = main_profile_access_key
[mainTesting]
source_profile = main
role_arn = Testing role arn
mfa_serial = mfa_arn_for_main_role
[mainProduction]
source_profile = main
role_arn = Production role arn
mfa_serial = mfa_arn_for_main_role
I don't know why but the mfa_serial key has to be on the roles for this to work instead of the source account which would make more sense.
Here's the code snippet I used
sts_client = boto3.client('sts')
assumed_role_object = sts_client.assume_role(
RoleArn=<arn of the role to assume>,
RoleSessionName="<role session name>"
)
print(assumed_role_object)
credentials = assumed_role_object['Credentials']
session = Session(
aws_access_key_id=credentials['AccessKeyId'],
aws_secret_access_key=credentials['SecretAccessKey'],
aws_session_token=credentials['SessionToken']
)
self.s3 = session.client('s3')
We are trying to read some partners data from their S3 using spark. We have the following code set up for that puprose:
S3_BUCKET = 'BUCKET_NAME'
ROLE_SESSION_NAME = 'SESSION_NAME'
BASE_ROLE_ARN = 'BASE_ROLE_ARN/'
ROLE_ARN = BASE_ROLE_ARN + ROLE_NAME
DURATION_SECONDS = 3600
client = boto3.client('sts')
role = client.assume_role(
RoleArn=ROLE_ARN,
RoleSessionName=ROLE_SESSION_NAME,
DurationSeconds=DURATION_SECONDS,
ExternalId=EXTERNAL_ID
)
s3_session = boto3.session.Session(
aws_access_key_id=role['Credentials']['AccessKeyId'],
aws_secret_access_key=role['Credentials']['SecretAccessKey'],
aws_session_token=role['Credentials']['SessionToken']
)
s3_credentials = s3_session.get_credentials().get_frozen_credentials()
s3_key = s3_credentials.access_key
s3_secret = s3_credentials.secret_key
s3_session_token = s3_credentials.token
We then use the following code to read the data:
input_path = 's3a://some_input_path/'
input_data = spark_sql_context.read.csv(input_path, header = True)
Also, we make sure that everything is set correctly as for spark config:
spark_context._jsc.hadoopConfiguration().set(
"fs.s3a.access.key", s3_key
)
spark_context._jsc.hadoopConfiguration().set(
"fs.s3a.secret.key", s3_secret
)
spark_context._jsc.hadoopConfiguration().set(
"fs.s3a.aws.credentials.provider", "org.apache.hadoop.fs.s3a.TemporaryAWSCredentialsProvider"
)
spark_context._jsc.hadoopConfiguration().set(
"fs.s3a.session.token", s3_session_token
)
But when trying to read the data, we see the following exception:
: java.nio.file.AccessDeniedException: s3a://the_input_path: getFileStatus on s3a://the_input_path: com.amazonaws.services.s3.model.AmazonS3Exception: Forbidden (Service: Amazon S3; Status Code: 403; Error Code: 403 Forbidden; Request ID: F08F9B987FF8DED9; S3 Extended Request ID: TRjfFjALAk7phRDxKdUlucY4yocQY2mNO4r7N6Qf9fSDzSa+TpZfimwbAzXdU+s11BBLBblfgik=), S3 Extended Request ID: TRjfFjALAk7phRDxKdUlucY4yocQY2mNO4r7N6Qf9fSDzSa+TpZfimwbAzXdU+s11BBLBblfgik=:403 Forbidden
at org.apache.hadoop.fs.s3a.S3AUtils.translateException(S3AUtils.java:218)
at org.apache.hadoop.fs.s3a.S3AUtils.translateException(S3AUtils.java:145)
at org.apache.hadoop.fs.s3a.S3AFileSystem.s3GetFileStatus(S3AFileSystem.java:2184)
at org.apache.hadoop.fs.s3a.S3AFileSystem.innerGetFileStatus(S3AFileSystem.java:2149)
at org.apache.hadoop.fs.s3a.S3AFileSystem.getFileStatus(S3AFileSystem.java:2088)
at org.apache.hadoop.fs.FileSystem.exists(FileSystem.java:1683)
at org.apache.hadoop.fs.s3a.S3AFileSystem.exists(S3AFileSystem.java:2976)
This gets more weird as the following code snippet works just fine:
import os
import boto3
import sys
ROLE_NAME = 'ROLE_NAME'
EXTERNAL_ID = 'EXTERNAL_ID'
S3_BUCKET = 'BUCKET_NAME'
# ------------------------------------------------ DO NOT ALTER BELOW ------------------------------------------------ #
ROLE_SESSION_NAME = 'SESSION_NAME'
BASE_ROLE_ARN = 'BASE_ROLE_ARN'
ROLE_ARN = BASE_ROLE_ARN + ROLE_NAME
DURATION_SECONDS = 3600
client = boto3.client('sts')
role = client.assume_role(
RoleArn=ROLE_ARN,
RoleSessionName=ROLE_SESSION_NAME,
DurationSeconds=DURATION_SECONDS,
ExternalId=EXTERNAL_ID
)
session = boto3.session.Session(
aws_access_key_id=role['Credentials']['AccessKeyId'],
aws_secret_access_key=role['Credentials']['SecretAccessKey'],
aws_session_token=role['Credentials']['SessionToken']
)
S3 = session.resource('s3')
my_bucket = S3.Bucket(S3_BUCKET)
for object_summary in my_bucket.objects.filter(Prefix='SOME_PREFIX'):
print (object_summary.key)
Any idea why we might be seeing that exception while trying to read files from the S3 path using spark? Are we missing something?
Thanks in advance, I have variable at the top of my code, LOCATION, VNET_NAME, SUBNET, SUBNETRANGE. I want to fill this information from the output of function List_VNET. Using this function I'm getting virtual network from resource group on azure (I've only single virtual network per resource group). And then wanted to populate it into the variable but it is giving output as paging container. I mostly work on powershell hence i know about arrays and we can get an instance using array[0].
from azure.common.credentials import ServicePrincipalCredentials
from azure.mgmt.resource import ResourceManagementClient
from azure.mgmt.compute import ComputeManagementClient
from azure.mgmt.network import NetworkManagementClient
from azure.mgmt.compute.models import DiskCreateOption
from azure.mgmt.network.v2017_03_01.models import NetworkSecurityGroup
from azure.mgmt.network.v2017_03_01.models import SecurityRule
import azure.mgmt.network.models
SUBSCRIPTION_ID = 'xxx'
GROUP_NAME = 'AQRG'
LOCATION = ''
VM_NAME = 'myVM'
VNET_NAME = ''
SUBNET_NAME = ''
SUBNETRANGE = ''
def List_VNET(network_client):
result_create = network_client.virtual_networks.list(
GROUP_NAME,
)
SUBNET_NAME = result_create
return SUBNET_NAME
def get_credentials():
credentials = ServicePrincipalCredentials(
client_id = 'xxxx',
secret = 'xxxx',
tenant = 'xxxx'
)
return credentials
if __name__ == "__main__":
credentials = get_credentials()
resource_group_client = ResourceManagementClient(
credentials,
SUBSCRIPTION_ID
)
network_client = NetworkManagementClient(
credentials,
SUBSCRIPTION_ID
)
creation_result = List_VNET(network_client)
print("------------------------------------------------------")
print(creation_result)
input('Press enter to continue...')
Getting output as below
<azure.mgmt.network.v2018_12_01.models.virtual_network_paged.VirtualNetworkPaged object at 0x0000023776C13908>
Update: Define the VNET_NAME as global in the function List_VNET:
SUBSCRIPTION_ID = 'xxx'
GROUP_NAME = 'AQRG'
LOCATION = ''
VM_NAME = 'myVM'
VNET_NAME = ''
SUBNET_NAME = ''
SUBNETRANGE = ''
def List_VNET(network_client):
result_create = network_client.virtual_networks.list(
GROUP_NAME
)
global VNET_NAME
for re in result_create:
VNET_NAME=re.name
return VNET_NAME
After the code: creation_result = List_VNET(network_client)
add the following code:
for re in creation_result:
print(re.name)
Then you can get all the virtual networks' name.
Thanks in advance, I wanted to get the region property of a vnet but using list function it only gives name property. Do we have to use another function to get the full details? currently i cannot do re.region. it only works with re.name
from azure.common.credentials import ServicePrincipalCredentials
from azure.mgmt.resource import ResourceManagementClient
from azure.mgmt.compute import ComputeManagementClient
from azure.mgmt.network import NetworkManagementClient
from azure.mgmt.compute.models import DiskCreateOption
from azure.mgmt.network.v2017_03_01.models import NetworkSecurityGroup
from azure.mgmt.network.v2017_03_01.models import SecurityRule
import azure.mgmt.network.models
SUBSCRIPTION_ID = 'xxxx'
GROUP_NAME = 'AQRG'
LOCATION = ''
VM_NAME = 'myVM'
VNET_NAME = ''
SUBNET = ''
def List_VNET(network_client):
result_create = network_client.virtual_networks.list(
GROUP_NAME,
)
global VNET_NAME
for re in result_create:
VNET_NAME = re.name
Region = re.region // This is not valid
return VNET_NAME
def get_credentials():
credentials = ServicePrincipalCredentials(
client_id = 'xxx',
secret = 'xxx',
tenant = 'xxxx'
)
return credentials
if __name__ == "__main__":
credentials = get_credentials()
resource_group_client = ResourceManagementClient(
credentials,
SUBSCRIPTION_ID
)
network_client = NetworkManagementClient(
credentials,
SUBSCRIPTION_ID
)
compute_client = ComputeManagementClient(
credentials,
SUBSCRIPTION_ID
)
creation_result_listvnet = List_VNET(network_client)
print("------------------------------------------------------")
print(creation_result_listvnet)
input('Press enter to continue...')
it should be re.location instead of re.region.
and I just found that you can fetch all the properties of virtual network with print(re). Then you can use any properties in the output.
FYI: The doc of VirtualNetwork class, which lists the properties.
I'm trying to create NSG and then attach it to a existing subnet.
I've successfully able to create the NSG but it throws an error while attaching it to subnet. Stating that the address prefix cannot be null. Do we have to pass the address prefix as well? in below function?
params_create = azure.mgmt.network.models.Subnet(
Below is the full code snippet.
from azure.common.credentials import ServicePrincipalCredentials
from azure.mgmt.resource import ResourceManagementClient
from azure.mgmt.compute import ComputeManagementClient
from azure.mgmt.network import NetworkManagementClient
from azure.mgmt.compute.models import DiskCreateOption
from azure.mgmt.network.v2017_03_01.models import NetworkSecurityGroup
from azure.mgmt.network.v2017_03_01.models import SecurityRule
import azure.mgmt.network.models
SUBSCRIPTION_ID = 'xxx'
GROUP_NAME = 'xxxx'
LOCATION = 'xxxx'
VM_NAME = 'myVM'
VNET = 'existingvnet'
SUBNET = 'default'
def get_credentials():
credentials = ServicePrincipalCredentials(
client_id = 'xxx',
secret = 'xxxx',
tenant = 'xxxx'
)
return credentials
def create_network_security_group(network_client):
params_create = azure.mgmt.network.models.NetworkSecurityGroup(
location=LOCATION,
security_rules=[
azure.mgmt.network.models.SecurityRule(
name='rdprule',
access=azure.mgmt.network.models.SecurityRuleAccess.allow,
description='test security rule',
destination_address_prefix='*',
destination_port_range='3389',
direction=azure.mgmt.network.models.SecurityRuleDirection.inbound,
priority=500,
protocol=azure.mgmt.network.models.SecurityRuleProtocol.tcp,
source_address_prefix='*',
source_port_range='*',
),
],
)
result_create_NSG = network_client.network_security_groups.create_or_update(
GROUP_NAME,
'nsg-vm',
params_create,
)
return result_create_NSG.result()
def attach_network_security_group(network_client,creation_result_nsg):
params_create = azure.mgmt.network.models.Subnet(
network_security_group= creation_result_nsg,
)
result_create = network_client.subnets.create_or_update(
GROUP_NAME,
VNET,
SUBNET,
params_create,
)
return result_create.result()
if __name__ == "__main__":
credentials = get_credentials()
resource_group_client = ResourceManagementClient(
credentials,
SUBSCRIPTION_ID
)
network_client = NetworkManagementClient(
credentials,
SUBSCRIPTION_ID
)
compute_client = ComputeManagementClient(
credentials,
SUBSCRIPTION_ID
)
creation_result_nsg = create_network_security_group(network_client)
print("------------------------------------------------------")
print(creation_result_nsg)
input('Press enter to continue...')
creation_result = attach_network_security_group(network_client,creation_result_nsg)
print("------------------------------------------------------")
print(creation_result)
input('Press enter to continue...')
that means you are not passing it the address prefix it should use. According to the docs you need to pass in address_prefix parameter. so add it to your params_create, something like this:
params_create = Subnet(
address_prefix = "10.0.0.0/24",
network_security_group = azure.mgmt.network.models.NetworkSecurityGroup(xxx)
)