The account that is running cloudrun has been linked to the billing account, but still the cloud billing python apis throw errors. What else needs to be done in the service account ?
#1. Created json token for service account and passed into the access_bills() method
#2. The service account has role for Billing Access View
#3. Copied these methods as advised in comments from John Hanley's blog:
def load_private_key(json_cred):
''' Return the private key from the json credentials '''
return json_cred['private_key']
def create_signed_jwt(pkey, pkey_id, email, scope):
'''
Create a Signed JWT from a service account Json credentials file
This Signed JWT will later be exchanged for an Access Token
'''
# Google Endpoint for creating OAuth 2.0 Access Tokens from Signed-JWT
auth_url = "https://www.googleapis.com/oauth2/v4/token"
expires_in = 3600
issued = int(time.time())
expires = issued + expires_in # expires_in is in seconds
# Note: this token expires and cannot be refreshed. The token must be recreated
# JWT Headers
additional_headers = {
'kid': pkey_id,
"alg": "RS256", # Google uses SHA256withRSA
"typ": "JWT"
}
# JWT Payload
payload = {
"iss": email, # Issuer claim
"sub": email, # Issuer claim
"aud": auth_url, # Audience claim
"iat": issued, # Issued At claim
"exp": expires, # Expire time
"scope": scope # Permissions
}
# Encode the headers and payload and sign creating a Signed JWT (JWS)
sig = jwt.encode(payload, pkey, algorithm="RS256", headers=additional_headers)
return sig
def exchangeJwtForAccessToken(signed_jwt):
'''
This function takes a Signed JWT and exchanges it for a Google OAuth Access Token
'''
auth_url = "https://www.googleapis.com/oauth2/v4/token"
params = {
"grant_type": "urn:ietf:params:oauth:grant-type:jwt-bearer",
"assertion": signed_jwt
}
r = requests.post(auth_url, data=params)
if r.ok:
return(r.json()['access_token'], '')
return None, r.text
def access_bills(sa_json):
cred = json.loads(sa_json)
private_key = load_private_key(cred)
# scopes = "https://www.googleapis.com/auth/cloud-platform" # this does not work, gets 404
scopes = "https://www.googleapis.com/auth/cloud-billing.readonly"
s_jwt = create_signed_jwt(
private_key,
cred['private_key_id'],
cred['client_email'],
scopes)
token, err = exchangeJwtForAccessToken(s_jwt)
if token is None:
logger.error("Error: {}".format(err))
exit(1)
logger.info("Token response: {}".format(token))
# the token is obtained and prints in the log
headers = {
"Host": "www.googleapis.com",
"Authorization": "Bearer " + token,
"Content-Type": "application/json"
}
try:
url = "https://cloudbilling.googleapis.com/v1/billingAccounts/01C8DC-336472-E177E1" # account name is "Billing Core"
response = requests.get(url=url, headers=headers)
logger.info("Response: {}".format(response))
# logs -> app - INFO - Response: <Response [404]>
return {
'statusCode': 200,
'body': 'Success'
}
except Exception as e:
logger.error("Error")
raise e
It gives 404 error as shown in the comment log after trying on that url.
Okay I found that the only way it works as of now is via big query export from billing account, sort out dataViewer permission and run the corresponding sql from python application, it works.
Related
In Google Cloud documentation, the command line to get the list and number of instances that are running on a project in Google Cloud is given as following :
gcloud compute instances list
or
GET https://compute.googleapis.com/compute/v1/projects/{project}/zones/{zone}/instances
How can I get the equivalent function in Python using Google Cloud ?
There is documentation on this here: https://cloud.google.com/compute/docs/tutorials/python-guide
In short:
import googleapiclient.discovery
project="your project"
zone="your zone"
compute = googleapiclient.discovery.build('compute', 'v1')
instances = compute.instances().list(project=project, zone=zone).execute()
for instance in instances:
print(' - ' + instance['name'])
Below is an example I wrote that uses the REST API and does not use one of the Google Cloud SDKs.
This example will teach you the low level details of services accounts, authorization, access tokens and the Compute Engine REST API.
'''
This program lists lists the Google Compute Engine Instances in one zone
'''
import time
import json
import jwt
import requests
import httplib2
# Project ID for this request.
project = 'development-123456'
# The name of the zone for this request.
zone = 'us-west1-a'
# Service Account Credentials, Json format
json_filename = 'service-account.json'
# Permissions to request for Access Token
scopes = "https://www.googleapis.com/auth/cloud-platform"
# Set how long this token will be valid in seconds
expires_in = 3600 # Expires in 1 hour
def load_json_credentials(filename):
''' Load the Google Service Account Credentials from Json file '''
with open(filename, 'r') as f:
data = f.read()
return json.loads(data)
def load_private_key(json_cred):
''' Return the private key from the json credentials '''
return json_cred['private_key']
def create_signed_jwt(pkey, pkey_id, email, scope):
'''
Create a Signed JWT from a service account Json credentials file
This Signed JWT will later be exchanged for an Access Token
'''
# Google Endpoint for creating OAuth 2.0 Access Tokens from Signed-JWT
auth_url = "https://www.googleapis.com/oauth2/v4/token"
issued = int(time.time())
expires = issued + expires_in # expires_in is in seconds
# Note: this token expires and cannot be refreshed. The token must be recreated
# JWT Headers
additional_headers = {
'kid': pkey_id,
"alg": "RS256",
"typ": "JWT" # Google uses SHA256withRSA
}
# JWT Payload
payload = {
"iss": email, # Issuer claim
"sub": email, # Issuer claim
"aud": auth_url, # Audience claim
"iat": issued, # Issued At claim
"exp": expires, # Expire time
"scope": scope # Permissions
}
# Encode the headers and payload and sign creating a Signed JWT (JWS)
sig = jwt.encode(payload, pkey, algorithm="RS256", headers=additional_headers)
return sig
def exchangeJwtForAccessToken(signed_jwt):
'''
This function takes a Signed JWT and exchanges it for a Google OAuth Access Token
'''
auth_url = "https://www.googleapis.com/oauth2/v4/token"
params = {
"grant_type": "urn:ietf:params:oauth:grant-type:jwt-bearer",
"assertion": signed_jwt
}
r = requests.post(auth_url, data=params)
if r.ok:
return(r.json()['access_token'], '')
return None, r.text
def gce_list_instances(accessToken):
'''
This functions lists the Google Compute Engine Instances in one zone
'''
# Endpoint that we will call
url = "https://www.googleapis.com/compute/v1/projects/" + project + "/zones/" + zone + "/instances"
# One of the headers is "Authorization: Bearer $TOKEN"
headers = {
"Host": "www.googleapis.com",
"Authorization": "Bearer " + accessToken,
"Content-Type": "application/json"
}
h = httplib2.Http()
resp, content = h.request(uri=url, method="GET", headers=headers)
status = int(resp.status)
if status < 200 or status >= 300:
print('Error: HTTP Request failed')
return
j = json.loads(content.decode('utf-8').replace('\n', ''))
print('Compute instances in zone', zone)
print('------------------------------------------------------------')
for item in j['items']:
print(item['name'])
if __name__ == '__main__':
cred = load_json_credentials(json_filename)
private_key = load_private_key(cred)
s_jwt = create_signed_jwt(
private_key,
cred['private_key_id'],
cred['client_email'],
scopes)
token, err = exchangeJwtForAccessToken(s_jwt)
if token is None:
print('Error:', err)
exit(1)
gce_list_instances(token)
I am trying to do the authorization code flow using Spotify's API to ultimately add songs to a playlist. I am building this from scratch, and not using any libraries such as Spotipy.
I am able to successfully hit the authorize endpoint, but I am having some issues with the token endpoint. Here is the code I have so far:
# URLS
AUTH_URL = 'https://accounts.spotify.com/authorize'
TOKEN_URL = 'https://accounts.spotify.com/api/token'
BASE_URL = 'https://api.spotify.com/v1/'
# Make a request to the /authorize endpoint to get an authorization code
auth_code = requests.get(AUTH_URL, {
'client_id': CLIENT_ID,
'response_type': 'code',
'redirect_uri': 'https://open.spotify.com/collection/playlists',
'scope': 'playlist-modify-private',
})
print(auth_code)
auth_header = base64.urlsafe_b64encode((CLIENT_ID + ':' + CLIENT_SECRET).encode('ascii'))
headers = {
'Content-Type': 'application/x-www-form-urlencoded',
'Authorization': 'Basic %s' % auth_header.decode('ascii')
}
payload = {
'grant_type': 'authorization_code',
'code': auth_code,
'redirect_uri': 'https://open.spotify.com/collection/playlists',
#'client_id': CLIENT_ID,
#'client_secret': CLIENT_SECRET,
}
# Make a request to the /token endpoint to get an access token
access_token_request = requests.post(url=TOKEN_URL, data=payload, headers=headers)
# convert the response to JSON
access_token_response_data = access_token_request.json()
print(access_token_response_data)
# save the access token
access_token = access_token_response_data['access_token']
When I run my script, I get this output in Terminal:
{'error': 'invalid_grant', 'error_description': 'Invalid authorization code'}
Traceback (most recent call last):
File "auth.py", line 48, in <module>
access_token = access_token_response_data['access_token']
KeyError: 'access_token'```
Can anyone explain to me what I might be doing wrong here?
By now, you probably have figure out what the problem was, but I am answering for future readers as well.
When you encode the auth_header, you should encode it to bytes not ascii. The str.encode() does this by default, so you can call it without arguments. Your code changes to this:
auth_header = base64.urlsafe_b64encode((CLIENT_ID + ':' + CLIENT_SECRET).encode()).
Now you must have a working example.
The code below worked for me to create a playlist and add a song.
The prerequisite is that you have a client_id and client_secret.
Run the code below and browse to your base URL (in my case, that was localhost), then you will see an authorize hyperlink that forwards you to the spotify authorization page.
Note that the redirect_url needs to be whitelisted in the spotify developer dashboard.
from fastapi import FastAPI
from fastapi.responses import HTMLResponse
import requests
import uvicorn
# client_id = "YOUR_CLIENT_ID"
# client_secret = "YOUR_CLIENT_SECRET"
# redirect_uri = "YOUR_REDIRECT_URI" # e.g. http://localhost:8000/callback/ --> you will have to whitelist this url in the spotify developer dashboard
app = FastAPI()
def get_access_token(auth_code: str):
response = requests.post(
"https://accounts.spotify.com/api/token",
data={
"grant_type": "authorization_code",
"code": auth_code,
"redirect_uri": redirect_uri,
},
auth=(client_id, client_secret),
)
access_token = response.json()["access_token"]
return {"Authorization": "Bearer " + access_token}
#app.get("/")
async def auth():
scope = ["playlist-modify-private", "playlist-modify-public"]
auth_url = f"https://accounts.spotify.com/authorize?response_type=code&client_id={client_id}&redirect_uri={redirect_uri}&scope={' '.join(scope)}"
return HTMLResponse(content=f'Authorize')
#app.get("/callback")
async def callback(code):
headers = get_access_token(code)
response = requests.get("https://api.spotify.com/v1/me", headers=headers)
user_id = response.json()["id"]
name = "Name of your playlist"
description = "Description of your playlist"
params = {
"name": name,
"description": description,
"public": True,
}
url = f"https://api.spotify.com/v1/users/{user_id}/playlists"
response = requests.post(url=url, headers=headers, json=params)
playlist_id = response.json()["id"]
track_uri = "spotify:track:319eU2WvplIHzjvogpnNc6"
response = requests.post(
f"https://api.spotify.com/v1/playlists/{playlist_id}/tracks",
headers=headers,
json={"uris": [track_uri]},
)
if response.status_code == 201:
return {"message": "Track added successfully!"}
else:
return {"error": response.json()}
if __name__ == "__main__":
uvicorn.run(app, debug=True)
You're missing the CLIENT_ID and CLIENT_SECRET set within the code if I'm not mistaken.
This means Spotify will return invalid access token, which results in you not being able to continue.
You can also use Spotipy library for Python to make things easier.
https://spotipy.readthedocs.io/en/2.16.1/
My goal is to get token then querying graph resources.
I wrote these codes to get token from my app. First code, acquire token with login/pass:
from adal import AuthenticationContext
auth_context = AuthenticationContext('https://login.microsoftonline.com/g***************r.onmicrosoft.com')
token = auth_context.acquire_token_with_username_password('https://graph.microsoft.com',
't***************n.fr',
'***************', 'de8bc8b5-***************-b748da725064')
and got these :
adal.adal_error.AdalError: Get Token request returned http error: 401 and server response: {"error":"invalid_client","error_description":"AADSTS7000218: The request body must contain the following parameter: 'client_assertion' or 'client_secret'.\r\nTrace ID: e9ffa752-a258-4e32-9164-156e50912a00\r\nCorrelation ID: afb3c52f-8f36-4904-9926-90ba2aaba0ab\r\nTimestamp: 2020-12-08 14:02:58Z","error_codes":[7000218],"timestamp":"2020-12-08 14:02:58Z","trace_id":"e9ffa752-a258-4e32-9164-156e50912a00","correlation_id":"afb3c52f-8f36-4904-9926-90ba2aaba0ab","error_uri":"https://login.microsoftonline.com/error?code=7000218"}
Second code, acquire token with client credentials:
from adal import AuthenticationContext
auth_context = AuthenticationContext('https://login.microsoftonline.com/g*************r.onmicrosoft.com')
token = auth_context.acquire_token_with_client_credentials('https://graph.microsoft.com',
'de8bc8b5-************-b748da725064',
'****************************')
And got these:
adal.adal_error.AdalError: Get Token request returned http error: 401 and server response: {"error":"invalid_client","error_description":"AADSTS7000215: Invalid client secret is provided.\r\nTrace ID: 79fe51c5-2987-4855-acf5-88fee0592a00\r\nCorrelation ID: cdfc9215-b7a6-4ceb-9d90-2de371c3178c\r\nTimestamp: 2020-12-08 14:21:13Z","error_codes":[7000215],"timestamp":"2020-12-08 14:21:13Z","trace_id":"79fe51c5-2987-4855-acf5-88fee0592a00","correlation_id":"cdfc9215-b7a6-4ceb-9d90-2de371c3178c","error_uri":"https://login.microsoftonline.com/error?code=7000215"}
I don't know if the client secret is wrong or something I did?
I didn't need those libs.
Lib requests was enough.
def authenticate_oauth():
validation_url = 'https://login.microsoftonline.com/******************.com/oauth2/v2.0/token'
client_id = '1a******************************b7'
client_secret = '-*****************************r'
grant_type = 'client_credentials'
scope = 'https://graph.microsoft.com/.default'
params = {
'client_id': client_id,
'client_secret': client_secret,
'grant_type': grant_type,
'scope': scope,
}
response = requests.post(validation_url, data=params)
return json.loads(response.content)
if __name__ == '__main__':
token = authenticate_oauth()
if 'access_token' in token:
graph_data = requests.get(
'https://graph.microsoft.com/v1.0/users',
headers={'Authorization': 'Bearer ' + token['access_token']})
data_users = json.loads(graph_data.content)
for data in data_users['value']:
print(data)
My goal is to reproduce/replicate the functionality of gcloud compute addresses create without depending on the gcloud binary.
I am trying to use python to authenticate a POST to a googleapis compute endpoint per the documentation at https://cloud.google.com/compute/docs/ip-addresses/reserve-static-external-ip-address about reserving a static external ip address
But my POST's return 401 every time.
I have created a JWT from google.auth.jwt python module and when I decode it the JWT has all the strings embedded that I would expect to be there.
I've also tried combinations of the following OAuth scopes to be included in the JWT:
- "https://www.googleapis.com/auth/userinfo.email"
- "https://www.googleapis.com/auth/compute"
- "https://www.googleapis.com/auth/cloud-platform"
this is my function for getting a JWT using the information in my service account's JSON key file
def _generate_jwt( tokenPath, expiry_length=3600 ):
now = int(time.time())
tokenData = load_json_data( tokenPath )
sa_email = tokenData['client_email']
payload = {
'iat': now,
# expires after 'expiry_length' seconds.
"exp": now + expiry_length,
'iss': sa_email,
"scope": " ".join( [
"https://www.googleapis.com/auth/cloud-platform",
"https://www.googleapis.com/auth/compute",
"https://www.googleapis.com/auth/userinfo.email"
] ),
'aud': "https://www.googleapis.com/oauth2/v4/token",
'email': sa_email
}
# sign with keyfile
signer = google.auth.crypt.RSASigner.from_service_account_file( tokenPath )
jwt = google.auth.jwt.encode(signer, payload)
return jwt
once I have the JWT then I make the following post that fails, 401, ::
gapiURL = 'https://www.googleapis.com/compute/v1/projects/' + projectID + '/regions/' + region + '/addresses'
jwtToken = _generate_jwt( servicetoken )
headers = {
'Authorization': 'Bearer {}'.format( jwtToken ),
'content-type' : 'application/json',
}
post = requests.post( url=gapiURL, headers=headers, data=data )
post.raise_for_status()
return post.text
I received a 401 no matter how many combinations of scopes I used in the JWT or permissions I provided to my service account. What am I doing wrong?
edit: many thanks to #JohnHanley for pointing out that I'm missing the next/second POST to https://www.googleapis.com/oauth2/v4/token URL in GCP's auth sequence. So, you get a JWT to get an 'access token.'
I've changed my calls to use the python jwt module rather than the google.auth.jwt module in-combo with the google.auth.crypt.RSASigner. So the code is a bit simpler and I put it in a single method
## serviceAccount auth sequence for google :: JWT -> accessToken
def gke_get_token( serviceKeyDict, expiry_seconds=3600 ):
epoch_time = int(time.time())
# Generate a claim from the service account file.
claim = {
"iss": serviceKeyDict["client_email"],
"scope": " ".join([
"https://www.googleapis.com/auth/cloud-platform",
"https://www.googleapis.com/auth/userinfo.email"
]),
"aud": "https://www.googleapis.com/oauth2/v4/token",
"exp": epoch_time + expiry_seconds,
"iat": epoch_time
}
# Sign claim with JWT.
assertion = jwt.encode( claim, serviceKeyDict["private_key"], algorithm='RS256' ).decode()
data = urllib.urlencode( {
"grant_type": "urn:ietf:params:oauth:grant-type:jwt-bearer",
"assertion": assertion
} )
# Request the access token.
result = requests.post(
url="https://www.googleapis.com/oauth2/v4/token",
headers={
"Content-Type": "application/x-www-form-urlencoded"
},
data=data
)
result.raise_for_status()
return loadJsonData(result.text)["access_token"]
In Google Cloud there are three types of "tokens" that grant access:
Signed JWT
Access Token
Identity Token
In your case you created a Signed JWT. A few Google services accept this token. Most do not.
Once you create a Signed JWT, then next step is to call a Google OAuth endpoint and exchange for an Access Token. I wrote an article that describes this in detail:
Google Cloud – Creating OAuth Access Tokens for REST API Calls
Some Google services now accept Identity Tokens. This is called Identity Based Access Control (IBAC). This does not apply to your question but is the trend for the future in Google Cloud Authorization. An example is my article on Cloud Run + Cloud Storage + KMS:
Google Cloud – Go – Identity Based Access Control
The following example Python code shows how to exchange tokens:
def exchangeJwtForAccessToken(signed_jwt):
'''
This function takes a Signed JWT and exchanges it for a Google OAuth Access Token
'''
auth_url = "https://www.googleapis.com/oauth2/v4/token"
params = {
"grant_type": "urn:ietf:params:oauth:grant-type:jwt-bearer",
"assertion": signed_jwt
}
r = requests.post(auth_url, data=params)
if r.ok:
return(r.json()['access_token'], '')
return None, r.text
I'm trying to creating a URL that enables federated users to access the AWS Management Console following the
[officlal documentation][1]. I'm using Cognito with [enhanced authflow][2] in order to authenticate user with username and password. This is the code:
################## 1. LOGIN ####################
cognito = boto3.client('cognito-idp', aws_access_key_id='', aws_secret_access_key='')
response = cognito.initiate_auth(
ClientId = app_client_id,
AuthFlow = 'USER_PASSWORD_AUTH',
AuthParameters = {
"USERNAME": username,
"PASSWORD": password
},
ClientMetadata = { 'UserPoolId': user_pool_id }
)
id_token = response['AuthenticationResult']['IdToken']
################## 2. GET ID ####################
cognito_identity = boto3.client('cognito-identity', aws_access_key_id='', aws_secret_access_key='', region_name=region)
response = cognito_identity.get_id(
IdentityPoolId = identity_pool_id,
Logins = {
'cognito-idp.{}.amazonaws.com/{}'.format(region, user_pool_id) : id_token
}
)
identity_id = response['IdentityId']
################## 3. RETRIEVE CREDENTIALS ####################
response = cognito_identity.get_credentials_for_identity(
IdentityId = identity_id,
Logins = {
'cognito-idp.{}.amazonaws.com/{}'.format(region, user_pool_id) : id_token
}
)
access_key_id = response['Credentials']['AccessKeyId']
secret_key = response['Credentials']['SecretKey']
session_token = response['Credentials']['SessionToken']
For the next step (assume role and call federation endpoint) i'm not using the example in the official documentation linked above because it use boto rather than boto3. This is the code:
sts_boto_3 = boto3.client('sts', aws_access_key_id = access_key_id,
aws_secret_access_key = secret_key,
aws_session_token = session_token,
region_name = region)
response = sts_boto_3.assume_role(
RoleArn = role,
RoleSessionName = role_session_name,
)
session_id = response['Credentials']['AccessKeyId']
session_key = response['Credentials']['SecretAccessKey']
session_token = response['Credentials']['SessionToken']
session_string = '{{"sessioId" : "{}" , "sessionKey": "{}", "sessionToken" : "{}"}}'.format(session_id, session_key, session_token)
req_url = 'https://signin.aws.amazon.com/federation?Action=getSigninToken&SessionDuration={}&Session={}'.format(3600, urllib.quote_plus(session_string))
r = requests.get(req_url)
print r
The result is
<Response [503]>
What i'm wrong?
[EDIT]
There wasn't an error in session_string (sessioId instead of sessionId)
session_string = '{{"sessionId" : "{}" , "sessionKey": "{}", "sessionToken" : "{}"}}'.format(session_id, session_key, session_token)
Now the response is 400 BAD REQUEST
<Response [400]>
I've added a full example of how to set up credentials and construct a URL that gives federated users direct access to the AWS Management Console on GitHub.
Here's the salient part of the code that constructs the URL:
def construct_federated_url(assume_role_arn, session_name, issuer, sts_client):
"""
Constructs a URL that gives federated users direct access to the AWS Management
Console.
1. Acquires temporary credentials from AWS Security Token Service (AWS STS) that
can be used to assume a role with limited permissions.
2. Uses the temporary credentials to request a sign-in token from the
AWS federation endpoint.
3. Builds a URL that can be used in a browser to navigate to the AWS federation
endpoint, includes the sign-in token for authentication, and redirects to
the AWS Management Console with permissions defined by the role that was
specified in step 1.
For more information, see Enabling Custom Identity Broker Access to the AWS Console
[https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_providers_enable-console-custom-url.html]
in the AWS Identity and Access Management User Guide.
:param assume_role_arn: The role that specifies the permissions that are granted.
The current user must have permission to assume the role.
:param session_name: The name for the STS session.
:param issuer: The organization that issues the URL.
:param sts_client: A Boto3 STS instance that can assume the role.
:return: The federated URL.
"""
response = sts_client.assume_role(
RoleArn=assume_role_arn, RoleSessionName=session_name)
temp_credentials = response['Credentials']
print(f"Assumed role {assume_role_arn} and got temporary credentials.")
session_data = {
'sessionId': temp_credentials['AccessKeyId'],
'sessionKey': temp_credentials['SecretAccessKey'],
'sessionToken': temp_credentials['SessionToken']
}
aws_federated_signin_endpoint = 'https://signin.aws.amazon.com/federation'
# Make a request to the AWS federation endpoint to get a sign-in token.
# The requests.get function URL-encodes the parameters and builds the query string
# before making the request.
response = requests.get(
aws_federated_signin_endpoint,
params={
'Action': 'getSigninToken',
'SessionDuration': str(datetime.timedelta(hours=12).seconds),
'Session': json.dumps(session_data)
})
signin_token = json.loads(response.text)
print(f"Got a sign-in token from the AWS sign-in federation endpoint.")
# Make a federated URL that can be used to sign into the AWS Management Console.
query_string = urllib.parse.urlencode({
'Action': 'login',
'Issuer': issuer,
'Destination': 'https://console.aws.amazon.com/',
'SigninToken': signin_token['SigninToken']
})
federated_url = f'{aws_federated_signin_endpoint}?{query_string}'
return federated_url