I am trying to use the python sample code from amazon docs to do a "GET with an Authorization Header" mechanism on a s3 bucket. The source code which I am using is below:
# AWS Version 4 signing example
# EC2 API (DescribeRegions)
# See: http://docs.aws.amazon.com/general/latest/gr/sigv4_signing.html
# This version makes a GET request and passes the signature
# in the Authorization header.
import sys, os, base64, datetime, hashlib, hmac
import requests # pip install requests
# ************* REQUEST VALUES *************
method = 'GET'
service = 's3'
host = 's3.amazonaws.com'
region = 'us-east-1'
endpoint = 'http://s3.amazonaws.com/sample_object/foo'
request_parameters = ''
# Key derivation functions. See:
# http://docs.aws.amazon.com/general/latest/gr/signature-v4-examples.html#signature-v4-examples-python
def sign(key, msg):
return hmac.new(key, msg.encode('utf-8'), hashlib.sha256).digest()
def getSignatureKey(key, dateStamp, regionName, serviceName):
kDate = sign(('AWS4' + key).encode('utf-8'), dateStamp)
kRegion = sign(kDate, regionName)
kService = sign(kRegion, serviceName)
kSigning = sign(kService, 'aws4_request')
return kSigning
# Read AWS access key from env. variables or configuration file. Best practice is NOT
# to embed credentials in code.
access_key = os.environ.get('AWS_ACCESS_KEY_ID')
secret_key = os.environ.get('AWS_SECRET_ACCESS_KEY')
if access_key is None or secret_key is None:
print 'No access key is available.'
sys.exit()
# Create a date for headers and the credential string
t = datetime.datetime.utcnow()
amzdate = t.strftime('%Y%m%dT%H%M%SZ')
datestamp = t.strftime('%Y%m%d') # Date w/o time, used in credential scope
# ************* TASK 1: CREATE A CANONICAL REQUEST *************
# http://docs.aws.amazon.com/general/latest/gr/sigv4-create-canonical-request.html
# Step 1 is to define the verb (GET, POST, etc.)--already done.
# Step 2: Create canonical URI--the part of the URI from domain to query
# string (use '/' if no path)
canonical_uri = '/sample_object/foo'
# Step 3: Create the canonical query string. In this example (a GET request),
# request parameters are in the query string. Query string values must
# be URL-encoded (space=%20). The parameters must be sorted by name.
# For this example, the query string is pre-formatted in the request_parameters variable.
canonical_querystring = request_parameters
# Step 4: Create the canonical headers and signed headers. Header names
# and value must be trimmed and lowercase, and sorted in ASCII order.
# Note that there is a trailing \n.
# amz_content_sha256 = hashlib.sha256('').hexdigest()
amz_content_sha256 = 'UNSIGNED-PAYLOAD'
canonical_headers = 'host:' + host + '\n' + 'x-amz-content-sha256:' + amz_content_sha256 + '\n' + 'x-amz-date:' + amzdate + '\n'
# Step 5: Create the list of signed headers. This lists the headers
# in the canonical_headers list, delimited with ";" and in alpha order.
# Note: The request can include any headers; canonical_headers and
# signed_headers lists those that you want to be included in the
# hash of the request. "Host" and "x-amz-date" are always required.
signed_headers = 'host;x-amz-content-sha256;x-amz-date'
# Step 6: Create payload hash (hash of the request body content). For GET
# requests, the payload is an empty string ("").
payload_hash = hashlib.sha256('').hexdigest()
# Step 7: Combine elements to create create canonical request
canonical_request = method + '\n' + canonical_uri + '\n' + canonical_headers + '\n' + signed_headers + '\n' + payload_hash
# ************* TASK 2: CREATE THE STRING TO SIGN*************
# Match the algorithm to the hashing algorithm you use, either SHA-1 or
# SHA-256 (recommended)
algorithm = 'AWS4-HMAC-SHA256'
credential_scope = datestamp + '/' + region + '/' + service + '/' + 'aws4_request'
string_to_sign = algorithm + '\n' + amzdate + '\n' + credential_scope + '\n' + hashlib.sha256(canonical_request).hexdigest()
# ************* TASK 3: CALCULATE THE SIGNATURE *************
# Create the signing key using the function defined above.
signing_key = getSignatureKey(secret_key, datestamp, region, service)
# Sign the string_to_sign using the signing_key
signature = hmac.new(signing_key, (string_to_sign).encode('utf-8'), hashlib.sha256).hexdigest()
# ************* TASK 4: ADD SIGNING INFORMATION TO THE REQUEST *************
# The signing information can be either in a query string value or in
# a header named Authorization. This code shows how to use a header.
# Create authorization header and add to request headers
authorization_header = algorithm + ' ' + 'Credential=' + access_key + '/' + credential_scope + ', ' + 'SignedHeaders=' + signed_headers + ', ' + 'Signature=' + signature
# The request can include any headers, but MUST include "host", "x-amz-date",
# and (for this scenario) "Authorization". "host" and "x-amz-date" must
# be included in the canonical_headers and signed_headers, as noted
# earlier. Order here is not significant.
# Python note: The 'host' header is added automatically by the Python 'requests' library.
headers = {'x-amz-date':amzdate, 'Authorization':authorization_header, 'x-amz-content-sha256':amz_content_sha256}
# ************* SEND THE REQUEST *************
request_url = endpoint
print '\nBEGIN REQUEST++++++++++++++++++++++++++++++++++++'
print 'Request URL = ' + request_url
r = requests.get(request_url, headers=headers)
print '\nRESPONSE++++++++++++++++++++++++++++++++++++'
print 'Response code: %d\n' % r.status_code
print r.text
I have correctly set the access id and secret access key. I used the same with boto python library and it seems to work when I list buckets, put objects etc. However when I use the above method I get back a 403 error saying "The request signature we calculated does not match the signature you provided. Check your key and signing method." I am not sure what is wrong with the above method. Any pointers.?
The canonical headers should have a trailing newline and canonical_request needs the canonical_querystring line even if it is blank. Also, use UNSIGNED_PAYLOAD for amz_content_sha256 for GET requests.
Here's a working example. It gets a file called key on an S3 bucket in the us-west-2 region called bucket:
import sys, os, base64, datetime, hashlib, hmac
import requests
method = 'GET'
service = 's3'
host = 's3-us-west-2.amazonaws.com'
region = 'us-west-2'
endpoint = 'http://s3-us-west-2.amazonaws.com'
request_parameters = ''
def sign(key, msg):
return hmac.new(key, msg.encode('utf-8'), hashlib.sha256).digest()
def getSignatureKey(key, dateStamp, regionName, serviceName):
kDate = sign(('AWS4' + key).encode('utf-8'), dateStamp)
kRegion = sign(kDate, regionName)
kService = sign(kRegion, serviceName)
kSigning = sign(kService, 'aws4_request')
return kSigning
access_key = os.environ.get('AWS_ACCESS_KEY_ID')
secret_key = os.environ.get('AWS_SECRET_ACCESS_KEY')
if access_key is None or secret_key is None:
print 'No access key is available.'
sys.exit()
t = datetime.datetime.utcnow()
amzdate = t.strftime('%Y%m%dT%H%M%SZ')
datestamp = t.strftime('%Y%m%d') # Date w/o time, used in credential scope
canonical_uri = '/bucket/key'
canonical_querystring = request_parameters
canonical_headers = 'host:' + host + '\n' + 'x-amz-content-sha256:UNSIGNED-PAYLOAD' + '\n' + 'x-amz-date:' + amzdate + '\n'
signed_headers = 'host;x-amz-content-sha256;x-amz-date'
payload_hash = 'UNSIGNED-PAYLOAD'
canonical_request = method + '\n' + canonical_uri + '\n' + canonical_querystring + '\n' + canonical_headers + '\n' + signed_headers + '\n' + payload_hash
algorithm = 'AWS4-HMAC-SHA256'
credential_scope = datestamp + '/' + region + '/' + service + '/' + 'aws4_request'
string_to_sign = algorithm + '\n' + amzdate + '\n' + credential_scope + '\n' + hashlib.sha256(canonical_request).hexdigest()
signing_key = getSignatureKey(secret_key, datestamp, region, service)
signature = hmac.new(signing_key, (string_to_sign).encode('utf-8'), hashlib.sha256).hexdigest()
authorization_header = algorithm + ' ' + 'Credential=' + access_key + '/' + credential_scope + ', ' + 'SignedHeaders=' + signed_headers + ', ' + 'Signature=' + signature
headers = {'x-amz-date':amzdate, 'x-amz-content-sha256': 'UNSIGNED-PAYLOAD', 'Authorization':authorization_header}
request_url = endpoint + canonical_uri
print '\nBEGIN REQUEST++++++++++++++++++++++++++++++++++++'
print 'Request URL = ' + request_url
r = requests.get(request_url, headers=headers)
print '\nRESPONSE++++++++++++++++++++++++++++++++++++'
print 'Response code: %d\n' % r.status_code
print r.text
Related
I am a newbie in python and aws.I dont know much how to ask questions in stackoverflow.
Please do not block me.
I am trying to send a HTTP Post Request to put record into Amazon Kinesis Stream.
I have created a stream mystream in kinesis. I use method post.
I tried the following link to set up gateway api and it worked fine.
https://docs.aws.amazon.com/kinesis/latest/APIReference/API_CreateStream.html
I am trying to do it with python code using requests.
But i am getting the below mentioned error:
The following is my code:
import sys, os, base64, datetime, hashlib, hmac
import requests # pip install requests
# ************* REQUEST VALUES *************
method = 'POST'
service = 'kinesis'
host = 'kinesis.eu-west-1.amazonaws.com'
region = 'eu-west-1'
endpoint = 'https://kinesis.eu-west-1.amazonaws.com'
content_type = 'application/x-amz-json-1.1'
amz_target = 'Kinesis_20181114.PutRecord'
request_parameters = '{'
request_parameters += '"StreamName": mystream,'
request_parameters += '"Data": + base64.b64encode(test) + ,'
request_parameters += '"PartitionKey": 1234 '
request_parameters += '}'
# Key derivation functions. See:
# http://docs.aws.amazon.com/general/latest/gr/signature-v4-
examples.html#signature-v4-examples-python
def sign(key, msg):
return hmac.new(key, msg.encode('utf-8'), hashlib.sha256).digest()
def getSignatureKey(key,datestamp,region,service):
kDate = sign(('AWS4' +key ).encode('utf-8'), datestamp)
kRegion = sign(kDate,region)
kService = sign(kRegion,service)
kSigning = sign(kService, 'aws4_request')
return kSigning
# Read AWS access key from env. variables or configuration file. Best
practice is NOT
# to embed credentials in code.
with open ('C:\\Users\\Connectm\\Desktop\\acesskeyid.txt') as f:
contents = f.read().split('\n')
access_key = contents[0]
secret_key = contents[1]
if access_key is None or secret_key is None:
print('No access key is available.')
sys.exit()
# Create a date for headers and the credential string
t = datetime.datetime.utcnow()
amzdate = t.strftime('%Y%m%dT%H%M%SZ')
datestamp = t.strftime('%Y%m%d') # Date w/o time, used in credential scope
canonical_uri = '/'
canonical_querystring = ''
canonical_headers = 'content-type:' + content_type + '\n' + 'host:' + host +
'\n' + 'x-amz-date:' + amzdate + '\n' + 'x-amz-target:' + amz_target + '\n'
signed_headers = 'content-type;host;x-amz-date;x-amz-target'
payload_hash = hashlib.sha256(request_parameters).hexdigest()
canonical_request = method + '\n' + canonical_uri + '\n' +
canonical_querystring + '\n' + canonical_headers + '\n' + signed_headers +
'\n' + payload_hash
algorithm = 'AWS4-HMAC-SHA256'
credential_scope = datestamp + '/' + region + '/' + service + '/' +
'aws4_request'
string_to_sign = algorithm + '\n' + amzdate + '\n' + credential_scope +
'\n' + hashlib.sha256(canonical_request).hexdigest()
signing_key = getSignatureKey(secret_key, datestamp, region, service)
signature = hmac.new(signing_key, (string_to_sign).encode('utf-8'),
hashlib.sha256).hexdigest()
authorization_header = algorithm + ' ' + 'Credential=' + access_key + '/' +
credential_scope + ', ' + 'SignedHeaders=' + signed_headers + ', ' +
'Signature=' + signature
print authorization_header;
headers = {'Content-Type':content_type,
'X-Amz-Date':amzdate,
'X-Amz-Target':amz_target,
'Authorization':authorization_header}
# ************* SEND THE REQUEST *************
print '\nBEGIN REQUEST++++++++++++++++++++++++++++++++++++'
print 'Request URL = ' + endpoint
r = requests.post(endpoint, data=request_parameters, headers=headers)
print '\nRESPONSE++++++++++++++++++++++++++++++++++++'
print 'Response code: %d\n' % r.status_code
print r.text
The following error i am getting
AWS4-HMAC-SHA256 Credential=AKIAI5C357A6YSKQFXEA/20181114/eu-west-
1/kinesis/aws4_request, SignedHeaders=content-type;host;x-amz-date;x-amz-
target,
Signature=1d7d463e77beaf86930806812188180db9cc7cff082663ad547f647a9c6d545a
BEGIN REQUEST++++++++++++++++++++++++++++++++++++
Request URL = https://kinesis.eu-west-1.amazonaws.com
RESPONSE++++++++++++++++++++++++++++++++++++
Response code: 400
{"__type":"SerializationException"}
Please can someone explain me how i can rectify the above error?
Is the code connecting to the stream?Is there a problem regarding
serialization of data?
The fact that you're getting a SerializationException means your code is working to talk to kinesis but the data you're giving in test is likely not valid JSON.
That said:
I strongly recommend not doing the requests logic stuff yourself but use the software development kit (SDK) for AWS, called boto3.
import json
import boto3
kinesis = boto3.client("kinesis")
response = kinesis.put_record(
StreamName="my-fancy-kinesis-stream",
Data=json.dumps({
'example': 'payload',
'yay': 'data',
'hello': 'world'
}),
PartitionKey="AdjustAsNeeded"
)
print response
This will instantiate a kinesis client using the credentials on your machine (either via instance metadata or ~/.aws/config) or environment variables.
Then it takes a simple dictionary and dumps it into a JSON string for the data.
Lots to say on partition keys that you can find out here.
Also, check out boto3!
I'm trying to recognize faces in a video stream using these instructions but I couldn't find any help to implement PutMedia operation using Python. I'm using Ubuntu 16.04 and Python 3.6. Any hint please so I can implement it using Python.
maybe it is too late, but hope it can help you.
the answer is from this article, and i have changed just a little. to use the code below, you need add your own configurations shown in "<>"
Amazon Kinesis Video PutMedia Using Python
import requests
import sys
import os
import datetime
import hashlib
import hmac
import time
def sign(key, msg):
return hmac.new(key, msg.encode("utf-8"), hashlib.sha256).digest()
def get_signature_key(key, date_stamp, regionName, serviceName):
kDate = sign(('AWS4' + key).encode('utf-8'), date_stamp)
kRegion = sign(kDate, regionName)
kService = sign(kRegion, serviceName)
kSigning = sign(kService, 'aws4_request')
return kSigning
# ************* REQUEST VALUES *************
method = 'POST'
service = 'kinesisvideo'
host = 'kinesisvideo.us-east-1.amazonaws.com'
region = 'us-east-1'
endpoint = 'https://**<the endpoint you get with get_data_endpoint>**/PutMedia'
# POST requests use a content type header. For DynamoDB,
# the content is JSON.
content_type = 'application/json'
start_tmstp = repr(time.time())
localfile = 'test_1.png'
with open(localfile, 'rb') as image:
request_parameters = image.read()
# Read AWS access key from env. variables or configuration file. Best practice is NOT
# to embed credentials in code.
access_key = '*************************'
secret_key = '*************************'
if access_key is None or secret_key is None:
print('No access key is available.')
sys.exit()
# Create a date for headers and the credential string
t = datetime.datetime.utcnow()
amz_date = t.strftime('%Y%m%dT%H%M%SZ')
date_stamp = t.strftime('%Y%m%d') # Date w/o time, used in credential scope
# ************* TASK 1: CREATE A CANONICAL REQUEST *************
# http://docs.aws.amazon.com/general/latest/gr/sigv4-create-canonical-request.html
# Step 1 is to define the verb (GET, POST, etc.)--already done.
# Step 2: Create canonical URI--the part of the URI from domain to query
# string (use '/' if no path)
canonical_uri = '/'
## Step 3: Create the canonical query string. In this example, request
# parameters are passed in the body of the request and the query string
# is blank.
canonical_querystring = ''
# Step 4: Create the canonical headers. Header names must be trimmed
# and lowercase, and sorted in code point order from low to high.
# Note that there is a trailing \n.
canonical_headers = 'content-type:' + content_type + '\n' + 'host:' + host + '\n' + 'x-amz-content-sha256:' + 'UNSIGNED-PAYLOAD' + '\n' + 'x-amz-date:' + amz_date + '\n' + 'x-amzn-fragment-timecode-type:' + 'ABSOLUTE' + '\n' + 'x-amzn-producer-start-timestamp:' + start_tmstp + '\n' + 'x-amzn-stream-name:' + <your_stream_name> + '\n'
# Step 5: Create the list of signed headers. This lists the headers
# in the canonical_headers list, delimited with ";" and in alpha order.
# Note: The request can include any headers; canonical_headers and
# signed_headers include those that you want to be included in the
# hash of the request. "Host" and "x-amz-date" are always required.
# For DynamoDB, content-type and x-amz-target are also required.
signed_headers = 'content-type;host;x-amz-content-sha256;x-amz-date;x-amz-target;x-amzn-fragment-timecode-type;x-amzn-producer-start-timestamp;x-amzn-stream-name'
# Step 6: Create payload hash. In this example, the payload (body of
# the request) contains the request parameters.
# Step 7: Combine elements to create canonical request
canonical_request = method + '\n' + canonical_uri + '\n' + canonical_querystring + '\n' + canonical_headers + '\n' + signed_headers
# ************* TASK 2: CREATE THE STRING TO SIGN*************
# Match the algorithm to the hashing algorithm you use, either SHA-1 or
# SHA-256 (recommended)
algorithm = 'AWS4-HMAC-SHA256'
credential_scope = date_stamp + '/' + region + '/' + service + '/' + 'aws4_request'
string_to_sign = algorithm + '\n' + amz_date + '\n' + credential_scope + '\n' + hashlib.sha256(
canonical_request.encode('utf-8')).hexdigest()
# ************* TASK 3: CALCULATE THE SIGNATURE *************
# Create the signing key using the function defined above.
signing_key = get_signature_key(secret_key, date_stamp, region, service)
# Sign the string_to_sign using the signing_key
signature = hmac.new(signing_key, (string_to_sign).encode('utf-8'),
hashlib.sha256).hexdigest()
# ************* TASK 4: ADD SIGNING INFORMATION TO THE REQUEST *************
# Put the signature information in a header named Authorization.
authorization_header = algorithm + ' ' + 'Credential=' + access_key + '/' + credential_scope + ', ' + 'SignedHeaders=' + signed_headers + ', ' + 'Signature=' + signature
# # Python note: The 'host' header is added automatically by the Python 'requests' library.
headers = {
'Content-Type': content_type,
'x-amzn-fragment-timecode-type': 'ABSOLUTE',
'x-amzn-producer-start-timestamp': start_tmstp,
'x-amzn-stream-name': <your_stream_name>,
# 'X-Amz-Target': amz_target,
# 'x-amz-content-sha256': 'UNSIGNED-PAYLOAD',
'Authorization': authorization_header
}
# ************* SEND THE REQUEST *************
print('\nBEGIN REQUEST++++++++++++++++++++++++++++++++++++')
print('Request URL = ' + endpoint)
r = requests.post(endpoint, data=request_parameters, headers=headers)
print('\nRESPONSE++++++++++++++++++++++++++++++++++++')
print('Response code: %d\n' % r.status_code)
print(r.text)
I'm getting following response when using putMedia api using python.
Response code: 200
{"Output":{"__type":"com.amazon.coral.service#SerializationException","Message":null},"Version":"1.0"}
I have saved video frames locally and trying to post first frame on the amazon video stream. I have signed using Signing AWS Requests with Signature Version 4 instructions.
Below is my code -
import sys, os, base64, datetime, hashlib, hmac, time
import requests # pip install requests
# ************* REQUEST VALUES *************
method = 'POST'
service = 'kinesisvideo'
host = 's-1110bf70.kinesisvideo.us-west-2.amazonaws.com'
region = 'us-west-2'
endpoint = 'https://s-1110bf70.kinesisvideo.us-west-2.amazonaws.com'
# POST requests use a content type header. For DynamoDB,
# the content is JSON.
content_type = 'application/json'
start_tmstp = repr(time.time())
localfile = 'mkv-0.jpg';
with open(localfile,'rb') as image:
request_parameters = image.read()
# Key derivation functions. See:
# http://docs.aws.amazon.com/general/latest/gr/signature-v4-examples.html#signature-v4-examples-python
def sign(key, msg):
return hmac.new(key, msg.encode("utf-8"), hashlib.sha256).digest()
def getSignatureKey(key, date_stamp, regionName, serviceName):
kDate = sign(('AWS4' + key).encode('utf-8'), date_stamp)
kRegion = sign(kDate, regionName)
kService = sign(kRegion, serviceName)
kSigning = sign(kService, 'aws4_request')
return kSigning
# Read AWS access key from env. variables or configuration file. Best practice is NOT
# to embed credentials in code.
access_key = '***********'
secret_key = '*************'
if access_key is None or secret_key is None:
print('No access key is available.')
sys.exit()
# Create a date for headers and the credential string
t = datetime.datetime.utcnow()
amz_date = t.strftime('%Y%m%dT%H%M%SZ')
date_stamp = t.strftime('%Y%m%d') # Date w/o time, used in credential scope
# ************* TASK 1: CREATE A CANONICAL REQUEST *************
# http://docs.aws.amazon.com/general/latest/gr/sigv4-create-canonical-request.html
# Step 1 is to define the verb (GET, POST, etc.)--already done.
# Step 2: Create canonical URI--the part of the URI from domain to query
# string (use '/' if no path)
canonical_uri = '/putMedia'
## Step 3: Create the canonical query string. In this example, request
# parameters are passed in the body of the request and the query string
# is blank.
canonical_querystring = ''
# Step 4: Create the canonical headers. Header names must be trimmed
# and lowercase, and sorted in code point order from low to high.
# Note that there is a trailing \n.
canonical_headers = 'content-type:' + content_type + '\n' + 'host:' + host + '\n' + 'x-amz-content-sha256:' + 'UNSIGNED-PAYLOAD' + '\n' + 'x-amz-date:' + amz_date + '\n' + 'x-amz-target:' + amz_target + '\n' + 'x-amzn-fragment-timecode-type:' + 'ABSOLUTE' + '\n' + 'x-amzn-producer-start-timestamp:' + start_tmstp + '\n' + 'x-amzn-stream-name:' + 'ExampleStream' + '\n'
# Step 5: Create the list of signed headers. This lists the headers
# in the canonical_headers list, delimited with ";" and in alpha order.
# Note: The request can include any headers; canonical_headers and
# signed_headers include those that you want to be included in the
# hash of the request. "Host" and "x-amz-date" are always required.
# For DynamoDB, content-type and x-amz-target are also required.
signed_headers = 'content-type;host;x-amz-content-sha256;x-amz-date;x-amz-target;x-amzn-fragment-timecode-type;x-amzn-producer-start-timestamp;x-amzn-stream-name'
# Step 6: Create payload hash. In this example, the payload (body of
# the request) contains the request parameters.
# Step 7: Combine elements to create canonical request
canonical_request = method + '\n' + canonical_uri + '\n' + canonical_querystring + '\n' + canonical_headers + '\n' + signed_headers
# ************* TASK 2: CREATE THE STRING TO SIGN*************
# Match the algorithm to the hashing algorithm you use, either SHA-1 or
# SHA-256 (recommended)
algorithm = 'AWS4-HMAC-SHA256'
credential_scope = date_stamp + '/' + region + '/' + service + '/' + 'aws4_request'
string_to_sign = algorithm + '\n' + amz_date + '\n' + credential_scope + '\n' + hashlib.sha256(canonical_request.encode('utf-8')).hexdigest()
# ************* TASK 3: CALCULATE THE SIGNATURE *************
# Create the signing key using the function defined above.
signing_key = getSignatureKey(secret_key, date_stamp, region, service)
# Sign the string_to_sign using the signing_key
signature = hmac.new(signing_key, (string_to_sign).encode('utf-8'), hashlib.sha256).hexdigest()
# ************* TASK 4: ADD SIGNING INFORMATION TO THE REQUEST *************
# Put the signature information in a header named Authorization.
authorization_header = algorithm + ' ' + 'Credential=' + access_key + '/' + credential_scope + ', ' + 'SignedHeaders=' + signed_headers + ', ' + 'Signature=' + signature
# # Python note: The 'host' header is added automatically by the Python 'requests' library.
headers = {
'Content-Type':content_type,
'x-amzn-fragment-timecode-type': 'ABSOLUTE',
'x-amzn-producer-start-timestamp': start_tmstp,
'x-amzn-stream-name': 'ExampleStream',
'X-Amz-Target':amz_target,
'x-amz-content-sha256':'UNSIGNED-PAYLOAD',
'Authorization':authorization_header
}
# ************* SEND THE REQUEST *************
print('\nBEGIN REQUEST++++++++++++++++++++++++++++++++++++')
print('Request URL = ' + endpoint)
r = requests.post(endpoint, data=request_parameters, headers=headers)
print('\nRESPONSE++++++++++++++++++++++++++++++++++++')
print('Response code: %d\n' % r.status_code)
print(r.text)
I have spent around a week on this issue. Please help me.
The input of PutMedia is not frames, you will need to provide MKV format input to it. You can refer to a similar example for PutMedia in java here: https://docs.aws.amazon.com/kinesisvideostreams/latest/dg/examples-putmedia.html
I am getting Invalid Request with below error message when trying to create a thing-type using Rest API. I am not using aws-sdk rather python rest client.
I followed the steps given here to sign the request using sigv4. The example given there is working for listing the Thing-Types. So the credentials I am using, are correct. Error Message :
Response code: 403
{"message":"The request signature we calculated does not match the signature you provided. Check your AWS Secret Access Key and signing
method. Consult the service documentation for details."}
The code I am using :
# See: http://docs.aws.amazon.com/general/latest/gr/sigv4_signing.html
# This version makes a POST request and passes request parameters
# in the body (payload) of the request. Auth information is passed in
# an Authorization header.
import sys, os, base64, datetime, hashlib, hmac
import requests # pip install requests
# ************* REQUEST VALUES *************
method = 'POST'
service = 'execute-api'
api = 'https://iot.eu-west-2.amazonaws.com/thing-types/GasSensor'
request_parameter='{"thingTypeProperties":{"searchableAttributes": ["gas"],"thingTypeDescription":"Devices that reports gas level readings"}}'
host = api.split('/')[2] if api.count('/') >= 2 else ''
region = api.split('.')[1] if api.count('.') >= 1 else ''
endpoint = api.split('?')[0] if api.count('?') >= 0 else api
#request_parameters = api.split('?')[1] if api.__contains__('?') else ''
# Request parameters for CreateTable--passed in a JSON block.
request_parameters = payload
print(host + ' $ ' + region + ' $ ' + endpoint + ' $ ' + request_parameters)
# Key derivation functions. See:
# http://docs.aws.amazon.com/general/latest/gr/signature-v4-examples.html#signature-v4-examples-python
def sign(key, msg):
return hmac.new(key, msg.encode("utf-8"), hashlib.sha256).digest()
def getSignatureKey(key, date_stamp, regionName, serviceName):
kDate = sign(('AWS4' + key).encode('utf-8'), date_stamp)
kRegion = sign(kDate, regionName)
kService = sign(kRegion, serviceName)
kSigning = sign(kService, 'aws4_request')
return kSigning
# Read AWS access key from env. variables or configuration file. Best practice is NOT
# to embed credentials in code.
access_key = 'XXXXXXXXXXXXX'
secret_key = 'XXXXXXXXXXXXXX'
if access_key is None or secret_key is None:
print
'No access key is available.'
sys.exit()
# Create a date for headers and the credential string
t = datetime.datetime.utcnow()
amz_date = t.strftime('%Y%m%dT%H%M%SZ')
date_stamp = t.strftime('%Y%m%d') # Date w/o time, used in credential scope
# ************* TASK 1: CREATE A CANONICAL REQUEST *************
# http://docs.aws.amazon.com/general/latest/gr/sigv4-create-canonical-request.html
# Step 1 is to define the verb (GET, POST, etc.)--already done.
# Step 2: Create canonical URI--the part of the URI from domain to query
# string (use '/' if no path)
canonical_uri = '/'
## Step 3: Create the canonical query string. In this example, request
# parameters are passed in the body of the request and the query string
# is blank.
canonical_querystring = ''
# Step 4: Create the canonical headers. Header names must be trimmed
# and lowercase, and sorted in code point order from low to high.
# Note that there is a trailing \n.
canonical_headers = 'content-type:' + content_type + '\n' + 'host:' + host + '\n' + 'x-amz-date:' + amz_date + '\n'
# Step 5: Create the list of signed headers. This lists the headers
# in the canonical_headers list, delimited with ";" and in alpha order.
# Note: The request can include any headers; canonical_headers and
# signed_headers include those that you want to be included in the
# hash of the request. "Host" and "x-amz-date" are always required.
# For DynamoDB, content-type and x-amz-target are also required.
signed_headers = 'content-type;host;x-amz-date'
# Step 6: Create payload hash. In this example, the payload (body of
# the request) contains the request parameters.
payload_hash = hashlib.sha256(request_parameters.encode('utf-8')).hexdigest()
# Step 7: Combine elements to create create canonical request
canonical_request = method + '\n' + canonical_uri + '\n' + canonical_querystring + '\n' + canonical_headers + '\n' + signed_headers + '\n' + payload_hash
# ************* TASK 2: CREATE THE STRING TO SIGN*************
# Match the algorithm to the hashing algorithm you use, either SHA-1 or
# SHA-256 (recommended)
algorithm = 'AWS4-HMAC-SHA256'
credential_scope = date_stamp + '/' + region + '/' + service + '/' + 'aws4_request'
string_to_sign = algorithm + '\n' + amz_date + '\n' + credential_scope + '\n' + hashlib.sha256(
canonical_request.encode('utf-8')).hexdigest()
# ************* TASK 3: CALCULATE THE SIGNATURE *************
# Create the signing key using the function defined above.
signing_key = getSignatureKey(secret_key, date_stamp, region, service)
# Sign the string_to_sign using the signing_key
signature = hmac.new(signing_key, (string_to_sign).encode('utf-8'), hashlib.sha256).hexdigest()
# ************* TASK 4: ADD SIGNING INFORMATION TO THE REQUEST *************
# Put the signature information in a header named Authorization.
authorization_header = algorithm + ' ' + 'Credential=' + access_key + '/' + credential_scope + ', ' + 'SignedHeaders=' + signed_headers + ', ' + 'Signature=' + signature
# # Python note: The 'host' header is added automatically by the Python 'requests' library.
headers = {'Content-Type': content_type,
'X-Amz-Date': amz_date,
'Authorization': authorization_header}
# ************* SEND THE REQUEST *************
print('\nBEGIN REQUEST++++++++++++++++++++++++++++++++++++')
print( 'Request URL = ' + endpoint)
r = requests.post(endpoint, data=request_parameters, headers=headers)
print('\nRESPONSE++++++++++++++++++++++++++++++++++++')
print('Response code: %d\n' % r.status_code)
print(r.text)
For most use cases, I'd recommend leaning on a library for this. Since you're using the requests library, aws-requests-auth is an attractive choice.
If you really are interested in implementing the signing process yourself, this test suite may help you figure out what's going wrong.
The only thing I noticed looking at your code:
When I use aws-requests-auth, my call looks like this:
response = requests.post('https://<api-endpoint>',
auth=auth,
data= json.dumps({'payload': 'data'}))
Just wanted to call out the json.dumps(), which stringifies my params object. Amazon's sample also uses a string for the request_parameters.
I'm not sure where payload comes from in your code, but make sure you're turning it into a string if it isn't already one.
Update
I looked at your code more closely, and you need to implement Step 2 under Task 1 - Create canonical URI. This URI is indeed "/" for the dynamo call, but yours should look something like this:
# Step 2: Create canonical URI--the part of the URI from domain to query
# string (use '/' if no path)
canonical_uri = '/thing-types/GasSensor'
Also, this request will succeed with or without a Content-Type header. So this:
content_type='application/json'
canonical_headers = 'content-type:' + content_type + '\n' + 'host:' + host + '\n' + 'x-amz-date:' + amz_date + '\n'
signed_headers = 'content-type;host;x-amz-date'
And this:
canonical_headers = 'host:' + host + '\n' + 'x-amz-date:' + amz_date + '\n'
signed_headers = 'host;x-amz-date'
Both work for me.
Sample code
import boto3
glacier = boto3.resource('glacier')
job = glacier.Job(accountID, vaultlist[0], id=joblist[0])
r = job.get_output()
print(r0['body'])
That print only yields botocore.response.StreamingBody at 0xsnip
r0['body'] should be the inventory in CSV format, but I can't figure out how to get to it. I spent a bit of time trying to us io to read in the steam and either that is not the right way or I did it wrong. Can you point me in the right direction?
Thanks!
Here's a solution that worked for me to save a glacier archive that showed up as a StreamingBody to a file. This in particular was an mp3 file.
import boto3
glacier = boto3.resource('glacier')
job = glacier.Job(accountID, vaultName, jobID)
r = job.get_output()
f1 = open('my file',"wb")
f1.write(r['body'].read())
f1.close
OK I couldn't get the other way to work at all, mostly my own lack of skills I'm sure. But I was able to use the HTTP GET to download the inventory into a file. This is how I did that. You will see lots of I had two vaults, one job each, you could modify this and loop in other ways or just use [0] for both lists if you have one vault and one job, but the important part is the sample from Amazon EC2 that I modified to retrieve the Inventory from a completed Glacier Job.
I know my code it not very well written, but it worked for my one-shot need.
Hope this is helpful to others.
import requests, sys, os, hashlib, hmac, json
from datetime import datetime
# ************* REQUEST VALUES *************
method = 'GET'
service = 'glacier'
region = '<YOUR_REGION'
host = 'glacier.' + region + '.amazonaws.com'
endpoint = 'https://glacier.' + region + '.amazonaws.com'
request_parameters = ''
accountid = '<YOUR_ACCOUNT_ID'
vaultlist = ["VAULT_ONE", "VAULT_TWO"]
joblist = ['JOB_ID_ONE',
'JOB_ID_TWO']
rangelist = ['JOB_SIZE_ONE',
'JOB_SIZE_TWO',]
url0 = "/" + accountid + "/vaults/" + vaultlist[0] + "/jobs/" + joblist[0] + "/output"
url1 = "/" + accountid + "/vaults/" + vaultlist[1] + "/jobs/" + joblist[1] + "/output"
filename =['archive0.json', 'archive1.json'] #filenames
# Key derivation functions. See:
# http://docs.aws.amazon.com/general/latest/gr/signature-v4-examples.html#signature-v4-examples-python
def sign(key, msg):
return hmac.new(key, msg.encode('utf-8'), hashlib.sha256).digest()
def getSignatureKey(key, dateStamp, regionName, serviceName):
kDate = sign(('AWS4' + key).encode('utf-8'), dateStamp)
kRegion = sign(kDate, regionName)
kService = sign(kRegion, serviceName)
kSigning = sign(kService, 'aws4_request')
return kSigning
# Read AWS access key from env. variables or configuration file. Best practice is NOT
# to embed credentials in code.
access_key = os.environ.get('AWS_ACCESS_KEY')
secret_key = os.environ.get('AWS_SECRET_KEY')
if access_key is None or secret_key is None:
print('No access key is available via your environment variables.')
sys.exit()
# Create a date for headers and the credential string
t = datetime.utcnow()
amzdate = t.strftime('%Y%m%dT%H%M%SZ')
datestamp = t.strftime('%Y%m%d') # Date w/o time, used in credential scope
# ************* TASK 1: CREATE A CANONICAL REQUEST *************
# http://docs.aws.amazon.com/general/latest/gr/sigv4-create-canonical-request.html
# Step 1 is to define the verb (GET, POST, etc.)--already done.
# Step 2: Create canonical URI--the part of the URI from domain to query
# string (use '/' if no path)
canonical_uri = url1
# Step 3: Create the canonical query string. In this example (a GET request),
# request parameters are in the query string. Query string values must
# be URL-encoded (space=%20). The parameters must be sorted by name.
# For this example, the query string is pre-formatted in the request_parameters variable.
canonical_querystring = request_parameters
# Step 4: Create the canonical headers and signed headers. Header names
# and value must be trimmed and lowercase, and sorted in ASCII order.
# Note that there is a trailing \n.
canonical_headers = 'host:' + host + '\n' + 'x-amz-date:' + amzdate + '\n'
# Step 5: Create the list of signed headers. This lists the headers
# in the canonical_headers list, delimited with ";" and in alpha order.
# Note: The request can include any headers; canonical_headers and
# signed_headers lists those that you want to be included in the
# hash of the request. "Host" and "x-amz-date" are always required.
signed_headers = 'host;x-amz-date'
# Step 6: Create payload hash (hash of the request body content). For GET
# requests, the payload is an empty string ("").
payload_hash = hashlib.sha256("".encode()).hexdigest()
# Step 7: Combine elements to create create canonical request
canonical_request = method + '\n' + canonical_uri + '\n' + canonical_querystring + '\n' + canonical_headers +\
'\n' + signed_headers + '\n' + payload_hash
# ************* TASK 2: CREATE THE STRING TO SIGN*************
# Match the algorithm to the hashing algorithm you use, either SHA-1 or
# SHA-256 (recommended)
algorithm = 'AWS4-HMAC-SHA256'
credential_scope = datestamp + '/' + region + '/' + service + '/' + 'aws4_request'
string_to_sign = algorithm + '\n' + amzdate + '\n' + credential_scope + '\n' + \
hashlib.sha256(canonical_request.encode()).hexdigest()
# ************* TASK 3: CALCULATE THE SIGNATURE *************
# Create the signing key using the function defined above.
signing_key = getSignatureKey(secret_key, datestamp, region, service)
# Sign the string_to_sign using the signing_key
signature = hmac.new(signing_key, string_to_sign.encode('utf-8'), hashlib.sha256).hexdigest()
# ************* TASK 4: ADD SIGNING INFORMATION TO THE REQUEST *************
# The signing information can be either in a query string value or in
# a header named Authorization. This code shows how to use a header.
# Create authorization header and add to request headers
authorization_header = algorithm + ' ' + 'Credential=' + access_key + '/' + credential_scope + ', ' +\
'SignedHeaders=' + signed_headers + ', ' + 'Signature=' + signature
# The request can include any headers, but MUST include "host", "x-amz-date",
# and (for this scenario) "Authorization". "host" and "x-amz-date" must
# be included in the canonical_headers and signed_headers, as noted
# earlier. Order here is not significant.
# Python note: The 'host' header is added automatically by the Python 'requests' library.
# headers = {'x-amz-date':amzdate, 'Authorization':authorization_header}
headers0 = {'x-amz-date': amzdate,
'Authorization': authorization_header,
'x-amz-glacier-version': '2012-06-01',
'Range': '0 - ' + rangelist[0],
}
headers1 = {'x-amz-date': amzdate,
'Authorization': authorization_header,
'x-amz-glacier-version': '2012-06-01',
'Range': rangelist[1],
}
headers = headers1
# ************* SEND THE REQUEST *************
request_url = endpoint + url1
print(url0)
print('\nBEGIN REQUEST++++++++++++++++++++++++++++++++++++')
print('Request URL: ' + request_url + '\n')
print('Headers: ' + json.dumps(headers))
print('Auth : ' + authorization_header + '\n' )
r = requests.get(request_url, headers=headers, stream = True)
print('\nRESPONSE++++++++++++++++++++++++++++++++++++')
print('Response code: %d\n' % r.status_code)
# print(r.text) #This is in the original Sample and useful for debugging. But not if your inventory is large.
# *********** Write it to file ***********
f = open(filename[1], mode='w')
f.write(r.text)
f.close()