We Keep Coding

Amazon Selling Partner API Signature

method = 'GET'
service = 'execute-api'
user_agent = 'My Selling Tool/2.0 (Language=Python3; Platform=Windows/10)'
region = 'us-east-1'
host = 'sellingpartnerapi-na.amazon.com'
endpoint = 'https://sellingpartnerapi-na.amazon.com'
request_parameters = '/fba/inbound/v0/shipments/shipmentId1/preorder/confirm?MarketplaceId=ATVPDKIKX0DER&NeedByDate=2020-10-10'
acess_token = 'xxx'
access_key = 'xxx'
secret_key = 'xxx'
def sign(key, msg):
return hmac.new(key, msg.encode('utf-8'), hashlib.sha256).digest()
def getSignatureKey(key, dateStamp, regionName, serviceName):
kDate = sign(('AWS4' + key).encode('utf-8'), dateStamp)
kRegion = sign(kDate, regionName)
kService = sign(kRegion, serviceName)
kSigning = sign(
def getSignatureKey(key, dateStamp, regionName, serviceName):
t = datetime.datetime.utcnow()
amzdate = t.strftime('%Y%m%dT%H%M%SZ')
datestamp = t.strftime('%Y%m%d')
canonical_uri = '/'
canonical_querystring = request_parameters
canonical_headers = 'host:' + host + '\n' + 'user-agent:' + user_agent + '\n' + 'x-amz-access-token:' + acess_token + '\n' + 'x-amz-date:' + amzdate
signed_headers = 'host;user-agent;x-amz-access-token'
payload_hash = hashlib.sha256(('').encode('utf-8')).hexdigest()
canonical_request = method + '\n' + canonical_uri + '\n' + canonical_querystring + '\n' + canonical_headers + '\n' + signed_headers + '\n' + payload_hash
algorithm = 'AWS4-HMAC-SHA256'
credential_scope = datestamp + '/' + region + '/' + service + '/' + 'aws4_request'
string_to_sign = algorithm + '\n' + amzdate + '\n' + credential_scope + '\n' + hashlib.sha256(canonical_request.encode('utf-8')).hexdigest()
signing_key = getSignatureKey(secret_key, datestamp, region, service)
signature = hmac.new(signing_key, (string_to_sign).encode(
'utf-8'), hashlib.sha256).hexdigest()
authorization_header = algorithm + ' ' + 'Credential=' + access_key + '/' + credential_scope + ', ' + 'SignedHeaders=' + signed_headers + ', ' + 'Signature=' + signature
headers = {'Authorization': authorization_header,
'host':host,
'user-agent': user_agent,
'x-amz-access-token': acess_token,
'x-amz-date': amzdate}
request_url = endpoint + '' + canonical_querystring
r = requests.get(request_url, headers=headers)
I follow sp-api and signatureV4 but got response "InvalidSignature"
this error is occur on Step 4. Create and sign your request.
I have no idea for 2 months, I guess the problem is about headers ?
Does my code need to be corrected or the problem is not here ?

You can use pip install aws-requests-auth There is Pip install link
from aws_requests_auth.aws_auth import AWSRequestsAuth
import json
import sys
AWS_AUTH = AWSRequestsAuth(
aws_access_key=config[0]["aws_access_key"],
aws_secret_access_key=config[0]["aws_secret_access_key"],
aws_host=config[0]["aws_host"],
aws_region=config[0]["aws_region"],
aws_service=config[0]["aws_service"],
)
AUTH_URL = config[0]["AUTH_URL"]
REFRESH_TOKEN = config[0]["REFRESH_TOKEN"]
CLIENT_ID = config[0]["CLIENT_ID"]
CLIENT_SECRET = config[0]["CLIENT_SECRET"]
AUTH_BODY = {
"grant_type": "refresh_token",
"refresh_token": REFRESH_TOKEN,
"client_id": CLIENT_ID,
"client_secret": CLIENT_SECRET,
}
headers = {
"content-type": "application/x-www-form-urlencoded;charset=UTF-8",
"host": "sellingpartnerapi-fe.amazon.com",
}
auth_login = requests.post(AUTH_URL, headers=headers, data=AUTH_BODY)
response = auth_login.json()
reponseCode = auth_login.status_code
if reponseCode == 200:
print("====== AWS AUTH SIGN ====== OK", reponseCode)
return response["access_token"], AWS_AUTH, config[0]
else:
print("AUTH SIGN Error", response)
return False

"AWS4 Signing request signature does not match" error

I followed the steps from here for generating a canonical string and string to sign for the AWS4 SDK. And I am getting 403. This is what I get when I print the response.text:
{\"message\":\"The request signature we calculated does not match the signature you provided. Check your AWS Secret Access Key and signing method. Consult the service documentation for details.
The Canonical String for this request should have been
\\'PUT
/dev/trial
content-type:application/x-www-form-urlencoded
host:myendpoint.execute-api.us-west-2.amazonaws.com
x-amz-date:20190918T002703Z
content-type;host;x-amz-date
402d04afaaf71664b4820123456789bda0df4601423fe13cc851b475798016b5\\'
The String-to-Sign should have been
\\'AWS4-HMAC-SHA256
20190918T002703Z
20190918/us-west-2/execute-api/aws4_request
55f919eb5d745c06760eea01da0123456789b3b1ac1cf2bf0627701d06db0780\\'
\"}
I also tried to print canonical string and string-to-sign which I calculate and they are as follows:
Actual canonical request:
PUT
/dev/trial
content-type:application/x-www-form-urlencoded
host:myendpoint.execute-api.us-west-2.amazonaws.com
x-amz-date:20190918T002703Z
content-type;host;x-amz-date
402d04afaaf71664b4820123456789bda0df4601423fe13cc851b475798016b5
Actual string to sign:
AWS4-HMAC-SHA256
20190918T002703Z
20190918/us-west-2/execute-api/aws4_request
55f919eb5d745c06760eea01da0123456789b3b1ac1cf2bf0627701d06db0780
To me at least, what AWS has calculated and what I am calculating are exactly the same. Then I am still getting the "signature does not match" error?
I did take a look at similar question on SO. All of them had some issue with the request, e.g. missing \n or different date formats. I don't believe I have those issues.
If it helps, here is how I am creating some of the components of signature:
canonical_request = method + '\n' + canonical_uri + '\n' + canonical_querystring + '\n' + canonical_headers + '\n' + signed_headers + '\n' + payload_hash
algorithm = 'AWS4-HMAC-SHA256'
credential_scope = datestamp + '/' + region + '/' + service + '/' + 'aws4_request'
string_to_sign = algorithm + '\n' + amzdate + '\n' + credential_scope + '\n' + hashlib.sha256(
canonical_request.encode('utf-8')).hexdigest()
authorization_header = algorithm + ' ' + 'Credential=' + access_key + '/' + credential_scope + ', ' + 'SignedHeaders=' + signed_headers + ', ' + 'Signature=' + signature
headers = {'x-amz-date': amzdate, 'Authorization': authorization_header, 'host': host,
'content-type': content_type, 'x-api-key': api_key}
Would be happy to include more details if that helps solving this issue.

Error when connecting to azure-datalakes using continuation token

Im currently trying list files/directories inside of adls2 using a continuation token (currently our folder has over 5000 files). I am able to send my first request, however receive a 403 error (presumably meaning incorrect formatting) when trying to connect with the continuation token in the response, and unsure what formatting problems could be causing this error.
I have currently tried removing the = sign at the end of the key for the uri, seeing as that was the problem for someone else. I had also tried creating a header for the continuation inside of my request with no luck.
adls_request is the main function. This gets run twice, once for the initial request, second for the continuation. Currently I have the continuation set up inside the uri and signature.
def gen_signature(request_time, api_version, storage_account_name, file_system_name, storage_account_key, signature_params):
string_params = {
'verb': 'GET',
'Content-Encoding': '',
'Content-Language': '',
'Content-Length': '',
'Content-MD5': '',
'Content-Type': '',
'Date': '',
'If-Modified-Since': '',
'If-Match': '',
'If-None-Match': '',
'If-Unmodified-Since': '',
'Range': '',
'CanonicalizedHeaders': 'x-ms-date:' + request_time + '\nx-ms-version:' + api_version,
'CanonicalizedResource': '/' + storage_account_name+'/'+file_system_name+signature_params
}
string_to_sign = (string_params['verb'] + '\n'
+ string_params['Content-Encoding'] + '\n'
+ string_params['Content-Language'] + '\n'
+ string_params['Content-Length'] + '\n'
+ string_params['Content-MD5'] + '\n'
+ string_params['Content-Type'] + '\n'
+ string_params['Date'] + '\n'
+ string_params['If-Modified-Since'] + '\n'
+ string_params['If-Match'] + '\n'
+ string_params['If-None-Match'] + '\n'
+ string_params['If-Unmodified-Since'] + '\n'
+ string_params['Range'] + '\n'
+ string_params['CanonicalizedHeaders']+'\n'
+ string_params['CanonicalizedResource'])
signed_string = base64.b64encode(hmac.new(base64.b64decode(storage_account_key), msg=string_to_sign.encode('utf-8'), digestmod=hashlib.sha256).digest()).decode()
return signed_string
def create_headers(request_time, api_version, storage_account_name, signed_string):
headers = {
'x-ms-date' : request_time,
'x-ms-version' : api_version,
'Authorization' : ('SharedKey ' + storage_account_name + ':' + signed_string)
}
return headers
def create_url(storage_account_name, file_system_name, url_params):
url = ('https://' + storage_account_name + '.dfs.core.windows.net/'+file_system_name+url_params)
return url
def set_optional_params(list_dir, file_dir, token_continuation):
if token_continuation != '':
token_continuation_sig = '\ncontinuation:'+token_continuation
token_continuation_url = '&continuation='+token_continuation[:-1]
else:
token_continuation_sig = ''
token_continuation_url = ''
print token_continuation_sig
print token_continuation_url
if list_dir:
print type(token_continuation)
signature_params = '\ndirectory:'+file_dir+'\nrecursive:true'+token_continuation_sig+'\nresource:filesystem'
url_params = '?directory='+file_dir+'&recursive=true'+token_continuation_url+'&resource=filesystem'
return signature_params, url_params
else:
signature_params = ''
url_params = ''
return signature_params, url_params
def get_request_time():
return datetime.datetime.utcnow().strftime('%a, %d %b %Y %H:%M:%S GMT')
def adls_request(list_dir,
file_system_name,
file_dir = '',
storage_account_name = 'account_name',
storage_account_key = '123456789==',
api_version = '2018-11-09',
token_continuation = ''):
signature_params, url_params = set_optional_params(list_dir, file_dir, token_continuation)
request_time = get_request_time()
signature = gen_signature(request_time, api_version, storage_account_name, file_system_name, storage_account_key, signature_params)
headers = create_headers(request_time, api_version, storage_account_name, signature)
url = create_url(storage_account_name, file_system_name, url_params)
r = requests.get(url, headers = headers)
return r
I expect the response output to come up 200, containing the rest of the files inside the directory, but still am currently receiving 403 error.

Please try the code below, I use python 3.7 for the test:
import requests
import datetime
import hmac
import hashlib
import base64
import urllib.parse
def gen_signature(request_time, api_version, storage_account_name, file_system_name, storage_account_key, signature_params):
string_params = {
'verb': 'GET',
'Content-Encoding': '',
'Content-Language': '',
'Content-Length': '',
'Content-MD5': '',
'Content-Type': '',
'Date': '',
'If-Modified-Since': '',
'If-Match': '',
'If-None-Match': '',
'If-Unmodified-Since': '',
'Range': '',
'CanonicalizedHeaders': 'x-ms-date:' + request_time + '\nx-ms-version:' + api_version,
'CanonicalizedResource': '/' + storage_account_name + '/' + file_system_name + signature_params,
}
string_to_sign = (string_params['verb'] + '\n'
+ string_params['Content-Encoding'] + '\n'
+ string_params['Content-Language'] + '\n'
+ string_params['Content-Length'] + '\n'
+ string_params['Content-MD5'] + '\n'
+ string_params['Content-Type'] + '\n'
+ string_params['Date'] + '\n'
+ string_params['If-Modified-Since'] + '\n'
+ string_params['If-Match'] + '\n'
+ string_params['If-None-Match'] + '\n'
+ string_params['If-Unmodified-Since'] + '\n'
+ string_params['Range'] + '\n'
+ string_params['CanonicalizedHeaders']+'\n'
+ string_params['CanonicalizedResource'])
signed_string = base64.b64encode(hmac.new(base64.b64decode(storage_account_key), msg=string_to_sign.encode('utf-8'), digestmod=hashlib.sha256).digest()).decode()
return signed_string
def create_headers(request_time, api_version, storage_account_name, signed_string):
headers = {
'x-ms-date' : request_time,
'x-ms-version' : api_version,
'Authorization' : ('SharedKey ' + storage_account_name + ':' + signed_string)
}
return headers
def create_url(storage_account_name, file_system_name, url_params):
url = ('https://' + storage_account_name + '.dfs.core.windows.net/'+file_system_name+url_params)
return url
def set_optional_params(list_dir, file_dir, token_continuation):
if token_continuation != '':
token_continuation_sig = '\ncontinuation:'+ token_continuation
#Note that since the continuation token ended with =, you should encode the token, then add to url.
token_continuation_url = '&continuation='+urllib.parse.quote_plus(token_continuation)
else:
token_continuation_sig = ''
token_continuation_url = ''
#print(token_continuation_sig)
#print(token_continuation_url)
if list_dir:
if token_continuation !='':
signature_params = token_continuation_sig + '\ndirectory:' + file_dir + '\nrecursive:true' + '\nresource:filesystem'
url_params = '?directory=' + file_dir + '&recursive=true' + token_continuation_url + '&resource=filesystem'
return signature_params, url_params
else:
signature_params = '\ndirectory:' + file_dir + '\nrecursive:true' + '\nresource:filesystem'
url_params = '?directory=' + file_dir + '&recursive=true' + '&resource=filesystem'
return signature_params, url_params
else:
signature_params = ''
url_params = ''
return signature_params, url_params
def get_request_time():
return datetime.datetime.utcnow().strftime('%a, %d %b %Y %H:%M:%S GMT')
def adls_request(list_dir,
file_system_name,
file_dir = '',
storage_account_name = 'account_name',
storage_account_key = '123456789==',
api_version = '2018-11-09',
token_continuation = ''):
signature_params,url_params = set_optional_params(list_dir, file_dir, token_continuation)
request_time = get_request_time()
signature = gen_signature(request_time, api_version, storage_account_name, file_system_name, storage_account_key, signature_params)
headers = create_headers(request_time, api_version, storage_account_name, signature)
url = create_url(storage_account_name, file_system_name, url_params)
print(url)
r = requests.get(url, headers = headers)
return r
if __name__ == '__main__':
list_dir = True
file_system_name ="dd1"
file_dir="testfile"
storage_account_name = 'xxx'
storage_account_key = 'xxxx'
api_version = '2018-11-09'
token_continuation = ''
print("******First Time without continuation token******")
#The 1st time to get files which can be up to 5000
r = adls_request(list_dir,file_system_name,file_dir,storage_account_name,storage_account_key,api_version,token_continuation)
print(r)
print("\n\n******Sencond Time with continuation token******")
#Then 2nd time to get files with continuation token
#when files are more than 5000, you will get a continuation token
if 'x-ms-continuation' in r.headers:
token_continuation=r.headers["x-ms-continuation"]
print("continuation token: "+token_continuation)
r = adls_request(list_dir,file_system_name,file_dir,storage_account_name,storage_account_key,api_version,token_continuation)
print(r)
Test result:
I have 6000 files in the directory, and note that if you get the continuation token(if files in directory are more than 5000, otherwise no token returned), you should encode the token, then add the encoded token to the url.
This is just a simple test, and please feel free to change the code to meet your need.

Initiate Multipart upload to S3 Version 4 Method Not allowed err.code 405

I have tried everything. It works well for GET AND PUT requests. But with POST (to initiate multipart request) it throws various error like 405 Method not allowed or Signature Mismatch error. Any help is appreciated and welcome. Please if anyone have any idea throw some hints.
If anyone can edit this python code to get it working i would really appreciate that. Thanks in advance.
import sys, os, base64, datetime, hashlib, hmac
import requests # pip install requests
method = 'POST'
service = 's3'
host = 'freemedianews.s3.amazonaws.com'
region = 'ap-southeast-1'
endpoint = 'https://freemedianews.s3-ap-southeast-1.amazonaws.com'
request_parameters = ''
def sign(key, msg):
return hmac.new(key, msg.encode('utf-8'), hashlib.sha256).digest()
def getSignatureKey(key, dateStamp, regionName, serviceName):
kDate = sign(('AWS4' + key).encode('utf-8'), dateStamp)
kRegion = sign(kDate, regionName)
kService = sign(kRegion, serviceName)
kSigning = sign(kService, 'aws4_request')
return kSigning
access_key = '*******'
secret_key = '*******'
if access_key is None or secret_key is None:
print 'No access key is available.'
sys.exit()
t = datetime.datetime.utcnow()
amzdate = '20180621T151517Z'
datestamp = '20180621' # Date w/o time, used in credential scope
request.html
canonical_uri = '/a/message/1200/1200.png'
content_type = "multipart/form-data"
request_parameters variable.
canonical_querystring = request_parameters
payload_hash = hashlib.sha256('').hexdigest()
canonical_headers ='content-type:' + content_type + '\n' + 'host:' + host + '\n' + 'x-amz-content-sha256:' + payload_hash + '\n' + 'x-amz-date:' + amzdate + '\n'
signed_headers = 'content-type;host;x-amz-content-sha256;x-amz-date'
canonical_request = method + '\n' + canonical_uri + '\n' + canonical_querystring + '\n' + canonical_headers + '\n' + signed_headers + '\n' + payload_hash
algorithm = 'AWS4-HMAC-SHA256'
credential_scope = datestamp + "/" + region + '/' + service + '/' + 'aws4_request'
credential_scope_final = access_key + "/" + datestamp + "/" + region + '/' + service + '/' + 'aws4_request'
string_to_sign = algorithm + '\n' + amzdate + '\n' + credential_scope + '\n' + hashlib.sha256(canonical_request).hexdigest()
signing_key = getSignatureKey(secret_key, datestamp, region, service)
print "signing_key -----" + signing_key
signature = hmac.new(signing_key, (string_to_sign).encode('utf-8'), hashlib.sha256).hexdigest()
print "signature -----" + signature
authorization_header = algorithm + ' ' + 'Credential=' + credential_scope_final + ', ' + 'SignedHeaders=' + signed_headers + ', ' + 'Signature=' + signature
headers = {'Content-Type': content_type, 'Host': host, 'X-Amz-Content-Sha256': payload_hash, 'X-Amz-Date':amzdate, 'Authorization':authorization_header}
print authorization_header + "---this"
request_url = endpoint + canonical_uri
r = requests.post(request_url, data=request_parameters, headers=headers)
print r
print '\nRESPONSE++++++++++++++++++++++++++++++++++++'
print 'Response code: %d\n' % r.status_code
print r.text

Upload Files to S3 using POST from different servers with python

I wan to achieve below. I have 10 log servers and 10 web servers in different locations. Every location has a pair i.e 1 log server and 1 web server. In S3 for every pair of servers, there is a bucket for storing logs like Location 1, Location 2, location 3.
I want to upload logs from every location to its respective buckets. I can do that with awscli but for that i have to create Iam user for every location and attach a s3 policy and put in access keys and secret keys in every location. I do not want this approach.
Instead, i was thinking that i would embed my access keys and secret access keys in every web server and then using AWS Signature version 4 , i would generate a signature for every file with respect to its bucket and upload to S3.
import sys, os, base64, datetime, hashlib, hmac
from cassandra.cluster import Cluster
from datetime import datetime, timedelta
import json
import requests
import logging
LOGGER = None
def sign(secret_key, msg):
return hmac.new(secret_key, msg.encode("utf-8"),hashlib.sha256).digest()
def getSignatureKey(secret_key, date_stamp, regionName, serviceName):
kDate = sign(('AWS4' + secret_key).encode('utf-8'), date_stamp)
kRegion = sign(kDate, regionName)
kService = sign(kRegion, serviceName)
kSigning = sign(kService, 'aws4_request')
return kSigning
def S3UploadPolicy(date_stampfiso,customer_name, amz_date,credential):
params = {}
params['expiration'] = date_stampfiso
params['conditions'] = [{'bucket': customer_name},
{'acl': 'private'},
{'success_action_status': '201'},
['starts_with', '', ''],
{'x-amz-algorithm': 'AWS4-HMAC-SHA256'},
{'x-amz-credential': credential},
{'x-amz-date': amz_date}]
params = json.dumps(params)
return params
def S3Upload(access_key,date_stamp,date_stampfiso,customer_name, amz_date, regionName, secret_key, serviceName, filename):
host = '<bucketname>.s3.amazonaws.com'
endpoint_url = 'http://' + customer_name + '.s3.amazonaws.com'
#content_type = 'multipart/form-data; charset=UTF-8'
content_type = 'text/plain'
#method = 'POST'
method = 'PUT'
canonical_uri = '/' + customer_name
canonical_querystring = filename
canonical_headers = 'content-type:' + content_type + '\n' + 'host:' + host + '\n' + 'x-amz-date:' + amz_date + '\n'
credential_scope = date_stamp + '/' + regionName + '/' + serviceName + '/' + 'aws4_request'
signed_headers = 'content-type;host;x-amz-content-sha256;x-amz-date'
policy = S3UploadPolicy(date_stampfiso,customer_name, amz_date,credential_scope)
policyBase64 = base64.b64encode(policy)
payload_hash = hashlib.sha256(policyBase64).hexdigest()
canonical_request = method + '\n' + canonical_uri + '\n' + canonical_querystring + '\n' + canonical_headers + '\n' + signed_headers + '\n' + payload_hash
xamzalgorithm = 'AWS4-HMAC-SHA256'
algorithm = xamzalgorithm
string_to_sign = algorithm + '\n' + amz_date + '\n' + credential_scope + '\n' + hashlib.sha256(canonical_request).hexdigest()
signing_key = getSignatureKey(secret_key, date_stamp, regionName, serviceName)
signature = hmac.new(signing_key, (string_to_sign).encode('utf-8'), hashlib.sha256).hexdigest()
authorization_header = algorithm + ' ' + 'Credential=' + access_key + '/' + credential_scope + ', ' + 'SignedHeaders=' + signed_headers + ', ' + 'Signature=' + signature
print authorization_header
#sys.exit(1)
headers = {
'content-type': content_type,
'x-amz-date': amz_date,
'authorization': authorization_header,
'x-amz-content-sha256': payload_hash
}
try:
print '\nBEGIN REQUEST++++++++++++++++++++++++++++++++++++'
print 'Request URL = ' + endpoint_url
r = requests.put(endpoint_url, headers=headers)
print '\nRESPONSE++++++++++++++++++++++++++++++++++++'
print 'Response code: %d\n' % r.status_code
print r.text
except Exception as e:
LOGGER.error(e)
def main():
global LOGGER
msg = ''
access_key = 'xxxxxxxxxxxx'
secret_key = 'xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx'
customer_name = 'abc-test2'
regionName = 'us-west-2'
serviceName = 's3'
filename = '/home/abc/abc.pem'
logging.basicConfig(
level=logging.DEBUG,
format='%(asctime)s | %(levelname)s | %(module)s [%(process)d %(thread)d] | [%(filename)s:%(lineno)s - %(funcName)s() ] | \n%(message)s')
LOGGER = logging.getLogger(__name__)
## Calculate Date
t = datetime.utcnow()
amz_date = t.strftime('%Y%m%dT%H%M%SZ')
date_stamp = t.strftime('%Y%m%d')
date_stampf = datetime.now() + timedelta(hours=24)
amz_date_future = date_stampf.strftime('%Y%m%dT%H%M%SZ')
date_stampfiso = date_stampf.isoformat()
S3Upload(access_key, date_stamp, date_stampfiso, customer_name, amz_date, regionName, secret_key, serviceName,
filename)
if __name__ == '__main__':
main()