Related
Hoping to get some help on making calls to the Coinbase Pro API.
I created a key, noted my phasephrase, key and secret, and am running the below Python script. The response I get is "invalid signature".
On the CBPro documentation site, when I try running it with my credentials on this page, I get a "Sorry, you couldn't be authenticated with those credentails" message.
I've seen some sources that encode to base64 and have success, and others that don't, but neither works for me. What am I don't wrong?
Code:
import requests
import time
import base64
import json
url = "https://api.exchange.coinbase.com/accounts/account_id/transfers?limit=100"
key = "key"
secret = "secret"
passphrase = "pass"
timestamp = str(time.time())
headers = {
"Accept": "application/json",
"cb-access-key": key,
"cb-access-passphrase": passphrase,
"cb-access-sign": encodedData,
"cb-access-timestamp": timestamp
}
response = requests.request("GET", url, headers=headers)
print(response.text)
Signing a request is probably the worst part of the coinbase api.
Here is the documentation for it. some things to note:
signature is only good for 30 seconds so you have to hurry and copy / paste the encoded data and timestamp into the docs form.
the timestamp has to be the same value you use in the encodedData.
the non-pro api signs a little different so make sure you're on the right set of docs. This one I don't think works for pro.
If you're still having trouble authenticating your requests to coinbase here's an example of what i'm using. You might have to change a few things but does the job.
What you're looking for in this example are the HMACAuth & CoinbaseSession classes for your particular need.
# -*- coding: UTF-8 -*-
from base64 import b64encode, b64decode
from collections import namedtuple
from datetime import datetime, timezone
from hashlib import sha256
from hmac import HMAC
from json import loads, JSONDecodeError
from types import SimpleNamespace
from typing import Union, Generator
from requests import Session
from requests.adapters import HTTPAdapter
from requests.auth import AuthBase
from requests.exceptions import HTTPError
from requests.models import PreparedRequest
from requests.utils import to_native_string
from urllib3.util.retry import Retry
EXCHANGE: str = r"https://api.exchange.coinbase.com"
ACCOUNT = namedtuple(
"ACCOUNT",
(
"id",
"currency",
"balance",
"hold",
"available",
"profile_id",
"trading_enabled",
)
)
PRODUCT = namedtuple(
"PRODUCT",
(
"id",
"base_currency",
"quote_currency",
"base_min_size",
"base_max_size",
"quote_increment",
"base_increment",
"display_name",
"min_market_funds",
"max_market_funds",
"margin_enabled",
"fx_stablecoin",
"max_slippage_percentage",
"post_only",
"limit_only",
"cancel_only",
"trading_disabled",
"status",
"status_message",
"auction_mode",
)
)
def encode(value: Union[str, bytes]) -> bytes:
"""Encode the string `value` with UTF-8."""
if isinstance(value, str) is True:
value = value.encode("UTF-8")
return value
def decode(value: Union[bytes, str]) -> str:
"""Decode the bytes-like object `value` with UTF-8."""
if isinstance(value, bytes) is True:
value = value.decode("UTF-8")
return value
def req_time():
"""POSIX timestamp as float. Number of seconds since Unix Epoch in UTC."""
utc = get_utc()
return utc.timestamp()
def get_utc() -> datetime:
"""Construct a datetime object with UTC time zone info."""
return datetime.now(timezone.utc)
class TimeoutHTTPAdapter(HTTPAdapter):
"""Custom HTTP adapter with timeout capability."""
def __init__(self, *args, **kwargs):
self._timeout = kwargs.pop("timeout")
super(TimeoutHTTPAdapter, self).__init__(*args, **kwargs)
def send(self, request, **kwargs):
kwargs.update({"timeout": self._timeout})
return super(TimeoutHTTPAdapter, self).send(request, **kwargs)
class CoinbaseSession(Session):
"""Coinbase Session handle."""
_headers: dict = {
"Accept": "application/json",
"Content-Type": "application/json",
"Accept-Charset": "utf-8",
}
#staticmethod
def http_adapter(retries: int = 3, backoff: int = 1, timeout: int = 30):
return TimeoutHTTPAdapter(
max_retries=Retry(total=retries, backoff_factor=backoff),
timeout=timeout
)
def __init__(self):
super(CoinbaseSession, self).__init__()
self.headers.update(self._headers)
self.auth = HMACAuth()
self.mount("https://", self.http_adapter())
self.mount("http://", self.http_adapter())
class HMACAuth(AuthBase):
"""Requests signing handle."""
#staticmethod
def __pre_hash(timestamp: Union[str, int, float], request: PreparedRequest) -> bytes:
"""
Create the pre-hash string by concatenating the timestamp with
the request method, path_url and body if exists.
"""
message = f"{timestamp}{request.method.upper()}{request.path_url}"
body = request.body
if body is not None:
message = f"{message}{decode(body)}"
return encode(message)
#staticmethod
def __sign(message: bytes) -> bytes:
"""Create a sha256 HMAC and sign the required `message`."""
key = b64decode(encode(API.SECRET)) # be careful were you keep this!
hmac = HMAC(key=key, msg=message, digestmod=sha256).digest()
return b64encode(hmac)
def __call__(self, request: PreparedRequest):
timestamp = req_time()
message = self.__pre_hash(timestamp, request)
cb_access_sign = self.__sign(message)
request.headers.update(
{
to_native_string('CB-ACCESS-KEY'): API.KEY, # be careful where you keep this!
to_native_string('CB-ACCESS-SIGN'): cb_access_sign,
to_native_string('CB-ACCESS-TIMESTAMP'): str(timestamp),
to_native_string('CB-ACCESS-PASSPHRASE'): API.PASSPHRASE, # be careful where you keep this!
}
)
return request
class CoinbaseAPI(object):
"""Coinbase API handle."""
def __init__(self):
self._session = CoinbaseSession()
def request(self, **kwargs):
"""
Send HTTP requests to the Coinbase API.
Raises HTTPError if response is not 200.
"""
print(f"DEBUG: Requesting resource (url = {kwargs.get('url')})")
try:
results = self.__request(**kwargs)
except HTTPError as http_error:
print(f"ERROR: Resource not found!", f"Cause: {http_error}")
else:
print(f"DEBUG: Resource found (url = {kwargs.get('url')})")
return results
def __request(self, **kwargs):
"""
Send HTTP requests to the Coinbase API.
Raises HTTPError if response is not 200.
"""
method = getattr(self._session, kwargs.pop("method"))
response = method(**kwargs)
if response.status_code != 200:
response.raise_for_status()
else:
try:
results = loads(response.text)
except JSONDecodeError as json_error:
print("WARNING: Decoding JSON object failed!", f"Cause: {json_error}")
kwargs.update({"method": method.__name__})
return self.__request(**kwargs)
else:
return results
class Endpoints(object):
"""Coinbase server endpoints."""
_server = None
#staticmethod
def join(*path, **params) -> str:
"""
Construct the resource url by appending all `path`
items to base url and join `params` if any.
"""
url = "/".join(path)
if len(params) > 0:
params = [f"{key}={value}" for key, value in params.items()]
url = f"{url}?{'&'.join(params)}"
return url
class ExchangeEndpoints(Endpoints):
"""Coinbase exchange server endpoints."""
_server = EXCHANGE
def __init__(self):
self.products = self.join(self._server, "products")
self.accounts = self.join(self._server, "accounts")
class Exchange(CoinbaseAPI):
"""Coinbase exchange API client handle."""
_endpoints = ExchangeEndpoints()
def __init__(self):
super(Exchange, self).__init__()
def get_accounts(self) -> Generator:
"""Get a list of trading accounts from the profile of the API key."""
response = self._accounts()
for item in response:
yield ACCOUNT(**item)
def get_account(self, account_id: str) -> ACCOUNT:
"""
Information for a single account. Use this endpoint when you know the account_id.
API key must belong to the same profile as the account.
"""
response = self._accounts(account_id)
return ACCOUNT(**response)
def get_products(self, query: str = None) -> Generator:
"""
Gets a list of available currency pairs for trading.
:param query: `type` query parameter (unknown).
"""
response = self._products(type=query)
for item in response:
yield PRODUCT(**item)
def get_product(self, product_id: str) -> PRODUCT:
"""
Get information on a single product.
:param product_id: The `id` string of the product/currency pair (ex: BTC-USD).
"""
result = self._products(product_id)
return PRODUCT(**result)
def _accounts(self, *path, **params):
"""Access resources from the `accounts` endpoint of the exchange API."""
params = self.clean_params(**params)
_url = self._endpoints.join(
self._endpoints.accounts,
*path,
**params
)
return self.request(method="get", url=_url)
def _products(self, *path, **params):
"""Access resources from the `products` endpoint of the exchange API."""
params = self.clean_params(**params)
_url = self._endpoints.join(
self._endpoints.products,
*path,
**params
)
return self.request(method="get", url=_url)
def clean_params(self, **params) -> dict:
"""Clean `params` by removing None values."""
temp = dict()
for key, value in params.items():
if value is None:
continue
if isinstance(value, dict) is True:
value = self.clean_params(**value)
temp.update({key: value})
return temp
if __name__ == '__main__':
# we're using PBKDF2HMAC (with symmetrically derived encryption key)
# not included in this example
key_vault = KeyVault() # custom class for encrypting and storing secrets to keyring
key_vault.cypher.password(value="your_password", salt="salty_password")
api_credentials = loads(
key_vault.get_password("coinbase", "pro-coinbase-api")
)
# accounts:
ADA: str = "your_crypto_account_id" # ex: 8b9806a4-7395-11ec-9b1a-f02f74d9105d
API = SimpleNamespace(
NAME="pro-coinbase-api",
VERSION="2021-08-27",
KEY=api_credentials.get("key"),
PASSPHRASE=api_credentials.get("passphrase"),
SECRET=api_credentials.get("secret"),
) # not the best example but does the job as long as you don't invite hackers in your PC :)
api = Exchange()
accounts = api.get_accounts()
for account in accounts:
print(account)
account = api.get_account(account_id=ADA)
print(account)
products = api.get_products()
for product in products:
print(product)
product = api.get_product("ATOM-EUR")
print(product)
I'm trying to write a Python wrapper for a cryptocurrency exchange.
#!/usr/bin/python2.7
import hashlib
import hmac
import time
base_url = 'https://api.coinnest.co.kr'
class Coinnest():
def __init__(self, key, secret):
self.key = key
self.secret = secret
def get_balance(self):
url = base_url + '/api/account/balance'
nonce = str(int(time.time())*1000)
key = hashlib.md5(self.secret).hexdigest()
message = 'key={}&nonce={}'.format(self.key, nonce)
signature = hmac.new(key, message, hashlib.sha256).hexdigest()
payload = {'key': key, 'nonce': nonce, 'signature': signature}
r = requests.post(url, data=payload)
return r.json()
coinnest = Coinnest('','')
print coinnest.get_order_history()
Response: u'status': 102, u'msg': u'', u'data': u''
According to the API response description: Code 102 means
Parameter error. Required parameters are missing or in wrong format.
I believe I have all the required parameters of
key
nonce
signature.
Am I delivering the payload in the wrong location or in the wrong format? Unfortunately, their documentation is not very clear and I am a beginner.
Thank you.
The docs are terrible, but it looks like you should be signing your message with md5(secret), and set key to your public key, which is different from md5(secret).
from collections import OrderedDict
key = self.key
secret_md5 = hashlib.md5(self.secret).hexdigest()
signature = hmac.new(secret_md5, message, hashlib.sha256).hexdigest()
payload = OrderedDict([('key', key), ('nonce', nonce), ('signature', signature)])
I also recommend using an ordered dict to enforce parameter order.
I wanna make python client for Last.fm API. I wanna build kind of library.
I managed to get and set a session by getting a session key. Afterwards, I try to call a POST method that requires API_key, api_signature and session key. So I use the APi key I have, same api_signature I used to get the session key and the session key itself.
But I get an "invalid method signature" even though I use the same api_signature for the POST calls.
import json
import webbrowser
from hashlib import md5
import urllib3
class PyLast():
def __init__(self, API_KEY, SECRET, SESSION_KEY=None):
self.__API_KEY__ = API_KEY
self.__SECRET__ = SECRET
self.__SESSION_KEY__ = SESSION_KEY
self.__api_signature__ = None
if SESSION_KEY is None:
self.__is_authorized__ = False
else:
self.__is_authorized__ = True
self.__http__ = urllib3.PoolManager()
def request_token(self):
print("Getting the token...")
url = 'http://ws.audioscrobbler.com/2.0/?method=auth.gettoken&api_key={}&format=json'.format(self.__API_KEY__)
req_response = self.__http__.request('GET', url, headers={'User-Agent' : 'Mozilla/5.0'})
if req_response.status == 200:
json_data = json.loads(req_response.data.decode('utf-8'))
TOKEN = json_data['token']
self.__TOKEN__ = TOKEN
return TOKEN
else:
print("Error with code " + req_response.status)
def authorize(self):
if not self.__is_authorized__:
url = 'http://www.last.fm/api/auth/?api_key={}&token={}'.format(self.__API_KEY__, self.__TOKEN__)
# open browser to authorize app
webbrowser.open(url, new=0, autoraise=True)
# Make sure authorized
self.__is_authorized__ = True
def start_session(self):
if self.__is_authorized__:
data = "api_key{}methodauth.getSessiontoken{}{}" \
.format(self.__API_KEY__, self.__TOKEN__, self.__SECRET__).encode(
encoding='utf-8')
self.__api_signature__ = md5(data).hexdigest()
url = 'http://ws.audioscrobbler.com/2.0/?method=auth.getSession&api_key={}&token={}&api_sig={}&format=json'.format(
self.__API_KEY__, self.__TOKEN__, self.__api_signature__)
req_response = self.__http__.request('GET', url)
if req_response.status == 200:
json_data = json.loads(req_response.data.decode('utf-8'))
session_key = json_data['session']['key']
self.__SESSION_KEY__ = session_key
url = 'http://ws.audioscrobbler.com/2.0/?method=track.love&api_key={}&api_sig={}&sk={}&artist=cher&track=believe&format=json'.format(
self.__API_KEY__, self.__api_signature__, self.__SESSION_KEY__)
req_response = self.__http__.request('POST', url)
return self.__SESSION_KEY__
else:
print("Error with code " + str(req_response.status))
else:
print("Not authorized!")
I found a solution. The problem was that I was using the same parameters used to generate session key to make a POST call. The right way to sign a method for Last.fm API is to build the api_sig from the POST method we want to use. for example, to generate api_sig for track.love we use these parameters:
data = {"api_key": API_KEY,
"method": "track.love",
"track" : "yellow",
"artist" :"coldplay",
"sk" : SESSION_KEY
}
keys = sorted(data.keys())
param = [k+data[k] for k in keys]
param = "".join(param) + SECRET
api_sig = md5(param.encode()).hexdigest() # this api_sig used to sign track.love call.
I've been trying to get this to work for hours now. None of the solutions on other questions are working.
What I'm trying to do is get my list of orders on Amazon. To do this, I am making a call to Amazon MWS. However, this is the error message I am getting:
<?xml version="1.0"?>
<ErrorResponse xmlns="https://mws.amazonservices.com/Orders/2013-09-01">
<Error>
<Type>Sender</Type>
<Code>SignatureDoesNotMatch</Code>
<Message>The request signature we calculated does not match the signature you provided. Check your AWS Secret Access Key and signing method. Consult the service documentation for details.</Message>
</Error>
<RequestID>03686743-15a6-4207-b0b7-316d1e4e5c8f</RequestID>
</ErrorResponse>
To find out what was wrong, I went onto Amazon MWS Scratchpad and used the same values, even the timestamp (I think you have 15 minutes until it expires), and compared the signatures. They were the same. However, my program still got the same error while the scratchpad worked perfectly.
Here is all the code:
import sys, os, base64, datetime, hashlib, hmac, urllib
from time import gmtime, strftime
from requests import request
import xml.etree.ElementTree as ET
def get_timestamp():
"""Return correctly formatted timestamp"""
return strftime("%Y-%m-%dT%H:%M:%SZ", gmtime())
def calc_signature():
"""Calculate signature to send with request"""
sig_data = method + '\n' + domain.replace('https://', '').lower() + '\n' + URI + '\n' + request_description
hmac_obj = hmac.new(str(SECRET_KEY), sig_data, hashlib.sha256)
return urllib.quote(base64.b64encode(hmac_obj.digest()), safe='-_.~')
SECRET_KEY = 'xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx'
AWS_ACCESS_KEY = 'xxxxxxxxxxxxxxxxxxx'
SELLER_ID = 'xxxxxxxxxxxxxxxxxxxxxxx'
MARKETPLACE_ID = 'xxxxxxxxxxxxxxx'
Action = 'ListOrders'
SignatureMethod = 'HmacSHA256'
SignatureVersion = '2'
Timestamp = get_timestamp()
Version = '2013-09-01'
CreatedAfter = '2014-08-26T23:00:57Z' # TODO -1 day
URI = '/Orders/2013-09-01'
domain = 'https://mws.amazonservices.co.uk'
method = 'POST'
payload = {'AWSAccessKeyId': AWS_ACCESS_KEY,
'Action': Action,
'SellerId': SELLER_ID,
'SignatureVersion': SignatureVersion,
'Timestamp': Timestamp,
'Version': Version,
'SignatureMethod': SignatureMethod,
'CreatedAfter': '2014-08-26T23:00:00Z',
'MarketplaceId.Id.1': MARKETPLACE_ID
}
request_description = '&'.join(['%s=%s' % (k, urllib.quote(payload[k], safe='-_.~').encode('utf-8')) for k in sorted(payload)])
sig = calc_signature()
url = '%s%s?%s&Signature=%s' % (domain, URI, request_description, urllib.quote(sig))
headers = {'Host': 'amazonwebservices.co.uk', 'Content-Type': 'text/xml', 'x-amazon-user-agent': 'python-requests/1.2.0 (Language=Python)'}
response = request(method, url, headers=headers)
print response.content
Here's a print of sig_data in calc_signature():
POST
mws.amazonservices.co.uk
/Orders/2013-09-01
AWSAccessKeyId=xxxxxxxxxxxxx&Action=ListOrders&CreatedAfter=2014-08-26T23%3A00%3A00Z&MarketplaceId.Id.1=xxxxxxxxxxxxx&SellerId=xxxxxxxxxxxxxxx&SignatureMethod=HmacSHA256&SignatureVersion=2&Timestamp=2014-08-28T15%3A50%3A34Z&Version=2013-09-01
Here's a print of url:
https://mws.amazonservices.co.uk/Orders/2013-09-01?AWSAccessKeyId=xxxxxxxxxxxx&Action=ListOrders&CreatedAfter=2014-08-26T23%3A00%3A00Z&MarketplaceId.Id.1=xxxxxxxxxxxx&SellerId=xxxxxxxxxxxxx&SignatureMethod=HmacSHA256&SignatureVersion=2&Timestamp=2014-08-28T15%3A50%3A34Z&Version=2013-09-01&Signature=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
I am completely out of ideas at this point. I have triple checked that my secret key, access key, seller ID and marketplace ID are correct.
Any help would be massively, massively appreciated!
I know this is a few months late but I had the exact same issue despite knowing my signature was correct and found out it was in fact the POST execution that was the problem and not the signature. If Amazon can't understand your params, it will throw an HTTP 403 error and tell you your signature is wrong even though it's not. I can't tell you why it works this way but try skipping the requests library and doing the following using urllib.request after building your url instead which worked for me:
#using python 3.4
import urllib.request
#... your code from before...
headers = {'Host': 'mws.amazonservices.com', 'Content-Type': 'text/xml', 'x-amazon-user-agent': 'SomeApp/1.1 (Language=Python)'}
req = urllib.request.Request(method=method,url=url,headers=headers)
response = urllib.request.urlopen(req)
the_page = response.read()
print(the_page)
#As seen here: https://docs.python.org/3/howto/urllib2.html#data
If that doesn't do it, take a second look at how you encoded everything and maybe dabble with urllib.parse.
Happy Amazoning!
Inside your calc_signature function, your safe parameter isn't allowing + and / which are allowed in the base64 output character set. As a result, it is percent-encoding them, and then those percents get further perecnt-encoded as %25s in the query string.
I made some additional changes, so here's all the code:
import sys, os, base64, datetime, hashlib, hmac, urllib
from time import gmtime, strftime
from requests import request
import xml.etree.ElementTree as ET
def get_timestamp():
"""Return correctly formatted timestamp"""
return strftime("%Y-%m-%dT%H:%M:%SZ", gmtime())
def calc_signature(method, domain, URI, request_description, key):
"""Calculate signature to send with request"""
sig_data = method + '\n' + \
domain.lower() + '\n' + \
URI + '\n' + \
request_description
hmac_obj = hmac.new(key, sig_data, hashlib.sha256)
digest = hmac_obj.digest()
return urllib.quote(base64.b64encode(digest), safe='-_+=/.~')
SECRET_KEY = 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX'
AWS_ACCESS_KEY = 'XXXXXXXXXXXXXX'
SELLER_ID = 'XXXXXXXXXXXXX'
MARKETPLACE_ID = 'XXXXXXXXXXXX'
Action = 'ListOrders'
SignatureMethod = 'HmacSHA256'
SignatureVersion = '2'
Timestamp = get_timestamp()
Version = '2013-09-01'
CreatedAfter = '2013-08-26T23:00:57Z'
URI = '/Orders/2013-09-01'
domain = 'mws.amazonservices.com'
proto = 'https://'
method = 'POST'
payload = {
'AWSAccessKeyId': AWS_ACCESS_KEY,
'Action': Action,
'SellerId': SELLER_ID,
'SignatureVersion': SignatureVersion,
'Timestamp': Timestamp,
'Version': Version,
'SignatureMethod': SignatureMethod,
'CreatedAfter': CreatedAfter,
'MarketplaceId.Id.1': MARKETPLACE_ID
}
request_description = '&'.join(['%s=%s' % (k, urllib.quote(payload[k], safe='-_.~').encode('utf-8')) for k in sorted(payload)])
sig = calc_signature(method, domain, URI, request_description, SECRET_KEY)
url = '%s%s?%s&Signature=%s' % \
(proto+domain, URI, request_description, urllib.quote(sig))
headers = {
'Host': domain,
'Content-Type': 'text/xml',
'x-amazon-user-agent': 'python-requests/1.2.0 (Language=Python)'
}
response = request(method, url, headers=headers)
print response.content
I'm having an awfully hard time with Yahoo's authentication/authorization. I've enabled BOSS in my account, set up a payment method, and now I'm trying to run a search using some python code:
import urllib2
import oauth2 as oauth
import time
OAUTH_CONSUMER_KEY = "blahblahblah"
OAUTH_CONSUMER_SECRET = "blah"
def oauth_request(url, params, method="GET"):
params['oauth_version'] = "1.0",
params['oauth_nonce'] = oauth.generate_nonce(),
params['oauth_timestamp'] = int(time.time())
consumer = oauth.Consumer(key=OAUTH_CONSUMER_KEY,
secret=OAUTH_CONSUMER_SECRET)
params['oauth_consumer_key'] = consumer.key
req = oauth.Request(method=method, url=url, parameters=params)
req.sign_request(oauth.SignatureMethod_HMAC_SHA1(), consumer, None)
return req
if __name__ == "__main__":
url = "http://yboss.yahooapis.com/ysearch/web"
req = oauth_request(url, params={"q": "cats dogs"})
req_url = req.to_url()
print req_url
result = urllib2.urlopen(req_url)
I keep getting a urllib2.HTTPError: HTTP Error 401: Unauthorized exception. I can't figure out whether there's something wrong with my key, or the method of signing, or if I'm somehow tampering with my data after signing, or what the deal is. Anyone have suggestions?
I made some small changes to make your example work. See code for comments.
import urllib2
import oauth2 as oauth
import time
OAUTH_CONSUMER_KEY = "blahblahblah"
OAUTH_CONSUMER_SECRET = "blah"
def oauth_request(url, params, method="GET"):
# Removed trailing commas here - they make a difference.
params['oauth_version'] = "1.0" #,
params['oauth_nonce'] = oauth.generate_nonce() #,
params['oauth_timestamp'] = int(time.time())
consumer = oauth.Consumer(key=OAUTH_CONSUMER_KEY,
secret=OAUTH_CONSUMER_SECRET)
params['oauth_consumer_key'] = consumer.key
req = oauth.Request(method=method, url=url, parameters=params)
req.sign_request(oauth.SignatureMethod_HMAC_SHA1(), consumer, None)
return req
if __name__ == "__main__":
url = "http://yboss.yahooapis.com/ysearch/web"
req = oauth_request(url, params={"q": "cats dogs"})
# This one is a bit nasty. Apparently the BOSS API does not like
# "+" in its URLs so you have to replace "%20" manually.
# Not sure if the API should be expected to accept either.
# Not sure why to_url does not just return %20 instead...
# Also, oauth2.Request seems to store parameters as unicode and forget
# to encode to utf8 prior to percentage encoding them in its to_url
# method. However, it's handled correctly for generating signatures.
# to_url fails when query parameters contain non-ASCII characters. To
# work around, manually utf8 encode the request parameters.
req['q'] = req['q'].encode('utf8')
req_url = req.to_url().replace('+', '%20')
print req_url
result = urllib2.urlopen(req_url)
Here is a Python code snippet that works for me against Yahoo! BOSS:
import httplib2
import oauth2
import time
OAUTH_CONSUMER_KEY = "Blah"
OAUTH_CONSUMER_SECRET = "Blah"
if __name__ == "__main__":
url = "http://yboss.yahooapis.com/ysearch/web?q=cats%20dogs"
consumer = oauth2.Consumer(key=OAUTH_CONSUMER_KEY,secret=OAUTH_CONSUMER_SECRET)
params = {
'oauth_version': '1.0',
'oauth_nonce': oauth2.generate_nonce(),
'oauth_timestamp': int(time.time()),
}
oauth_request = oauth2.Request(method='GET', url=url, parameters=params)
oauth_request.sign_request(oauth2.SignatureMethod_HMAC_SHA1(), consumer, None)
oauth_header=oauth_request.to_header(realm='yahooapis.com')
# Get search results
http = httplib2.Http()
resp, content = http.request(url, 'GET', headers=oauth_header)
print resp
print content
Im using an Authenticate Header to submit the OAuth signature.
So I decided to ditch Python and try Perl, and it Just Worked. Here's a minimal code sample:
use strict;
use Net::OAuth;
use LWP::UserAgent;
my $CC_KEY = "blahblahblah";
my $CC_SECRET = "blah";
my $url = 'http://yboss.yahooapis.com/ysearch/web';
print make_request($url, {q => "cat dog", format => "xml", count => 5});
sub make_request {
my ($url, $args) = #_;
my $request = Net::OAuth->request("request token")
->new(
consumer_key => $CC_KEY,
consumer_secret => $CC_SECRET,
request_url => $url,
request_method => 'GET',
signature_method => 'HMAC-SHA1',
timestamp => time,
nonce => int(rand 10**6),
callback => 'oob',
extra_params => $args,
protocol_version => Net::OAuth::PROTOCOL_VERSION_1_0A,
);
$request->sign;
my $res = LWP::UserAgent->new(env_proxy=>1)->get($request->to_url);
return $res->content if $res->is_success;
die $res->status_line;
}
Here's another solution, this time back in python-land. This was put together by Tom De Smedt, author of the Pattern web-mining kit.
I'll communicate with the author of python-oauth2 to see if it can be fixed.
OAUTH_CONSUMER_KEY = "blahblahblah"
OAUTH_CONSUMER_SECRET = "blah"
import urllib
import hmac
import time
import random
import base64
try:
from hashlib import sha1
from hashlib import md5
except:
import sha as sha1
import md5; md5=md5.new
def hmac_sha1(key, text):
return hmac.new(key, text, sha1).digest()
def oauth_nonce(length=40):
h = "".join([str(random.randint(0, 9)) for i in range(length)])
h = md5(str(time.time()) + h).hexdigest()
return h
def oauth_timestamp():
return str(int(time.time()))
def oauth_encode(s):
return urllib.quote(s, "~")
def oauth_signature(url, data={}, method="get", secret="", token=""):
# Signature base string: http://tools.ietf.org/html/rfc5849#section-3.4.1
base = oauth_encode(method.upper()) + "&"
base += oauth_encode(url.rstrip("?")) + "&"
base += oauth_encode("&".join(["%s=%s" % (k, v) for k, v in sorted(data.items())]))
# HMAC-SHA1 signature algorithm: http://tools.ietf.org/html/rfc5849#section-3.4.2
signature = hmac_sha1(oauth_encode(secret) + "&" + token, base)
signature = base64.b64encode(signature)
return signature
q = "cat"
url = "http://yboss.yahooapis.com/ysearch/" + "web" # web | images | news
data = {
"q": q,
"start": 0,
"count": 50, # 35 for images
"format": "xml",
"oauth_version": "1.0",
"oauth_nonce" : oauth_nonce(),
"oauth_timestamp" : oauth_timestamp(),
"oauth_consumer_key" : OAUTH_CONSUMER_KEY,
"oauth_signature_method" : "HMAC-SHA1",
}
data["oauth_signature"] = oauth_signature(url, data, secret=OAUTH_CONSUMER_SECRET)
complete_url = url + "?" + urllib.urlencode(data)
response = urllib.urlopen(complete_url)
print response.read()
Here is sample code to access Yahoo! BOSS API v2 using with python-oauth as oauth liberary.
OAUTH_CONSUMER_KEY = "<oauth consumer key>"
OAUTH_CONSUMER_SECRET = "<oauth consumer secret>"
URL = "http://yboss.yahooapis.com/ysearch/web"
import urllib
import oauth.oauth as oauth
data = {
"q": "yahoo boss search",
"start":0,
"count":2,
"format":"json"
}
consumer = oauth.OAuthConsumer(OAUTH_CONSUMER_KEY, OAUTH_CONSUMER_SECRET)
signature_method_plaintext = oauth.OAuthSignatureMethod_PLAINTEXT()
signature_method_hmac_sha1 = oauth.OAuthSignatureMethod_HMAC_SHA1()
oauth_request = oauth.OAuthRequest.from_consumer_and_token(consumer, token=None, http_method='GET', http_url=URL, parameters=data)
oauth_request.sign_request(signature_method_hmac_sha1, consumer, "")
complete_url = oauth_request.to_url()
response = urllib.urlopen(complete_url)
print "REQUEST URL => %s" % complete_url
print ""
print "RESPONSE =>"
print response.read()
I stepped into the urllib2.open code using the debugger, and found that the response has this header:
WWW-Authenticate: OAuth oauth_problem="version_rejected", realm="yahooapis.com"
So I guess I'm having some kind of version mismatch of OAuth.