I'm trying to retrieve the information within the non_public_metrics field in twitter API (i.e, "impression_count", "url_link_clicks", "user_profile_clicks"). I was able to access the public_metrics field using only the Bearer Token. But, when I include the non_public_metrics in my query params I got the error Field Authorization Error. Here is my code:
import requests
import collections
import os
from dotenv import load_dotenv
load_dotenv()
def auth():
return os.getenv('TWITTER_TOKEN')
def create_headers(bearer_token):
headers = {"Authorization": "Bearer {}".format(bearer_token)}
return headers
def create_url(keyword, start_date, end_date, max_results = 10):
ttid = 1184334528837574656
search_url = f"https://api.twitter.com/2/users/{ttid}/tweets" #Change to the endpoint you want to collect data from
#change params based on the endpoint you are using
query_params = {'start_time': start_date,
'end_time': end_date,
'max_results': max_results,
'tweet.fields': 'public_metrics,created_at,non_public_metric',#remove non_public_metric and the code will work
'next_token': {}}
return (search_url, query_params)
def connect_to_endpoint(url, headers, params, next_token = None):
params['next_token'] = next_token #params object received from create_url function
response = requests.request("GET", url, headers = headers, params = params)
print("Endpoint Response Code: " + str(response.status_code))
if response.status_code != 200:
raise Exception(response.status_code, response.text)
return response.json()
def flatten(d, parent_key='', sep='_'):
items = []
for k, v in d.items():
new_key = parent_key + sep + k if parent_key else k
if isinstance(v, collections.MutableMapping):
items.extend(flatten(v, new_key, sep=sep).items())
else:
items.append((new_key, v))
return dict(items)
#Inputs for the request
bearer_token = auth()
headers = create_headers(bearer_token)
keyword = "xbox lang:en"
start_time = "2021-12-01T00:00:00.000Z"
end_time = "2021-12-22T00:00:00.000Z"
max_results = 100
url = create_url(keyword, start_time,end_time, max_results)
json_response = connect_to_endpoint(url[0], headers, url[1])
print(json_response['data']) #if non_public_metrics is included, this throws a error
Then I read in Twitter Docs that I need to use OAuth1.0 authorization in order to access the field non_public_metrics. I tried to use one of the sample codes available in twitter-dev GH'page that uses OAuth1.0 authentication. Here is the snippet I used:
from requests_oauthlib import OAuth1Session
import os
import json
from dotenv import load_dotenv
load_dotenv()
consumer_key = os.getenv("CONSUMER_KEY")
consumer_secret = os.getenv("CONSUMER_SECRET")
#I actually used an ID associate to my account, not this one
params = {"ids": "1184334528837574656", "tweet.fields": "public_metrics,created_at,non_public_metrics"}
request_token_url = "https://api.twitter.com/oauth/request_token"
oauth = OAuth1Session(consumer_key, client_secret=consumer_secret)
try:
fetch_response = oauth.fetch_request_token(request_token_url)
except ValueError:
print(
"There may have been an issue with the consumer_key or consumer_secret you entered."
)
resource_owner_key = fetch_response.get("oauth_token")
resource_owner_secret = fetch_response.get("oauth_token_secret")
print("Got OAuth token: %s" % resource_owner_key)
# Get authorization
base_authorization_url = "https://api.twitter.com/oauth/authorize"
authorization_url = oauth.authorization_url(base_authorization_url)
print("Please go here and authorize: %s" % authorization_url)
verifier = input("Paste the PIN here: ")
# Get the access token
access_token_url = "https://api.twitter.com/oauth/access_token"
oauth = OAuth1Session(
consumer_key,
client_secret=consumer_secret,
resource_owner_key=resource_owner_key,
resource_owner_secret=resource_owner_secret,
verifier=verifier,
)
oauth_tokens = oauth.fetch_access_token(access_token_url)
access_token = oauth_tokens["oauth_token"]
access_token_secret = oauth_tokens["oauth_token_secret"]
# Make the request
oauth = OAuth1Session(
consumer_key,
client_secret=consumer_secret,
resource_owner_key=access_token,
resource_owner_secret=access_token_secret,
)
response = oauth.get(
"https://api.twitter.com/2/tweets", params=params
)
if response.status_code != 200:
raise Exception(
"Request returned an error: {} {}".format(response.status_code, response.text)
)
print("Response code: {}".format(response.status_code))
json_response = response.json()
print(json.dumps(json_response, indent=4, sort_keys=True))
This snippet, however, leads me to a similar error "Sorry, you are not authorized to access 'non_public_metrics.impression_count' on the Tweet with ids. Besides, this snippet has the huge incovenient of ask me to click a link and generate a PIN every time I need to request information for a particular tweet.
How can I properly request information on non_public_metrics field for my tweets?
One can retrieve the information within the non_public_metrics field using the url https://api.twitter.com/2/tweets/[YOU_TWEET_ID]?tweet.fields=non_public_metrics in Postman. To do the same in python just use the following snippet:
import os
from requests_oauthlib import OAuth1
import requests
from dotenv import load_dotenv
load_dotenv()
YOUR_TWEET_ID = ''
url = f'https://api.twitter.com/2/tweets/{YOUR_TWEET_ID}?tweet.fields=public_metrics,non_public_metrics'
CONSUMER_KEY=os.getenv('CONSUMER_KEY')
CONSUMER_SECRET=os.getenv('CONSUMER_SECRET')
ACCESS_TOKEN=os.getenv('ACCESS_TOKEN')
ACCESS_SECRET=os.getenv('ACCESS_SECRET')
headeroauth = OAuth1(CONSUMER_KEY, CONSUMER_SECRET,ACCESS_TOKEN, ACCESS_SECRET, signature_type='auth_header')
r = requests.get(url, auth=headeroauth)
print(r.json())
Related
I’ve quite recently found this feature on Bitget which enables users to essentially copy other ranked traders. This feature comes with a corresponding api documentation. But after going through it im more confused than ever. Firstly, im trying to obtain the historical data trading data of specific traders which are available data on their “orders tab” from the website (shown in excerpt above). I reckon this is possible from the following get request from the documentation: “GET /api/mix/v1/trace/waitProfitDateList”.
Based on the above http request from i have produced the following python code below. The request response is 403. Help a fellow novice
import requests
import hmac
import base64
import hashlib
import json
import time
def sign(message, secret_key):
mac = hmac.new(bytes(secret_key, encoding='utf8'), bytes(message, encoding='utf-8'), digestmod='sha256')
d = mac.digest()
return base64.b64encode(d).decode('utf-8')
def pre_hash(timestamp, method, request_path, query_string, body):
return str(timestamp) + str.upper(method) + request_path + query_string + body
if __name__ == '__main__':
params = {
"pageSize": 10,
"pageNo": 1
}
rest_url = "https://api.bitget.com"
secret_key = ""
api_key = ""
passphrase = ""
timestamp = int(time.time_ns() / 1000000);
query_string = '&'.join([f'{k}={v}' for k, v in params.items()])
message = pre_hash(timestamp, 'GET', '/api/mix/v1/trace/waitProfitDateList', "?"+query_string,"")
sign = sign(message, secret_key)
headers = {
"ACCESS-KEY":api_key,
"ACCESS-SIGN":sign,
"ACCESS-TIMESTAMP":str(timestamp),
"ACCESS-PASSPHRASE":passphrase,
"Content-Type":"application/json",
"locale":"en-US"
}
response = requests.get(rest_url, headers=headers, params=params)
if response.status_code == 200:
result = response.json()
print(result)
else:
print(response.status_code)
The below is a result of this question How to sign an OKEx API request? and some of the answers:
import hmac
import base64
import requests
import datetime
import json
from config import KEY, SECRET, PASS, ROOT_URL
def get_time():
now = datetime.datetime.utcnow()
t = now.isoformat("T", "milliseconds")
return t + "Z"
def signature(timestamp, request_type, endpoint, body, secret):
if body != '':
body = json.dumps(body)
message = str(timestamp) + str.upper(request_type) + endpoint + body
print(message)
mac = hmac.new(bytes(secret, encoding='utf-8'), bytes(message, encoding='utf-8'), digestmod='sha256')
d = mac.digest()
return base64.b64encode(d)
def get_header(request_type, endpoint, body):
time = get_time()
header = dict()
header['CONTENT-TYPE'] = 'application/json'
header['OK-ACCESS-KEY'] = KEY
header['OK-ACCESS-SIGN'] = signature(time, request_type, endpoint, body, SECRET)
header['OK-ACCESS-TIMESTAMP'] = str(time)
header['OK-ACCESS-PASSPHRASE'] = PASS
return header
def get(endpoint, body=''):
url = ROOT_URL + endpoint
header = get_header('GET', endpoint, body)
return requests.get(url, headers=header)
def post(endpoint, body=''):
url = ROOT_URL + endpoint
header = get_header('POST', endpoint, body)
return requests.post(url, headers=header)
where KEY, SECRET, PASS are the API key, secret key, and pass phrase respectively; The ROOT_URL is 'https://www.okex.com'.
The Problem
GET requests work absolutely fine, so when I run the following, there are no issues:
ENDPOINT = '/api/v5/account/balance'
BODY = ''
response = get(ENDPOINT)
response.json()
However, when I try to place an order via a POST request, like so:
ENDPOINT = '/api/v5/trade/order'
BODY = {"instId":"BTC-USDT",
"tdMode":"cash",
"side":"buy",
"ordType":"market",
"sz":"1"}
response = post(ENDPOINT, body=BODY)
response.json()
I get the following output, i.e. it won't accept the signature:
{'msg': 'Invalid Sign', 'code': '50113'}
Related Questions
In this one Can't figure out how to send a signed POST request to OKEx an answer was provided, but it does not work for me as I was already using the suggested URL. More or less the same question was asked here Unable to send a post requests OKEX Invalid Signature, but no activity likely due to the format, so I thought I would repost and elaborate.
OKEX Docs
The docs simply specify that The API endpoints of Trade require authentication (https://www.okex.com/docs-v5/en/?python#rest-api-authentication-signature). But they make no reference to there being any difference between the two methods. Away from that, I am including all required parameters in the body of the post request as far as I can see.
I would appreciate any input on this.
Many thanks!
I ran into the same POST problem and figured it out. I used new domain name okex.com. Here is my code.
def set_userinfo(self):
position_path = "/api/v5/account/set-position-mode"
try:
self.get_header("POST", position_path, {"posMode":"net_mode"})
resp = requests.post(url=self.base_url+position_path, headers=self.headers, json={"posMode":"long_short_mode"}).json()
except Exception as e:
log.error("OK set_userinfo error={} type={}".format(f'{e}', f'{type(e)}'))
def get_header(self, request_type, endpoint, body=''):
timestamp = self.get_time()
self.headers["OK-ACCESS-TIMESTAMP"] = timestamp
self.headers["OK-ACCESS-SIGN"] = self.signature(timestamp, request_type, endpoint, body)
def signature(self, timestamp, request_type, endpoint, body):
if body != '':
body = json.dumps(body)
message = str(timestamp) + str.upper(request_type) + endpoint + body
mac = hmac.new(bytes(self.secret_key, encoding='utf-8'), bytes(message, encoding='utf-8'), digestmod='sha256').digest()
return base64.b64encode(mac)
I have fix the same problem.
Both of the 'body' in signature() and in get_header() should be json.
So you should add following code:
if str(body) == '{}' or str(body) == 'None':
body = ''
else:
body = json.dumps(body)
I ran into the same problem and solved it using below code snippet, the idea is from https://stackoverflow.com/a/68115787/20497127, but I modified a little by adding POST functionality
APIKEY = "" # input key
APISECRET = "" #input secret
PASS = "" #input passphrase
BASE_URL = 'https://www.okx.com'
def send_signed_request(http_method, url_path, payload={}):
def get_time():
return dt.datetime.utcnow().isoformat()[:-3]+'Z'
def signature(timestamp, method, request_path, body, secret_key):
if str(body) == '{}' or str(body) == 'None':
body = ''
message = str(timestamp) + str.upper(method) + request_path + str(body)
mac = hmac.new(bytes(secret_key, encoding='utf8'), bytes(message, encoding='utf-8'), digestmod='sha256')
d = mac.digest()
return base64.b64encode(d)
# set request header
def get_header(request='GET', endpoint='', body:dict=dict()):
cur_time = get_time()
header = dict()
header['CONTENT-TYPE'] = 'application/json'
header['OK-ACCESS-KEY'] = APIKEY
header['OK-ACCESS-SIGN'] = signature(cur_time, request, endpoint , body, APISECRET)
header['OK-ACCESS-TIMESTAMP'] = str(cur_time)
header['OK-ACCESS-PASSPHRASE'] = PASS
# demo trading: need to set x-simulated-trading=1, live trading is 0
header['x-simulated-trading'] = '1'
return header
url = BASE_URL + url_path
header = get_header(http_method, url_path, payload)
print(url)
print(header)
if http_method == 'GET':
response = requests.get(url, headers=header)
elif http_method == 'POST':
response = requests.post(url, headers=header, data=payload)
return response.json()
# this will run get requests
res = send_signed_request("GET", "/api/v5/account/balance", payload={})
# this will run post requests
data = {
"instId": "BTC-USDT",
"tdMode": "cross",
"side": "sell",
"ccy":"USDT",
"ordType": "limit",
"px": "100000",
"sz": "0.01"
}
res = send_signed_request("POST", "/api/v5/trade/order", payload=json.dumps(data))
I am trying to make a basic authenticated api call to their new v2 api and getting an invalid api key error returned.
I reissued the api key just to verify, same error.
from time import time
import urllib.request
import urllib.parse
import hashlib
import hmac
APIkey = b'myapikeyyouarenotsupposedtosee'
secret = b'myceeeeecretkeyyyy'
url = 'https://api.bitfinex.com/v2/auth/r/wallets'
payload = {
#'request':'/auth/r/wallets',
'nonce': int(time() * 1000),
}
paybytes = urllib.parse.urlencode(payload).encode('utf8')
print(paybytes)
sign = hmac.new(secret, paybytes, hashlib.sha512).hexdigest()
print(sign)
headers = {
'Key': APIkey,
'Sign': sign
}
req = urllib.request.Request(url, headers=headers, data=paybytes)
with urllib.request.urlopen(req) as response:
the_page = response.read()
print(the_page)
How do I make an authenticated api call to the new v2 API for bitfinex?
Your headers are wrong. I was also trying to do this and tried using the example code from the bitfinex v2 api docs, however their example contained a bug in that they needed to encode the strings into UTF-8 byte arrays first. So I've fixed it and posting the entire example below.
#
# Example Bitfinex API v2 Auth Python Code
#
import requests # pip install requests
import json
import base64
import hashlib
import hmac
import os
import time #for nonce
class BitfinexClient(object):
BASE_URL = "https://api.bitfinex.com/"
KEY = "API_KEY_HERE"
SECRET = "API_SECRET_HERE"
def _nonce(self):
# Returns a nonce
# Used in authentication
return str(int(round(time.time() * 10000)))
def _headers(self, path, nonce, body):
secbytes = self.SECRET.encode(encoding='UTF-8')
signature = "/api/" + path + nonce + body
sigbytes = signature.encode(encoding='UTF-8')
h = hmac.new(secbytes, sigbytes, hashlib.sha384)
hexstring = h.hexdigest()
return {
"bfx-nonce": nonce,
"bfx-apikey": self.KEY,
"bfx-signature": hexstring,
"content-type": "application/json"
}
def req(self, path, params = {}):
nonce = self._nonce()
body = params
rawBody = json.dumps(body)
headers = self._headers(path, nonce, rawBody)
url = self.BASE_URL + path
resp = requests.post(url, headers=headers, data=rawBody, verify=True)
return resp
def active_orders(self):
# Fetch active orders
response = self.req("v2/auth/r/orders")
if response.status_code == 200:
return response.json()
else:
print('error, status_code = ', response.status_code)
return ''
# fetch all your orders and print out
client = BitfinexClient()
result = client.active_orders()
print(result)
Why not use one of the open source api clients out there ? and you can compare to your work .
https://github.com/scottjbarr/bitfinex
https://github.com/tuberculo/bitfinex
I've been trying to use the API of the website Idealista (https://www.idealista.com/) to retrieve information of real estate data.
Since I'm not familiarized with OAuth2 I haven't been able to obtain the token so far. I have just been provided with the api key, the secret and some basic info of how to mount the http request.
I would appreciate an example (preferably in Python) of the functioning of this API, or else some more generic info about dealing with OAuth2 and Python.
After some days of research I came up with a basic python code to retrieve real estate data from the Idealista API.
def get_oauth_token():
http_obj = Http()
url = "https://api.idealista.com/oauth/token"
apikey= urllib.parse.quote_plus('Provided_API_key')
secret= urllib.parse.quote_plus('Provided_API_secret')
auth = base64.encode(apikey + ':' + secret)
body = {'grant_type':'client_credentials'}
headers = {'Content-Type': 'application/x-www-form-urlencoded;charset=UTF-8','Authorization' : 'Basic ' + auth}
resp, content = http_obj.request(url,method='POST',headers=headers, body=urllib.parse.urlencode(body))
return content
This function would return a JSON with the OAuth2 token and the session time in seconds. Afterwards, to query the API, it would be as simple as:
def search_api(token):
http_obj = Http()
url = "http://api.idealista.com/3.5/es/search?center=40.42938099999995,-3.7097526269835726&country=es&maxItems=50&numPage=1&distance=452&propertyType=bedrooms&operation=rent"
headers = {'Authorization' : 'Bearer ' + token}
resp, content = http_obj.request(url,method='POST',headers=headers)
return content
This time the we would find in the content var the data we were looking for, again as a JSON.
That can't be marked as correct answer since
auth = base64.encode(apikey + ':' + secret)
body = {'grant_type':'client_credentials'}
headers = {'Content-Type': 'application/x-www-form-urlencoded;charset=UTF-8','Authorization' : 'Basic ' + auth}
Will give you TypeError:
can only concatenate str (not "bytes") to str
Since base64encode returns a byte type object...
It's true Idealista API is very limited about documentation, but I think this is a better approach since I don't use unnecesary libs (Only native):
#first request
message = API_KEY + ":" + SECRET
auth = "Basic " + base64.b64encode(message.encode("ascii")).decode("ascii")
headers_dic = {"Authorization" : auth,
"Content-Type" : "application/x-www-form-urlencoded;charset=UTF-8"}
params_dic = {"grant_type" : "client_credentials",
"scope" : "read"}
r = requests.post("https://api.idealista.com/oauth/token",
headers = headers_dic,
params = params_dic)
This works flawless with only python requests and base64 module...
regards
This is my code, improving #3... this run ok! for me!!!!
only put your apikey and your password (secret)...
import pandas as pd
import json
import urllib
import requests as rq
import base64
def get_oauth_token():
url = "https://api.idealista.com/oauth/token"
apikey= 'your_api_key' #sent by idealista
secret= 'your_password' #sent by idealista
auth = base64.b64encode(apikey + ':' + secret)
headers = {'Content-Type': 'application/x-www-form-urlencoded;charset=UTF-8' ,'Authorization' : 'Basic ' + auth}
params = urllib.urlencode({'grant_type':'client_credentials'})
content = rq.post(url,headers = headers, params=params)
bearer_token = json.loads(content.text)['access_token']
return bearer_token
def search_api(token, url):
headers = {'Content-Type': 'Content-Type: multipart/form-data;', 'Authorization' : 'Bearer ' + token}
content = rq.post(url, headers = headers)
result = json.loads(content.text)['access_token']
return result
country = 'es' #values: es, it, pt
locale = 'es' #values: es, it, pt, en, ca
language = 'es' #
max_items = '50'
operation = 'sale'
property_type = 'homes'
order = 'priceDown'
center = '40.4167,-3.70325'
distance = '60000'
sort = 'desc'
bankOffer = 'false'
df_tot = pd.DataFrame()
limit = 10
for i in range(1,limit):
url = ('https://api.idealista.com/3.5/'+country+'/search?operation='+operation+#"&locale="+locale+
'&maxItems='+max_items+
'&order='+order+
'¢er='+center+
'&distance='+distance+
'&propertyType='+property_type+
'&sort='+sort+
'&numPage=%s'+
'&language='+language) %(i)
a = search_api(get_oauth_token(), url)
df = pd.DataFrame.from_dict(a['elementList'])
df_tot = pd.concat([df_tot,df])
df_tot = df_tot.reset_index()
I found some mistakes. At least, I cannot run it.
I believe, I improved with this:
import pandas as pd
import json
import urllib
import requests as rq
import base64
def get_oauth_token():
url = "https://api.idealista.com/oauth/token"
apikey= 'your_api_key' #sent by idealist
secret= 'your_password' #sent by idealista
apikey_secret = apikey + ':' + secret
auth = str(base64.b64encode(bytes(apikey_secret, 'utf-8')))[2:][:-1]
headers = {'Authorization' : 'Basic ' + auth,'Content-Type': 'application/x-www-form-
urlencoded;charset=UTF-8'}
params = urllib.parse.urlencode({'grant_type':'client_credentials'}) #,'scope':'read'
content = rq.post(url,headers = headers, params=params)
bearer_token = json.loads(content.text)['access_token']
return bearer_token
def search_api(token, URL):
headers = {'Content-Type': 'Content-Type: multipart/form-data;', 'Authorization' : 'Bearer ' + token}
content = rq.post(url, headers = headers)
result = json.loads(content.text)
return result
The Twitter v1 API is now defunct so I've been trying to use the Search and Streaming APIs to collate hashtag information. The Search API is rate limited, so if there are a lot of entries on a hashtag you will probably miss some. Streaming seemed like the way to go.
Using OAuth2 here is my (anonymized) code:
import oauth2 as oauth
import json
consumer_key = "<consumer key from twitter developer site>"
consumer_secret = "<consumer secret>"
oauth_token = "<access token>"
oauth_token_secret = "<access token secret>"
consumer = oauth.Consumer(key=consumer_key, secret=consumer_secret)
access_token = oauth.Token(key=oauth_token, secret=oauth_token_secret)
client = oauth.Client(consumer, access_token)
terms = json.dumps({'track' : 'twitter'})
stream_endpoint = "https://stream.twitter.com/1.1/statuses/filter.json"
response, data = client.request(stream_endpoint,"POST", body=terms, headers={'Content-Type':'application/json'})
The issue I run into is this always returns the following message:
>>>'No filter parameters found. Expect at least one parameter: follow track locations\r\n'
I think your error is becouse using JSON data on
terms = json.dumps({'track' : 'twitter'})
You should write your code just like this
terms = 'track=twitter'
USER = request.params.get('username', '00000')
LIMIT = request.params.get('limit', '50')
REQUEST_TOKEN_URL = 'https://api.twitter.com/oauth/request_token'
consumer_key ='424245wfdsfa4'
consumer_secret ='afar234252523adsasd'
if consumer_key is None or consumer_secret is None:
print 'you need consumer_key & consumer_secret key'
sys.exit(1)
signature_method_hmac_sha1 = oauth.SignatureMethod_HMAC_SHA1()
oauth_consumer = oauth.Consumer(key=consumer_key, secret=consumer_secret)
oauth_client = oauth.Client(oauth_consumer)
response, content = oauth_client.request(REQUEST_TOKEN_URL, 'POST')
if response['status'] == '200':
request_token = dict(parse_qsl(content))
else:
print 'Invalid response from Twitter requesting token.........: %s' % response['status']
endpoint = 'https://api.twitter.com/1.1/statuses/user_timeline.json?screen_name=' + USER + '&count=' + LIMIT
response, content = oauth_client.request(endpoint, 'GET')
url = response['content-location']
f = urllib2.urlopen(url)
response = f.read()
return simplejson.loads(response)