We Keep Coding

Python api for copy trading on bitget

I’ve quite recently found this feature on Bitget which enables users to essentially copy other ranked traders. This feature comes with a corresponding api documentation. But after going through it im more confused than ever. Firstly, im trying to obtain the historical data trading data of specific traders which are available data on their “orders tab” from the website (shown in excerpt above). I reckon this is possible from the following get request from the documentation: “GET /api/mix/v1/trace/waitProfitDateList”.
Based on the above http request from i have produced the following python code below. The request response is 403. Help a fellow novice
import requests
import hmac
import base64
import hashlib
import json
import time
def sign(message, secret_key):
mac = hmac.new(bytes(secret_key, encoding='utf8'), bytes(message, encoding='utf-8'), digestmod='sha256')
d = mac.digest()
return base64.b64encode(d).decode('utf-8')
def pre_hash(timestamp, method, request_path, query_string, body):
return str(timestamp) + str.upper(method) + request_path + query_string + body
if __name__ == '__main__':
params = {
"pageSize": 10,
"pageNo": 1
}
rest_url = "https://api.bitget.com"
secret_key = ""
api_key = ""
passphrase = ""
timestamp = int(time.time_ns() / 1000000);
query_string = '&'.join([f'{k}={v}' for k, v in params.items()])
message = pre_hash(timestamp, 'GET', '/api/mix/v1/trace/waitProfitDateList', "?"+query_string,"")
sign = sign(message, secret_key)
headers = {
"ACCESS-KEY":api_key,
"ACCESS-SIGN":sign,
"ACCESS-TIMESTAMP":str(timestamp),
"ACCESS-PASSPHRASE":passphrase,
"Content-Type":"application/json",
"locale":"en-US"
}
response = requests.get(rest_url, headers=headers, params=params)
if response.status_code == 200:
result = response.json()
print(result)
else:
print(response.status_code)

How to access non_public_metrics field in twitter API?

I'm trying to retrieve the information within the non_public_metrics field in twitter API (i.e, "impression_count", "url_link_clicks", "user_profile_clicks"). I was able to access the public_metrics field using only the Bearer Token. But, when I include the non_public_metrics in my query params I got the error Field Authorization Error. Here is my code:
import requests
import collections
import os
from dotenv import load_dotenv
load_dotenv()
def auth():
return os.getenv('TWITTER_TOKEN')
def create_headers(bearer_token):
headers = {"Authorization": "Bearer {}".format(bearer_token)}
return headers
def create_url(keyword, start_date, end_date, max_results = 10):
ttid = 1184334528837574656
search_url = f"https://api.twitter.com/2/users/{ttid}/tweets" #Change to the endpoint you want to collect data from
#change params based on the endpoint you are using
query_params = {'start_time': start_date,
'end_time': end_date,
'max_results': max_results,
'tweet.fields': 'public_metrics,created_at,non_public_metric',#remove non_public_metric and the code will work
'next_token': {}}
return (search_url, query_params)
def connect_to_endpoint(url, headers, params, next_token = None):
params['next_token'] = next_token #params object received from create_url function
response = requests.request("GET", url, headers = headers, params = params)
print("Endpoint Response Code: " + str(response.status_code))
if response.status_code != 200:
raise Exception(response.status_code, response.text)
return response.json()
def flatten(d, parent_key='', sep='_'):
items = []
for k, v in d.items():
new_key = parent_key + sep + k if parent_key else k
if isinstance(v, collections.MutableMapping):
items.extend(flatten(v, new_key, sep=sep).items())
else:
items.append((new_key, v))
return dict(items)
#Inputs for the request
bearer_token = auth()
headers = create_headers(bearer_token)
keyword = "xbox lang:en"
start_time = "2021-12-01T00:00:00.000Z"
end_time = "2021-12-22T00:00:00.000Z"
max_results = 100
url = create_url(keyword, start_time,end_time, max_results)
json_response = connect_to_endpoint(url[0], headers, url[1])
print(json_response['data']) #if non_public_metrics is included, this throws a error
Then I read in Twitter Docs that I need to use OAuth1.0 authorization in order to access the field non_public_metrics. I tried to use one of the sample codes available in twitter-dev GH'page that uses OAuth1.0 authentication. Here is the snippet I used:
from requests_oauthlib import OAuth1Session
import os
import json
from dotenv import load_dotenv
load_dotenv()
consumer_key = os.getenv("CONSUMER_KEY")
consumer_secret = os.getenv("CONSUMER_SECRET")
#I actually used an ID associate to my account, not this one
params = {"ids": "1184334528837574656", "tweet.fields": "public_metrics,created_at,non_public_metrics"}
request_token_url = "https://api.twitter.com/oauth/request_token"
oauth = OAuth1Session(consumer_key, client_secret=consumer_secret)
try:
fetch_response = oauth.fetch_request_token(request_token_url)
except ValueError:
print(
"There may have been an issue with the consumer_key or consumer_secret you entered."
)
resource_owner_key = fetch_response.get("oauth_token")
resource_owner_secret = fetch_response.get("oauth_token_secret")
print("Got OAuth token: %s" % resource_owner_key)
# Get authorization
base_authorization_url = "https://api.twitter.com/oauth/authorize"
authorization_url = oauth.authorization_url(base_authorization_url)
print("Please go here and authorize: %s" % authorization_url)
verifier = input("Paste the PIN here: ")
# Get the access token
access_token_url = "https://api.twitter.com/oauth/access_token"
oauth = OAuth1Session(
consumer_key,
client_secret=consumer_secret,
resource_owner_key=resource_owner_key,
resource_owner_secret=resource_owner_secret,
verifier=verifier,
)
oauth_tokens = oauth.fetch_access_token(access_token_url)
access_token = oauth_tokens["oauth_token"]
access_token_secret = oauth_tokens["oauth_token_secret"]
# Make the request
oauth = OAuth1Session(
consumer_key,
client_secret=consumer_secret,
resource_owner_key=access_token,
resource_owner_secret=access_token_secret,
)
response = oauth.get(
"https://api.twitter.com/2/tweets", params=params
)
if response.status_code != 200:
raise Exception(
"Request returned an error: {} {}".format(response.status_code, response.text)
)
print("Response code: {}".format(response.status_code))
json_response = response.json()
print(json.dumps(json_response, indent=4, sort_keys=True))
This snippet, however, leads me to a similar error "Sorry, you are not authorized to access 'non_public_metrics.impression_count' on the Tweet with ids. Besides, this snippet has the huge incovenient of ask me to click a link and generate a PIN every time I need to request information for a particular tweet.
How can I properly request information on non_public_metrics field for my tweets?

One can retrieve the information within the non_public_metrics field using the url https://api.twitter.com/2/tweets/[YOU_TWEET_ID]?tweet.fields=non_public_metrics in Postman. To do the same in python just use the following snippet:
import os
from requests_oauthlib import OAuth1
import requests
from dotenv import load_dotenv
load_dotenv()
YOUR_TWEET_ID = ''
url = f'https://api.twitter.com/2/tweets/{YOUR_TWEET_ID}?tweet.fields=public_metrics,non_public_metrics'
CONSUMER_KEY=os.getenv('CONSUMER_KEY')
CONSUMER_SECRET=os.getenv('CONSUMER_SECRET')
ACCESS_TOKEN=os.getenv('ACCESS_TOKEN')
ACCESS_SECRET=os.getenv('ACCESS_SECRET')
headeroauth = OAuth1(CONSUMER_KEY, CONSUMER_SECRET,ACCESS_TOKEN, ACCESS_SECRET, signature_type='auth_header')
r = requests.get(url, auth=headeroauth)
print(r.json())

Fetch metadata for tweet ids using tweepy is not working anymore

I used the tweepy library (for twitter api-v1.1) to get some metadata (e.g., tweet text, #retweets, userid, etc.) for a list of tweet ids. Here is my code:
consumer_key = 'xxxxxxxxxxxx'
consumer_key_secret = 'xxxxxxxxxxxx'
access_token = 'xxxxxxxxxxxxxxxxxx'
access_token_secret = 'xxxxxxxxxxxxxxxxxx'
auth = tweepy.OAuthHandler(consumer_key, consumer_key_secret)
auth.set_access_token(access_token, access_token_secret)
api = tweepy.API(auth)
def createTrainingSet(corpusFile, tweetContent):
import csv
import time
import json
counter = 0
corpus = []
with open(corpusFile, 'r') as csvfile:
lineReader = csv.reader(csvfile, delimiter=',')
for row in lineReader:
corpus.append({"tweet_id": row[0], "unreliable": row[1], "conspiracy": row[2],\
"clickbait": row[3], "political/biased": row[4], "date": row[5]})
sleepTime = 2
trainingDataSet = []
for tweet in corpus:
try:
tweetFetched = api.get_status(tweet["tweet_id"])
print("Tweet fetched" + tweetFetched.text)
print("followers_count: "+ str(tweetFetched.user.followers_count))
print("friends_count: " + str(tweetFetched.user.friends_count))
tweet["text"] = tweetFetched.text
tweet["retweet_count"] = tweetFetched.retweet_count
tweet["favorite_count"] = tweetFetched.favorite_count
tweet["created_at"] = tweetFetched.created_at
tweet["user_id"] = tweetFetched.user.id_str
tweet["user_created_at"] = tweetFetched.user.created_at
trainingDataSet.append(tweet)
time.sleep(sleepTime)
except:
print("Inside the exception - no:2")
continue
# This is corpus dataset
corpusFile = "sample.csv"
# This is my target file
tweetContent = "tweetContent.csv"
# Call the method
resultFile = createTrainingSet(corpusFile, tweetContent)
I don't know why this code doesn't work any more (the last time it worked was a bout a couple of months ago). However, when I run it now, it returns "Inside the exception - no:2". Why is that?

Here is the two lines of code that helped me find the erros:
except tweepy.TweepError as e:
print ('the error code:', e.args[0][0]['code'])
print ('the error message:', e.args[0][0]['message'])
Also, thanks to Jeyekomon's answer in this post, I found that the e.message[0]['code'] is not working anymore:
The error code used to be accessed using e.message[0]['code'] which no longer works. The message attribute has been deprecated in Python 2.6 and removed in Python 3.0. Currently you get an error 'TweepError' object has no attribute 'message'
In addition, it seems there are some other helpful attributes (api_code, reason and response) in TweepError exception class that are not in the documentation.

Twitter API code 130 error (request for code test)

I've written folowing code in python for followers scraping:
import tweepy
import time
import csv
import sys
import random
consumer_key = ''
consumer_secret = ''
access_token = ''
access_token_secret = ''
auth = tweepy.OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_token, access_token_secret)
api = tweepy.API(auth, wait_on_rate_limit=True, wait_on_rate_limit_notify=True, retry_errors=set([401, 404, 500, 502, 503, 504]))
account = 'setavakfi'
log_file = '{}_followers.csv'.format(account)
retry = 0
errorCount = 0
tweepy_cursor = tweepy.Cursor(api.followers, screen_name=account, count=200, cursor=1574812962976647290).pages()
followers_count = []
while True:
try:
retry =0
user = tweepy_cursor.next()
cursor = tweepy_cursor.next_cursor
followers_count += user
print 'Retrieved {} followers accounts'.format(len(followers_count))
print 'Current cursor: {}'.format(cursor)
with open(log_file, 'ab') as fd:
writer = csv.writer(fd)
for i, user in enumerate(user):
writer.writerow([str("#"+user.screen_name), unicode(user.name).encode('utf-8'), str(user.lang), unicode(user.location).encode('utf-8')])
print "Resting..."
time.sleep(random.randint(60, 70)
except tweepy.TweepError as e:
print "Error code: {} with message: {}".format(e.api_code, e.message[0]['message'])
errorCount +=1
retry += 1
print 'Retrying in {} seconds'.format(60+retry*5)
time.sleep(60+retry*5)
if retry == 10:
break
except StopIteration:
break
print 'Done with {} errors'.format(errorCount)
Problem is that with given cursor (1574812962976647290) and account (#setavakfi) I'm only getting error 503 (code 130). This cursor is stuck exacly at page with 8000 folllowers. Whole account is >60,000 followers. I've tried this code on different accounts with more than 60,000 folowers and it works. Have tried to change ISP, IP address and twitter dev account. Nothing change.
Can you see what could be wrong with this code? Is it a problem with this single account? Is there a way to automaticly jump over problematic cursor to see if other cursors will have same problem?
Thanks in advance.
M.

I think that error is because a server overload on the twitter side with this account... I don't think it's an error with your code.

Using Python and OAuth2 with Twitter streaming API

The Twitter v1 API is now defunct so I've been trying to use the Search and Streaming APIs to collate hashtag information. The Search API is rate limited, so if there are a lot of entries on a hashtag you will probably miss some. Streaming seemed like the way to go.
Using OAuth2 here is my (anonymized) code:
import oauth2 as oauth
import json
consumer_key = "<consumer key from twitter developer site>"
consumer_secret = "<consumer secret>"
oauth_token = "<access token>"
oauth_token_secret = "<access token secret>"
consumer = oauth.Consumer(key=consumer_key, secret=consumer_secret)
access_token = oauth.Token(key=oauth_token, secret=oauth_token_secret)
client = oauth.Client(consumer, access_token)
terms = json.dumps({'track' : 'twitter'})
stream_endpoint = "https://stream.twitter.com/1.1/statuses/filter.json"
response, data = client.request(stream_endpoint,"POST", body=terms, headers={'Content-Type':'application/json'})
The issue I run into is this always returns the following message:
>>>'No filter parameters found. Expect at least one parameter: follow track locations\r\n'

I think your error is becouse using JSON data on
terms = json.dumps({'track' : 'twitter'})
You should write your code just like this
terms = 'track=twitter'

USER = request.params.get('username', '00000')
LIMIT = request.params.get('limit', '50')
REQUEST_TOKEN_URL = 'https://api.twitter.com/oauth/request_token'
consumer_key ='424245wfdsfa4'
consumer_secret ='afar234252523adsasd'
if consumer_key is None or consumer_secret is None:
print 'you need consumer_key & consumer_secret key'
sys.exit(1)
signature_method_hmac_sha1 = oauth.SignatureMethod_HMAC_SHA1()
oauth_consumer = oauth.Consumer(key=consumer_key, secret=consumer_secret)
oauth_client = oauth.Client(oauth_consumer)
response, content = oauth_client.request(REQUEST_TOKEN_URL, 'POST')
if response['status'] == '200':
request_token = dict(parse_qsl(content))
else:
print 'Invalid response from Twitter requesting token.........: %s' % response['status']
endpoint = 'https://api.twitter.com/1.1/statuses/user_timeline.json?screen_name=' + USER + '&count=' + LIMIT
response, content = oauth_client.request(endpoint, 'GET')
url = response['content-location']
f = urllib2.urlopen(url)
response = f.read()
return simplejson.loads(response)