I am downloading files from One Drive with the following piece of code:
if response.status_code == 200:
print('\n> Response Success')
with open('Book2.xlsx', 'wb') as File:
File.write(response.content)
print('\n> File Downloaded')
else:
print('\n> Failed:', response.status_code)
print(response.content)
The code is from:This post here
The "File" is coming from One Drive with the use of the following code:
import sys, os, time, requests
import pandas as pd
import urllib.parse
OneDrive_FilePath = 'Book2.xlsx'
OneDrive_FileURL = 'https://graph.microsoft.com/v1.0/me/drive/root:/' + OneDrive_FilePath + ':/content'
OneDrive_FileURL = urllib.parse.quote(OneDrive_FileURL, safe=':/')
print(OneDrive_FileURL)
Client_Id = 'XXXX'
Tenant_Id = 'YYYYY'
Refresh_Token_First = 'ZZZZZ'
PostStr = {'grant_type': 'refresh_token', 'client_id': Client_Id, 'refresh_token': Refresh_Token_First}
Token_Response = requests.post('https://login.microsoftonline.com/' + Tenant_Id + '/oauth2/v2.0/token', data=PostStr)
Access_Token = Token_Response.json()['access_token']
New_Refresh_Token = Token_Response.json()['refresh_token']
if Access_Token is None or New_Refresh_Token is None:
print('\n> Failed: Access_Token NOT Retrieved')
sys.exit()
Response = requests.get(OneDrive_FileURL, headers={'Authorization': 'Bearer ' + Access_Token})
The "File" which is getting downloaded is in the form of "io.BufferedWriter". I want to actually load the "File" as a data frame so that I can do certain operations on that and upload it to AWS.
How can I do that, please help.
Thanks
As explained in the comments, the type of File is mostly irrelevant. All you need to do is read the excel directly from the response, as in :
url = "https://go.microsoft.com/fwlink/?LinkID=521962"
res = requests.get(url)
pd.read_excel(res.content)
The last statement produces a regular pandas dataframe. You can use that however you want.
Related
I’ve quite recently found this feature on Bitget which enables users to essentially copy other ranked traders. This feature comes with a corresponding api documentation. But after going through it im more confused than ever. Firstly, im trying to obtain the historical data trading data of specific traders which are available data on their “orders tab” from the website (shown in excerpt above). I reckon this is possible from the following get request from the documentation: “GET /api/mix/v1/trace/waitProfitDateList”.
Based on the above http request from i have produced the following python code below. The request response is 403. Help a fellow novice
import requests
import hmac
import base64
import hashlib
import json
import time
def sign(message, secret_key):
mac = hmac.new(bytes(secret_key, encoding='utf8'), bytes(message, encoding='utf-8'), digestmod='sha256')
d = mac.digest()
return base64.b64encode(d).decode('utf-8')
def pre_hash(timestamp, method, request_path, query_string, body):
return str(timestamp) + str.upper(method) + request_path + query_string + body
if __name__ == '__main__':
params = {
"pageSize": 10,
"pageNo": 1
}
rest_url = "https://api.bitget.com"
secret_key = ""
api_key = ""
passphrase = ""
timestamp = int(time.time_ns() / 1000000);
query_string = '&'.join([f'{k}={v}' for k, v in params.items()])
message = pre_hash(timestamp, 'GET', '/api/mix/v1/trace/waitProfitDateList', "?"+query_string,"")
sign = sign(message, secret_key)
headers = {
"ACCESS-KEY":api_key,
"ACCESS-SIGN":sign,
"ACCESS-TIMESTAMP":str(timestamp),
"ACCESS-PASSPHRASE":passphrase,
"Content-Type":"application/json",
"locale":"en-US"
}
response = requests.get(rest_url, headers=headers, params=params)
if response.status_code == 200:
result = response.json()
print(result)
else:
print(response.status_code)
I'm using a python script to send an API request to get the attachments of an email. The email I'm using has 4 attachments (plus pictures in the signature in the email). The python request only retrieves 1 attachment along with the pics from the signature. When using Postman with the exact same information, it retrieves all attachments along with the pics.
Any ideas on how I can get the other attachments?
import requests
url = 'https://graph.microsoft.com/v1.0/users/{{users email}}/messages/{{messageID}}/attachments'
body = None
head = {"Content-Type": "application/json;charset=UFT-8", "Authorization": "Bearer " + accessToken}
response1 = requests.get(url, data=body, headers=head)
response = response1.text
Below shows the response from the python script, with only 7 items, and the Postman response with 10 items.
Below code retrieves multiple attachments
(files being an array of attachment names)
def execute(accessToken, messageID, files, noAttachments):
import os
from os import path
import requests
import base64
import json
if noAttachments == "False":
url = 'https://graph.microsoft.com/v1.0/users/{{users email}}/messages/{{messageID}}/attachments'
body = {}
head = {"Authorization": "Bearer " + accessToken}
responseCode = requests.request("GET", url, headers=head, data = body)
response = responseCode.text
test = json.loads(responseCode.text.encode('utf8'))
x, contentBytes = response.split('"contentBytes":"',1)
if len(files) == 1:
imgdata = base64.b64decode(str(contentBytes))
filename = "C:/Temp/SAPCareAttachments/" + files[0]
with open(filename, 'wb') as f:
f.write(imgdata)
else:
for file in test["value"]:
imgdata = base64.b64decode(file["contentBytes"])
if file["name"] in files:
filename = "C:/Temp/" + file["name"]
with open(filename, 'wb') as f:
f.write(imgdata)
print(responseCode)
I am trying to make a basic authenticated api call to their new v2 api and getting an invalid api key error returned.
I reissued the api key just to verify, same error.
from time import time
import urllib.request
import urllib.parse
import hashlib
import hmac
APIkey = b'myapikeyyouarenotsupposedtosee'
secret = b'myceeeeecretkeyyyy'
url = 'https://api.bitfinex.com/v2/auth/r/wallets'
payload = {
#'request':'/auth/r/wallets',
'nonce': int(time() * 1000),
}
paybytes = urllib.parse.urlencode(payload).encode('utf8')
print(paybytes)
sign = hmac.new(secret, paybytes, hashlib.sha512).hexdigest()
print(sign)
headers = {
'Key': APIkey,
'Sign': sign
}
req = urllib.request.Request(url, headers=headers, data=paybytes)
with urllib.request.urlopen(req) as response:
the_page = response.read()
print(the_page)
How do I make an authenticated api call to the new v2 API for bitfinex?
Your headers are wrong. I was also trying to do this and tried using the example code from the bitfinex v2 api docs, however their example contained a bug in that they needed to encode the strings into UTF-8 byte arrays first. So I've fixed it and posting the entire example below.
#
# Example Bitfinex API v2 Auth Python Code
#
import requests # pip install requests
import json
import base64
import hashlib
import hmac
import os
import time #for nonce
class BitfinexClient(object):
BASE_URL = "https://api.bitfinex.com/"
KEY = "API_KEY_HERE"
SECRET = "API_SECRET_HERE"
def _nonce(self):
# Returns a nonce
# Used in authentication
return str(int(round(time.time() * 10000)))
def _headers(self, path, nonce, body):
secbytes = self.SECRET.encode(encoding='UTF-8')
signature = "/api/" + path + nonce + body
sigbytes = signature.encode(encoding='UTF-8')
h = hmac.new(secbytes, sigbytes, hashlib.sha384)
hexstring = h.hexdigest()
return {
"bfx-nonce": nonce,
"bfx-apikey": self.KEY,
"bfx-signature": hexstring,
"content-type": "application/json"
}
def req(self, path, params = {}):
nonce = self._nonce()
body = params
rawBody = json.dumps(body)
headers = self._headers(path, nonce, rawBody)
url = self.BASE_URL + path
resp = requests.post(url, headers=headers, data=rawBody, verify=True)
return resp
def active_orders(self):
# Fetch active orders
response = self.req("v2/auth/r/orders")
if response.status_code == 200:
return response.json()
else:
print('error, status_code = ', response.status_code)
return ''
# fetch all your orders and print out
client = BitfinexClient()
result = client.active_orders()
print(result)
Why not use one of the open source api clients out there ? and you can compare to your work .
https://github.com/scottjbarr/bitfinex
https://github.com/tuberculo/bitfinex
I have the below code and want to write the stream of tweets to a text file. Is there a way to include the output to text file within the same code and save it in the working directory? I am an IDE lover and really don't like using the console. I am new to python (2 weeks), I am an R / R Studio user.
I know I could use:
filename.py > output.txt
I am currently using Rodeo, Python 3.6.1.
import oauth2 as oauth
import urllib.request as urllib
api_key = "##"
api_secret = "##"
access_token_key = "##-##"
access_token_secret = "##"
_debug = 0
oauth_token = oauth.Token(key=access_token_key, secret=access_token_secret)
oauth_consumer = oauth.Consumer(key=api_key, secret=api_secret)
signature_method_hmac_sha1 = oauth.SignatureMethod_HMAC_SHA1()
http_method = "GET"
http_handler = urllib.HTTPHandler(debuglevel=_debug)
https_handler = urllib.HTTPSHandler(debuglevel=_debug)
'''
Construct, sign, and open a twitter request
using the hard-coded credentials above.
'''
def twitterreq(url, method, parameters):
req = oauth.Request.from_consumer_and_token(oauth_consumer,
token=oauth_token,
http_method=http_method,
http_url=url,
parameters=parameters)
req.sign_request(signature_method_hmac_sha1, oauth_consumer, oauth_token)
headers = req.to_header()
if http_method == "POST":
encoded_post_data = req.to_postdata()
else:
encoded_post_data = None
url = req.to_url()
opener = urllib.OpenerDirector()
opener.add_handler(http_handler)
opener.add_handler(https_handler)
response = opener.open(url, encoded_post_data)
f = open("output.txt", "wb")
def fetchsamples():
url = "https://stream.twitter.com/1.1/statuses/sample.json"
parameters = []
response = twitterreq(url, "GET", parameters)
for line in response:
f.write(line)
if __name__ == '__main__':
fetchsamples()
# f.close()
Besides the comment I made previously, I would suggesting checking out this stack overflow question: how to direct output into a txt file in python in windows
To quote:
If you want to do it in Python then you would write:
with open('out.txt', 'w') as f:
f.write(something)`
Obviously this is just a trivial example. You'd clearly do more inside the with block.
I've been trying to use the API of the website Idealista (https://www.idealista.com/) to retrieve information of real estate data.
Since I'm not familiarized with OAuth2 I haven't been able to obtain the token so far. I have just been provided with the api key, the secret and some basic info of how to mount the http request.
I would appreciate an example (preferably in Python) of the functioning of this API, or else some more generic info about dealing with OAuth2 and Python.
After some days of research I came up with a basic python code to retrieve real estate data from the Idealista API.
def get_oauth_token():
http_obj = Http()
url = "https://api.idealista.com/oauth/token"
apikey= urllib.parse.quote_plus('Provided_API_key')
secret= urllib.parse.quote_plus('Provided_API_secret')
auth = base64.encode(apikey + ':' + secret)
body = {'grant_type':'client_credentials'}
headers = {'Content-Type': 'application/x-www-form-urlencoded;charset=UTF-8','Authorization' : 'Basic ' + auth}
resp, content = http_obj.request(url,method='POST',headers=headers, body=urllib.parse.urlencode(body))
return content
This function would return a JSON with the OAuth2 token and the session time in seconds. Afterwards, to query the API, it would be as simple as:
def search_api(token):
http_obj = Http()
url = "http://api.idealista.com/3.5/es/search?center=40.42938099999995,-3.7097526269835726&country=es&maxItems=50&numPage=1&distance=452&propertyType=bedrooms&operation=rent"
headers = {'Authorization' : 'Bearer ' + token}
resp, content = http_obj.request(url,method='POST',headers=headers)
return content
This time the we would find in the content var the data we were looking for, again as a JSON.
That can't be marked as correct answer since
auth = base64.encode(apikey + ':' + secret)
body = {'grant_type':'client_credentials'}
headers = {'Content-Type': 'application/x-www-form-urlencoded;charset=UTF-8','Authorization' : 'Basic ' + auth}
Will give you TypeError:
can only concatenate str (not "bytes") to str
Since base64encode returns a byte type object...
It's true Idealista API is very limited about documentation, but I think this is a better approach since I don't use unnecesary libs (Only native):
#first request
message = API_KEY + ":" + SECRET
auth = "Basic " + base64.b64encode(message.encode("ascii")).decode("ascii")
headers_dic = {"Authorization" : auth,
"Content-Type" : "application/x-www-form-urlencoded;charset=UTF-8"}
params_dic = {"grant_type" : "client_credentials",
"scope" : "read"}
r = requests.post("https://api.idealista.com/oauth/token",
headers = headers_dic,
params = params_dic)
This works flawless with only python requests and base64 module...
regards
This is my code, improving #3... this run ok! for me!!!!
only put your apikey and your password (secret)...
import pandas as pd
import json
import urllib
import requests as rq
import base64
def get_oauth_token():
url = "https://api.idealista.com/oauth/token"
apikey= 'your_api_key' #sent by idealista
secret= 'your_password' #sent by idealista
auth = base64.b64encode(apikey + ':' + secret)
headers = {'Content-Type': 'application/x-www-form-urlencoded;charset=UTF-8' ,'Authorization' : 'Basic ' + auth}
params = urllib.urlencode({'grant_type':'client_credentials'})
content = rq.post(url,headers = headers, params=params)
bearer_token = json.loads(content.text)['access_token']
return bearer_token
def search_api(token, url):
headers = {'Content-Type': 'Content-Type: multipart/form-data;', 'Authorization' : 'Bearer ' + token}
content = rq.post(url, headers = headers)
result = json.loads(content.text)['access_token']
return result
country = 'es' #values: es, it, pt
locale = 'es' #values: es, it, pt, en, ca
language = 'es' #
max_items = '50'
operation = 'sale'
property_type = 'homes'
order = 'priceDown'
center = '40.4167,-3.70325'
distance = '60000'
sort = 'desc'
bankOffer = 'false'
df_tot = pd.DataFrame()
limit = 10
for i in range(1,limit):
url = ('https://api.idealista.com/3.5/'+country+'/search?operation='+operation+#"&locale="+locale+
'&maxItems='+max_items+
'&order='+order+
'¢er='+center+
'&distance='+distance+
'&propertyType='+property_type+
'&sort='+sort+
'&numPage=%s'+
'&language='+language) %(i)
a = search_api(get_oauth_token(), url)
df = pd.DataFrame.from_dict(a['elementList'])
df_tot = pd.concat([df_tot,df])
df_tot = df_tot.reset_index()
I found some mistakes. At least, I cannot run it.
I believe, I improved with this:
import pandas as pd
import json
import urllib
import requests as rq
import base64
def get_oauth_token():
url = "https://api.idealista.com/oauth/token"
apikey= 'your_api_key' #sent by idealist
secret= 'your_password' #sent by idealista
apikey_secret = apikey + ':' + secret
auth = str(base64.b64encode(bytes(apikey_secret, 'utf-8')))[2:][:-1]
headers = {'Authorization' : 'Basic ' + auth,'Content-Type': 'application/x-www-form-
urlencoded;charset=UTF-8'}
params = urllib.parse.urlencode({'grant_type':'client_credentials'}) #,'scope':'read'
content = rq.post(url,headers = headers, params=params)
bearer_token = json.loads(content.text)['access_token']
return bearer_token
def search_api(token, URL):
headers = {'Content-Type': 'Content-Type: multipart/form-data;', 'Authorization' : 'Bearer ' + token}
content = rq.post(url, headers = headers)
result = json.loads(content.text)
return result