I'm using a python script, to pull data from https://graph.microsoft.com. The output that is delivered duplicates itself (10) times for each parsed user. What step is missing to only capture the requested once?
import requests
import urllib
import json
import csv
import os
client_id = urllib.parse.quote_plus("#######################")
client_secret = urllib.parse.quote_plus("######################")
tenant = urllib.parse.quote_plus("#########################")
auth_uri = "https://login.microsoftonline.com/" + tenant + "/oauth2/v2.0/token"
auth_body = "grant_type=client_credentials&client_id=" + client_id + "&client_secret=" + client_secret + "&scope=https%3A%2F%2Fgraph.microsoft.com%2F.default"
authorization = requests.post(auth_uri, data=auth_body, headers={"Content-Type": "application/x-www-form-urlencoded"})
token = json.loads(authorization.content)['access_token']
graph_uri = "https://graph.microsoft.com/v1.0/reports/getEmailActivityUserDetail(period=%27D30%27)"
response = requests.get(graph_uri, data=auth_body, headers={'Content-Type': "application/json", 'Authorization': 'Bearer ' + token})
print(response.text)
temp_usr_list = [
'User.One#domain.com',
'User.Two#domain.com'
]
report_user_list = []
for line in response.iter_lines():
line_fields = line.decode("utf-8").split(',')
for entry in line_fields:
if len(entry) < 1:
continue
if line_fields[1] in temp_usr_list:
d = dict(
user_principle_name = line_fields[1],
send_count = line_fields[6],
recv_count = line_fields[7],
read_count = line_fields[8],
assigned_products = line_fields[9]
)
report_user_list.append(d)
print(report_user_list)
OUTPUT:
{'user_principle_name': 'User.One#domain.com', 'send_count': '0', 'recv_count': '0', 'read_count': '0', 'assigned_products': 'MICROSOFT'},...
{'user_principle_name': 'User.Two#domain.com', 'send_count': '0', 'recv_count': '0', 'read_count': '0', 'assigned_products': 'MICROSOFT'},...
Try running it once in this format and let me know if you get the same output.
import requests
import urllib
import json
import csv
import os
# Parms
client_id = urllib.parse.quote_plus('#######################')
client_secret = urllib.parse.quote_plus('######################')
tenant = urllib.parse.quote_plus('#########################')
auth_uri = 'https://login.microsoftonline.com/' + tenant \
+ '/oauth2/v2.0/token'
auth_body = 'grant_type=client_credentials&client_id=' + client_id \
+ '&client_secret=' + client_secret \
+ '&scope=https%3A%2F%2Fgraph.microsoft.com%2F.default'
authorization = requests.post(auth_uri, data=auth_body,
headers={'Content-Type': 'application/x-www-form-urlencoded'
})
token = json.loads(authorization.content)['access_token']
graph_uri = \
'https://graph.microsoft.com/v1.0/reports/getEmailActivityUserDetail(period=%27D30%27)'
response = requests.get(graph_uri, data=auth_body,
headers={'Content-Type': 'application/json',
'Authorization': 'Bearer ' + token})
print response.text
temp_usr_list = ['User.One#domain.com', 'User.Two#domain.com']
report_user_list = []
for line in response.iter_lines():
line_fields = line.decode('utf-8').split(',')
for entry in line_fields:
if len(entry) < 1:
continue
if line_fields[1] in temp_usr_list:
d = dict(user_principle_name=line_fields[1],
send_count=line_fields[6],
recv_count=line_fields[7],
read_count=line_fields[8],
assigned_products=line_fields[9])
report_user_list.append(d)
print report_user_list
Related
The working SHA-1 code is as so.
import oauth2 as oauth
import requests
import hmac
import hashlib
from base64 import b64encode #To encode binary data into Base64
import binascii
import requests
import random
import time
import urllib.parse
url = apiURL
token = oauth.Token(key='x')
consumer = oauth.Consumer(key='x')
http_method = 'POST'
realm="xxx"
params = {
'oauth_version': '1.0',
'oauth_nonce': oauth.generate_nonce(),
'oauth_timestamp': str(int(time.time())),
'oauth_token': token.key,
'oauth_consumer_key': consumer.key
}
req = oauth.Request(method=http_method, url=url, parameters=params)
signature_method = oauth.SignatureMethod_HMAC_SHA1()
req.sign_request(signature_method, consumer, token)
header = req.to_header(realm)
print(header['Authorization'])
headery = header['Authorization'].encode('ascii', 'ignore')
headerx = {"Authorization": headery, "Content-Type": "application/json"}
#print(headerx)
payload = {"operation": "moveFile", "id": "1450"}
conn = requests.post(url, data=netSuiteData, headers=headerx)
# print("\n\nResult: " + conn.text)
# print("\n\n\n")
# print(conn.headers)
After researching the issue I found a similar problem from a post over a year old. I implemented the solution suggested as so:
import oauth2 as oauth
import requests
import hmac
import hashlib
from base64 import b64encode #To encode binary data into Base64
import binascii
import requests
import random
import time
import urllib.parse
url = apiURL
method = 'POST'
access_token = "x"
token_secret = "x"
oauth_consumer_key = "x"
consumer_secret = "x"
oauth_signature_method = "HMAC-SHA256"
oauth_version = "1.0"
oauth_nonce = ''.join(random.choices("0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ", k=11))
oauth_timestamp = str(int(time.time()))
account = 'x'
body = netSuiteData
def create_parameter_string(oauth_consumer_key,oauth_nonce,oauth_signature_method,oauth_timestamp,oauth_version, token_id):
parameter_string = ''
parameter_string = parameter_string + 'oauth_consumer_key=' + oauth_consumer_key
parameter_string = parameter_string + '&oauth_nonce=' + oauth_nonce
parameter_string = parameter_string + '&oauth_signature_method=' + oauth_signature_method
parameter_string = parameter_string + '&oauth_timestamp=' + oauth_timestamp
parameter_string = parameter_string + '&oauth_token=' + token_id
parameter_string = parameter_string + '&oauth_version=' + oauth_version
return parameter_string
parameter_string = create_parameter_string(oauth_consumer_key,oauth_nonce,oauth_signature_method,oauth_timestamp,oauth_version,access_token)
encoded_parameter_string = urllib.parse.quote(parameter_string, safe='')
encoded_base_string = method + '&' + urllib.parse.quote(url, safe='')
encoded_base_string = encoded_base_string + '&' + encoded_parameter_string
signing_key = consumer_secret + '&' + token_secret
def create_signature(secret_key, signature_base_string):
encoded_string = signature_base_string.encode()
encoded_key = secret_key.encode()
temp = hmac.new(encoded_key, encoded_string, hashlib.sha256).hexdigest()
byte_array = b64encode(binascii.unhexlify(temp))
return byte_array.decode()
oauth_signature = create_signature(signing_key, encoded_base_string)
encoded_oauth_signature = urllib.parse.quote(oauth_signature, safe='')
print("param" + parameter_string )
headers = {
'Content-Type':'application/x-www-form-urlencoded',
'Authorization':'OAuth realm="{0}",oauth_consumer_key="{1}",oauth_token="{2}",oauth_signature_method="{3}",oauth_timestamp="{4}",oauth_nonce="{5}",oauth_version="{6}",oauth_signature="{7}"'.format(
str(account),oauth_consumer_key,access_token,oauth_signature_method, oauth_timestamp ,oauth_nonce,oauth_version ,encoded_oauth_signature)
}
print("headers***************")
print(headers)
response = requests.post(url, data=body, headers=headers)
print(response.text)
The first block of code works just fine, however when testing the new block of code (when running the script which needs signatures generated) - the console is telling me access was denied, and the authentication failed.
Hi I'm trying to fetch data from NetSuite but unable to do so because of INVALID LOGIN ATTEMPT. I tried a lot of stuff but noting happened interestingly in my POSTMAN everything is working perfectly. And Following are the some documents out of dozens which I referred.
Doc1: Netsuite OAuth Not Working
Doc2: https://netsuite.custhelp.com/app/answers/detail/a_id/44241/kw/SuiteTalk%20Authentication
Doc3: https://www.endyourif.com/netsuite-api-setting-up-access-token-authentication/
Doc4: https://github.com/NetSweet/netsuite/blob/master/lib/netsuite/passports/token.rb
Doc5: Implementation HMAC-SHA1 in python
And is my code.
nsAccountID = "1059967"
consumerKey = "434545646123fdgty7565g2bd1a71f0a2ae2badbeda67771a"
consumerSecret = "cdnc87rrth34ut4346wvnhrfg84fhf8923945u48r42fhcedw78df4"
token = "43t43f7hefc7h34fh34789fwf234rf90e89cf4h98f234"
tokenSecret = "78hf487rfy478fhc478fh34f34f3434t4yhbwae21443665u"
Nonce = self._generateNonce(length=11)
currentTime = self._generateTimestamp()
signature_method = 'HMAC-SHA256'
version = '1.0'
method = 'GET'
base_url = "https://1056867.suitetalk.api.netsuite.com/services/rest/record/v1/customer"
encoded_url = urllib.parse.quote(base_url)
collected_string = '&'.join(['oauth_consumer_key='+consumerKey, 'oauth_nonce='+Nonce,
'oauth_signature_method='+signature_method, 'oauth_timestamp='+currentTime,
'oauth_token='+token, 'oauth_version='+version])
encoded_string = urllib.parse.quote(collected_string)
base = '&'.join([method, encoded_url, encoded_string])
key = '&'.join([consumerSecret, tokenSecret])
digest = hmac.new(key=str.encode(key), msg=str.encode(base), digestmod=hashlib.sha256).digest()
signature = base64.b64encode(digest).decode()
url = "https://1059967.suitetalk.api.netsuite.com/services/rest/record/v1/customer"
payload = ""
headers = {
'Content-Type': "application/json",
'Authorization': "OAuth realm=\"1059967\","
"oauth_consumer_key=\"434545646123fdgty7565g2bd1a71f0a2ae2badbeda67771a\","
"oauth_token=\"43t43f7hefc7h34fh34789fwf234rf90e89cf4h98f234\","
"oauth_signature_method=\"HMAC-SHA256\","
"oauth_timestamp=\"" + currentTime + "\","
"oauth_nonce=\""+Nonce+"\","
"oauth_version=\"1.0\","
"oauth_signature=\"" + signature + "\"",
'cache-control': "no-cache",
}
response = requests.request("GET", url, data=payload, headers=headers)
I also tried sdk i.e netsuitesdk but it was giving me an error
unable to open database file
Note: Upper given credentials are dummy
Here you can see my code.
def _generateTimestamp(self):
return str(int(time.time()))
def _generateNonce(self, length=11):
"""Generate pseudorandom number
"""
return ''.join([str(random.randint(0, 9)) for i in range(length)])
def _generateSignature(self, method, url, consumerKey, Nonce, currentTime, token, consumerSecret,
tokenSecret, offset):
signature_method = 'HMAC-SHA256'
version = '1.0'
base_url = url
encoded_url = urllib.parse.quote_plus(base_url)
collected_string = None
if type(offset) == int:
collected_string = '&'.join(['oauth_consumer_key=' + consumerKey, 'oauth_nonce=' + Nonce,
'oauth_signature_method=' + signature_method, 'oauth_timestamp=' + currentTime,
'oauth_token=' + token, 'oauth_version=' + version, 'offset=' + str(offset)])
else:
collected_string = '&'.join(['oauth_consumer_key=' + consumerKey, 'oauth_nonce=' + Nonce,
'oauth_signature_method=' + signature_method, 'oauth_timestamp=' + currentTime,
'oauth_token=' + token, 'oauth_version=' + version])
encoded_string = urllib.parse.quote_plus(collected_string)
base = '&'.join([method, encoded_url, encoded_string])
key = '&'.join([consumerSecret, tokenSecret])
digest = hmac.new(key=str.encode(key), msg=str.encode(base), digestmod=hashlib.sha256).digest()
signature = base64.b64encode(digest).decode()
return urllib.parse.quote_plus(signature)
def _create_oauth(self, base_url):
nsAccountID = 'YOUR_NETSUITE_ACCOUNT_ID'
consumerKey = 'YOUR_NETSUITE_CONSUMER_KEY'
consumerSecret = 'YOUR_NETSUITE_CONSUMER_SECRET'
token = 'YOUR_NETSUITE_TOKEN'
tokenSecret = 'YOUR_NETSUITE_TOKEN_SECRET'
Nonce = self._generateNonce(length=11)
currentTime = self._generateTimestamp()
signature = self._generateSingleSignature('GET', base_url, consumerKey, Nonce, currentTime, token,
consumerSecret, tokenSecret)
oauth = "OAuth realm=\"" + nsAccountID + "\"," \
"oauth_consumer_key=\"" + consumerKey + "\"," \
"oauth_token=\"" + token + "\"," \
"oauth_signature_method=\"HMAC-SHA256\"," \
"oauth_timestamp=\"" + currentTime + "\"," \
"oauth_nonce=\"" + Nonce + "\"," \
"oauth_version=\"1.0\"," \
"oauth_signature=\"" + signature + "\""
headers = {
'Content-Type': "application/json",
'Authorization': oauth,
'cache-control': "no-cache",
}
return headers
"""Here is my API call"""
base_url = "https://REALM.suitetalk.api.netsuite.com/services/rest/record/v1/salesorder/"
payload = ""
data = {}
response = requests.request("GET", base_url, data=payload, headers=self._create_oauth(base_url))
import requests
import json
import pandas as pd
CSV_output_df = pd.read_csv('output/DEMO_CSV.csv', index_col=None)
payload = {}
headers = {
'Authorization': 'Basic ***********************************************************'
}
for index, row in CSV_output_df.iterrows():
Package_label = CSV_output_df.loc[index, "Package Id"]
licenseNumber = CSV_output_df.loc[index, "licenseNumber"]
Package_label = str(Package_label)
licenseNumber = str(licenseNumber)
url = ("https://api-mi.metrc.com/packages/v1/" + Package_label + "?licenseNumber=" + licenseNumber)
response = requests.request("GET", url, headers=headers, data=payload)
json_data = (response.text.encode('utf8'))
json_data = str(json_data)
json_data = (json_data.strip('b'))
json_data = (json_data.strip("'"))
json_data = (json_data.strip('{'))
json_data = (json_data.strip('}'))
json_data = (json_data.replace('"Item":{', ''))
json_data = (json_data.split(','))
json_data_df = pd.DataFrame(json_data)
Id = json_data_df.loc[0, 0]
Id = Id.replace('"Id":', '')
CSV_output_df.loc[index, "api_id"] = Id
for index, row in CSV_output_df.iterrows():
api_id = CSV_output_df.loc[index, "api_id"]
licenseNumber = CSV_output_df.loc[index, "licenseNumber"]
api_id = str(api_id)
licenseNumber = str(licenseNumber)
url0 = ("https://api-mi.metrc.com/labtests/v1/results?packageId=" + api_id + "&licenseNumber=" + licenseNumber)
response0 = requests.request("GET", url0, headers=headers, data=payload)
json_data0 = (response0.text.encode('utf8'))
json_data0 = str(json_data0)
json_data0 = (json_data0.strip('b'))
json_data0 = (json_data0.strip("'"))
json_data0 = (json_data0.strip('{'))
json_data0 = (json_data0.strip('}'))
json_data0 = (json_data0.strip('['))
json_data0 = (json_data0.strip(']'))
json_data0 = (json_data0.split(','))
json_data_df0 = pd.DataFrame(json_data0)
data_point = (json_data_df0.loc[1187, 0])
Python noobie here and 1st-time poster. So the issue is, below command not working in my for loop but is working as a standalone command.
data_point = (json_data_df0.loc[1187, 0])
The traceback log is telling me
ValueError: 1187 is not in range
but there are 1326 rows in json_data_df0 and all values except 0, 0 do not work in the loop.
I think you are supposed to use .iloc if you want to access the columns/rows using integer. .loc is for accessing columns/rows using the label.
For your reference: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html
import requests
if __name__ == '__main__':
API = 'https://api.twitch.tv/helix/streams?user_login=rediban'
Client_ID = "myid"
OAuth = "mytoken"
head = {
'Client-ID': Client_ID,
'OAuth': OAuth,
}
rq = requests.get(url=API, headers=head)
print(rq.text)
I would like to have the currently live viewer in a stream.
When i start the script it says {"error":"Unauthorized","status":401,"message":"OAuth token is missing"}.
Hope you can help me :)
Here is an example, you'll need to provide your own ClientID and Secret and the stream_name you want to lookup. You can sign up for CID at https://dev.twitch.tv/console/apps
This example will generate a Client Credentials token each time which is bad practice. You should (normally) generate and store then resuse a token till it expires.
It generates a token, then calls the streams API to lookup the live status
import requests
client_id = ''
client_secret = ''
streamer_name = ''
body = {
'client_id': client_id,
'client_secret': client_secret,
"grant_type": 'client_credentials'
}
r = requests.post('https://id.twitch.tv/oauth2/token', body)
#data output
keys = r.json();
print(keys)
headers = {
'Client-ID': client_id,
'Authorization': 'Bearer ' + keys['access_token']
}
print(headers)
stream = requests.get('https://api.twitch.tv/helix/streams?user_login=' + streamer_name, headers=headers)
stream_data = stream.json();
print(stream_data);
if len(stream_data['data']) == 1:
print(streamer_name + ' is live: ' + stream_data['data'][0]['title'] + ' playing ' + stream_data['data'][0]['game_name']);
else:
print(streamer_name + ' is not live');
I've been trying to use the API of the website Idealista (https://www.idealista.com/) to retrieve information of real estate data.
Since I'm not familiarized with OAuth2 I haven't been able to obtain the token so far. I have just been provided with the api key, the secret and some basic info of how to mount the http request.
I would appreciate an example (preferably in Python) of the functioning of this API, or else some more generic info about dealing with OAuth2 and Python.
After some days of research I came up with a basic python code to retrieve real estate data from the Idealista API.
def get_oauth_token():
http_obj = Http()
url = "https://api.idealista.com/oauth/token"
apikey= urllib.parse.quote_plus('Provided_API_key')
secret= urllib.parse.quote_plus('Provided_API_secret')
auth = base64.encode(apikey + ':' + secret)
body = {'grant_type':'client_credentials'}
headers = {'Content-Type': 'application/x-www-form-urlencoded;charset=UTF-8','Authorization' : 'Basic ' + auth}
resp, content = http_obj.request(url,method='POST',headers=headers, body=urllib.parse.urlencode(body))
return content
This function would return a JSON with the OAuth2 token and the session time in seconds. Afterwards, to query the API, it would be as simple as:
def search_api(token):
http_obj = Http()
url = "http://api.idealista.com/3.5/es/search?center=40.42938099999995,-3.7097526269835726&country=es&maxItems=50&numPage=1&distance=452&propertyType=bedrooms&operation=rent"
headers = {'Authorization' : 'Bearer ' + token}
resp, content = http_obj.request(url,method='POST',headers=headers)
return content
This time the we would find in the content var the data we were looking for, again as a JSON.
That can't be marked as correct answer since
auth = base64.encode(apikey + ':' + secret)
body = {'grant_type':'client_credentials'}
headers = {'Content-Type': 'application/x-www-form-urlencoded;charset=UTF-8','Authorization' : 'Basic ' + auth}
Will give you TypeError:
can only concatenate str (not "bytes") to str
Since base64encode returns a byte type object...
It's true Idealista API is very limited about documentation, but I think this is a better approach since I don't use unnecesary libs (Only native):
#first request
message = API_KEY + ":" + SECRET
auth = "Basic " + base64.b64encode(message.encode("ascii")).decode("ascii")
headers_dic = {"Authorization" : auth,
"Content-Type" : "application/x-www-form-urlencoded;charset=UTF-8"}
params_dic = {"grant_type" : "client_credentials",
"scope" : "read"}
r = requests.post("https://api.idealista.com/oauth/token",
headers = headers_dic,
params = params_dic)
This works flawless with only python requests and base64 module...
regards
This is my code, improving #3... this run ok! for me!!!!
only put your apikey and your password (secret)...
import pandas as pd
import json
import urllib
import requests as rq
import base64
def get_oauth_token():
url = "https://api.idealista.com/oauth/token"
apikey= 'your_api_key' #sent by idealista
secret= 'your_password' #sent by idealista
auth = base64.b64encode(apikey + ':' + secret)
headers = {'Content-Type': 'application/x-www-form-urlencoded;charset=UTF-8' ,'Authorization' : 'Basic ' + auth}
params = urllib.urlencode({'grant_type':'client_credentials'})
content = rq.post(url,headers = headers, params=params)
bearer_token = json.loads(content.text)['access_token']
return bearer_token
def search_api(token, url):
headers = {'Content-Type': 'Content-Type: multipart/form-data;', 'Authorization' : 'Bearer ' + token}
content = rq.post(url, headers = headers)
result = json.loads(content.text)['access_token']
return result
country = 'es' #values: es, it, pt
locale = 'es' #values: es, it, pt, en, ca
language = 'es' #
max_items = '50'
operation = 'sale'
property_type = 'homes'
order = 'priceDown'
center = '40.4167,-3.70325'
distance = '60000'
sort = 'desc'
bankOffer = 'false'
df_tot = pd.DataFrame()
limit = 10
for i in range(1,limit):
url = ('https://api.idealista.com/3.5/'+country+'/search?operation='+operation+#"&locale="+locale+
'&maxItems='+max_items+
'&order='+order+
'¢er='+center+
'&distance='+distance+
'&propertyType='+property_type+
'&sort='+sort+
'&numPage=%s'+
'&language='+language) %(i)
a = search_api(get_oauth_token(), url)
df = pd.DataFrame.from_dict(a['elementList'])
df_tot = pd.concat([df_tot,df])
df_tot = df_tot.reset_index()
I found some mistakes. At least, I cannot run it.
I believe, I improved with this:
import pandas as pd
import json
import urllib
import requests as rq
import base64
def get_oauth_token():
url = "https://api.idealista.com/oauth/token"
apikey= 'your_api_key' #sent by idealist
secret= 'your_password' #sent by idealista
apikey_secret = apikey + ':' + secret
auth = str(base64.b64encode(bytes(apikey_secret, 'utf-8')))[2:][:-1]
headers = {'Authorization' : 'Basic ' + auth,'Content-Type': 'application/x-www-form-
urlencoded;charset=UTF-8'}
params = urllib.parse.urlencode({'grant_type':'client_credentials'}) #,'scope':'read'
content = rq.post(url,headers = headers, params=params)
bearer_token = json.loads(content.text)['access_token']
return bearer_token
def search_api(token, URL):
headers = {'Content-Type': 'Content-Type: multipart/form-data;', 'Authorization' : 'Bearer ' + token}
content = rq.post(url, headers = headers)
result = json.loads(content.text)
return result