I'm getting the data from API using requests library doing like this:
import requests
url = "Some String"
headers = {
'Authorization':"Some Token"}
response = requests.request("GET", url, headers=headers)
But the file that I'm trying to get is very large so I receive the time exceptions error. How can I get it using chunks in request?
Thanks!
import requests
import datetime
import pandas as pd
url = "some URL"
headers = {
'Authorization':"Some Token"}
start_date = datetime.datetime(2018, 6, 1)
end_date = datetime.datetime.now()
temp_end_date = start_date + datetime.timedelta(days=7)
output = dict()
while temp_end_date <= end_date:
temp_url = url % (start_date.timestamp()*1000, temp_end_date.timestamp()*1000)
response = requests.get(temp_url, headers=headers)
temp_data = response.json()
for key, value in temp_data.items():
output_arr = output.get(key, [])
output_arr.extend(value)
output[key] = output_arr
start_date = temp_end_date + datetime.timedelta(seconds=1)
temp_end_date += datetime.timedelta(days=7)
data=output
df=pd.DataFrame(data)
df.head()
Related
I am trying to executing rest api get using python script.Below the code I am using.But api response doesn't have proper Json format I am getting error.
import requests
# api-endpoint
URL = "end point url"
# sending get request and saving the response as response object
r = requests.get(url = URL)
# extracting data in json format
data = r.json()
print(data)
My api response below
[{:mac_address=>"10:55", :Parameter=>"Device.Info", :status=>"Success", :response_code=>"200", :value=>"2.4Ghz"}]
I need to change response as below
[{"mac_address":"10:55","Parameter":"Device.Info","status":"Success","response_code":"200","value":"2.4Ghz"}]
How to achieve this in python? I am new to python.
test = '[{:mac_address=>"10:55", :Parameter=>"Device.Info", :status=>"Success", :response_code=>"200", :value=>"2.4Ghz"}]'
def to_json(val: str):
val = val.replace("{:", '{"').replace(" :", '"').replace("=>", '":')
return val
res = to_json(test)
print(res)
result:
[{"mac_address":"10:55","Parameter":"Device.Info","status":"Success","response_code":"200","value":"2.4Ghz"}]
in your case:
import requests
import json
def to_json(val: str):
val = val.replace("{:", '{"').replace(" :", '"').replace("=>", '":')
return val
# api-endpoint
URL = "end point url"
# sending get request and saving the response as response object
r = requests.get(url = URL)
# extracting data in string format
data = r.text
# converting string to json object
res = json.loads(to_json(data))
print(res)
Assuming you get a plain text response, you can manually re-format the response using String.replace():
response = '[{:mac_address=>"10:55", :Parameter=>"Device.Info", :status=>"Success", :response_code=>"200", :value=>"2.4Ghz"}]'
desired = {}
response = response[2:-3].replace('"', '').split(', ')
for r in response:
key, value = r.split('=>')
key = key.replace(':','')
desired[key] = value
i have a json response as a string inside a json list
as you in the picture
enter image description here
i trying to get the value inside the string i tired to use eval()
but output shows me this error NameError: name 'null' is not defined
i can't read the json values when they are a string
enter image description here
this is my code :
url = "https://api.pipedream.com/v1/sources/code/event_summaries?
expand=event"
headers = {"Authorization": "Bearer hash "}
response = requests.get(url, headers=headers)
data = response.text
datas = json.loads(data)
darts = datas['data']
for i in darts:
trake = i['event']['body']
for docz in trake:
open_time = open_time = docz['open_time']
print(open_time)
enter image description here
the problem is the json values are string i cannot detect values
By the way the Bearer Authorization is just a demo
The data you needed is inside a dict key. So, you need to use .keys() attribute to retrieve it and then you have to use json.loads() to convert it to a dictionary.
Please check the below code:
import requests
import http.client
import json
from ast import literal_eval as evall
url = "https://api.pipedream.com/v1/sources/code/event_summaries?expand=event"
headers = {"Authorization": "Bearer hash"}
response = requests.get(url, headers=headers)
data = response.text
datas = json.loads(data)
darts = datas['data']
for i in darts:
trake = i['event']['body']
for docz in trake:
print(docz)
for tracks in darts:
tracks = json.loads(list(tracks['event']['body'].keys())[0])
print(tracks)
open_time = tracks['event']['trade'].get('open_time', '')
close_time = tracks['event']['trade'].get('close_time', '')
Lots = tracks['event']['trade'].get('lots', '')
balance = tracks['event']['account'].get('balance', '')
symbol = tracks['event']['trade'].get('symbol', '')
profit = tracks['event']['trade'].get('profit', '')
total_profit = tracks['event']['trade'].get('total_profit', '')
msg = """
Open time : """ +open_time + """
Close time : """ +close_time + """
Symbol : """ +symbol + """
lots : """ +Lots + """
Balance : """ +balance + """
"""
print(msg)
print("success")
import requests
import json
import pandas as pd
CSV_output_df = pd.read_csv('output/DEMO_CSV.csv', index_col=None)
payload = {}
headers = {
'Authorization': 'Basic ***********************************************************'
}
for index, row in CSV_output_df.iterrows():
Package_label = CSV_output_df.loc[index, "Package Id"]
licenseNumber = CSV_output_df.loc[index, "licenseNumber"]
Package_label = str(Package_label)
licenseNumber = str(licenseNumber)
url = ("https://api-mi.metrc.com/packages/v1/" + Package_label + "?licenseNumber=" + licenseNumber)
response = requests.request("GET", url, headers=headers, data=payload)
json_data = (response.text.encode('utf8'))
json_data = str(json_data)
json_data = (json_data.strip('b'))
json_data = (json_data.strip("'"))
json_data = (json_data.strip('{'))
json_data = (json_data.strip('}'))
json_data = (json_data.replace('"Item":{', ''))
json_data = (json_data.split(','))
json_data_df = pd.DataFrame(json_data)
Id = json_data_df.loc[0, 0]
Id = Id.replace('"Id":', '')
CSV_output_df.loc[index, "api_id"] = Id
for index, row in CSV_output_df.iterrows():
api_id = CSV_output_df.loc[index, "api_id"]
licenseNumber = CSV_output_df.loc[index, "licenseNumber"]
api_id = str(api_id)
licenseNumber = str(licenseNumber)
url0 = ("https://api-mi.metrc.com/labtests/v1/results?packageId=" + api_id + "&licenseNumber=" + licenseNumber)
response0 = requests.request("GET", url0, headers=headers, data=payload)
json_data0 = (response0.text.encode('utf8'))
json_data0 = str(json_data0)
json_data0 = (json_data0.strip('b'))
json_data0 = (json_data0.strip("'"))
json_data0 = (json_data0.strip('{'))
json_data0 = (json_data0.strip('}'))
json_data0 = (json_data0.strip('['))
json_data0 = (json_data0.strip(']'))
json_data0 = (json_data0.split(','))
json_data_df0 = pd.DataFrame(json_data0)
data_point = (json_data_df0.loc[1187, 0])
Python noobie here and 1st-time poster. So the issue is, below command not working in my for loop but is working as a standalone command.
data_point = (json_data_df0.loc[1187, 0])
The traceback log is telling me
ValueError: 1187 is not in range
but there are 1326 rows in json_data_df0 and all values except 0, 0 do not work in the loop.
I think you are supposed to use .iloc if you want to access the columns/rows using integer. .loc is for accessing columns/rows using the label.
For your reference: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html
I'm having trouble generating the authorization token for cosmos db for a simple get databases request. Here is my python code:
import requests
import hmac
import hashlib
import base64
from datetime import datetime
key = 'AG . . .EZPcZBKz7gvrKiXKsuaPA=='
now = datetime.utcnow().strftime('%a, %d %b %Y %H:%M:00 GMT')
payload = ('get\ndbs\n\n' + now + '\n\n').lower()
signature = base64.b64encode(hmac.new(key, msg = payload, digestmod = hashlib.sha256).digest()).decode()
url = 'https://myacct.documents.azure.com/dbs'
headers = {
'Authorization': "type=master&ver=1.0&sig=" + signature,
"x-ms-date": now,
"x-ms-version": "2017-02-22"
}
res = requests.get(url, headers = headers)
print res.content
Which produces this error:
{"code":"Unauthorized","message":"The input authorization token can't serve the request. Please check that the expected payload is built as per the protocol, and check the key being used. Server used the following payload to sign: 'get\ndbs\n\nsun, 08 apr 2018 02:39:00 gmt\n\n'\r\nActivityId: 5abe59d8-f44e-42c1-9380-5cf4e63425ec, Microsoft.Azure.Documents.Common/1.21.0.0"}
Greg. Per my observe, the miss of your code is url encode. You could find the sample code here.
Please refer to my code which was made a slight adjustment to your code.
import requests
import hmac
import hashlib
import base64
from datetime import datetime
import urllib
key = '***'
now = datetime.utcnow().strftime('%a, %d %b %Y %H:%M:00 GMT')
print now
payload = ('get\ndbs\n\n' + now + '\n\n').lower()
payload = bytes(payload).encode('utf-8')
key = base64.b64decode(key.encode('utf-8'))
signature = base64.b64encode(hmac.new(key, msg = payload, digestmod = hashlib.sha256).digest()).decode()
print signature
authStr = urllib.quote('type=master&ver=1.0&sig={}'.format(signature))
print authStr
headers = {
'Authorization': authStr,
"x-ms-date": now,
"x-ms-version": "2017-02-22"
}
url = 'https://***.documents.azure.com/dbs'
res = requests.get(url, headers = headers)
print res.content
Execute result:
Hope it helps you.
I know this has been asked before .. but my code doesn't seem to work for users other than the one using whom accesstoken is created
.. i.e I am able to access the profile of only one user.(using whom I created accesstoken)
from urllib2 import urlopen
from json import load
import json
import json
import sys
import time
import requests
userid="userid_of_some_instagram_user"
access_token="mytoken"
url = "https://api.instagram.com/v1/users/%s/?access_token=%s"%(userid,access_token)
response=urlopen(url)
print(response)
jsonobj=load(response)
print "name:",jsonobj["data"]["username"]
print "follows ",jsonobj["data"]["counts"]["follows"]," people"
print "followed by ",jsonobj["data"]["counts"]["followed_by"]," people"
Error I am getting when I am trying to access other profiles is
HTTPError: HTTP Error 400: BAD REQUEST
Edit : **Voila ! I have found the bug . It is the access key. I have tried with a different access key . It worked**
If you don't wanna fight the new InstagramAPI you can use a workaround like this:
import requests
import json
import time
import random
user = 'google'
url = 'https://www.instagram.com/'
url_user = '%s%s%s' % (url, user, '/')
url_login = 'https://www.instagram.com/accounts/login/ajax/'
s = requests.Session()
s.cookies.update ({'sessionid' : '', 'mid' : '', 'ig_pr' : '1',
'ig_vw' : '1920', 'csrftoken' : '',
's_network' : '', 'ds_user_id' : ''})
login_post = {'username' : 'your_login',
'password' : 'your_pw'}
s.headers.update ()
r = s.get(url)
s.headers.update({'X-CSRFToken' : r.cookies['csrftoken']})
time.sleep(5 * random.random())
login = s.post(url_login, data=login_post,
allow_redirects=True)
s.headers.update({'X-CSRFToken' : login.cookies['csrftoken']})
if login.status_code == 200:
r = s.get('https://www.instagram.com/')
finder = r.text.find('your_login')
r = s.get(url_user)
text = r.text
finder_text_start = ('<script type="text/javascript">'
'window._sharedData = ')
finder_text_start_len = len(finder_text_start)-1
finder_text_end = ';</script>'
all_data_start = text.find(finder_text_start)
all_data_end = text.find(finder_text_end, all_data_start + 1)
json_str = text[(all_data_start + finder_text_start_len + 1) \
: all_data_end]
user_info = json.loads(json_str)
follower_count = user_info['entry_data']['ProfilePage'][0]['user']["followed_by"]['count']
print(follower_count)
Try this:
import json
import requests
access_token = "mytoken"
url = "https://api.instagram.com/v1/users/self/?access_token={0}".format(access_token)
response = requests.get(url)
j = json.loads(response.text)
if 'data' in j:
print("handle correct response")
else:
print("handle incorrect response")