i have a json response as a string inside a json list
as you in the picture
enter image description here
i trying to get the value inside the string i tired to use eval()
but output shows me this error NameError: name 'null' is not defined
i can't read the json values when they are a string
enter image description here
this is my code :
url = "https://api.pipedream.com/v1/sources/code/event_summaries?
expand=event"
headers = {"Authorization": "Bearer hash "}
response = requests.get(url, headers=headers)
data = response.text
datas = json.loads(data)
darts = datas['data']
for i in darts:
trake = i['event']['body']
for docz in trake:
open_time = open_time = docz['open_time']
print(open_time)
enter image description here
the problem is the json values are string i cannot detect values
By the way the Bearer Authorization is just a demo
The data you needed is inside a dict key. So, you need to use .keys() attribute to retrieve it and then you have to use json.loads() to convert it to a dictionary.
Please check the below code:
import requests
import http.client
import json
from ast import literal_eval as evall
url = "https://api.pipedream.com/v1/sources/code/event_summaries?expand=event"
headers = {"Authorization": "Bearer hash"}
response = requests.get(url, headers=headers)
data = response.text
datas = json.loads(data)
darts = datas['data']
for i in darts:
trake = i['event']['body']
for docz in trake:
print(docz)
for tracks in darts:
tracks = json.loads(list(tracks['event']['body'].keys())[0])
print(tracks)
open_time = tracks['event']['trade'].get('open_time', '')
close_time = tracks['event']['trade'].get('close_time', '')
Lots = tracks['event']['trade'].get('lots', '')
balance = tracks['event']['account'].get('balance', '')
symbol = tracks['event']['trade'].get('symbol', '')
profit = tracks['event']['trade'].get('profit', '')
total_profit = tracks['event']['trade'].get('total_profit', '')
msg = """
Open time : """ +open_time + """
Close time : """ +close_time + """
Symbol : """ +symbol + """
lots : """ +Lots + """
Balance : """ +balance + """
"""
print(msg)
print("success")
Related
I am trying to executing rest api get using python script.Below the code I am using.But api response doesn't have proper Json format I am getting error.
import requests
# api-endpoint
URL = "end point url"
# sending get request and saving the response as response object
r = requests.get(url = URL)
# extracting data in json format
data = r.json()
print(data)
My api response below
[{:mac_address=>"10:55", :Parameter=>"Device.Info", :status=>"Success", :response_code=>"200", :value=>"2.4Ghz"}]
I need to change response as below
[{"mac_address":"10:55","Parameter":"Device.Info","status":"Success","response_code":"200","value":"2.4Ghz"}]
How to achieve this in python? I am new to python.
test = '[{:mac_address=>"10:55", :Parameter=>"Device.Info", :status=>"Success", :response_code=>"200", :value=>"2.4Ghz"}]'
def to_json(val: str):
val = val.replace("{:", '{"').replace(" :", '"').replace("=>", '":')
return val
res = to_json(test)
print(res)
result:
[{"mac_address":"10:55","Parameter":"Device.Info","status":"Success","response_code":"200","value":"2.4Ghz"}]
in your case:
import requests
import json
def to_json(val: str):
val = val.replace("{:", '{"').replace(" :", '"').replace("=>", '":')
return val
# api-endpoint
URL = "end point url"
# sending get request and saving the response as response object
r = requests.get(url = URL)
# extracting data in string format
data = r.text
# converting string to json object
res = json.loads(to_json(data))
print(res)
Assuming you get a plain text response, you can manually re-format the response using String.replace():
response = '[{:mac_address=>"10:55", :Parameter=>"Device.Info", :status=>"Success", :response_code=>"200", :value=>"2.4Ghz"}]'
desired = {}
response = response[2:-3].replace('"', '').split(', ')
for r in response:
key, value = r.split('=>')
key = key.replace(':','')
desired[key] = value
I am trying to extract data from a JSON API but I get an error string indices must be integers i couldn't find anything about this here is my code:
import requests
import json
name = input('input a name: ')
server = input('input a server: ')
response = requests.get('https://api.battlemetrics.com/players?fields[server]=name&filter[search]=' + name + '&filter[servers]=' + server + '&page[size]=10&include=server')
def jprint(obj):
#create a formatted string of the Python JSON onject
text = json.dumps(obj, sort_keys=True, indent=4)
print (text)
pass_times = response.json()
jprint(pass_times)
status = []
for d in pass_times:
time = d["online"]
status.append(time)
print (status)
import requests
import json
name = "master oogway"
server = "6354292"
response = requests.get('https://api.battlemetrics.com/players?fields[server]=name&filter[search]=' + name + '&filter[servers]=' + server + '&page[size]=10&include=server')
def jprint(obj):
#create a formatted string of the Python JSON onject
text = json.dumps(obj, sort_keys=True, indent=4)
print (text)
pass_times = response.json()
#jprint(pass_times)
status = []
for data in pass_times["data"]:
status.append(data["relationships"]["servers"]["data"][0]["meta"]["online"])
print(status)
I am calling an API to get a list of properties. I obtain the jsons in the 'listings' file (https://api.nestoria.es/show_example?name=search_listings_es&syntax=1).
I am using zip codes to find each property and I need to add to each property its zip code. At the end I transform the json to a csv. I don't know how to add to each property the zip code (so I would need a new key-value to each result from the API call)...
Thanks!!!!!!!!!
Here it is the code:
from requests import get
import json
import pandas as pd
import time
import datetime
import csv
def get_nestoria(type):
#call the api
api = 'http://api.nestoria.es/api?action=search_listings'
place = '&place_name=' + area_name
listing_type = '&listing_type=' + type
json_es = '&encoding=json&pretty=1&country=es'
page = '&page='
api_input = api + place + listing_type + json_es
response = get(api_input)
# Check if the API has worked
if response.status_code == 200:
print("API called successfully")
elif response.status_code == 400:
print("Wrong request for" + area_name + ". Chechk this area is searchable")
elif response.status_code == 403:
print("Forbidden API call. Maximum number of calls reached.")
else:
print("Wrong code", response.status_code)
content_as_string = response.content.decode()
# Decode JSON
content = json.loads(content_as_string)
content_response = content['response']
# Number of total web pages neded for the area
web_pages = content_response['total_pages']
print('Number of pages in that area: ', web_pages)
print("Numer of total properties " + area_name, content_response['total_results'])
#2nd call to the API
homes = pd.DataFrame()
headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 6.0; WOW64; rv:24.0) Gecko/20100101 Firefox/24.0'}
for i in range(1, web_pages+1):
api_input = api + place + listing_type + json_es + page + str(i)
response = get(api_input, headers=headers)
content_as_string = response.content.decode()
content = json.loads(content_as_string)
content_response = content['response']
listings = content_response['listings']
listings = pd.DataFrame(listings)
if i==1:
homes = listings
else:
homes = homes.append(listings, sort=True)
time.sleep(3)
if homes.empty:
homes = homes
else:
homes = homes[['bathroom_number','bedroom_number','car_spaces','commission','construction_year','datasource_name', 'img_height','img_url','img_width', 'floor',
'keywords','latitude','lister_url','listing_type','location_accuracy','longitude','price','price_currency','price_formatted','price_high','price_low',
'property_type','room_number','size','size_type','size_unit','summary','thumb_height','thumb_url','thumb_width','title','updated_in_days','updated_in_days_formatted']]
return homes
homes = pd.DataFrame()
codigos_postales = ['01008']
today=datetime.date.today() #to change the name of the file
for i in codigos_postales:
area_name = i
temp = get_nestoria('buy')
if i == 0:
homes =temp
else:
homes = homes.append(temp,sort=True)
print('Number of extracted properties ', len(homes))
print(homes.head())
homes.to_csv('D:\\a000Master Big Data\\Prácticas\\Web scrapping\\Nestoria\\GranadaVenta'+str(today)+'.csv')
data = response.json()
Here data is the parsed response you get from sending the request.
You can then update the listings like this:
for i in data['listings']:
i['ZipCode'] = zipcode
zipcode being the one you want to assign i.e the one you sent in the request.
You can either convert the data object into dataframe and then call pd.to_csv or use python in-built csv.writer
If you don't have the ZipCode you can use the google maps API to get that
http://maps.googleapis.com/maps/api/geocode/json?address=valencia&sensor=true_or_false&key=YOUR_API_KEY
You will have to Sign Up to get your API key and then you will get zip code from the JSON response.
UPDATE:
Here is an example on how to use it.
api = 'http://api.nestoria.es/api?action=search_listings'
place = '&place_name=' + area_name
listing_type = '&listing_type=' + type
json_es = '&encoding=json&pretty=1&country=es'
page = '&page='
api_input = api + place + listing_type + json_es
response = get(api_input)
update = response.json()['response']['listings']
for i in update:
i['Zipcode'] = zipcode
I've been trying to use the API of the website Idealista (https://www.idealista.com/) to retrieve information of real estate data.
Since I'm not familiarized with OAuth2 I haven't been able to obtain the token so far. I have just been provided with the api key, the secret and some basic info of how to mount the http request.
I would appreciate an example (preferably in Python) of the functioning of this API, or else some more generic info about dealing with OAuth2 and Python.
After some days of research I came up with a basic python code to retrieve real estate data from the Idealista API.
def get_oauth_token():
http_obj = Http()
url = "https://api.idealista.com/oauth/token"
apikey= urllib.parse.quote_plus('Provided_API_key')
secret= urllib.parse.quote_plus('Provided_API_secret')
auth = base64.encode(apikey + ':' + secret)
body = {'grant_type':'client_credentials'}
headers = {'Content-Type': 'application/x-www-form-urlencoded;charset=UTF-8','Authorization' : 'Basic ' + auth}
resp, content = http_obj.request(url,method='POST',headers=headers, body=urllib.parse.urlencode(body))
return content
This function would return a JSON with the OAuth2 token and the session time in seconds. Afterwards, to query the API, it would be as simple as:
def search_api(token):
http_obj = Http()
url = "http://api.idealista.com/3.5/es/search?center=40.42938099999995,-3.7097526269835726&country=es&maxItems=50&numPage=1&distance=452&propertyType=bedrooms&operation=rent"
headers = {'Authorization' : 'Bearer ' + token}
resp, content = http_obj.request(url,method='POST',headers=headers)
return content
This time the we would find in the content var the data we were looking for, again as a JSON.
That can't be marked as correct answer since
auth = base64.encode(apikey + ':' + secret)
body = {'grant_type':'client_credentials'}
headers = {'Content-Type': 'application/x-www-form-urlencoded;charset=UTF-8','Authorization' : 'Basic ' + auth}
Will give you TypeError:
can only concatenate str (not "bytes") to str
Since base64encode returns a byte type object...
It's true Idealista API is very limited about documentation, but I think this is a better approach since I don't use unnecesary libs (Only native):
#first request
message = API_KEY + ":" + SECRET
auth = "Basic " + base64.b64encode(message.encode("ascii")).decode("ascii")
headers_dic = {"Authorization" : auth,
"Content-Type" : "application/x-www-form-urlencoded;charset=UTF-8"}
params_dic = {"grant_type" : "client_credentials",
"scope" : "read"}
r = requests.post("https://api.idealista.com/oauth/token",
headers = headers_dic,
params = params_dic)
This works flawless with only python requests and base64 module...
regards
This is my code, improving #3... this run ok! for me!!!!
only put your apikey and your password (secret)...
import pandas as pd
import json
import urllib
import requests as rq
import base64
def get_oauth_token():
url = "https://api.idealista.com/oauth/token"
apikey= 'your_api_key' #sent by idealista
secret= 'your_password' #sent by idealista
auth = base64.b64encode(apikey + ':' + secret)
headers = {'Content-Type': 'application/x-www-form-urlencoded;charset=UTF-8' ,'Authorization' : 'Basic ' + auth}
params = urllib.urlencode({'grant_type':'client_credentials'})
content = rq.post(url,headers = headers, params=params)
bearer_token = json.loads(content.text)['access_token']
return bearer_token
def search_api(token, url):
headers = {'Content-Type': 'Content-Type: multipart/form-data;', 'Authorization' : 'Bearer ' + token}
content = rq.post(url, headers = headers)
result = json.loads(content.text)['access_token']
return result
country = 'es' #values: es, it, pt
locale = 'es' #values: es, it, pt, en, ca
language = 'es' #
max_items = '50'
operation = 'sale'
property_type = 'homes'
order = 'priceDown'
center = '40.4167,-3.70325'
distance = '60000'
sort = 'desc'
bankOffer = 'false'
df_tot = pd.DataFrame()
limit = 10
for i in range(1,limit):
url = ('https://api.idealista.com/3.5/'+country+'/search?operation='+operation+#"&locale="+locale+
'&maxItems='+max_items+
'&order='+order+
'¢er='+center+
'&distance='+distance+
'&propertyType='+property_type+
'&sort='+sort+
'&numPage=%s'+
'&language='+language) %(i)
a = search_api(get_oauth_token(), url)
df = pd.DataFrame.from_dict(a['elementList'])
df_tot = pd.concat([df_tot,df])
df_tot = df_tot.reset_index()
I found some mistakes. At least, I cannot run it.
I believe, I improved with this:
import pandas as pd
import json
import urllib
import requests as rq
import base64
def get_oauth_token():
url = "https://api.idealista.com/oauth/token"
apikey= 'your_api_key' #sent by idealist
secret= 'your_password' #sent by idealista
apikey_secret = apikey + ':' + secret
auth = str(base64.b64encode(bytes(apikey_secret, 'utf-8')))[2:][:-1]
headers = {'Authorization' : 'Basic ' + auth,'Content-Type': 'application/x-www-form-
urlencoded;charset=UTF-8'}
params = urllib.parse.urlencode({'grant_type':'client_credentials'}) #,'scope':'read'
content = rq.post(url,headers = headers, params=params)
bearer_token = json.loads(content.text)['access_token']
return bearer_token
def search_api(token, URL):
headers = {'Content-Type': 'Content-Type: multipart/form-data;', 'Authorization' : 'Bearer ' + token}
content = rq.post(url, headers = headers)
result = json.loads(content.text)
return result
I have the following code
import requests
import sys
import json
arg_count = 0
#check that correct number of arguments are passed to utility
for arg in sys.argv:
arg_count = arg_count + 1
assert arg_count == 4
#parse the input arguments to the utility
connectplus_group_title = sys.argv[1]
connectplus_uname = sys.argv[2]
connectplus_passwords = sys.argv[3]
#find the group id from the input group name i.e connectplus_group_title
response = requests.get("https://connectplus.geometricglobal.com/_vti_bin/beezy/v2/api.svc//Groups/Availability?title={title})", auth=(connectplus_uname, connectplus_passwords))
#assert response.status_code == response.codes.ok
group_data = json.loads(response)
group_id = group_data[connectplus_group_title]
connectplus_group_contributors = "https://connectplus.geometricglobal.com/_vti_bin/beezy/v2/api.svc//Groups/{id}/Contributors?skip={skip}&take={take}"
response = requests.get(connectplus_group_contributors,auth=(connectplus_uname, connectplus_passwords))
data = response.json()
I need to get the json data which is returned by the URL , I have tried using json.loads(response.text) or simple json.loads(response) with no luck I keep getting the above error.
What is the correct way to get the data from a REST url in the from of dictionary.
You have to pass response.content, response is a Response type not a simple string (try type(response)) and json.loads needs (as the error highlights) a string or buffer.
Replace your call with:
json.loads(response.content)