I have a paginated API and I'm trying to go through all the data available and save it into a list. However, the nature of my API is that it's nested here's an example of how it looks like.
"data": [{"type": "general-Type", "id": 1, "attributes": {"firstname": "Kevin", "lastname": "Wolf", "emailaddress": "kevinwolf#gmail.com"}}]
Thus when I save it into a list the last part of the data aka "attributes" looks like a dictionary causing the following error:
sample_data.extend(sample_data['data'])
AttributeError: 'dict' object has no attribute 'extend'
I'm new to this so any help on how to successfully do this request would be helpful
Thank you in advance
If it helps here's my code:
the request limit is 10,000 that's why I set the limit to 10,000 increments
sample_data = []
offset = 0
limit = 10000
while True:
print("----")
url = f"https://results.us.sampledata.com/api/reporting/v0.1.0/samples?offset={offset}&page[size]={limit}"
headers = {"Content-Type": "application/json", "Accept-Charset": "UTF-8", "x-apikey-token": "sampletoken"}
print("Requesting", url)
response = requests.get(url, data={"sample": "data"}, headers=headers)
sample_data = response.json()
if len(sample_data['data']) == 0:
# If not, exit the loop
break
# If we did find records, add them
# to our list and then move on to the next offset
sample_data.extend(sample_data['data'])
offset = offset + 10000
As #8349697 already said your problem is that you use the same name sample_data to keep two different structures.
First you create list sample_data = [] but later you overwrite it with dictionary sample_data = response.json() but next you want to use original list sample_data to add values from dict sample_data
You should use different names like
page_data = response.json()
if not page_data['data']: # if len(page_data['data']) == 0:
break
sample_data.extend(page_data['data'])
Minimal code with other changes - but I couldn't test it with your url.
import requests
sample_data = []
headers = {
"Content-Type": "application/json",
"Accept-Charset": "UTF-8",
"x-apikey-token": "sampletoken"
}
data = {
"sample": "data"
}
params = {
"offset": 0,
"page[size]": 10000,
}
url = "https://results.us.sampledata.com/api/reporting/v0.1.0/samples"
while True:
print("----")
#url = f"https://results.us.sampledata.com/api/reporting/v0.1.0/samples?offset={offset}&page[size]={limit}"
#print("Requesting", url)
print('Offset:', params['offset'])
response = requests.get(url, params=params, data=data, headers=headers)
page_data = response.json()
if (not 'data' in page_data) or (not page_data['data']):
break
sample_data.extend(page_data['data'])
params['offset'] += 10000
Related
def test3(address):
url = 'https://apilist.tronscan.org/api/transfer?'
payload = {
"sort": 'timestamp',
'count':True,
'limit':20,
'start':0,
'token':'_',
'address':address
}
res = requests.get(url, params=payload)
obj = json.loads(res.text)
print(res)
test3('TQAuZ2YsGgPRPNpHXxoJZLHsvuxrLrnVEM')
I am trying to query the transfer record of TRC20 of an address. According to the API, I only found the record of TRX. But I need to check the USDT transfer record of TRC20
How should I modify it?
It's supposed to help
def test3(address):
url = f'https://api.trongrid.io/v1/accounts/{address}/transactions/trc20?'
payload = {
"sort": 'blockNumber',
'limit':10,
}
res = requests.get(url,params=payload)
obj = json.loads(res.text)
return obj
Source=["SGD"]
Destination=["USD"]
Amount=[5000]
```import requests
url = "https://api.currencyfair.com/comparisonQuotes"
payload = "{\"currencyFrom\":\"SGD\",\"currencyTo\":\"EUR\",\"type\":\"SELL\",\"amountInfo\":
{\"amount\":50000,\"scale\":2}}"
headers = {
'user-agent': "vscode-restclient",
'content-type': "application/json",
'accept': "application/json"
}
response = requests.request("POST", url, data=payload, headers=headers)
print(response.text)
I need to pass values to payload string -
payload = "{"currencyFrom":"SGD","currencyTo":"EUR","type":"SELL","amountInfo":{"amount":50000,"scale":2}}"
Need to pass values to payload using 3 list created above```
Could you please try to explain a bit more what you're after?
I suspect what you mean is that you'd like to dynamically update the values in the text payload every time you call the function to post data.
I'd usually go about doing this by creating a placeholder string then updating that by replacing placeholder values at runtime.
payload = "{\"currencyFrom\":\"#currencyFrom\",\"currencyTo\":\"#currencyTo\",\"type\":\"SELL\",\"amountInfo\": {\"amount\":#amountInfo,\"scale\":2}}"
currencyFrom = 'USD'
currencyTo = 'EUR'
amountInfo = 50000
payload = payload.replace('#currencyFrom', currencyFrom).replace('#currencyTo', currencyTo).replace('#amountInfo', amountInfo)
Looking at the API you're trying to interact with, this is a sample of what it expects:
{"currencyFrom": "EUR",
"currencyTo": "GBP",
"type": "SELL",
"amountInfo": {"amount": 100},
"ignoreFee": false}
This is a JSON object that follows a specific format, if you try and pass a list as opposed to a string in the "currencyFrom", "currencyTo" fields etc you'll get an error.
To get multiple values as responses simply conduct multiple requests to the API, for example:
payload = "{\"currencyFrom\":\"#currencyFrom\",\"currencyTo\":\"#currencyTo\",\"type\":\"SELL\",\"amountInfo\": {\"amount\":#amountInfo,\"scale\":2}}"
currencyFrom = ['USD', 'GBP']
currencyTo = ['EUR', 'CHF']
amountInfo = 50000
payload = payload.replace('#currencyFrom', currencyFrom).replace('#currencyTo', currencyTo).replace('#amountInfo', amountInfo)
for currFrom in currencyFrom:
for currTo in currencyTo:
for amount in amountInfo:
payload = payload.replace('#currencyFrom', currFrom ).replace('#currencyTo', currTo ).replace('#amountInfo', amount )
response = requests.request("POST", url, data=payload, headers=headers)
Hope this makes sense!
Edit: Updating code as per your comments.
Problem: The output of this code seems to be repeating alot of the same entries in the final list, thus making it exponentially longer.
The goal would be complete the query and the print the final list with all city within the region
[
{
"name": "Herat",
"id": "AF~HER~Herat"
}
]
[
{
"name": "Herat",
"id": "AF~HER~Herat"
},
{
"name": "Kabul",
"id": "AF~KAB~Kabul"
}
]
[
{
"name": "Herat",
"id": "AF~HER~Herat"
},
{
"name": "Kabul",
"id": "AF~KAB~Kabul"
},
{
"name": "Kandahar",
"id": "AF~KAN~Kandahar"
}
]
My goal is to to a get a list with cityID. I first to a GET request and parse the JSON response to get the country IDs to a list,
Second: I have a for loop, which will make another GET request for the region id, but i now need to add the country IDs to the api url. I do that by adding .format on the GET request. and iterate trough all the countries and there respective region IDs, i parse them and store them in a list.
Third: i have another for loop, which will make another GET request for the cityID that will loop trough all cities with the above Region ID list, and the respectively collect the cityID that i really need.
Code :
from requests.auth import HTTPBasicAuth
import requests
import json
import urllib3
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
def countries():
data = requests.get("https://localhost/api/netim/v1/countries/", verify=False, auth=HTTPBasicAuth("admin", "admin"))
rep = data.json()
a = []
for elem in rep['items']:
a.extend([elem.get("id","")])
print(a)
return a
def regions():
ids = []
for c in countries():
url = requests.get("https://localhost/api/netim/v1/countries/{}/regions".format(c), verify=False, auth=HTTPBasicAuth("admin", "admin"))
response = url.json()
for cid in response['items']:
ids.extend([cid.get("id","")])
data = []
for r in ids:
url = requests.get("https://localhost/api/netim/v1/regions/{}/cities".format(r), verify=False, auth=HTTPBasicAuth("admin", "admin"))
response = url.json()
data.extend([{"name":r.get("name",""),"id":r.get("id", "")} for r in response['items']])
print(json.dumps(data, indent=4))
return data
regions()
print(regions())
You will see thou output contains several copies of the same entry.
Not a programmer, not sure where am i getting it wrong
It looks as though the output you're concerned with might be due to the fact that you're printing data as you iterate through it in the regions() method.
Try to remove the line:
print(json.dumps(data, indent=4))?
Also, and more importantly - you're setting data to an empty list every time you iterate on an item in Countries. You should probably declare that variable before the initial loop.
You're already printing the final result when you call the function. So printing as you iterate only really makes sense if you're debugging & needing to review the data as you go through it.
I want to make a list from api json response as shown for each ticket in jira and remove any duplicates
I can get the values for each ticket but not able to make it as list and remove duplicates from it to process
Here is the api json response for each ticket
response = {
"expand": "renderedFields,names,schema,operations,editmeta,changelog,versionedRepresentations",
"id": "1831845",
"self": "https://jira.com/login/rest/api/latest/issue/1845",
"key": "pc-1002",
"fields": {
"customfield_1925": {
"self": "https://jira.com/login/rest/api/2/customFieldOption/1056",
"value": "windows",
"id": "101056"
}
so i have script like this:
import requests, json
tick = """jira: pc-1002,pc-1003,pc-1005
env"""
ticks = tick.replace(' ','').split(':')[1].split('\n')[0].split(',')
print(ticks)
for i in ticks:
url = "https://jira.com/login/rest/api/latest/issue/" + str(i)
print(url)
response = requests.request("GET", url, verify=False)
response = json.loads(response.text)
resp = response['fields']['customfield_1925']['value']
print(resp)
so it prints all the values like below :
output:
windows1
windows2
windows1
I want the output values to be unique and as it may end up having duplicates.
I wanted output as below
['windows1', 'windows2']
Simply add each response to a list of responses, and use Python's convenient "in" operator to check if each response is already in the list. Something along the lines of:
allResponses = []
for i in ticks:
url = "https://jira.com/login/rest/api/latest/issue/" + str(i)
print(url)
response = requests.request("GET", url, verify=False)
response = json.loads(response.text)
resp = response['fields']['customfield_1925']['value']
if resp not in allResponses:
print(resp)
allResponses.append(resp)
Am working with an API that asks to specify a file (Excel in my case) to upload to the API. Documentation specifies: JSON Property = file, Data Type = FileUpload. My question: What is FileUpload. I tried simply specifying the name of the file (e.g. c:\test\Data.xls) but obviously that does not work.
I am working with an API for FastField Mobile Forms (www.fastfield.com). Documentation shown in screen shot, as is code and result. Somehow, I am not posting the file data to the API correctly.
So this is the Python code I am attempting to run
import requests
import json
from requests.auth import HTTPBasicAuth
import base64
# Get session token, this must be specified in header of subsequent request and returns a JSON object ready for insertion into header
rqstResponse = requests.post('https://manage.fastfieldforms.com/api/authenticate', auth=HTTPBasicAuth('***', '***'))
jsonObj = json.loads(rqstResponse.content)
sessionToken = jsonObj['data']['sessionToken']
headers = {'X-Gatekeeper-SessionToken': sessionToken}
# run this code to get listIds - which are hard coded further down
rqstResponse = requests.get("https://manage.fastfieldforms.com/api/globallists", headers=headers)
print (rqstResponse.content)
del rqstResponse
# Read file and convert to binary string
filePath = r"J:\Properties\PropGIS\proj\20150820140457_TelecoMapping\data\Survey_Feb17\FastField_Test01.xlsx"
with open(filePath, 'r') as f:
filecontents = f.read()
fileDataEncoded = base64.b64encode(filecontents)
# create JSON
payloadDictObj = {}
payloadDictObj['file'] = fileDataEncoded
payloadDictObj['id'] = "03c804cb-b983-4e4c-956b-96ac23da16b2"
#payloadDictObj['listname'] = "Test02"
serializedJsonStr = json.dumps(payloadDictObj)
print serializedJsonStr
# Update Global List
rqstResponse = requests.post("https://manage.fastfieldforms.com/api//globallist", data=serializedJsonStr, headers=headers)
print (rqstResponse.content)
del rqstResponse
# --------------------
# Response
{
"code": 200,
"data": {
"searchResults": [
{
"id": 7793,
"accountId": 43600,
"name": "Test01",
"active": true,
"createdAt": "2017-05-24T06:37:28.49Z",
"updatedAt": "2017-05-24T06:37:28.49Z",
"version": 1,
"listId": "03c804cb-b983-4e4c-956b-96ac23da16b2",
"path": "{ bucket:'fastfield-globallists', key:'43600/ca4b89df75db4ef8b513d15d59f654d8.csv'}"
}
]
}
}
{"id": "03c804cb-b983-4e4c-956b-96ac23da16b2", "file": "UEsDB...qaJXQ=="}
{
"code": 403,
"error": "listname name is required",
"data": {}
}
OK. For what it's worth. This is how I finally got this to work. I now realise that this question was more about my understanding, or lack thereof, of the Python request module.
import requests
import json
from requests.auth import HTTPBasicAuth
import base64
# Get session token, this must be specified in header of subsequent request and returns a JSON object ready for insertion into header
rqstResponse = requests.post('https://manage.fastfieldforms.com/api/authenticate', auth=HTTPBasicAuth('XXX', 'XXX'))
jsonObj = json.loads(rqstResponse.content)
sessionToken = jsonObj['data']['sessionToken']
headers = {'X-Gatekeeper-SessionToken': sessionToken}
def getGloballistsDetails():
# run this code to get listIds - which are hard coded further down
rqstResponse = requests.get("https://manage.fastfieldforms.com/api/globallists", headers=headers)
print (rqstResponse.content)
del rqstResponse
def createGloballist(listname, filePath):
# usage example createGloballist("test01", r"c:\temp\test.xlsx")
files = {'file': open(filePath, 'rb')}
data = {'listname': listname}
rqstResponse = requests.post("https://manage.fastfieldforms.com/api//globallist", files=files, data=data, headers=headers)
print (rqstResponse.content)
del rqstResponse
def updateGloballist(id, filePath):
# usage example createGloballist("f03c7db1-cfea-4486-8350-53381ac048b4", r"c:\temp\test.xlsx")
files = files = {'file': open(filePath, 'rb')}
data = {'id': id}
rqstResponse = requests.post("https://manage.fastfieldforms.com/api//globallist", files=files, data=data, headers=headers)
print (rqstResponse.content)
del rqstResponse
filePath = r"J:\Properties\PropGIS\proj\20150820140457_TelecoMapping\data\Survey_Feb17\FastField_Test01.xlsx"
getGloballistsDetails()
#createGloballist("Test02", filePath)
updateGloballist('f03c7db1-cfea-4486-8350-53381ac048b4', filePath)