I'm trying to get every page and write it to a JSON. This is what I have and it writes fine to a JSON but it only writes the first 100 and doesn't cursor to each page. This is the api https://developers.gfycat.com/api/#user-feeds
import requests
import json
url = "https://api.gfycat.com/v1/users/brandonedora/gfycats"
params = {"count": "100", "cursor": "YnJhbmRvbmVkb3JhfDEsdmlnb3JvdXNkZWZpYW50ZG9iZXJtYW5waW5zY2hlcixicmFuZG9uZWRvcmEsMTU4NDYxNDQ5Mg=="}
r = requests.get(url, params=params)
text_json = json.loads(r.content)
json.dump(text_json, open("testdump.json", "w"), indent = 2)
import requests
import json
params = {
"count": "100",
"cursor": "YnJhbmRvbmVkb3JhfDEsdmlnb3JvdXNkZWZpYW50ZG9iZXJtYW5waW5zY2hlcixicmFuZG9uZWRvcmEsMTU4NDYxNDQ5Mg=="
}
def main(url):
with requests.Session() as req:
allin = []
for _ in range(1, 4):
r = req.get(url, params=params).json()
params['cursor'] = r['cursor']
print(params)
allin.append(r)
goal = json.dumps(allin, indent=4)
with open("data.txt", 'w') as f:
f.write(goal)
main("https://api.gfycat.com/v1/users/brandonedora/gfycats")
Related
My Json is like this
[
{
"name":"abc"
"emai":"hotmail"
},
{
"name":"def"
"emai":"gmail"
}
]
I have to read one record at time and post request
in 1st post request with data
'{
"name":"abc"
"emai":"hotmail"
}'
in 2nd post request with data
'{
"name":"def"
"emai":"gmail"
}'
i tried with this code,
import json
import requests
from requests.structures import CaseInsensitiveDict
url = "https://testurl.com"
headers = CaseInsensitiveDict()
headers["accept"] = "text/plain"
headers["username"] = "username"
headers["password"] = "password"
headers["Content-Type"] = "application/json"
with open("data.json", encoding='utf-8', errors='ignore') as json_data:
for i in json_data:
data = json.load(i, strict=False)
idata = json.dumps(data) ## converting to string
fData = idata.replace("'", '"') # replacing ' with "
output = "'"+fData+"'" ## placing ' before and after json string which is mandatory in my case
print(fData)
# resp = requests.post(url, headers=headers, data=fData)
# print(f"status_code: : {resp.status_code}, response : {resp.json}")
I'm getting error 'str' object has no attribute error, can you please suggest me what i have to change ?
load all json, then loop through it
import json
import requests
from requests.structures import CaseInsensitiveDict
url = "https://testurl.com"
headers = CaseInsensitiveDict()
headers["accept"] = "text/plain"
headers["username"] = "username"
headers["password"] = "password"
headers["Content-Type"] = "application/json"
with open("data.json", encoding='utf-8', errors='ignore') as json_string:
json_data = json.load(json_string.read(), strict=False)
for i in json_data:
data = i
idata = json.dumps(data) ## converting to string
fData = idata.replace("'", '"') # replacing ' with "
output = "'"+fData+"'" ## placing ' before and after json string which is mandatory in my case
print(fData)
# resp = requests.post(url, headers=headers, data=fData)
# print(f"status_code: : {resp.status_code}, response : {resp.json}")
I guess you want to convert your JSON data to python structure first and then process it, something like:
import json
with open("data.json", encoding='utf-8', errors='ignore') as json_data:
data = json.loads(json_data.read())
for obj in data:
resp = requests.post(url, headers=headers, data=obj)
print(f"status_code: : {resp.status_code}, response : {resp.json}")
I have a paginated API and I'm trying to go through all the data available and save it into a list. However, the nature of my API is that it's nested here's an example of how it looks like.
"data": [{"type": "general-Type", "id": 1, "attributes": {"firstname": "Kevin", "lastname": "Wolf", "emailaddress": "kevinwolf#gmail.com"}}]
Thus when I save it into a list the last part of the data aka "attributes" looks like a dictionary causing the following error:
sample_data.extend(sample_data['data'])
AttributeError: 'dict' object has no attribute 'extend'
I'm new to this so any help on how to successfully do this request would be helpful
Thank you in advance
If it helps here's my code:
the request limit is 10,000 that's why I set the limit to 10,000 increments
sample_data = []
offset = 0
limit = 10000
while True:
print("----")
url = f"https://results.us.sampledata.com/api/reporting/v0.1.0/samples?offset={offset}&page[size]={limit}"
headers = {"Content-Type": "application/json", "Accept-Charset": "UTF-8", "x-apikey-token": "sampletoken"}
print("Requesting", url)
response = requests.get(url, data={"sample": "data"}, headers=headers)
sample_data = response.json()
if len(sample_data['data']) == 0:
# If not, exit the loop
break
# If we did find records, add them
# to our list and then move on to the next offset
sample_data.extend(sample_data['data'])
offset = offset + 10000
As #8349697 already said your problem is that you use the same name sample_data to keep two different structures.
First you create list sample_data = [] but later you overwrite it with dictionary sample_data = response.json() but next you want to use original list sample_data to add values from dict sample_data
You should use different names like
page_data = response.json()
if not page_data['data']: # if len(page_data['data']) == 0:
break
sample_data.extend(page_data['data'])
Minimal code with other changes - but I couldn't test it with your url.
import requests
sample_data = []
headers = {
"Content-Type": "application/json",
"Accept-Charset": "UTF-8",
"x-apikey-token": "sampletoken"
}
data = {
"sample": "data"
}
params = {
"offset": 0,
"page[size]": 10000,
}
url = "https://results.us.sampledata.com/api/reporting/v0.1.0/samples"
while True:
print("----")
#url = f"https://results.us.sampledata.com/api/reporting/v0.1.0/samples?offset={offset}&page[size]={limit}"
#print("Requesting", url)
print('Offset:', params['offset'])
response = requests.get(url, params=params, data=data, headers=headers)
page_data = response.json()
if (not 'data' in page_data) or (not page_data['data']):
break
sample_data.extend(page_data['data'])
params['offset'] += 10000
I've below codes to collect the json output from my storage device by using restAPI.
The code is working fine when I pass 2 query parameters (portId and hostGroupName) manually, but currently I want to pass all value from csv file to the query parameters and combine all output under the same json structure. Is is possible to do that? Thanks
import json
from pathlib import Path
import requests
path = Path(r"C:\Python\Code")
url = "http://192.168.1.30:23451/ConfigurationManager/v1/objects/storages/800000010888/host-wwns"
pwd = "129yZm1vhjqidGFzc3dvcmRAZW50MQ=="
headers = {
"Accept": "application/json",
"Content-Type": "application/json",
"Authorization": "Basic " + pwd
}
params = {
"portId": "CL3-C",
"hostGroupName": "expdb1a"
##### Expected to take value from list.csv file and pass it to below query parameters ######
"portId": "portID from list.csv",
"hostGroupName": "hostGroupName from list.csv"
##########################################################################################
}
response = requests.get(url, headers=headers, params=params)
stg = response.json()
json_data_w = json.dumps(stg, indent=2)
Path(path/"output.json").write_text(json_data_w)
list.csv
portId,hostGroupName
CL3-C,expdb1a
CL4-C,expdb1a
CL3-A,expdb2a
CL4-A,expdb2a
CL3-E,expapp1a
CL4-E,expapp1a
CL3-J,expapp2a
CL4-J,expapp2a
Example of output.json I pass query parameters manually:
{
"data" : [ {
"hostWwnId" : "CL3-C,10,c050760996aa003a",
"portId" : "CL3-C",
"hostGroupNumber" : 10,
"hostGroupName" : "expdb1a",
"hostWwn" : "1090384996aa003a",
"wwnNickname" : "expdb1a_v1"
}, {
"hostWwnId" : "CL3-C,10,c050760996aa003b",
"portId" : "CL3-C",
"hostGroupNumber" : 10,
"hostGroupName" : "expdb1a",
"hostWwn" : "1090384996aa003b",
"wwnNickname" : "expdb1a_v2"
} ]
}
maybe this:
with open('list.csv') as f:
lines = f.read().splitlines() #list of portId,hostGroupName
json_list = []
for l in lines[1:]: #start at 1 to skip headers
# print(l.split(',')[0],l.split(',')[1] )
params = {
"portId": l.split(',')[0],
"hostGroupName": l.split(',')[1]
}
response = requests.get(url, headers=headers, params=params)
stg = response.json()
json_data_w = json.dumps(stg, indent=2)
json_list.append(json_data_w)
Path(path/"output.json").write_text(json_list) #this writes a list of your json outputs
How to send the below request using python requests library?
Request :
I have tried
with requests.Session() as session:
// Some login action
url = f'http://somewebsite.com/lib/ajax/service.php?key={key}&info=get_enrolled'
json_data = {
"index": 0,
"methodname": "get_enrolled",
// And so on, from Request Body
}
r = session.post(url, json=json_data)
But it doesn't give the output I want.
1.Define a POST request method
import urllib3
import urllib.parse
def request_with_url(url_str, parameters=None):
"""
https://urllib3.readthedocs.io/en/latest/user-guide.html
"""
http = urllib3.PoolManager()
response = http.request("POST",
url_str,
headers={
'Content-Type' : 'application/json'
},
body=parameters)
resp_data = str(response.data, encoding="utf-8")
return resp_data
2.Call the function with your specific url and parameters
json_data = {
"index": 0,
"methodname": "get_enrolled",
// And so on, from Request Body
}
key = "123456"
url = "http://somewebsite.com/lib/ajax/service.php?key={0}&info=get_enrolled".format(key)
request_with_url(url, json_data)
With no more info what you want and from what url it is hard to help but.
But try adding headers with the user-agent to the post. More headers may be needed, but User-Agent is a header that is often required.
with requests.Session() as session:
// Some login action
url = f'http://somewebsite.com/lib/ajax/service.php?key={key}&info=get_enrolled'
json_data = {
"index": 0,
"methodname": "get_enrolled",
// And so on, from Request Body
}
headers = {'User-Agent': 'Mozilla/5.0'}
r = session.post(url, json=json_data, headers=headers)
Am working with an API that asks to specify a file (Excel in my case) to upload to the API. Documentation specifies: JSON Property = file, Data Type = FileUpload. My question: What is FileUpload. I tried simply specifying the name of the file (e.g. c:\test\Data.xls) but obviously that does not work.
I am working with an API for FastField Mobile Forms (www.fastfield.com). Documentation shown in screen shot, as is code and result. Somehow, I am not posting the file data to the API correctly.
So this is the Python code I am attempting to run
import requests
import json
from requests.auth import HTTPBasicAuth
import base64
# Get session token, this must be specified in header of subsequent request and returns a JSON object ready for insertion into header
rqstResponse = requests.post('https://manage.fastfieldforms.com/api/authenticate', auth=HTTPBasicAuth('***', '***'))
jsonObj = json.loads(rqstResponse.content)
sessionToken = jsonObj['data']['sessionToken']
headers = {'X-Gatekeeper-SessionToken': sessionToken}
# run this code to get listIds - which are hard coded further down
rqstResponse = requests.get("https://manage.fastfieldforms.com/api/globallists", headers=headers)
print (rqstResponse.content)
del rqstResponse
# Read file and convert to binary string
filePath = r"J:\Properties\PropGIS\proj\20150820140457_TelecoMapping\data\Survey_Feb17\FastField_Test01.xlsx"
with open(filePath, 'r') as f:
filecontents = f.read()
fileDataEncoded = base64.b64encode(filecontents)
# create JSON
payloadDictObj = {}
payloadDictObj['file'] = fileDataEncoded
payloadDictObj['id'] = "03c804cb-b983-4e4c-956b-96ac23da16b2"
#payloadDictObj['listname'] = "Test02"
serializedJsonStr = json.dumps(payloadDictObj)
print serializedJsonStr
# Update Global List
rqstResponse = requests.post("https://manage.fastfieldforms.com/api//globallist", data=serializedJsonStr, headers=headers)
print (rqstResponse.content)
del rqstResponse
# --------------------
# Response
{
"code": 200,
"data": {
"searchResults": [
{
"id": 7793,
"accountId": 43600,
"name": "Test01",
"active": true,
"createdAt": "2017-05-24T06:37:28.49Z",
"updatedAt": "2017-05-24T06:37:28.49Z",
"version": 1,
"listId": "03c804cb-b983-4e4c-956b-96ac23da16b2",
"path": "{ bucket:'fastfield-globallists', key:'43600/ca4b89df75db4ef8b513d15d59f654d8.csv'}"
}
]
}
}
{"id": "03c804cb-b983-4e4c-956b-96ac23da16b2", "file": "UEsDB...qaJXQ=="}
{
"code": 403,
"error": "listname name is required",
"data": {}
}
OK. For what it's worth. This is how I finally got this to work. I now realise that this question was more about my understanding, or lack thereof, of the Python request module.
import requests
import json
from requests.auth import HTTPBasicAuth
import base64
# Get session token, this must be specified in header of subsequent request and returns a JSON object ready for insertion into header
rqstResponse = requests.post('https://manage.fastfieldforms.com/api/authenticate', auth=HTTPBasicAuth('XXX', 'XXX'))
jsonObj = json.loads(rqstResponse.content)
sessionToken = jsonObj['data']['sessionToken']
headers = {'X-Gatekeeper-SessionToken': sessionToken}
def getGloballistsDetails():
# run this code to get listIds - which are hard coded further down
rqstResponse = requests.get("https://manage.fastfieldforms.com/api/globallists", headers=headers)
print (rqstResponse.content)
del rqstResponse
def createGloballist(listname, filePath):
# usage example createGloballist("test01", r"c:\temp\test.xlsx")
files = {'file': open(filePath, 'rb')}
data = {'listname': listname}
rqstResponse = requests.post("https://manage.fastfieldforms.com/api//globallist", files=files, data=data, headers=headers)
print (rqstResponse.content)
del rqstResponse
def updateGloballist(id, filePath):
# usage example createGloballist("f03c7db1-cfea-4486-8350-53381ac048b4", r"c:\temp\test.xlsx")
files = files = {'file': open(filePath, 'rb')}
data = {'id': id}
rqstResponse = requests.post("https://manage.fastfieldforms.com/api//globallist", files=files, data=data, headers=headers)
print (rqstResponse.content)
del rqstResponse
filePath = r"J:\Properties\PropGIS\proj\20150820140457_TelecoMapping\data\Survey_Feb17\FastField_Test01.xlsx"
getGloballistsDetails()
#createGloballist("Test02", filePath)
updateGloballist('f03c7db1-cfea-4486-8350-53381ac048b4', filePath)