I want to take the json response data from a REST request and create an actual json file. I tried something like this, but it did not work. Essentially, it just prints the headers. Any suggestions?
params = {'f': 'json', 'where': '1=1', 'geometryType': 'esriGeometryPolygon', 'spatialRel': 'esriSpatialRelIntersects','outFields': '*', 'returnGeometry': 'true'}
r = requests.get('https://hazards.fema.gov/gis/nfhl/rest/services/CSLF/Prelim_CSLF/MapServer/3/query', params)
cslfJson = r.json()
path = r"C:/Workspace/Sandbox/ScratchTests/cslf.json"
file = open(path, 'w')
for line in cslfJson:
file.write(line + "\r\n")
file.close()
use the json module
my_data = json.loads(r.json())
# my_data is a dict mapping the JSON
with open(path, 'w') as f:
json.dump(my_data, f)
if you want, you can print in pretty way using the indent parameter
json.dump(my_data, f, indent=2)
Related
Hi I would like to save my python requests cookies into a text file as json like so:
{"Proxy": "Cookies"}
At the moment im using:
with open('data.txt') as f:
proxy_data = json.load(f)
r = requests.get(URL,proxies=proxies, cookies=proxy_data['127:000'])
proxy_data['127:000'] = dict(session.cookies)
with open('data.txt', 'w') as json_file:
json.dump(proxy_data, json_file)
My data.txt file looks like this after writing it:
{"127:000": "{\"__cf_bm\": \"628f4c77a5139148cf28961a73f4edffed52f8c1-1622925414-1800-AaLguAyS9Hfl8TXCaQYUjjZ5i3sDck15kn/vcPJCWZBxi+nM9xU0zd/GfyKJu6/3R+GQEFdk/DGHEV4mANvpG1UMvcsXA9G9l/AFNEnIknTv\", \"cf_clearance\": \"84bffd070fbafcb91edcfa82f8cc3ba5a60a618d-1622925414-0-150\", \"cf_chl_seq_b116a013f9e2c44\": \"d3ac4406fc56d30\"}"}
However I run into a number of errors here. How can I do the properly?
UPDATE
ERRORS I GET
Python\Python37\lib\site-packages\requests\cookies.py", line 524, in
cookiejar_from_dict
cookiejar.set_cookie(create_cookie(name, cookie_dict[name])) TypeError: string indices must be integers
Currently, the value of proxy_data["127:000"] is a string which you're passing into the cookies parameter.
with open("data.txt") as f:
proxy_data = json.load(f)
print(type(proxy_data["127:000"]))
# >> <class 'str'>
Two options:
Parse the value of proxy_data["127:000"] to a dictionary using json.loads
r = requests.get(URL, proxies=proxies, cookies=json.loads(proxy_data['127:000']))
Make sure to save the nested dicts correctly.
proxy_data = {"127:000": {"cookie_test": "value_test"}}
with open("data.txt", "w") as f:
f.write(json.dumps(proxy_data))
with open("data.txt", "r") as f:
proxy_data = json.loads(f.read())
print(type(proxy_data["127:000"]))
# >> <class 'dict'>
I have the following content:
{
"z":"[{\"ItemId\":\"1234\",\"a\":\"1234\",\"b\":\"4567\",\"c\":\"d\"}]"
}
This is a part of the json response I get from a certain API. I need to replace the \"s with 's. Unfortunately, that's where I got stuck!
Most of the answers I get are simply replacing the \ with "" or " " so that did not help me. So my question are the following:
How can I replace the \" with ':
in a file where I copy-pasted the content?
if I receive this as a response to a certain API call?
I tried the following to replace the content in a file but I am clearly only replacing the "s with ':
with open(file, "r") as f:
content = f.read()
new_content = content.replace("\"", "'")
with open(file, "w") as new_file:
new_file.write(new_content)
If what you're trying to do is transform each value from a JSON string to a Python repr() string, while keeping the wrapper format as JSON, that might look like:
with open(filename, "r") as old_file:
old_content = json.load(old_file)
new_content = {k: repr(json.loads(v)) for k, v in old_content.items()}
with open(filename, "w") as new_file:
json.dump(new_content, new_file)
If your old file contains:
{"z":"[{\"ItemId\":\"1234\",\"a\":\"1234\",\"b\":\"4567\",\"c\":\"d\"}]"}
...the new file will contain:
{"z": "[{'ItemId': '1234', 'a': '1234', 'b': '4567', 'c': 'd'}]"}
Note that in this new file, the inner fields are now in Python format, not JSON format; they can no longer be parsed by JSON parsers. Usually, I would suggest doing something different instead, as in:
with open(filename, "r") as old_file:
old_content = json.load(old_file)
new_content = {k: json.loads(v) for k, v in old_content.items()}
with open(filename, "w") as new_file:
json.dump(new_content, new_file)
...which would yield an output file with:
{"z": [{"ItemId": "1234", "a": "1234", "b": "4567", "c": "d"}]}
...which is both easy-to-read and easy to process with standard JSON-centric tools (jq, etc).
Using json module, you can dumps the data then loads it using the following:
import json
data = {
"z": "[{\"ItemId\":\"1234\",\"a\":\"1234\",\"b\":\"4567\",\"c\":\"d\"}]"
}
g = json.dumps(data)
c = json.loads(data)
print(c)
print(str(c).replace("\"","'"))
Output:
{'z': '[{"ItemId":"1234","a":"1234","b":"4567","c":"d"}]'}
{'z': '[{'ItemId':'1234','a':'1234','b':'4567','c':'d'}]'}
I want to create a json file like this:
{"946705035":4,"946706692":4 ...}
I am taking a column that only contains Unix Timestamp and group them.
result = data['Last_Modified_Date_unixtimestamp_no_time'].value_counts()
In [21]: result.head()
Out[21]:
1508284800 131
1508716800 106
1508371200 101
1508457600 99
1508630400 96
Name: Last_Modified_Date_unixtimestamp_no_time, dtype: int64
transform to a dict
result = result.to_dict()
result
'''
{1507161600: 1,
1507852800: 1,
1508198400: 64,
1508284800: 131,
...
1535155200: 1,
1535241600: 1}
'''
import json
result = json.dumps(result)
with open('result.json', 'w') as fp:
json.dump(result, fp, indent=4)
result
this is the data structure that I expected
{"946705035":4,"946706692":4}
You're dumping the JSON twice, which causes quotes to be escaped on the second dump. (After the first json.dumps the result is only a string, so you're just dumping a string instead of a dict again)
import json
# result = json.dumps(result)
with open('result.json', 'w') as fp:
json.dump(result, fp, indent=4)
Or remove the second dump:
import json
result = json.dumps(result)
with open('result.json', 'w') as fp:
# json.dump(result, fp, indent=4)
print(result, file=fp)
data_json=df.to_json(orient='records')
parsed=json.loads(data_json)
with open('my_data.json', 'w') as f:
json.dump(parsed, f, indent=4)
First convert the dataframe to dict
response = df.to_dict(orient="records")
And then encode the response to json
json_compatible_data = jsonable_encoder(response)
This should work well.
The simplest way to solve the above problem is to play with json.dumps() and json.loads().
import json
result = json.dumps(result)
with open('result.json', 'w') as fp:
json.loads(result, fp)
I am trying to run this code but it creates error.
import json
import requests
import pprint
data = []
with open('data.txt') as o1:
for line in o1:
data.append(json.loads(line))
print(data)
print(" \n")
print(data)
url = 'http://xyz.abcdfx.in/devicedata'
body_json=json.dumps(data)
headers = {'Content-Type':'application/json'}
d = requests.post(url, data = body_json, headers=headers)
pprint.pprint(d.json())
it shows
Value Error: No json object could be Decoded
I am new to programming and not able to figure out what is the problem.
It seems like you are trying to parse the json file line by line, but the json objects may (and usually are) span more than one line. You need to have the entire file in order to parse it:
with open('data.txt') as o1:
data = json.loads(o1.read()) # read ALL the file and parse. no loops
print(data)
i solved my problem using this:
data =[]
with open('data.txt') as f:
for line in f:
data = json.loads(line)
print(data)
url = 'http://xyz.abcdfx.cn/devicedata'
body_json=json.dumps(data)
headers = {'Content-Type':'application/json'}
d = requests.post(url, data = body_json, headers=headers)
pprint.pprint(d.json())
I have a JSON file containing various objects each containing elements. With my python script, I only keep the objects I want, and then put the elements I want in a list. But the element has a prefix, which I'd like to suppress form the list.
The post-script JSON looks like that:
{
"ip_prefix": "184.72.128.0/17",
"region": "us-east-1",
"service": "EC2"
}
The "IP/mask" is what I'd like to keep. The List looks like that:
'"ip_prefix": "23.20.0.0/14",'
So what can I do to only keep "23.20.0.0/14" in the list?
Here is the code:
json_data = open(jsonsourcefile)
data = json.load(json_data)
print (destfile)
d=[]
for objects in (data['prefixes']):
if servicerequired in json.dumps(objects):
#print(json.dumps(objects, sort_keys=True, indent=4))
with open(destfile, 'a') as file:
file.write(json.dumps(objects, sort_keys=True, indent=4 ))
with open(destfile, 'r') as reads:
liste = list()
for strip in reads:
if "ip_prefix" in strip:
strip = strip.strip()
liste.append(strip)
print(liste)
Thanks,
dersoi
Ok so i've went through your JSON object
import json, urllib2
url = 'https://ip-ranges.amazonaws.com/ip-ranges.json'
req = urllib2.Request(url)
res = urllib2.urlopen(req)
j = json.load(res)
print j['prefixes'][0]['ip_prefix']
prefixes = j['prefixes']
for i in prefixes:
print i['ip_prefix']
the result:
>>>
23.20.0.0/14
23.20.0.0/14
27.0.0.0/22
43.250.192.0/24
43.250.193.0/24
46.51.128.0/18
46.51.192.0/20
46.51.216.0/21
46.51.224.0/19
etc...
So now you want all into one txt file right?
So you do this:
import json, urllib2
url = 'https://ip-ranges.amazonaws.com/ip-ranges.json'
req = urllib2.Request(url)
res = urllib2.urlopen(req)
j = json.load(res)
#print j['prefixes'][0]['ip_prefix']
prefixes = j['prefixes']
destfile = 'destfile.txt'
with open('destfile.txt', 'w') as f:
for i in prefixes:
#print i['ip_prefix']
f.write(i['ip_prefix'])
f.write('\n')
f.close
Best regards,
Rizzit
I've refactored your code, try this out:
import json
with open('sample.json', 'r') as data:
json_data = json.loads(data.read())
print json_data.get('ip_prefix')
# Output: "184.72.128.0/17"
You can rewrite the second open block as:
with open(destfile, 'r') as reads:
data = json.load(reads)
liste = [i['ip_prefix'] for i in data]
Although, I don't think you need to write to an intermediate file anyway, you could combine both blocks.