I have a geojson feature collection dataset with a lot of features. I want to add/update the properties of each feature with properties of a json file. The unique identifier of both datasets is the "uuid" value.
This is the geojson format:
mtx = {
"type": "FeatureCollection",
"crs": {
"type": "name",
"properties": {
"name": "EPSG:4326"
}
},
"features": [
{
"type": "Feature",
"id": 1,
"geometry": {
"type": "Point",
"coordinates": [
5.36516933279853,
51.5510854507331
]
},
"properties": {
"OBJECTID": 1,
"PK_UID": 1,
"uuid": "1efa8916-c854-465b-80f5-1f02fd25fb31",
"road": "A2",
"lane": 1,
"km": 134.96,
"bearing": 148.02261,
"locid": "A2134.96"
}
},
{
"type": "Feature",
"id": 2,
"geometry": {
"type": "Point",
"coordinates": [
5.05380200345974,
52.3264095459638
]
},
"properties": {
"OBJECTID": 2,
"PK_UID": 2,
"uuid": "73bf3758-6754-433f-9896-d03c0673ae55",
"road": "A1",
"lane": 3,
"km": 11.593,
"bearing": 113.404253,
"locid": "A111.593"
}
}
]
}
And this is the json format:
msi= [
{
"uuid": "1efa8916-c854-465b-80f5-1f02fd25fb31",
"road": "A2",
"carriageway": "R",
"lane": "1",
"km": "134.960",
"display": "blank",
"display_attrs": "{'flashing': 'false'}",
"speedlimit": "null"
},
{
"uuid": "73bf3758-6754-433f-9896-d03c0673ae55",
"road": "A1",
"carriageway": "R",
"lane": "3",
"km": "11.593",
"display": "blank",
"display_attrs": "{'flashing': 'false'}",
"speedlimit": "null"
}
]
So how can I make a python script that loop through the geojson features and update each feature properties with the matching properties from the json based on the "uuid" value?
I tried something like this but this didn't give me the expected result:
#Loop over GeoJSON features and update the new properties from msi json
for feat in mtx['features']:
for i in range(len(msi)):
if mtx['features'][i]['properties']['uuid'] == msi[i]['uuid']:
feat ['properties'].update(msi[i])
Thanks for helping me out.
I'd do something like this (completely untested, so beware of mistakes...):
# Turn msi into a dictionary for easy access
msi_dict = {m['uuid']: m for m in msi}
for f in mtx['features']:
f['properties'].update(msi_dict.get(f['properties']['uuid'], {}))
Related
Here is a schema of the json output which I am trying to parse and write specific fields from it into a csv file (Example: cve id, description,....)
{
"$schema": "http://json-schema.org/draft-04/schema#",
"title": "JSON Schema for NVD Vulnerability Data Feed version 1.1",
"id": "https://scap.nist.gov/schema/nvd/feed/1.1/nvd_cve_feed_json_1.1.schema",
"definitions": {
"def_cpe_name": {
"description": "CPE name",
"type": "object",
"properties": {
"cpe22Uri": {
"type": "string"
},
"cpe23Uri": {
"type": "string"
},
"lastModifiedDate": {
"type": "string"
}
},
"required": [
"cpe23Uri"
]
},
"def_cpe_match": {
"description": "CPE match string or range",
"type": "object",
"properties": {
"vulnerable": {
"type": "boolean"
},
"cpe22Uri": {
"type": "string"
},
"cpe23Uri": {
"type": "string"
},
"versionStartExcluding": {
"type": "string"
},
"versionStartIncluding": {
"type": "string"
},
"versionEndExcluding": {
"type": "string"
},
"versionEndIncluding": {
"type": "string"
},
"cpe_name": {
"type": "array",
"items": {
"$ref": "#/definitions/def_cpe_name"
}
}
},
"required": [
"vulnerable",
"cpe23Uri"
]
},
"def_node": {
"description": "Defines a node or sub-node in an NVD applicability statement.",
"properties": {
"operator": {"type": "string"},
"negate": {"type": "boolean"},
"children": {
"type": "array",
"items": {"$ref": "#/definitions/def_node"}
},
"cpe_match": {
"type": "array",
"items": {"$ref": "#/definitions/def_cpe_match"}
}
}
},
"def_configurations": {
"description": "Defines the set of product configurations for a NVD applicability statement.",
"properties": {
"CVE_data_version": {"type": "string"},
"nodes": {
"type": "array",
"items": {"$ref": "#/definitions/def_node"}
}
},
"required": [
"CVE_data_version"
]
},
"def_subscore": {
"description": "CVSS subscore.",
"type": "number",
"minimum": 0,
"maximum": 10
},
"def_impact": {
"description": "Impact scores for a vulnerability as found on NVD.",
"type": "object",
"properties": {
"baseMetricV3": {
"description": "CVSS V3.x score.",
"type": "object",
"properties": {
"cvssV3": {"$ref": "cvss-v3.x.json"},
"exploitabilityScore": {"$ref": "#/definitions/def_subscore"},
"impactScore": {"$ref": "#/definitions/def_subscore"}
}
},
"baseMetricV2": {
"description": "CVSS V2.0 score.",
"type": "object",
"properties": {
"cvssV2": {"$ref": "cvss-v2.0.json"},
"severity": {"type": "string"},
"exploitabilityScore": {"$ref": "#/definitions/def_subscore"},
"impactScore": {"$ref": "#/definitions/def_subscore"},
"acInsufInfo": {"type": "boolean"},
"obtainAllPrivilege": {"type": "boolean"},
"obtainUserPrivilege": {"type": "boolean"},
"obtainOtherPrivilege": {"type": "boolean"},
"userInteractionRequired": {"type": "boolean"}
}
}
}
},
"def_cve_item": {
"description": "Defines a vulnerability in the NVD data feed.",
"properties": {
"cve": {"$ref": "CVE_JSON_4.0_min_1.1.schema"},
"configurations": {"$ref": "#/definitions/def_configurations"},
"impact": {"$ref": "#/definitions/def_impact"},
"publishedDate": {"type": "string"},
"lastModifiedDate": {"type": "string"}
},
"required": ["cve"]
}
},
"type": "object",
"properties": {
"CVE_data_type": {"type": "string"},
"CVE_data_format": {"type": "string"},
"CVE_data_version": {"type": "string"},
"CVE_data_numberOfCVEs": {
"description": "NVD adds number of CVE in this feed",
"type": "string"
},
"CVE_data_timestamp": {
"description": "NVD adds feed date timestamp",
"type": "string"
},
"CVE_Items": {
"description": "NVD feed array of CVE",
"type": "array",
"items": {"$ref": "#/definitions/def_cve_item"}
}
},
"required": [
"CVE_data_type",
"CVE_data_format",
"CVE_data_version",
"CVE_Items"
]
}
# -*- coding: utf-8 -*-
"""
Created on Thu Dec 3 17:08:51 2020
#author: Rajat Varshney
"""
import requests, json
api_url = 'https://services.nvd.nist.gov/rest/json/cve/1.0/'
cveid = input('Enter CVE ID: ')
api_call = requests.get(api_url+cveid)
print(api_call.content)
with open('cve details.txt', 'w') as outfile:
json.dump(api_call.content, outfile)
I am getting none when I try to validate my Json schema with my Json response using Validate from Jsonschema.validate, while it shows matched on https://www.jsonschemavalidator.net/
Json Schema
{
"KPI": [{
"KPIDefinition": {
"id": {
"type": "string"
},
"name": {
"type": "string"
},
"version": {
"type": "number"
},
"description": {
"type": "string"
},
"datatype": {
"type": "string"
},
"units": {
"type": "string"
}
},
"KPIGroups": [{
"id": {
"type": "number"
},
"name": {
"type": "string"
}
}]
}],
"response": [{
"Description": {
"type": "string"
}
}]
}
JSON Response
JSON Response
{
"KPI": [
{
"KPIDefinition": {
"id": "2",
"name": "KPI 2",
"version": 1,
"description": "This is KPI 2",
"datatype": "1",
"units": "perHour"
},
"KPIGroups": [
{
"id": 7,
"name": "Group 7"
}
]
},
{
"KPIDefinition": {
"id": "3",
"name": "Parameter 3",
"version": 1,
"description": "This is KPI 3",
"datatype": "1",
"units": "per Hour"
},
"KPIGroups": [
{
"id": 7,
"name": "Group 7"
}
]
}
],
"response": [
{
"Description": "RECORD Found"
}
]
}
Code
json_schema2 = {"KPI":[{"KPIDefinition":{"id_new":{"type":"number"},"name":{"type":"string"},"version":{"type":"number"},"description":{"type":"string"},"datatype":{"type":"string"},"units":{"type":"string"}},"KPIGroups":[{"id":{"type":"number"},"name":{"type":"string"}}]}],"response":[{"Description":{"type":"string"}}]}
json_resp = {"KPI":[{"KPIDefinition":{"id":"2","name":"Parameter 2","version":1,"description":"This is parameter 2 definition version 1","datatype":"1","units":"kN"},"KPIGroups":[{"id":7,"name":"Group 7"}]},{"KPIDefinition":{"id":"3","name":"Parameter 3","version":1,"description":"This is parameter 3 definition version 1","datatype":"1","units":"kN"},"KPIGroups":[{"id":7,"name":"Group 7"}]}],"response":[{"Description":"RECORD FETCHED"}]}
print(jsonschema.validate(instance=json_resp, schema=json_schema2))
Validation is not being done correctly, I changed the dataType and key name in my response but still, it is not raising an exception or error.
jsonschema.validate(..) is not supposed to return anything.
Your schema object and the JSON object are both okay and validation has passed if it didn't raise any exceptions -- which seems to be the case here.
That being said, you should wrap your call within a try-except block so as to be able to catch validation errors.
Something like:
try:
jsonschema.validate(...)
print("Validation passed!")
except ValidationError:
print("Validation failed")
# similarly catch SchemaError too if needed.
Update: Your schema is invalid. As it stands, it will validate almost all inputs. A schema JSON should be an object (dict) that should have fields like "type" and based on the type, it might have other required fields like "items" or "properties". Please read up on how to write JSONSchema.
Here's a schema I wrote for your JSON:
{
"type": "object",
"required": [
"KPI",
"response"
],
"properties": {
"KPI": {
"type": "array",
"items": {
"type": "object",
"required": ["KPIDefinition","KPIGroups"],
"properties": {
"KPIDefinition": {
"type": "object",
"required": ["id","name"],
"properties": {
"id": {"type": "string"},
"name": {"type": "string"},
"version": {"type": "integer"},
"description": {"type": "string"},
"datatype": {"type": "string"},
"units": {"type": "string"},
},
"KPIGroups": {
"type": "array",
"items": {
"type": "object",
"required": ["id", "name"],
"properties": {
"id": {"type": "integer"},
"name": {"type": "string"}
}
}
}
}
}
}
},
"response": {
"type": "array",
"items": {
"type": "object",
"required": ["Description"],
"properties": {
"Description": {"type": "string"}
}
}
}
}
}
I am working with one of my requirement
My requirement: I need to pick and print only 3rd "id" from "syrap" list from the nested json file. I am not getting desired output. Any help will be appreciated.
Test file:
{
"id": "0001",
"type": "donut",
"name": "Cake",
"ppu": 0.55,
"batters":
{ "process": "abc",
"mix": "0303",
"syrap":
[
{ "id": "1001", "type": "Regular" },
{ "id": "1002", "type": "Chocolate" },
{ "id": "1003", "type": "Blueberry" },
{ "id": "1004", "type": "Devil's Food" }
]
},
"rate": 0.55,
"topping":
[
{ "id": "5001", "type": "None" },
{ "id": "5002", "type": "Glazed" },
{ "id": "5005", "type": "Sugar" },
{ "id": "5007", "type": "Powdered Sugar" },
{ "id": "5006", "type": "Chocolate with Sprinkles" },
{ "id": "5003", "type": "Chocolate" },
{ "id": "5004", "type": "Maple" }
]
}
Expected output in a csv:
0001,donut,abc,0303,1003
My code:
import requests
import json
import csv
f = open('testdata.json')
data = json.load(f)
f.close()
f = csv.writer(open('testout.csv', 'wb+'))
for item in data:
f.writerow([item['id'], item[type], item['batters'][0]['process'],
item['batters'][0]['mix'],
item['batters'][0]['syrap'][0]['id'],
item['batters'][0]['syrap'][1]['id'],
item['batters'][0]['syrap'][2]['id'])
Here is some sample code showing how you can iterate through json content parsed as a dictionary:
import json
json_str = '''{
"id": "0001",
"type": "donut",
"name": "Cake",
"ppu": 0.55,
"batters":
{ "process": "abc",
"mix": "0303",
"syrap":
[
{ "id": "1001", "type": "Regular" },
{ "id": "1002", "type": "Chocolate" },
{ "id": "1003", "type": "Blueberry" },
{ "id": "1004", "type": "Devil's Food" }
]
},
"rate": 0.55,
"topping":
[
{ "id": "5001", "type": "None" },
{ "id": "5002", "type": "Glazed" },
{ "id": "5005", "type": "Sugar" },
{ "id": "5007", "type": "Powdered Sugar" },
{ "id": "5006", "type": "Chocolate with Sprinkles" },
{ "id": "5003", "type": "Chocolate" },
{ "id": "5004", "type": "Maple" }
]
}
'''
jsondict = json.loads(json_str)
syrap_node = jsondict['batters']['syrap']
for item in syrap_node:
print (f'id:{item["id"]} type: {item["type"]}')
Simply, data[“batters”][“syrap”][2][“id”]
Much better way to achieve this would be
f = open('testout.csv', 'wb+')
with f:
fnames = ['id','type','process','mix','syrap']
writer = csv.DictWriter(f, fieldnames=fnames)
writer.writeheader()
for item in data:
print item
writer.writerow({'id' : item['id'], 'type': item['type'],
'process' : item['batters']['process'],
'mix': item['batters']['mix'],
'syrap': item['batters']['syrap'][2]['id']})
You need to make sure that data is actually a list. if it is not a list, don't use for loop.
simply,
writer.writerow({'id' : data['id'], 'type': data['type'],
'process' : data['batters']['process'],
'mix': data['batters']['mix'],
'syrap': data['batters']['syrap'][2]['id']})
I am trying to use a JSON file in Mapbox studio for network analysis but it gives me an error:
Input failed. "type" member required on line 1.
The representative sample of JSON file is:
"version": 0.6,
"generator": "Overpass API 0.7.56.2 b688b00f",
"osm3s": {
"timestamp_osm_base": "2020-03-27T11:58:01Z",
"copyright": "The data included in this document is from www.openstreetmap.org. The data is made available under ODbL."
},
"elements": [
{
"type": "node",
"id": 123458059,
"lat": -38.3344495,
"lon": 143.5394486
},
{
"type": "node",
"id": 123458066,
"lat": -38.3394461,
"lon": 143.5923655,
"tags": {
"crossing": "traffic_signals",
"highway": "traffic_signals"
}
},
{
"type": "way",
"id": 769574290,
"nodes": [
7183581936,
681081177,
1561328098,
1539139562,
448021781
],
"tags": {
"highway": "trunk",
"lanes": "2",
"maxspeed": "80",
"name": "Princes Highway",
"ref": "A1",
"source:maxspeed:sign": "mapillary"
}
},
{
"type": "way",
"id": 776227225,
"nodes": [
1017428185,
317738200
],
"tags": {
"alt_name": "Princes Highway",
"highway": "trunk",
"lanes": "2",
"maxspeed": "50",
"name": "Murray Street",
"ref": "A1",
"source:maxspeed:sign": "OpenStreetCam",
"source:name": "services.land.vic.gov.au"
}
}
]
}
Does the error occur because of the specification of the format? Do we need to reformat the features or types?
To upload data to Mapbox you will need to convert your JSON file to GeoJSON, a subset of the JSON format. For example:
{"type": "FeatureCollection",
"features": [
{
"geometry": {
"type": "Point",
"coordinates": [
-76.9750541388,
38.8410857803
]
},
"type": "Feature",
"properties": {
"description": "Southern Ave",
"marker-symbol": "rail-metro",
"title": "Southern Ave",
"url": "http://www.wmata.com/rider_tools/pids/showpid.cfm?station_id=107",
"lines": [
"Green"
],
"address": "1411 Southern Avenue, Temple Hills, MD 20748"
}
},
{
"geometry": {
"type": "Point",
"coordinates": [
-76.935256783,
38.9081784965
]
},
"type": "Feature",
"properties": {
"description": "Deanwood",
"marker-symbol": "rail-metro",
"title": "Deanwood",
"url": "http://www.wmata.com/rider_tools/pids/showpid.cfm?station_id=65",
"lines": [
"Orange"
],
"address": "4720 Minnesota Avenue NE, Washington, DC 20019"
}
}
]}
Mapbox accepts GeoJSON, you'll need to ask Overpass to return data as GeoJSON.
Got a bit of a puzzle here, I'm trying to build a schema to use in my python app, But I can't figure out how to get this "we" field to be both required and contain a random string (ex: "QWERT1")
{
"we": [
{
"finished": "01.23.2020 12:56:31",
"run": "02611",
"scenarios": [
{
"name": "name",
"status": "failed",
"run_id": "42",
"tests": [
{
"test_id": "7",
"name": "TC29",
"status": "success",
"finished": "01.23.2020 12:56:31"
}
]
}
]
}
]
}
Rest of the fields should be also mandatory (name, status etc). If I exclude the "we" from the required the rest of the fields are treated as non-mandatory, and if I add the "we" as mandatory I can't then use there any other word :/
This my schema I've ended up with (with "we" mandatory):
{
"$schema": "http://json-schema.org/draft-04/schema#",
"type": "object",
"properties": {
"we": {
"type": "array",
"items": [
{
"type": "object",
"properties": {
"finished": {
"type": "string"
},
"run": {
"type": "string"
},
"scenarios": {
"type": "array",
"items": [
{
"type": "object",
"properties": {
"name": {
"type": "string"
},
"status": {
"type": "string"
},
"run_id": {
"type": "string"
},
"tests": {
"type": "array",
"items": [
{
"type": "object",
"properties": {
"test_id": {
"type": "string"
},
"name": {
"type": "string"
},
"status": {
"type": "string"
},
"finished": {
"type": "string"
}
},
"required": [
"test_id",
"name",
"status",
"finished"
]
}
]
}
},
"required": [
"name",
"status",
"run_id",
"tests"
]
}
]
}
},
"required": [
"finished",
"run",
"scenarios"
]
}
]
}
},
"required": [
"we"
]
}
Any ideas ?
If I understand correctly, the root object key could be any string.
First, you need to replace required with minProperties: 1. If you require only 1 property, you also need maxProperties: 1.
Next, you need to use additionalProperties rather than properties > we.
additionalProperties applies the value subschema to all property values at the JSON instance location object.
Here's a bare version of that schema...
{
"$schema": "http://json-schema.org/draft-07/schema#",
"minProperties": 1,
"additionalProperties": {}
}
You can test it with your schema and instance here: https://jsonschema.dev/s/2kE9y