Related
While trying to convert a JSON output below to CSV, getting error
Here is the JSON output
{
"data": [
{
"id": "-1000100591151294842",
"type": "fres",
"attributes": {
"operationState": "In Service",
"deploymentState": "discovered",
"displayData": {
"operationState": "Up",
"adminState": "Enabled",
"displayTopologySource": "Protocol,Derived",
"displayPhotonicSpectrumData": [
{
"frequency": "194.950000",
"wavelength": "1537.79",
"channel": "CH-20"
}
],
"displayDeploymentState": "Discovered",
"displayName": "J-BBEG-CHLC-P109"
},
"utilizationData": {
"totalCapacity": "100.0",
"usedCapacity": "100.0",
"utilizationPercent": "100",
"capacityUnits": "Gbps"
},
"resourceState": "discovered",
"serviceClass": "OTU",
"linkLabel": "BBEG-ROADM-0101:5-4-1,CHLC-ROADM-0401:7-35-1",
"lastUpdatedAdminStateTimeStamp": "2021-05-03T00:29:24.444Z",
"lastUpdatedOperationalStateTimeStamp": "2022-12-08T22:42:21.567Z",
"userLabel": "J-BBEG-CHLC-P109",
"mgmtName": "",
"nativeName": "",
"awarenessTime": "2022-12-08T22:42:22.123Z",
"layerRate": "OTU4",
"layerRateQualifier": "OTU4",
"supportedByLayerRatePackageList": [
{
"layerRate": "OTSi",
"layerRateQualifier": "100G"
}
],
"networkRole": "FREAP",
"directionality": "bidirectional",
"topologySources": [
"adjacency",
"stitched"
],
"adminState": "In Service",
"photonicSpectrumPackageList": [
{
"frequency": "194.950000",
"width": "37.5"
}
],
"active": true,
"additionalAttributes": {
"isActual": "true",
"hasLowerTopology": "true"
},
"reliability": "auto",
"resilienceLevel": "unprotected"
},
"relationships": {
"freDiscovered": {
"data": {
"type": "freDiscovered",
"id": "-1000100591151294842"
}
},
"supportedByServices": {
"data": [
{
"type": "fres",
"id": "6765278351459212874"
}
]
},
"endPoints": {
"data": [
{
"type": "endPoints",
"id": "-1000100591151294842:1"
},
{
"type": "endPoints",
"id": "-1000100591151294842:2"
}
]
},
"partitionFres": {
"data": [
{
"type": "fres",
"id": "7147507956181395827"
}
]
}
}
},
{
"id": "-1013895107051577774",
"type": "fres",
"attributes": {
"operationState": "In Service",
"deploymentState": "discovered",
"displayData": {
"operationState": "Up",
"adminState": "Enabled",
"displayTopologySource": "Protocol,Derived",
"displayPhotonicSpectrumData": [
{
"frequency": "191.600000",
"wavelength": "1564.68",
"channel": "CH-87"
}
],
"displayDeploymentState": "Discovered",
"displayName": "J-KFF9-PNTH-P101"
},
"utilizationData": {
"totalCapacity": "100.0",
"usedCapacity": "90.0",
"utilizationPercent": "90",
"capacityUnits": "Gbps"
},
"resourceState": "discovered",
"serviceClass": "OTU",
"tags": [
"J-KFF9-PNTH-P101"
],
"linkLabel": "KFF9-ROADM-0301:1-1-1,PNTH-ROADM-0101:1-1-1",
"lastUpdatedAdminStateTimeStamp": "2021-09-12T20:22:59.334Z",
"lastUpdatedOperationalStateTimeStamp": "2022-10-12T14:20:44.779Z",
"userLabel": "J-KFF9-PNTH-P101",
"mgmtName": "",
"nativeName": "",
"awarenessTime": "2022-10-12T14:20:45.417Z",
"layerRate": "OTU4",
"layerRateQualifier": "OTU4",
"supportedByLayerRatePackageList": [
{
"layerRate": "OTSi",
"layerRateQualifier": "100G"
}
],
"networkRole": "FREAP",
"directionality": "bidirectional",
"topologySources": [
"adjacency",
"stitched"
],
"adminState": "In Service",
"photonicSpectrumPackageList": [
{
"frequency": "191.600000",
"width": "37.5"
}
],
"active": true,
"additionalAttributes": {
"isActual": "true",
"hasLowerTopology": "true"
},
"reliability": "auto",
"resilienceLevel": "unprotected"
},
"relationships": {
"freDiscovered": {
"data": {
"type": "freDiscovered",
"id": "-1013895107051577774"
}
},
"supportedByServices": {
"data": [
{
"type": "fres",
"id": "6055685088078365419"
}
]
},
"endPoints": {
"data": [
{
"type": "endPoints",
"id": "-1013895107051577774:1"
},
{
"type": "endPoints",
"id": "-1013895107051577774:2"
}
]
},
"partitionFres": {
"data": [
{
"type": "fres",
"id": "-6727082893715936342"
}
]
}
}
}
] }
getting below error, not sure what is missing
Here is the python script I used. have been trying different variations but no luck getting different errors in all other instances
filename = Path('fre.json')
data = []
with open(filename,'r') as json_file:
data_str = json_file.read()
data_str = data_str.split('[',1)[-1]
data_str = data_str.rsplit(']',1)[0]
data_str = data_str.split('][')
for jsonStr in data_str:
jsonStr = '[' + jsonStr + ']'
temp_data = json.loads(jsonStr)
for each in temp_data:
data.append(each)
what is wrong?
I have two files:
kube-apiserver.json
{
"apiVersion": "v1",
"kind": "Pod",
"metadata": {
[...]
},
"spec": {
"containers": [
{
"command": [
"kube-apiserver",
"--advertise-address=192.168.49.2",
"--allow-privileged=true",
"--authorization-mode=Node,RBAC",
"--client-ca-file=/var/lib/minikube/certs/ca.crt",
"--enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota",
"--enable-bootstrap-token-auth=true",
"--etcd-cafile=/var/lib/minikube/certs/etcd/ca.crt",
"--etcd-certfile=/var/lib/minikube/certs/apiserver-etcd-client.crt",
"--etcd-keyfile=/var/lib/minikube/certs/apiserver-etcd-client.key",
"--etcd-servers=https://127.0.0.1:2379",
"--kubelet-client-certificate=/var/lib/minikube/certs/apiserver-kubelet-client.crt",
"--kubelet-client-key=/var/lib/minikube/certs/apiserver-kubelet-client.key",
"--kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname",
"--proxy-client-cert-file=/var/lib/minikube/certs/front-proxy-client.crt",
"--proxy-client-key-file=/var/lib/minikube/certs/front-proxy-client.key",
"--requestheader-allowed-names=front-proxy-client",
"--requestheader-client-ca-file=/var/lib/minikube/certs/front-proxy-ca.crt",
"--requestheader-extra-headers-prefix=X-Remote-Extra-",
"--requestheader-group-headers=X-Remote-Group",
"--requestheader-username-headers=X-Remote-User",
"--secure-port=8443",
"--service-account-issuer=https://kubernetes.default.svc.cluster.local",
"--service-account-key-file=/var/lib/minikube/certs/sa.pub",
"--service-account-signing-key-file=/var/lib/minikube/certs/sa.key",
"--service-cluster-ip-range=10.96.0.0/12",
"--tls-cert-file=/var/lib/minikube/certs/apiserver.crt",
"--tls-private-key-file=/var/lib/minikube/certs/apiserver.key"
],
[...]
"volumeMounts": [
{
"mountPath": "/etc/ssl/certs",
"name": "ca-certs",
"readOnly": true
},
{
"mountPath": "/etc/ca-certificates",
"name": "etc-ca-certificates",
"readOnly": true
},
{
"mountPath": "/var/lib/minikube/certs",
"name": "k8s-certs",
"readOnly": true
},
{
"mountPath": "/usr/local/share/ca-certificates",
"name": "usr-local-share-ca-certificates",
"readOnly": true
},
{
"mountPath": "/usr/share/ca-certificates",
"name": "usr-share-ca-certificates",
"readOnly": true
}
]
}
],
[...]
"volumes": [
{
"hostPath": {
"path": "/etc/ssl/certs",
"type": "DirectoryOrCreate"
},
"name": "ca-certs"
},
{
"hostPath": {
"path": "/etc/ca-certificates",
"type": "DirectoryOrCreate"
},
"name": "etc-ca-certificates"
},
{
"hostPath": {
"path": "/var/lib/minikube/certs",
"type": "DirectoryOrCreate"
},
"name": "k8s-certs"
},
{
"hostPath": {
"path": "/usr/local/share/ca-certificates",
"type": "DirectoryOrCreate"
},
"name": "usr-local-share-ca-certificates"
},
{
"hostPath": {
"path": "/usr/share/ca-certificates",
"type": "DirectoryOrCreate"
},
"name": "usr-share-ca-certificates"
}
]
},
"status": {
[...]
}
}
and patch.json
{
"apiVersion": "v1",
"kind": "Pod",
"metadata": {
},
"spec": {
"containers": [
{
"command": [
"--audit-policy-file=/etc/kubernetes/audit-policy.yaml",
"--audit-log-path=/var/log/kubernetes/audit/audit.log"
],
"volumeMounts": [
{
"mountPath": "/etc/kubernetes/audit-policy.yaml",
"name": "audit",
"readOnly": true
},
{
"mountPath": "/var/log/kubernetes/audit/",
"name": "audit-log",
"readOnly": true
}
]
}
],
"volumes": [
{
"hostPath": {
"path": "/etc/kubernetes/audit-policy.yaml",
"type": "FileOrCreate"
},
"name": "audit"
},
{
"hostPath": {
"path": "/var/log/kubernetes/audit/",
"type": "DirectoryOrCreate"
},
"name": "audit-log"
}
]
},
"status": {
}
}
When i try to do jq -s '.[0] * .[1]' kube-apiserver.json patch.json > patched-apiserver.json items that are in patch.json overrides items from kube-apiserver.json so it looks like this:
{
"apiVersion": "v1",
"kind": "Pod",
"metadata": {
[...]
"spec": {
"containers": [
{
"command": [
"--audit-policy-file=/etc/kubernetes/audit-policy.yaml",
"--audit-log-path=/var/log/kubernetes/audit/audit.log"
],
"volumeMounts": [
{
"mountPath": "/etc/kubernetes/audit-policy.yaml",
"name": "audit",
"readOnly": true
},
{
"mountPath": "/var/log/kubernetes/audit/",
"name": "audit-log",
"readOnly": true
}
]
}
],
[..].
"volumes": [
{
"hostPath": {
"path": "/etc/kubernetes/audit-policy.yaml",
"type": "FileOrCreate"
},
"name": "audit"
},
{
"hostPath": {
"path": "/var/log/kubernetes/audit/",
"type": "DirectoryOrCreate"
},
"name": "audit-log"
}
]
},
"status": {
[...]
}
}
and i would like my file to look like this:
{
"apiVersion": "v1",
"kind": "Pod",
"metadata": {
[...]
},
"spec": {
"containers": [
{
"command": [
"kube-apiserver",
"--advertise-address=192.168.49.2",
"--allow-privileged=true",
"--authorization-mode=Node,RBAC",
"--client-ca-file=/var/lib/minikube/certs/ca.crt",
"--enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota",
"--enable-bootstrap-token-auth=true",
"--etcd-cafile=/var/lib/minikube/certs/etcd/ca.crt",
"--etcd-certfile=/var/lib/minikube/certs/apiserver-etcd-client.crt",
"--etcd-keyfile=/var/lib/minikube/certs/apiserver-etcd-client.key",
"--etcd-servers=https://127.0.0.1:2379",
"--kubelet-client-certificate=/var/lib/minikube/certs/apiserver-kubelet-client.crt",
"--kubelet-client-key=/var/lib/minikube/certs/apiserver-kubelet-client.key",
"--kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname",
"--proxy-client-cert-file=/var/lib/minikube/certs/front-proxy-client.crt",
"--proxy-client-key-file=/var/lib/minikube/certs/front-proxy-client.key",
"--requestheader-allowed-names=front-proxy-client",
"--requestheader-client-ca-file=/var/lib/minikube/certs/front-proxy-ca.crt",
"--requestheader-extra-headers-prefix=X-Remote-Extra-",
"--requestheader-group-headers=X-Remote-Group",
"--requestheader-username-headers=X-Remote-User",
"--secure-port=8443",
"--service-account-issuer=https://kubernetes.default.svc.cluster.local",
"--service-account-key-file=/var/lib/minikube/certs/sa.pub",
"--service-account-signing-key-file=/var/lib/minikube/certs/sa.key",
"--service-cluster-ip-range=10.96.0.0/12",
"--tls-cert-file=/var/lib/minikube/certs/apiserver.crt",
"--tls-private-key-file=/var/lib/minikube/certs/apiserver.key",
"--audit-policy-file=/etc/kubernetes/audit-policy.yaml",
"--audit-log-path=/var/log/kubernetes/audit/audit.log"
],
[...]
"volumeMounts": [
{
"mountPath": "/etc/ssl/certs",
"name": "ca-certs",
"readOnly": true
},
{
"mountPath": "/etc/ca-certificates",
"name": "etc-ca-certificates",
"readOnly": true
},
{
"mountPath": "/var/lib/minikube/certs",
"name": "k8s-certs",
"readOnly": true
},
{
"mountPath": "/usr/local/share/ca-certificates",
"name": "usr-local-share-ca-certificates",
"readOnly": true
},
{
"mountPath": "/usr/share/ca-certificates",
"name": "usr-share-ca-certificates",
"readOnly": true
},
{
"mountPath": "/etc/kubernetes/audit-policy.yaml",
"name": "audit",
"readOnly": true
},
{
"mountPath": "/var/log/kubernetes/audit/",
"name": "audit-log",
"readOnly": true
}
]
}
],
[...]
"volumes": [
{
"hostPath": {
"path": "/etc/ssl/certs",
"type": "DirectoryOrCreate"
},
"name": "ca-certs"
},
{
"hostPath": {
"path": "/etc/ca-certificates",
"type": "DirectoryOrCreate"
},
"name": "etc-ca-certificates"
},
{
"hostPath": {
"path": "/var/lib/minikube/certs",
"type": "DirectoryOrCreate"
},
"name": "k8s-certs"
},
{
"hostPath": {
"path": "/usr/local/share/ca-certificates",
"type": "DirectoryOrCreate"
},
"name": "usr-local-share-ca-certificates"
},
{
"hostPath": {
"path": "/usr/share/ca-certificates",
"type": "DirectoryOrCreate"
},
"name": "usr-share-ca-certificates"
},
{
"hostPath": {
"path": "/etc/kubernetes/audit-policy.yaml",
"type": "FileOrCreate"
},
"name": "audit"
},
{
"hostPath": {
"path": "/var/log/kubernetes/audit/",
"type": "DirectoryOrCreate"
},
"name": "audit-log"
}
]
},
"status": {
[...]
}
}
Does anyone know how to solve it with jq/python/bash/whatever?
jq --slurpfile patch patch.json '
(.spec.containers |= map(.command |= (. + $patch[].spec.containers[0].command | unique) |
.volumeMounts |= (. + $patch[].spec.containers[0].volumeMounts | unique))) |
(.spec.volumes |= (. + $patch[].spec.volumes | unique))
' kube-apiserver.json
I have added unique to ensure that commands, volumeMounts and volumes only appear once. This has the side effect of sorting the arrays.
You can remove unique if you do not want it.
There is a problem in your question: spec.containers is an array. In your example, this array contains only one element and my code adds the first element from the patch to each container.
Or would you like the patch to be merged based on position? In that case this solution does not work.
I currently have two JSONS that I want to merge into one singular JSON, additionally I want to add in a slight change.
Firstly, these are the two JSONS in question.
An intents JSON:
[
{
"ID": "G1",
"intent": "password_reset",
"examples": [
{
"text": "I forgot my password"
},
{
"text": "I can't log in"
},
{
"text": "I can't access the site"
},
{
"text": "My log in is failing"
},
{
"text": "I need to reset my password"
}
]
},
{
"ID": "G2",
"intent": "account_closure",
"examples": [
{
"text": "I want to close my account"
},
{
"text": "I want to terminate my account"
}
]
},
{
"ID": "G3",
"intent": "account_creation",
"examples": [
{
"text": "I want to open an account"
},
{
"text": "Create account"
}
]
},
{
"ID": "G4",
"intent": "complaint",
"examples": [
{
"text": "A member of staff was being rude"
},
{
"text": "I have a complaint"
}
]
}
]
and an entities JSON:
[
{
"ID": "K1",
"entity": "account_type",
"values": [
{
"type": "synonyms",
"value": "business",
"synonyms": [
"corporate"
]
},
{
"type": "synonyms",
"value": "personal",
"synonyms": [
"vanguard",
"student"
]
}
]
},
{
"ID": "K2",
"entity": "beverage",
"values": [
{
"type": "synonyms",
"value": "hot",
"synonyms": [
"heated",
"warm"
]
},
{
"type": "synonyms",
"value": "cold",
"synonyms": [
"ice",
"freezing"
]
}
]
}
]
The expected outcome is to create a JSON file that mimics this structure:
{
"intents": [
{
"intent": "password_reset",
"examples": [
{
"text": "I forgot my password"
},
{
"text": "I want to reset my password"
}
],
"description": "Reset a user password"
}
],
"entities": [
{
"entity": "account_type",
"values": [
{
"type": "synonyms",
"value": "business",
"synonyms": [
"company",
"corporate",
"enterprise"
]
},
{
"type": "synonyms",
"value": "personal",
"synonyms": []
}
],
"fuzzy_match": true
}
],
"metadata": {
"api_version": {
"major_version": "v2",
"minor_version": "2018-11-08"
}
},
"dialog_nodes": [
{
"type": "standard",
"title": "anything_else",
"output": {
"generic": [
{
"values": [
{
"text": "I didn't understand. You can try rephrasing."
},
{
"text": "Can you reword your statement? I'm not understanding."
},
{
"text": "I didn't get your meaning."
}
],
"response_type": "text",
"selection_policy": "sequential"
}
]
},
"conditions": "anything_else",
"dialog_node": "Anything else",
"previous_sibling": "node_4_1655399659061",
"disambiguation_opt_out": true
},
{
"type": "event_handler",
"output": {
"generic": [
{
"title": "What type of account do you hold with us?",
"options": [
{
"label": "Personal",
"value": {
"input": {
"text": "personal"
}
}
},
{
"label": "Business",
"value": {
"input": {
"text": "business"
}
}
}
],
"response_type": "option"
}
]
},
"parent": "slot_9_1655398217028",
"event_name": "focus",
"dialog_node": "handler_6_1655398217052",
"previous_sibling": "handler_7_1655398217052"
},
{
"type": "event_handler",
"output": {},
"parent": "slot_9_1655398217028",
"context": {
"account_type": "#account_type"
},
"conditions": "#account_type",
"event_name": "input",
"dialog_node": "handler_7_1655398217052"
},
{
"type": "standard",
"title": "business_account",
"output": {
"generic": [
{
"values": [
{
"text": "We have notified your corporate security team, they will be in touch to reset your password."
}
],
"response_type": "text",
"selection_policy": "sequential"
}
]
},
"parent": "node_3_1655397279884",
"next_step": {
"behavior": "jump_to",
"selector": "body",
"dialog_node": "node_4_1655399659061"
},
"conditions": "#account_type:business",
"dialog_node": "node_1_1655399028379",
"previous_sibling": "node_3_1655399027429"
},
{
"type": "standard",
"title": "intent_collection",
"output": {
"generic": [
{
"values": [
{
"text": "Thank you for confirming that you want to reset your password."
}
],
"response_type": "text",
"selection_policy": "sequential"
}
]
},
"next_step": {
"behavior": "jump_to",
"selector": "body",
"dialog_node": "node_3_1655397279884"
},
"conditions": "#password_reset",
"dialog_node": "node_3_1655396920143",
"previous_sibling": "Welcome"
},
{
"type": "frame",
"title": "account_type_confirmation",
"output": {
"generic": [
{
"values": [
{
"text": "Thank you"
}
],
"response_type": "text",
"selection_policy": "sequential"
}
]
},
"parent": "node_3_1655396920143",
"context": {},
"next_step": {
"behavior": "skip_user_input"
},
"conditions": "#password_reset",
"dialog_node": "node_3_1655397279884"
},
{
"type": "standard",
"title": "personal_account",
"output": {
"generic": [
{
"values": [
{
"text": "We have sent you an email with a password reset link."
}
],
"response_type": "text",
"selection_policy": "sequential"
}
]
},
"parent": "node_3_1655397279884",
"next_step": {
"behavior": "jump_to",
"selector": "body",
"dialog_node": "node_4_1655399659061"
},
"conditions": "#account_type:personal",
"dialog_node": "node_3_1655399027429"
},
{
"type": "standard",
"title": "reset_confirmation",
"output": {
"generic": [
{
"values": [
{
"text": "Do you need assistance with anything else today?"
}
],
"response_type": "text",
"selection_policy": "sequential"
}
]
},
"digress_in": "does_not_return",
"dialog_node": "node_4_1655399659061",
"previous_sibling": "node_3_1655396920143"
},
{
"type": "slot",
"output": {},
"parent": "node_3_1655397279884",
"variable": "$account_type",
"dialog_node": "slot_9_1655398217028",
"previous_sibling": "node_1_1655399028379"
},
{
"type": "standard",
"title": "welcome",
"output": {
"generic": [
{
"values": [
{
"text": "Hello. How can I help you?"
}
],
"response_type": "text",
"selection_policy": "sequential"
}
]
},
"conditions": "welcome",
"dialog_node": "Welcome"
}
],
"counterexamples": [],
"system_settings": {
"off_topic": {
"enabled": true
},
"disambiguation": {
"prompt": "Did you mean:",
"enabled": true,
"randomize": true,
"max_suggestions": 5,
"suggestion_text_policy": "title",
"none_of_the_above_prompt": "None of the above"
},
"human_agent_assist": {
"prompt": "Did you mean:"
},
"intent_classification": {
"training_backend_version": "v2"
},
"spelling_auto_correct": true
},
"learning_opt_out": false,
"name": "Reset Password",
"language": "en",
"description": "Basic Password Reset Request"
}
So what I am missing in my original files, is essentially:
"intents":
and for the entities file:
"entities"
at the start of each list of dictionaries.
Additionally, I would need to wrap the whole thing in curly braces to comply with json formatting.
As seen, the final goal is not just appending these two to one another but the file technically continues with some other JSON code that I have yet to write and deal with.
My question now is as follows; by what method can I either add in these words and the braces to the individual files, then combine them into a singular JSON or alternatively by what method can I read in these files and combine them with the changes all in one go?
The new output file closing on a curly brace after the entities list of dicts is an acceptable outcome for me at the time, so that I can continue to make changes and hopefully further learn from this how to do these changes in future when I get there.
TIA
JSON is only a string format, you can it load in a language structure, in python that is list and dict, do what you need then dump it back, so you don't "add strings" and "add brackets", on modify the structure
file = 'intents.txt'
intents = json.load(open(file)) # load a list
file = 'entities.txt'
entities = json.load(open(file)) # load a list
# create a dict
content = {
"intents": intents,
"entities": entities
}
json.dump(content, open(file, "w"))
If you're reading all the json in as a string, you can just prepend "{'intents':" to the start and append a closing "}".
myJson = "your json string"
myWrappedJson = '{"intents":' + myJson + "}"
I am trying to customize json data using object_hook in Python 3, but do not know how to get started. Any pointers are much appreciated. I am trying to introduce a new key and move existing data into the new key in Python Object.
I am trying to convert below json text:
{
"output": [
{
"Id": "101",
"purpose": "xyz text",
"array": [
{
"data": "abcd"
},
{
"data": "ef gh ij"
}
]
},
{
"Id": "102",
"purpose": "11xyz text",
"array": [
{
"data": "abcd"
},
{
"data": "java"
},
{
"data": "ef gh ij"
}
]
}
]
}
to
{
"output": [
{
"Id": "101",
"mydata": {
"purpose": "xyz text",
"array": [
{
"data": "abcd"
},
{
"data": "ef gh ij"
}
]
}
},
{
"Id": "102",
"mydata": {
"purpose": "11xyz text",
"array": [
{
"data": "abcd"
},
{
"data": "java"
},
{
"data": "ef gh ij"
}
]
}
}
]
}
My Python JSON object hook is defined as:
class JSONObject:
def __init__( self, dict ):
vars(self).update( dict )
def toJSON(self):
return json.dumps(self, default=lambda o: o.__dict__,
sort_keys=True, indent=4)
You can specify a custom object_pairs_hook (input_json is the string with your input JSON).
def mydata_hook(obj):
obj_d = dict(obj)
if 'Id' in obj_d:
return {'Id': obj_d['Id'], 'mydata': {k: v for k, v in obj_d.items() if 'Id' not in k}}
else:
return obj_d
print(json.dumps(json.loads(input_json, object_pairs_hook=mydata_hook), indent=2))
And the output:
{
"output": [
{
"mydata": {
"array": [
{
"data": "abcd"
},
{
"data": "ef gh ij"
}
],
"purpose": "xyz text"
},
"Id": "101"
},
{
"mydata": {
"array": [
{
"data": "abcd"
},
{
"data": "java"
},
{
"data": "ef gh ij"
}
],
"purpose": "11xyz text"
},
"Id": "102"
}
]
}
I'm trying to insert a doc in ElasticSearch but every time i try to insert in python, its return me an error. But if i try to insert from Kibana or cUrl, its succeed.
I already tried the elasticserach-dsl but i've got the same error.
(Sorry for my bad english, i'm from brazil :D)
Error i've got:
elasticsearch.helpers.BulkIndexError: ((...)'status': 400, 'error': {'type':
'illegal_argument_exception', 'reason': "object mapping [prices] can't be changed from nested to non-nested"}}}])
My code:
from elasticsearch import Elasticsearch
from elasticsearch.helpers import bulk
doc = [{
"_index": "products",
"_type": "test_products",
"_source": {
[...]
"prices": {
"latest": {
"value": 89,
"when": 1502795602848
},
"old": [
{
"value": 0,
"when": 1502795602848
}
]
},
"sizes": [
{
"name": "P",
"available": True
},
{
"name": "M",
"available": True
}
],
"created": "2017-08-15T08:13:22.848284"
}
}]
bulk(self.es, doc, index="products")
My ES mapping:
{
"test_products": {
"mappings": {
"products": {
"properties": {
"approved": {
"type": "boolean"
},
"available": {
"type": "boolean"
},
"brand": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"buyClicks": {
"type": "integer"
},
"category": {
"type": "keyword"
},
"code": {
"type": "keyword"
},
"color": {
"type": "nested",
"properties": {
"name": {
"type": "keyword"
},
"value": {
"type": "keyword"
}
}
},
"created": {
"type": "date"
},
"description": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"gender": {
"type": "keyword"
},
"images": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"likes": {
"type": "integer"
},
"link": {
"type": "keyword"
},
"name": {
"type": "text",
"term_vector": "yes",
"analyzer": "nGram_analyzer",
"search_analyzer": "whitespace_analyzer"
},
"prices": {
"type": "nested",
"properties": {
"latest": {
"type": "nested",
"properties": {
"value": {
"type": "long"
},
"when": {
"type": "date",
"format": "dd-MM-yyyy||epoch_millis"
}
}
},
"old": {
"type": "nested",
"properties": {
"value": {
"type": "long"
},
"when": {
"type": "date",
"format": "dd-MM-yyyy||epoch_millis"
}
}
}
}
},
"redirectClicks": {
"type": "integer"
},
"sizes": {
"type": "nested",
"properties": {
"available": {
"type": "boolean"
},
"name": {
"type": "keyword"
},
"quantity": {
"type": "integer"
}
}
},
"slug": {
"type": "keyword"
},
"store": {
"type": "keyword"
},
"subCategories": {
"type": "nested",
"properties": {
"name": {
"type": "keyword"
},
"value": {
"type": "keyword"
}
}
},
"tags": {
"type": "text",
"fields": {
"raw": {
"type": "text",
"term_vector": "yes",
"analyzer": "nGram_analyzer",
"search_analyzer": "whitespace_analyzer"
}
}
},
"thumbnails": {
"type": "keyword"
}
}
}
}
}
}