We Keep Coding

json.decoder.JSONDecodeError - while converting JSON to CSV output

While trying to convert a JSON output below to CSV, getting error
Here is the JSON output
{
"data": [
{
"id": "-1000100591151294842",
"type": "fres",
"attributes": {
"operationState": "In Service",
"deploymentState": "discovered",
"displayData": {
"operationState": "Up",
"adminState": "Enabled",
"displayTopologySource": "Protocol,Derived",
"displayPhotonicSpectrumData": [
{
"frequency": "194.950000",
"wavelength": "1537.79",
"channel": "CH-20"
}
],
"displayDeploymentState": "Discovered",
"displayName": "J-BBEG-CHLC-P109"
},
"utilizationData": {
"totalCapacity": "100.0",
"usedCapacity": "100.0",
"utilizationPercent": "100",
"capacityUnits": "Gbps"
},
"resourceState": "discovered",
"serviceClass": "OTU",
"linkLabel": "BBEG-ROADM-0101:5-4-1,CHLC-ROADM-0401:7-35-1",
"lastUpdatedAdminStateTimeStamp": "2021-05-03T00:29:24.444Z",
"lastUpdatedOperationalStateTimeStamp": "2022-12-08T22:42:21.567Z",
"userLabel": "J-BBEG-CHLC-P109",
"mgmtName": "",
"nativeName": "",
"awarenessTime": "2022-12-08T22:42:22.123Z",
"layerRate": "OTU4",
"layerRateQualifier": "OTU4",
"supportedByLayerRatePackageList": [
{
"layerRate": "OTSi",
"layerRateQualifier": "100G"
}
],
"networkRole": "FREAP",
"directionality": "bidirectional",
"topologySources": [
"adjacency",
"stitched"
],
"adminState": "In Service",
"photonicSpectrumPackageList": [
{
"frequency": "194.950000",
"width": "37.5"
}
],
"active": true,
"additionalAttributes": {
"isActual": "true",
"hasLowerTopology": "true"
},
"reliability": "auto",
"resilienceLevel": "unprotected"
},
"relationships": {
"freDiscovered": {
"data": {
"type": "freDiscovered",
"id": "-1000100591151294842"
}
},
"supportedByServices": {
"data": [
{
"type": "fres",
"id": "6765278351459212874"
}
]
},
"endPoints": {
"data": [
{
"type": "endPoints",
"id": "-1000100591151294842:1"
},
{
"type": "endPoints",
"id": "-1000100591151294842:2"
}
]
},
"partitionFres": {
"data": [
{
"type": "fres",
"id": "7147507956181395827"
}
]
}
}
},
{
"id": "-1013895107051577774",
"type": "fres",
"attributes": {
"operationState": "In Service",
"deploymentState": "discovered",
"displayData": {
"operationState": "Up",
"adminState": "Enabled",
"displayTopologySource": "Protocol,Derived",
"displayPhotonicSpectrumData": [
{
"frequency": "191.600000",
"wavelength": "1564.68",
"channel": "CH-87"
}
],
"displayDeploymentState": "Discovered",
"displayName": "J-KFF9-PNTH-P101"
},
"utilizationData": {
"totalCapacity": "100.0",
"usedCapacity": "90.0",
"utilizationPercent": "90",
"capacityUnits": "Gbps"
},
"resourceState": "discovered",
"serviceClass": "OTU",
"tags": [
"J-KFF9-PNTH-P101"
],
"linkLabel": "KFF9-ROADM-0301:1-1-1,PNTH-ROADM-0101:1-1-1",
"lastUpdatedAdminStateTimeStamp": "2021-09-12T20:22:59.334Z",
"lastUpdatedOperationalStateTimeStamp": "2022-10-12T14:20:44.779Z",
"userLabel": "J-KFF9-PNTH-P101",
"mgmtName": "",
"nativeName": "",
"awarenessTime": "2022-10-12T14:20:45.417Z",
"layerRate": "OTU4",
"layerRateQualifier": "OTU4",
"supportedByLayerRatePackageList": [
{
"layerRate": "OTSi",
"layerRateQualifier": "100G"
}
],
"networkRole": "FREAP",
"directionality": "bidirectional",
"topologySources": [
"adjacency",
"stitched"
],
"adminState": "In Service",
"photonicSpectrumPackageList": [
{
"frequency": "191.600000",
"width": "37.5"
}
],
"active": true,
"additionalAttributes": {
"isActual": "true",
"hasLowerTopology": "true"
},
"reliability": "auto",
"resilienceLevel": "unprotected"
},
"relationships": {
"freDiscovered": {
"data": {
"type": "freDiscovered",
"id": "-1013895107051577774"
}
},
"supportedByServices": {
"data": [
{
"type": "fres",
"id": "6055685088078365419"
}
]
},
"endPoints": {
"data": [
{
"type": "endPoints",
"id": "-1013895107051577774:1"
},
{
"type": "endPoints",
"id": "-1013895107051577774:2"
}
]
},
"partitionFres": {
"data": [
{
"type": "fres",
"id": "-6727082893715936342"
}
]
}
}
}
] }
getting below error, not sure what is missing
Here is the python script I used. have been trying different variations but no luck getting different errors in all other instances
filename = Path('fre.json')
data = []
with open(filename,'r') as json_file:
data_str = json_file.read()
data_str = data_str.split('[',1)[-1]
data_str = data_str.rsplit(']',1)[0]
data_str = data_str.split('][')
for jsonStr in data_str:
jsonStr = '[' + jsonStr + ']'
temp_data = json.loads(jsonStr)
for each in temp_data:
data.append(each)
what is wrong?

Python Cubes OLAP Framework - How to sum a json column?

I started using Python Cubes Olap recently.
I'm trying to sum/avg a JSON postgres column, how can i do this?
my db structure:
events
id
object_type
sn_name
spectra
id
snx_wavelengths (json column)
event_id
my json:
{
"dimensions": [
{
"name": "event",
"levels": [
{
"name": "object_type",
"label": "Object Type",
"attributes": [
"object_type"
]
},
{
"name": "sn_name",
"label": "name",
"attributes": [
"sn_name"
]
}
]
},
{
"name": "spectra",
"levels": [
{
"name": "catalog_name",
"label": "Catalog Name",
"attributes": [
"catalog_name"
]
},
{
"name": "capture_date",
"label": "Capture Date",
"attributes": [
"capture_date"
]
}
]
},
{
"name": "date"
}
],
"cubes": [
{
"id": "uid",
"name": "14G31Yx98ZG8aEhFHjOWNNBmFOETg5APjZo5AiHaqog5YxLMK5",
"dimensions": [
"event",
"spectra",
"date"
],
"aggregates": [
{
"name": "event_snx_wavelengths_sum",
"function": "sum",
"measure": "event.snx_wavelengths"
},
{
"name": "record_count",
"function": "count"
}
],
"joins": [
{
"master": "14G31Yx98ZG8aEhFHjOWNNBmFOETg5APjZo5AiHaqog5YxLMK5.id",
"detail": "spectra.event_id"
},
],
"mappings": {
"event.sn_name": "sn_name",
"event.object_type": "object_type",
"spectra.catalog_name": "spectra.catalog_name",
"spectra.capture_date": "spectra.capture_date",
"event.snx_wavelengths": "spectra.snx_wavelengths",
"date": "spectra.capture_date"
},
}
]
}
I'm getting the follow error:
Unknown attribute ''event.snx_wavelengths''
Anyone can help?
I already tried use mongodb to do the sum, i didnt had success.

How to combine two JSON objects using jq

I have two files:
kube-apiserver.json
{
"apiVersion": "v1",
"kind": "Pod",
"metadata": {
[...]
},
"spec": {
"containers": [
{
"command": [
"kube-apiserver",
"--advertise-address=192.168.49.2",
"--allow-privileged=true",
"--authorization-mode=Node,RBAC",
"--client-ca-file=/var/lib/minikube/certs/ca.crt",
"--enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota",
"--enable-bootstrap-token-auth=true",
"--etcd-cafile=/var/lib/minikube/certs/etcd/ca.crt",
"--etcd-certfile=/var/lib/minikube/certs/apiserver-etcd-client.crt",
"--etcd-keyfile=/var/lib/minikube/certs/apiserver-etcd-client.key",
"--etcd-servers=https://127.0.0.1:2379",
"--kubelet-client-certificate=/var/lib/minikube/certs/apiserver-kubelet-client.crt",
"--kubelet-client-key=/var/lib/minikube/certs/apiserver-kubelet-client.key",
"--kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname",
"--proxy-client-cert-file=/var/lib/minikube/certs/front-proxy-client.crt",
"--proxy-client-key-file=/var/lib/minikube/certs/front-proxy-client.key",
"--requestheader-allowed-names=front-proxy-client",
"--requestheader-client-ca-file=/var/lib/minikube/certs/front-proxy-ca.crt",
"--requestheader-extra-headers-prefix=X-Remote-Extra-",
"--requestheader-group-headers=X-Remote-Group",
"--requestheader-username-headers=X-Remote-User",
"--secure-port=8443",
"--service-account-issuer=https://kubernetes.default.svc.cluster.local",
"--service-account-key-file=/var/lib/minikube/certs/sa.pub",
"--service-account-signing-key-file=/var/lib/minikube/certs/sa.key",
"--service-cluster-ip-range=10.96.0.0/12",
"--tls-cert-file=/var/lib/minikube/certs/apiserver.crt",
"--tls-private-key-file=/var/lib/minikube/certs/apiserver.key"
],
[...]
"volumeMounts": [
{
"mountPath": "/etc/ssl/certs",
"name": "ca-certs",
"readOnly": true
},
{
"mountPath": "/etc/ca-certificates",
"name": "etc-ca-certificates",
"readOnly": true
},
{
"mountPath": "/var/lib/minikube/certs",
"name": "k8s-certs",
"readOnly": true
},
{
"mountPath": "/usr/local/share/ca-certificates",
"name": "usr-local-share-ca-certificates",
"readOnly": true
},
{
"mountPath": "/usr/share/ca-certificates",
"name": "usr-share-ca-certificates",
"readOnly": true
}
]
}
],
[...]
"volumes": [
{
"hostPath": {
"path": "/etc/ssl/certs",
"type": "DirectoryOrCreate"
},
"name": "ca-certs"
},
{
"hostPath": {
"path": "/etc/ca-certificates",
"type": "DirectoryOrCreate"
},
"name": "etc-ca-certificates"
},
{
"hostPath": {
"path": "/var/lib/minikube/certs",
"type": "DirectoryOrCreate"
},
"name": "k8s-certs"
},
{
"hostPath": {
"path": "/usr/local/share/ca-certificates",
"type": "DirectoryOrCreate"
},
"name": "usr-local-share-ca-certificates"
},
{
"hostPath": {
"path": "/usr/share/ca-certificates",
"type": "DirectoryOrCreate"
},
"name": "usr-share-ca-certificates"
}
]
},
"status": {
[...]
}
}
and patch.json
{
"apiVersion": "v1",
"kind": "Pod",
"metadata": {
},
"spec": {
"containers": [
{
"command": [
"--audit-policy-file=/etc/kubernetes/audit-policy.yaml",
"--audit-log-path=/var/log/kubernetes/audit/audit.log"
],
"volumeMounts": [
{
"mountPath": "/etc/kubernetes/audit-policy.yaml",
"name": "audit",
"readOnly": true
},
{
"mountPath": "/var/log/kubernetes/audit/",
"name": "audit-log",
"readOnly": true
}
]
}
],
"volumes": [
{
"hostPath": {
"path": "/etc/kubernetes/audit-policy.yaml",
"type": "FileOrCreate"
},
"name": "audit"
},
{
"hostPath": {
"path": "/var/log/kubernetes/audit/",
"type": "DirectoryOrCreate"
},
"name": "audit-log"
}
]
},
"status": {
}
}
When i try to do jq -s '.[0] * .[1]' kube-apiserver.json patch.json > patched-apiserver.json items that are in patch.json overrides items from kube-apiserver.json so it looks like this:
{
"apiVersion": "v1",
"kind": "Pod",
"metadata": {
[...]
"spec": {
"containers": [
{
"command": [
"--audit-policy-file=/etc/kubernetes/audit-policy.yaml",
"--audit-log-path=/var/log/kubernetes/audit/audit.log"
],
"volumeMounts": [
{
"mountPath": "/etc/kubernetes/audit-policy.yaml",
"name": "audit",
"readOnly": true
},
{
"mountPath": "/var/log/kubernetes/audit/",
"name": "audit-log",
"readOnly": true
}
]
}
],
[..].
"volumes": [
{
"hostPath": {
"path": "/etc/kubernetes/audit-policy.yaml",
"type": "FileOrCreate"
},
"name": "audit"
},
{
"hostPath": {
"path": "/var/log/kubernetes/audit/",
"type": "DirectoryOrCreate"
},
"name": "audit-log"
}
]
},
"status": {
[...]
}
}
and i would like my file to look like this:
{
"apiVersion": "v1",
"kind": "Pod",
"metadata": {
[...]
},
"spec": {
"containers": [
{
"command": [
"kube-apiserver",
"--advertise-address=192.168.49.2",
"--allow-privileged=true",
"--authorization-mode=Node,RBAC",
"--client-ca-file=/var/lib/minikube/certs/ca.crt",
"--enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota",
"--enable-bootstrap-token-auth=true",
"--etcd-cafile=/var/lib/minikube/certs/etcd/ca.crt",
"--etcd-certfile=/var/lib/minikube/certs/apiserver-etcd-client.crt",
"--etcd-keyfile=/var/lib/minikube/certs/apiserver-etcd-client.key",
"--etcd-servers=https://127.0.0.1:2379",
"--kubelet-client-certificate=/var/lib/minikube/certs/apiserver-kubelet-client.crt",
"--kubelet-client-key=/var/lib/minikube/certs/apiserver-kubelet-client.key",
"--kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname",
"--proxy-client-cert-file=/var/lib/minikube/certs/front-proxy-client.crt",
"--proxy-client-key-file=/var/lib/minikube/certs/front-proxy-client.key",
"--requestheader-allowed-names=front-proxy-client",
"--requestheader-client-ca-file=/var/lib/minikube/certs/front-proxy-ca.crt",
"--requestheader-extra-headers-prefix=X-Remote-Extra-",
"--requestheader-group-headers=X-Remote-Group",
"--requestheader-username-headers=X-Remote-User",
"--secure-port=8443",
"--service-account-issuer=https://kubernetes.default.svc.cluster.local",
"--service-account-key-file=/var/lib/minikube/certs/sa.pub",
"--service-account-signing-key-file=/var/lib/minikube/certs/sa.key",
"--service-cluster-ip-range=10.96.0.0/12",
"--tls-cert-file=/var/lib/minikube/certs/apiserver.crt",
"--tls-private-key-file=/var/lib/minikube/certs/apiserver.key",
"--audit-policy-file=/etc/kubernetes/audit-policy.yaml",
"--audit-log-path=/var/log/kubernetes/audit/audit.log"
],
[...]
"volumeMounts": [
{
"mountPath": "/etc/ssl/certs",
"name": "ca-certs",
"readOnly": true
},
{
"mountPath": "/etc/ca-certificates",
"name": "etc-ca-certificates",
"readOnly": true
},
{
"mountPath": "/var/lib/minikube/certs",
"name": "k8s-certs",
"readOnly": true
},
{
"mountPath": "/usr/local/share/ca-certificates",
"name": "usr-local-share-ca-certificates",
"readOnly": true
},
{
"mountPath": "/usr/share/ca-certificates",
"name": "usr-share-ca-certificates",
"readOnly": true
},
{
"mountPath": "/etc/kubernetes/audit-policy.yaml",
"name": "audit",
"readOnly": true
},
{
"mountPath": "/var/log/kubernetes/audit/",
"name": "audit-log",
"readOnly": true
}
]
}
],
[...]
"volumes": [
{
"hostPath": {
"path": "/etc/ssl/certs",
"type": "DirectoryOrCreate"
},
"name": "ca-certs"
},
{
"hostPath": {
"path": "/etc/ca-certificates",
"type": "DirectoryOrCreate"
},
"name": "etc-ca-certificates"
},
{
"hostPath": {
"path": "/var/lib/minikube/certs",
"type": "DirectoryOrCreate"
},
"name": "k8s-certs"
},
{
"hostPath": {
"path": "/usr/local/share/ca-certificates",
"type": "DirectoryOrCreate"
},
"name": "usr-local-share-ca-certificates"
},
{
"hostPath": {
"path": "/usr/share/ca-certificates",
"type": "DirectoryOrCreate"
},
"name": "usr-share-ca-certificates"
},
{
"hostPath": {
"path": "/etc/kubernetes/audit-policy.yaml",
"type": "FileOrCreate"
},
"name": "audit"
},
{
"hostPath": {
"path": "/var/log/kubernetes/audit/",
"type": "DirectoryOrCreate"
},
"name": "audit-log"
}
]
},
"status": {
[...]
}
}
Does anyone know how to solve it with jq/python/bash/whatever?

jq --slurpfile patch patch.json '
(.spec.containers |= map(.command |= (. + $patch[].spec.containers[0].command | unique) |
.volumeMounts |= (. + $patch[].spec.containers[0].volumeMounts | unique))) |
(.spec.volumes |= (. + $patch[].spec.volumes | unique))
' kube-apiserver.json
I have added unique to ensure that commands, volumeMounts and volumes only appear once. This has the side effect of sorting the arrays.
You can remove unique if you do not want it.
There is a problem in your question: spec.containers is an array. In your example, this array contains only one element and my code adds the first element from the patch to each container.
Or would you like the patch to be merged based on position? In that case this solution does not work.

How to iterate through nested Dictionary in python boto3

Hi want to find only stopped instance and their ids below is the below-nested JSON
{
"Reservations": [
{
"Groups": [],
"Instances": [
{
"AmiLaunchIndex": 0,
"Architecture": "x86_64",
"BlockDeviceMappings": [
{
"DeviceName": "/dev/xvda",
"Ebs": {
"AttachTime": "2021-03-11 09:43:59+00:00",
"DeleteOnTermination": true,
"Status": "attached",
"VolumeId": "vol-0c49ff5c979904337"
}
}
],
"CapacityReservationSpecification": {
"CapacityReservationPreference": "open"
},
"ClientToken": "",
"CpuOptions": {
"CoreCount": 1,
"ThreadsPerCore": 1
},
"EbsOptimized": false,
"EnaSupport": true,
"EnclaveOptions": {
"Enabled": false
},
"HibernationOptions": {
"Configured": false
},
"Hypervisor": "xen",
"ImageId": "ami-038f1ca1bd58a5790",
"InstanceId": "i-041fb789f1554b7d5",
"InstanceType": "t2.micro",
"KeyName": "HadoopLearning",
"LaunchTime": "2021-03-11 09:43:58+00:00",
"MetadataOptions": {
"HttpEndpoint": "enabled",
"HttpPutResponseHopLimit": 1,
"HttpTokens": "optional",
"State": "applied"
},
"Monitoring": {
"State": "disabled"
},
"NetworkInterfaces": [
{
"Attachment": {
"AttachTime": "2021-03-11 09:43:58+00:00",
"AttachmentId": "eni-attach-022ac34489909cc78",
"DeleteOnTermination": true,
"DeviceIndex": 0,
"NetworkCardIndex": 0,
"Status": "attached"
},
"Description": "",
"Groups": [
{
"GroupId": "sg-5878120b",
"GroupName": "default"
}
],
"InterfaceType": "interface",
"Ipv6Addresses": [],
"MacAddress": "0a:15:95:72:23:6b",
"NetworkInterfaceId": "eni-0d356de72ae7e6bfc",
"OwnerId": "216808211954",
"PrivateDnsName": "ip-172-31-29-51.ec2.internal",
"PrivateIpAddress": "172.31.29.51",
"PrivateIpAddresses": [
{
"Primary": true,
"PrivateDnsName": "ip-172-31-29-51.ec2.internal",
"PrivateIpAddress": "172.31.29.51"
}
],
"SourceDestCheck": true,
"Status": "in-use",
"SubnetId": "subnet-e705ccaa",
"VpcId": "vpc-4401543e"
}
],
"Placement": {
"AvailabilityZone": "us-east-1d",
"GroupName": "",
"Tenancy": "default"
},
"PrivateDnsName": "ip-172-31-29-51.ec2.internal",
"PrivateIpAddress": "172.31.29.51",
"ProductCodes": [],
"PublicDnsName": "",
"RootDeviceName": "/dev/xvda",
"RootDeviceType": "ebs",
"SecurityGroups": [
{
"GroupId": "sg-5878120b",
"GroupName": "default"
}
],
"SourceDestCheck": true,
"State": {
"Code": 80,
"Name": "stopped" ####this one only which is stopped ###
},
"StateReason": {
"Code": "Client.UserInitiatedShutdown",
"Message": "Client.UserInitiatedShutdown: User initiated shutdown"
},
"StateTransitionReason": "User initiated (2021-03-11 10:13:00 GMT)",
"SubnetId": "subnet-e705ccaa",
"Tags": [
{
"Key": "Name",
"Value": "aviral"
}
],
"VirtualizationType": "hvm",
"VpcId": "vpc-4401543e"
},
{
"AmiLaunchIndex": 1,
"Architecture": "x86_64",
"BlockDeviceMappings": [
{
"DeviceName": "/dev/xvda",
"Ebs": {
"AttachTime": "2021-03-11 09:43:59+00:00",
"DeleteOnTermination": true,
"Status": "attached",
"VolumeId": "vol-069a9ca7f860fefd8"
}
}
],
"CapacityReservationSpecification": {
"CapacityReservationPreference": "open"
},
"ClientToken": "",
"CpuOptions": {
"CoreCount": 1,
"ThreadsPerCore": 1
},
"EbsOptimized": false,
"EnaSupport": true,
"EnclaveOptions": {
"Enabled": false
},
"HibernationOptions": {
"Configured": false
},
"Hypervisor": "xen",
"ImageId": "ami-038f1ca1bd58a5790",
"InstanceId": "i-0d0c876682eef71ae",
"InstanceType": "t2.micro",
"KeyName": "HadoopLearning",
"LaunchTime": "2021-03-11 09:43:58+00:00",
"MetadataOptions": {
"HttpEndpoint": "enabled",
"HttpPutResponseHopLimit": 1,
"HttpTokens": "optional",
"State": "applied"
},
"Monitoring": {
"State": "disabled"
},
"NetworkInterfaces": [
{
"Attachment": {
"AttachTime": "2021-03-11 09:43:58+00:00",
"AttachmentId": "eni-attach-0b0eba3fb5b3c3205",
"DeleteOnTermination": true,
"DeviceIndex": 0,
"NetworkCardIndex": 0,
"Status": "attached"
},
"Description": "",
"Groups": [
{
"GroupId": "sg-5878120b",
"GroupName": "default"
}
],
"InterfaceType": "interface",
"Ipv6Addresses": [],
"MacAddress": "0a:3d:3e:bc:96:e3",
"NetworkInterfaceId": "eni-0688951c8bd226676",
"OwnerId": "216808211954",
"PrivateDnsName": "ip-172-31-19-102.ec2.internal",
"PrivateIpAddress": "172.31.19.102",
"PrivateIpAddresses": [
{
"Primary": true,
"PrivateDnsName": "ip-172-31-19-102.ec2.internal",
"PrivateIpAddress": "172.31.19.102"
}
],
"SourceDestCheck": true,
"Status": "in-use",
"SubnetId": "subnet-e705ccaa",
"VpcId": "vpc-4401543e"
}
],
"Placement": {
"AvailabilityZone": "us-east-1d",
"GroupName": "",
"Tenancy": "default"
},
"PrivateDnsName": "ip-172-31-19-102.ec2.internal",
"PrivateIpAddress": "172.31.19.102",
"ProductCodes": [],
"PublicDnsName": "",
"RootDeviceName": "/dev/xvda",
"RootDeviceType": "ebs",
"SecurityGroups": [
{
"GroupId": "sg-5878120b",
"GroupName": "default"
}
],
"SourceDestCheck": true,
"State": {
"Code": 80,
"Name": "running" ###Not this one #####
},
"StateReason": {
"Code": "Client.UserInitiatedShutdown",
"Message": "Client.UserInitiatedShutdown: User initiated shutdown"
},
"StateTransitionReason": "User initiated (2021-03-11 10:13:00 GMT)",
"SubnetId": "subnet-e705ccaa",
"Tags": [
{
"Key": "Name",
"Value": "avinash"
}
],
"VirtualizationType": "hvm",
"VpcId": "vpc-4401543e"
}
],
"OwnerId": "216808211954",
"ReservationId": "r-0d646190512fec9f8"
}
],
"ResponseMetadata": {
"HTTPHeaders": {
"cache-control": "no-cache, no-store",
"content-type": "text/xml;charset=UTF-8",
"date": "Thu, 11 Mar 2021 10:14:39 GMT",
"server": "AmazonEC2",
"strict-transport-security": "max-age=31536000; includeSubDomains",
"transfer-encoding": "chunked",
"vary": "accept-encoding",
"x-amzn-requestid": "58065d1d-76fc-4f7a-9229-18584ee1031e"
},
"HTTPStatusCode": 200,
"RequestId": "58065d1d-76fc-4f7a-9229-18584ee1031e",
"RetryAttempts": 0
}
}
I only want an instance of those who are stopped and then I want to start them using their instance ID using boto3
I am using the below code
import boto3
import json
access_key = "XXXXXXXXXX"
secret_key = "XXXXXXXXXXXXXXXXXXXXX"
client = boto3.client('ec2', aws_access_key_id=access_key, aws_secret_access_key=secret_key, region_name='us-east-1')
ec2_result = client.describe_instances(
Filters=[
{
'Name': 'tag:Name',
'Values': ['?avi*']
}
]
)
#ids= [ec2_result['Reservations'][0]['Instances'][i]['InstanceId'] for i in range(len(ec2_result['Reservations'][0]['Instances']))]
#print(ids)
print(ec2_result)
using the above code I am getting first block JSON but I only want an instance which is stopped, as I have 4000 instances in my AWS

I think the following should work:
instances_stopped = []
for reservation in ec2_result['Reservations']:
for instance in reservation['Instances']:
instance_id = instance['InstanceId']
if 'State' not in instance:
print(f"Missing state for {instance_id}")
else:
state = instance['State']['Name']
if state == 'stopped':
instances_stopped.append(instance_id)
#print(instance_id, state)
print(instances_stopped)

singer.io incremental load is not working

Working with singer.io trying to get data from mysql using xampp on Ubuntu and able to fetch historical data but then i tried to get the incremental load the code is not working.
Not able to get the replication_key_value if anyone can help i will be really thankful to you.
config.json:
{
"host": "localhost",
"port": "3307",
"user": "root",
"password": ""
}
properties.json:
{
"streams": [
{
"tap_stream_id": "test_1-company",
"table_name": "company",
"schema": {
"properties": {
"COMPANY_ID": {
"inclusion": "automatic",
"maxLength": 6,
"type": [
"null",
"string"
]
},
"COMPANY_NAME": {
"inclusion": "available",
"maxLength": 25,
"type": [
"null",
"string"
]
},
"COMPANY_CITY": {
"inclusion": "available",
"maxLength": 25,
"type": [
"null",
"string"
]
}
},
"type": "object"
},
"stream": "company",
"metadata": [
{
"breadcrumb": [],
"metadata": {
"selected-by-default": false,
"database-name": "test_1",
"row-count": 7,
"is-view": false,
"table-key-properties": [
"COMPANY_ID"
],
"replication-method": "INCREMENTAL",
"replication-key": "COMPANY_ID"
}
},
{
"breadcrumb": [
"properties",
"COMPANY_ID"
],
"metadata": {
"selected-by-default": true,
"sql-datatype": "varchar(6)"
}
},
{
"breadcrumb": [
"properties",
"COMPANY_NAME"
],
"metadata": {
"selected-by-default": true,
"sql-datatype": "varchar(25)"
}
},
{
"breadcrumb": [
"properties",
"COMPANY_CITY"
],
"metadata": {
"selected-by-default": true,
"sql-datatype": "varchar(25)"
}
}
]
}
]
}
Console log:
$ tap-mysql -c config.json --properties properties.json
INFO Server Parameters: version: 10.4.14-MariaDB, wait_timeout: 2700, innodb_lock_wait_timeout: 2700, max_allowed_packet: 1048576, interactive_timeout: 28800
INFO Server SSL Parameters (blank means SSL is not active): [ssl_version: ], [ssl_cipher: ]
{"type": "STATE", "value": {"currently_syncing": null}}

Can you share your executable?
Your problem may be in the state.json file.
State.json needs the form:
{
"bookmarks":{
"123po-employee":{
"replication_key":"Id",
"replication_key_value":45,
"version":1609927544221
}
},
"currently_syncing":null
}