We Keep Coding

python iterate yaml and filter result

I have this yaml file
data:
- name: acme_aws1
source: aws
path: acme/acme_aws1.zip
- name: acme_gke1
source: gke
path: acme/acme_gke1.zip
- name: acme_oci
source: oci
path: acme/acme_oci1.zip
- name: acme_aws2
source: aws
path: acme/acme_aws2.zip
- name: acme_gke2
source: gke
path: acme/acme_gke2.zip
- name: acme_oci2
source: oci
path: acme/acme_oci2.zip
i want to filter out the data containing "source=gke" and for loop assign the value of path to variable., can any one please share how-to when using python with pyyaml as import module.

This code would do what you need, it just reads, and uses filter standard function to return an iterable with the elements passing a condition. Then such elements are put into a new list
import yaml
# for files you can use
# with open("data.yaml", "r") as file:
# yaml_data = yaml.safe_load(file)
yaml_data = yaml.safe_load("""
data:
- name: acme_aws1
source: aws
path: acme/acme_aws1.zip
- name: acme_gke1
source: gke
path: acme/acme_gke1.zip
- name: acme_oci
source: oci
path: acme/acme_oci1.zip
- name: acme_aws2
source: aws
path: acme/acme_aws2.zip
- name: acme_gke2
source: gke
path: acme/acme_gke2.zip
- name: acme_oci2
source: oci
path: acme/acme_oci2.zip
""")
data = yaml_data['data']
filtered = list(filter(lambda x: x.get('source') == 'gke', data))
print(filtered)
It prints
[{'name': 'acme_gke1', 'source': 'gke', 'path': 'acme/acme_gke1.zip'}, {'name': 'acme_gke2', 'source': 'gke', 'path': 'acme/acme_gke2.zip'}]

import yaml
# Read the file.
content = yaml.safe_load('your_file.yaml')
# Get rid of 'gke' elements.
not_gke_sources = [block for block in content if block.source != 'gke']
# Iterate over to access all 'path's.
for block in not_gke_sources:
path = block.path
# Some actions.

change a specific value in yaml file with lot of indentation using python

kind: Deployment
apiVersion: apps/v1
metadata:
name: websitemanager
namespace: white
selfLink: /apis/apps/v1/namespaces/white/deployments/websitemanager
generation: 89
labels:
app: websitemanager
app.kubernetes.io/instance: websitemanager
backup: kube-noah
annotations:
deployment.kubernetes.io/revision: '75'
kubectl.kubernetes.io/last-applied-configuration: >
{"apiVersion":"apps/v1","kind":"Deployment","metadata":{"annotations":{"deployment.kubernetes.io/revision":"75"},"labels":{"app":"websitemanager","app.kubernetes.io/instance":"websitemanager","backup":"kube-noah"},"name":"websitemanager","namespace":"white","selfLink":"/apis/apps/v1/namespaces/white/deployments/websitemanager"},"spec":{"progressDeadlineSeconds":600,"replicas":4,"revisionHistoryLimit":10,"selector":{"matchLabels":{"app":"websitemanager"}},"strategy":{"rollingUpdate":{"maxSurge":"25%","maxUnavailable":"25%"},"type":"RollingUpdate"},"template":{"metadata":{"annotations":{"co.elastic.logs.json-logging/json.add_error_key":"true","co.elastic.logs.json-logging/json.keys_under_root":"true","co.elastic.logs.json-logging/json.message_key":"message_key","co.elastic.logs/enabled":"true"},"creationTimestamp":null,"labels":{"app":"websitemanager"}},"spec":{"containers":[{"env":[{"name":"APP","value":"websitemanager"},{"name":"PORT","value":"80"},{"name":"ALLOW_SKIPS","valueFrom":{"secretKeyRef":{"key":"ALLOW_SKIPS","name":"env-vars"}}},{"name":"AWS_ACCESS_KEY_ID","valueFrom":{"secretKeyRef":{"key":"AWS_ACCESS_KEY_ID","name":"env-vars"}}},{"name":"AWS_REGION","valueFrom":{"secretKeyRef":{"key":"AWS_REGION","name":"env-vars"}}},{"name":"AWS_SECRET_ACCESS_KEY","valueFrom":{"secretKeyRef":{"key":"AWS_SECRET_ACCESS_KEY","name":"env-vars"}}},{"name":"BO_RENDERER_URL","valueFrom":{"secretKeyRef":{"key":"BO_RENDERER_URL","name":"env-vars"}}},{"name":"BO_S3_URL","valueFrom":{"secretKeyRef":{"key":"BO_S3_URL","name":"env-vars"}}},{"name":"CHAT_V3","valueFrom":{"secretKeyRef":{"key":"CHAT_V3","name":"env-vars"}}},{"name":"CMS_RENDERER_URL","valueFrom":{"secretKeyRef":{"key":"CMS_RENDERER_URL","name":"env-vars"}}},{"name":"CRON_EXPRESSION","valueFrom":{"secretKeyRef":{"key":"CRON_EXPRESSION","name":"env-vars"}}},{"name":"CRON_GROUP_SCHEDULE_HOURS","valueFrom":{"secretKeyRef":{"key":"CRON_GROUP_SCHEDULE_HOURS","name":"env-vars"}}},{"name":"CRON_SCHEDULE_HOURS","valueFrom":{"secretKeyRef":{"key":"CRON_SCHEDULE_HOURS","name":"env-vars"}}},{"name":"DYNAMIC_PAGE_URL","valueFrom":{"secretKeyRef":{"key":"DYNAMIC_PAGE_URL","name":"env-vars"}}},{"name":"ENVIRONMENT_BUCKET_ENDPOINT","valueFrom":{"secretKeyRef":{"key":"ENVIRONMENT_BUCKET_ENDPOINT","name":"env-vars"}}},{"name":"ENVIRONMENT_BUCKET_NAME","valueFrom":{"secretKeyRef":{"key":"ENVIRONMENT_BUCKET_NAME","name":"env-vars"}}},{"name":"FIREBASE_KEY","valueFrom":{"secretKeyRef":{"key":"FIREBASE_KEY","name":"env-vars"}}},{"name":"FO_RENDERER_URL","valueFrom":{"secretKeyRef":{"key":"FO_RENDERER_URL","name":"env-vars"}}},{"name":"FO_S3_URL","valueFrom":{"secretKeyRef":{"key":"FO_S3_URL","name":"env-vars"}}},{"name":"GEFEN_ENV","valueFrom":{"secretKeyRef":{"key":"GEFEN_ENV","name":"env-vars"}}},{"name":"LEADS_SQS_DL_QUEUE","valueFrom":{"secretKeyRef":{"key":"LEADS_SQS_DL_QUEUE","name":"env-vars"}}},{"name":"LEADS_SQS_QUEUE","valueFrom":{"secretKeyRef":{"key":"LEADS_SQS_QUEUE","name":"env-vars"}}},{"name":"LOGGLY_INPUT_TOKEN","valueFrom":{"secretKeyRef":{"key":"LOGGLY_INPUT_TOKEN","name":"env-vars"}}},{"name":"LOGGLY_SUBDOMAIN","valueFrom":{"secretKeyRef":{"key":"LOGGLY_SUBDOMAIN","name":"env-vars"}}},{"name":"LOKALISE_API_KEY","valueFrom":{"secretKeyRef":{"key":"LOKALISE_API_KEY","name":"env-vars"}}},{"name":"LOKALISE_PROJECT_ID","valueFrom":{"secretKeyRef":{"key":"LOKALISE_PROJECT_ID","name":"env-vars"}}},{"name":"LP_RENDERER_URL","valueFrom":{"secretKeyRef":{"key":"LP_RENDERER_URL","name":"env-vars"}}},{"name":"MONGO_URL","valueFrom":{"secretKeyRef":{"key":"MONGO_URL","name":"env-vars"}}},{"name":"MONGO_URLS","valueFrom":{"secretKeyRef":{"key":"MONGO_URLS","name":"env-vars"}}},{"name":"NEW_RELIC_LICENSE_KEY","valueFrom":{"secretKeyRef":{"key":"NEW_RELIC_LICENSE_KEY","name":"env-vars"}}},{"name":"NEXTGEN_S3_URL","valueFrom":{"secretKeyRef":{"key":"NEXTGEN_S3_URL","name":"env-vars"}}},{"name":"NODE_ENV","valueFrom":{"secretKeyRef":{"key":"NODE_ENV","name":"env-vars"}}},{"name":"OPERATION_BUCKET_ENV_PART","valueFrom":{"secretKeyRef":{"key":"OPERATION_BUCKET_ENV_PART","name":"env-vars"}}},{"name":"PG_URL","valueFrom":{"secretKeyRef":{"key":"PG_URL","name":"env-vars"}}},{"name":"QUICKSIGHT_ACCESS_KEY_ID","valueFrom":{"secretKeyRef":{"key":"QUICKSIGHT_ACCESS_KEY_ID","name":"env-vars"}}},{"name":"QUICKSIGHT_ACCOUNT_ID","valueFrom":{"secretKeyRef":{"key":"QUICKSIGHT_ACCOUNT_ID","name":"env-vars"}}},{"name":"QUICKSIGHT_SECRET_KEY_ID","valueFrom":{"secretKeyRef":{"key":"QUICKSIGHT_SECRET_KEY_ID","name":"env-vars"}}},{"name":"REDIS_URL","valueFrom":{"secretKeyRef":{"key":"REDIS_URL","name":"env-vars"}}},{"name":"ROLLBAR_API_TOKEN","valueFrom":{"secretKeyRef":{"key":"ROLLBAR_API_TOKEN","name":"env-vars"}}},{"name":"RS_URL","valueFrom":{"secretKeyRef":{"key":"RS_URL","name":"env-vars"}}},{"name":"SES_PASS","valueFrom":{"secretKeyRef":{"key":"SES_PASS","name":"env-vars"}}},{"name":"SES_USER","valueFrom":{"secretKeyRef":{"key":"SES_USER","name":"env-vars"}}},{"name":"STELLAR_REDIS_URL","valueFrom":{"secretKeyRef":{"key":"STELLAR_REDIS_URL","name":"env-vars"}}},{"name":"USE_HTTP_SERVER","valueFrom":{"secretKeyRef":{"key":"USE_HTTP_SERVER","name":"env-vars"}}},{"name":"USE_LEADS_V3","valueFrom":{"secretKeyRef":{"key":"USE_LEADS_V3","name":"env-vars"}}},{"name":"androidBuildName","valueFrom":{"secretKeyRef":{"key":"androidBuildName","name":"env-vars"}}},{"name":"iosAppStoreId","valueFrom":{"secretKeyRef":{"key":"iosAppStoreId","name":"env-vars"}}},{"name":"iosBuildName","valueFrom":{"secretKeyRef":{"key":"iosBuildName","name":"env-vars"}}}],"image":"gefenonline/websitemanager:develop-111","imagePullPolicy":"IfNotPresent","livenessProbe":{"failureThreshold":3,"httpGet":{"path":"/health","port":80,"scheme":"HTTP"},"initialDelaySeconds":5,"periodSeconds":3,"successThreshold":1,"timeoutSeconds":1},"name":"websitemanager","resources":{"limits":{"memory":"700Mi"},"requests":{"cpu":"150m","memory":"400Mi"}},"terminationMessagePath":"/dev/termination-log","terminationMessagePolicy":"File"}],"dnsPolicy":"ClusterFirst","imagePullSecrets":[{"name":"docker-registry"}],"restartPolicy":"Always","schedulerName":"default-scheduler","securityContext":{},"terminationGracePeriodSeconds":30}}},"status":null}
spec:
replicas: 4
selector:
matchLabels:
app: websitemanager
template:
metadata:
creationTimestamp: null
labels:
app: websitemanager
annotations:
co.elastic.logs.json-logging/json.add_error_key: 'true'
co.elastic.logs.json-logging/json.keys_under_root: 'true'
co.elastic.logs.json-logging/json.message_key: message_key
co.elastic.logs/enabled: 'true'
spec:
containers:
- name: websitemanager
image: 'gefenonline/websitemanager:develop-111'
resources:
limits:
memory: 700Mi
requests:
cpu: 150m
memory: 400Mi
livenessProbe:
httpGet:
path: /health
port: 80
scheme: HTTP
initialDelaySeconds: 5
timeoutSeconds: 1
periodSeconds: 3
successThreshold: 1
failureThreshold: 3
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: File
imagePullPolicy: IfNotPresent
restartPolicy: Always
terminationGracePeriodSeconds: 30
dnsPolicy: ClusterFirst
securityContext: {}
imagePullSecrets:
- name: docker-registry
schedulerName: default-scheduler
strategy:
type: RollingUpdate
rollingUpdate:
maxUnavailable: 25%
maxSurge: 25%
revisionHistoryLimit: 10
progressDeadlineSeconds: 600
status:
observedGeneration: 89
replicas: 4
updatedReplicas: 4
readyReplicas: 4
availableReplicas: 4
conditions:
- type: Progressing
status: 'True'
lastUpdateTime: '2021-08-03T06:34:01Z'
lastTransitionTime: '2021-08-03T06:33:40Z'
reason: NewReplicaSetAvailable
message: ReplicaSet "websitemanager-7cdbcf6488" has successfully progressed.
- type: Available
status: 'True'
lastUpdateTime: '2021-08-04T13:57:07Z'
lastTransitionTime: '2021-08-04T13:57:07Z'
reason: MinimumReplicasAvailable
message: Deployment has minimum availability.
I have a yaml file and i want to make change in the image key in the file which is under spec-containers by a python script.so far i have got this script but not able to access the key-value-pair and i am quite new to this so can anybody help me on this.i am not able to change the key-value pair in the deployments.yaml file
import yaml
with open('deployment.yaml', 'w') as f:
content = yaml.load(f)
# for k,v in content.items():
# print(k['selector'])
print(content['spec']['template']['spec']['containers'])
#doc['spec']['spec']['template']['spec']['containers']['image'] = 'letsencrypt-prod'
# for k,v in (content['spec']['template']['spec']['containers']):
# print(v)
yaml.dump(content, f)
# with open("deployment.yaml", "w") as f:
# yaml.dump(content, f)

You were just missing that the level at key "containers" is a list, so the zeroth index must be used to get to the image key:
import yaml
with open('deployment.yaml', 'r') as fin:
content = yaml.load(fin, Loader=yaml.FullLoader)
content['spec']['template']['spec']['containers'][0]['image'] = "letsencrypt-prod"
with open('deployment2.yaml', 'w') as fout:
yaml.dump(content, fout)

yaml params edit with python3

I have a list of yaml files which I would like to edit in the same way. Put the following block under spec.template.imagePullSecret and spec.template.container.imagePullPolicy in each file:
imagePullSecrets:
- dockerpullsecret
container:
imagePullPolicy: IfNotPresent
This is an example of one file:
apiVersion: argoproj.io/v1alpha1
kind: Sensor
meta data:
name: sensor-finanda-ci
namespace: argo-events
spec:
template:
serviceAccountName: argo-events-sa
dependencies:
- name: eventsource-iv
eventSourceName: eventsource-iv
eventName: iv
This is my desired output:
apiVersion: argoproj.io/v1alpha1
kind: Sensor
meta data:
name: sensor-finanda-ci
namespace: argo-events
spec:
template:
serviceAccountName: argo-events-sa
imagePullSecrets:
- dockerpullsecret
container:
imagePullPolicy: IfNotPresent
dependencies:
- name: eventsource-iv
eventSourceName: eventsource-iv
eventName: iv

If your heart is set on a Python solution, something like this might do:
import os
import yaml
for yaml_filename in [filename for filename in os.listdir('.') if filename.endswith('.yaml')]:
with open(yaml_filename, 'r') as yaml_file:
yaml_obj = yaml.safe_load(yaml_file)
if 'spec' not in yaml_obj:
yaml_obj['spec'] = {"template": {}}
if 'template' not in yaml_obj['spec']:
yaml_obj['spec']['template'] = {"imagePullSecrets": [], "container": {}}
if 'container' not in yaml_obj['spec']['template']:
yaml_obj['spec']['template']['container'] = {}
yaml_obj['spec']['template']['imagePullSecrets'] = ["dockerpullsecret"]
yaml_obj['spec']['template']['container']['imagePullPolicy'] = 'IfNotPresent'
with open(yaml_filename, 'w') as yaml_file:
yaml.dump(yaml_obj, yaml_file)
If you're open to command-line utils, this is more concise:
Loop over the files and use yq to edit them in-place.
for file in *.yaml; do
yq -i e '.spec.template.imagePullSecrets = ["dockerpullsecret"] | .spec.template.container.imagePullPolicy = "IfNotPresent"' "$file"
done

How to loop through mult level dict or yaml in Ansible 2.7 or 2.8?

I've found a few ansible looping examples through multi level yaml, but, I can't seem to get down to one more level.
I have a list of machines or vms I'm trying to create on a xenserver pool and I need to pick out hostnames, num of cpus, etc.
#Requires ansible dev version 2.8. 2.8 will be release summer 2019.
- hosts: xenservers
vars:
Machines:
Connectors:
Connector1:
hostname: CCConn-0001
num_cpus: 4
num_cpu_cores_per_socket: 2
memory_mb: 8192
Connector2:
hostname: CCConn-0002
num_cpus: 4
num_cpu_cores_per_socket: 2
memory_mb: 8192
#Storefronts:
# Storefront1:
# hostname: SFPrinci-0001
# num_cpus: 4
# num_cpu_cores_per_socket: 2
# memory_mb: 8192
tasks:
# - name: Create VMs from a template
# xenserver_guest:
# hostname: 10.8.47.11
# username:
# password:
# validate_certs: no
# #folder: /home/testvms
# name: '{{Need a hostname here}}'
# state: poweredon
# template: W2K16_RTM_64_EN_ans
# disks:
# - size_gb: 100
# name: ''
# sr: XenRTVol
# linked_clone: yes
# hardware:
# num_cpus: 'Need number of cpus here'
# num_cpu_cores_per_socket: 'Need cores per socket here'
# memory_mb: 'Need memory here'
# cdrom:
# type: iso
# iso_name: guest-tools.iso
# networks:
# - name: vlan40
# wait_for_ip_address: no
# delegate_to: localhost
# register: deploy
# with_subelements: '{{Machines}}'
- name:
debug:
msg: "{{ item.key }} - {{ item.value }}"
loop: "{{ Machines | dict2items }}"
I've been playing around with dict2items, but, I don't know how to get the values I need one level lower. If that makes sense.

You can throw away all key names on two levels and iterate over values only:
- debug:
msg: "{{ item }}"
loop: "{{ Machines | json_query('*.*[]') }}"
json_query filter uses JMESPath syntax.

Python nested dictionary lookups can be done as follows:
# Create test dictionary
test_dict = {'level_one': {'level_two': 'test'}}
# Print the nested value
print(test_dict['level_one']['level_two'])

How to preserve the order of Key Value in YAML file using python (not alphabetical order)

I have the python code below for generating (dumping) a YAML document to a file.
import yaml
import os
device_name = "NN41_R11"
ip = "10.110.11.11"
port = "2022"
def genyam():
data = {
"testbed" : {
"name" : "boot_ios"},
"devices" : {
device_name : {
"type" : "IOS",
"connections" : {
"defaults" : {
"class" : "con.con",
"a" : {
"protocol" : "telnet",
"ip" : ip,
"port" : port,
}
}
}
}
}
}
with open('/tmp/testbed.yaml', 'w') as outfile:
yaml.dump(data, outfile, default_flow_style=False)`
which generates the following YAML file
devices:
NN41_R11:
connections:
defaults:
a:
ip: 10.110.11.11
port: '2022'
protocol: telnet
class: con.con
type: IOS
testbed:
name: boot_ios
Though the key value indentation is correct it's not generating in right order. I would like to have testbed first & then devices however it's opposite now. I am suspecting it's dumping in alphabetical order. NN41_R11 is again a dictionary which contains type & connections (type & connections are generated at same level but need first type:IOS and under that connections). Looking for ordered dump basically
The generated YAML document should be like the following:
testbed:
name: "boot-ios"
devices:
NN41_R11:
type: IOS
connections:
defaults:
class: 'con.con'
a:
protocol: telnet
ip: 10.110.11.11
port: 2022

I recommend you look at ruamel.yaml (disclaimer: I am the author of that package), it is specifically designed to preserve order of keys when loading and dumping (i.e. round-tripping) YAML documents and can also easily be used to generate YAML documents with your specifics on the fly.
You'll have to somehow order your key-value pairs in your source, as although there is order in the Python source this is not preserved in the dict with name data. The omap type (i.e. ruamel.yaml.comments.CommentedMap) can be initialised with list of tuples, but I often find it easier to use step-by-step assignment.
To get double and single quotes around those strings that don't need it use the dq (i.e. ruamel.yaml.scalarstring.DoubleQuotedScalarString) resp. sq (i.e. ruamel.yaml.scalarstring.SingleQuotedScalarString)
You can get rid of the quotes around the port by specifying it as an int.
import sys
import ruamel.yaml
from ruamel.yaml.comments import CommentedMap as omap
from ruamel.yaml.scalarstring import DoubleQuotedScalarString as dq
from ruamel.yaml.scalarstring import SingleQuotedScalarString as sq
yaml = ruamel.yaml.YAML()
yaml.indent(mapping=4)
device_name = "NN41_R11"
ip = "10.110.11.11"
port = 2022
def genyam():
# initialise omap with list of tuples that are key-value-pairs
data = omap([
('testbed', omap([('name', dq('boot_ios'))])),
])
# or add in the order you want them in the YAML document
data['devices'] = devices = omap()
devices[device_name] = name = omap()
name['type'] = 'IOS'
name['connections'] = connections = omap()
connections['defaults'] = omap([('class', sq('con.con')),])
connections['a'] = a = omap()
a['protocol'] = 'telnet'
a['ip'] = ip
a['port'] = port
yaml.dump(data, sys.stdout)
genyam()
gives:
testbed:
name: "boot_ios"
devices:
NN41_R11:
type: IOS
connections:
defaults:
class: 'con.con'
a:
protocol: telnet
ip: 10.110.11.11
port: 2022
There is no way in ruamel.yaml (and even less so in PyYAML) to get different indents for different mappings as you have in your output (you have mostly four, but also five and two positions indent).
A completely different approach is to make a template for your YAML, and load and dump to make sure it is valid YAML (after filling out the template):
import sys
import ruamel.yaml
yaml_str = """\
testbed:
name: "boot_ios"
devices:
{device_name}:
type: IOS
connections:
defaults:
class: 'con.con'
a:
protocol: telnet
ip: {ip}
port: {port}
"""
yaml = ruamel.yaml.YAML()
yaml.indent(mapping=4)
yaml.preserve_quotes = True
def genyam2(device_name, ip, port):
data = yaml.load(yaml_str.format(device_name=device_name, ip=ip, port=port))
yaml.dump(data, sys.stdout)
genyam2(device_name = "NN41_R11", ip = "10.110.11.11", port = 2022)
This has the same output as the previous example, because on round-tripping order is preserved (and superfluous quotes as well if you specify yaml.preseve_quotes = True)