Max IOPS of EBS Volume from Last 2 Weeks - python

I want to retrieve max IOPS utilized by EBS volume in the last 2 weeks. I am using cloudwatch get_metric_data function to obtain data about metric VolumeReadOps and VolumeWriteOps. I am using following code to get VolumeReadOps and VolumeWriteOps and then trying to calculate MaxIOPS:
This is the function to get metric values:
def cloudwatch_metric_value(CWsession,NameSpace,ResourceIdentifier,vStat,vUnit,vMetricName,vPeriod):
"""
Function that returns metric value of cloudwatch for a given resource and metric Name
"""
if NameSpace=='EBS':
responseCW = CWsession.get_metric_data(
MetricDataQueries=[
{
'Id': 'string',
'MetricStat': {
'Metric': {
'Namespace': 'AWS/EBS',
'MetricName': vMetricName,
'Dimensions': [
{
'Name': 'VolumeId',
'Value': ResourceIdentifier
},
]
},
'Period': vPeriod,
'Stat': vStat,
'Unit': vUnit
},
'ReturnData': True
},
],
StartTime=vStartTime,
EndTime=vEndTime,
)
vValue=responseCW['MetricDataResults'][0]['Values']
vTimeStamps=responseCW['MetricDataResults'][0]['Timestamps']
index, value = max(enumerate(vValue), key=operator.itemgetter(1))
metric_value=value
metric_time=vTimeStamps[index]
return metric_time,metric_value
From main, it is called like following:
metric_time,metric_value = cloudwatch_metric_value(cloudwatch,'EBS',v['VolumeId'],'Sum','Count','VolumeReadOps',300)
vReadIOPS=metric_value
metric_time,metric_value = cloudwatch_metric_value(cloudwatch,'EBS',v['VolumeId'],'Sum','Count','VolumeWriteOps',300)
vWriteIOPS=metric_value
vTotalIOPS=round((vReadIOPS+vWriteIOPS)/300)
I understand that IOPS are calculated by diving the ReadOps/Write with duration. The values I get from this code for MaxIOPS for a given volume doesn't match with the values I see for same in cloudwatch console. Please advise if I am doing this in right way?
Thanks.

Ok, I was able to fix and here is the working function:
def cloudwatch_metric_value(CWsession,NameSpace,ResourceIdentifier,vStat,vUnit,vPeriod):
"""
Function that returns metric value of cloudwatch for a given resource and metric Name
"""
if NameSpace=='EBS':
responseCW = CWsession.get_metric_data(
MetricDataQueries=[
{
'Id': 'string1',
'MetricStat': {
'Metric': {
'Namespace': 'AWS/EBS',
'MetricName': 'VolumeReadOps',
'Dimensions': [
{
'Name': 'VolumeId',
'Value': ResourceIdentifier
},
]
},
'Period': vPeriod,
'Stat': vStat,
'Unit': vUnit
},
'ReturnData': True
},
{
'Id': 'string2',
'MetricStat': {
'Metric': {
'Namespace': 'AWS/EBS',
'MetricName': 'VolumeWriteOps',
'Dimensions': [
{
'Name': 'VolumeId',
'Value': ResourceIdentifier
},
]
},
'Period': vPeriod,
'Stat': vStat,
'Unit': vUnit
},
'ReturnData': True
},
],
StartTime=vStartTime,
EndTime=vEndTime,
)
vReadValue=responseCW['MetricDataResults'][0]['Values']
vReadTimeStamps=responseCW['MetricDataResults'][0]['Timestamps']
vWriteValue=responseCW['MetricDataResults'][1]['Values']
vWriteTimeStamps=responseCW['MetricDataResults'][1]['Timestamps']
vReadWriteValue = [vReadValue[i]+vWriteValue[i] for i in range(len(vWriteValue))]
if vReadWriteValue:
metric_value = max(vReadWriteValue)
metric_time = vReadTimeStamps[vReadWriteValue.index(metric_value)]
metric_value = metric_value / 300
else:
metric_value=1
metric_time=date_t
return metric_time,metric_value

Related

Twilio connect to call with python

15.3 from python, i need to replicate this scenario from python https://www.twilio.com/docs/studio/tutorials/how-to-forward-calls
i'm doing this.
flow = client.studio.v2.flows.create(
commit_message='Testing Flow',
friendly_name='Test',
status='published',
definition={
'description': 'A New Flow',
'states': [
{
'name': 'Trigger',
'type': 'trigger',
'transitions': [
],
'properties': {
'offset': {
'x': 0,
'y': 0
}
}
}
],
'initial_state': 'Trigger',
'flags': {
'allow_concurrent_calls': True
}
}
)
but just create the trigger, i need to create the widget with the forward call configuration. Thank!
i need the nex step to complete the scenario from code

Mongo messy DOB string field to Age

I have a collection with documents like:
{'state': 'NY', 'DOB': '2000-01-02'},
{'state': 'NY', 'DOB': '2002/03/04'},
{'state': 'NY', 'DOB': '00-00-00'},
{'state': 'NY', 'DOB': 'male'},
...
I want outputs like:
{'state': 'NY', 'DOB': '2000-01-02', 'Age': 21},
{'state': 'NY', 'DOB': '2002/03/04', 'Age': 19},
{'state': 'NY', 'DOB': '00-00-00', 'Age': None}, # or Mongo None equivalent
{'state': 'NY', 'DOB': 'male', 'Age': None}, # or Mongo None equivalent
...
I'm constructing aggregation queries in PyMongo, and I'm wondering if there's an aggregate way to try to convert a field to Mongo Date object and then extract Age from it, else (if a date cannot be extracted), return None. Some condition in the shell below?
def map_age(state, city)
db.aggregate([
{'$match': {
'state': state,
'DOB': {"$exists": True},
'Age': {"$exists": False}
}},
{...}
])
You can try,
$let to create variable for dob convert and do operation
$dateFromString to convert in to date from string if its in valid then replace with "None"
$subtract minus converted date from current date $$NOW you can use new Date() as well
$divide above subtract date by "31536000000" means "3652460601000"
$round to round age number
db.aggregate([
{
$set: {
Age: {
$let: {
vars: {
dob: {
$dateFromString: {
dateString: "$DOB",
onError: "None"
}
}
},
in: {
$cond: [
{ $eq: ["$$dob", "None"] },
"None",
{
$round: {
$divide: [
{ $subtract: ["$$NOW", "$$dob"] },
31536000000 // 365*24*60*60*1000
]
}
}
]
}
}
}
}
}
])
Playground
As suggested by #prasad_, you have to make use of the $dateFromString operator in either the $project or $addFields stage.
db.collection.aggregate([
{
"$project": {
"age": {
"$dateFromString": {
dateString: "$DOB",
onError: null,
onNull: null,
}
}
}
}
])

Python list to list of map

ENV: python 3.7+, boto3
I have list of private IP address ['10.0.3.11', '10.0.2.22']
response = route53_client.change_resource_record_sets(
HostedZoneId="ABDCEFGH",
ChangeBatch={
'Comment': 'Dns to ec2 instance',
'Changes': [
{
'Action': 'UPSERT',
'ResourceRecordSet': {
'Name': "ts-uat",
'Type': 'A',
'TTL': 120,
'ResourceRecords': [
{
'Value': record['Value']
},
]
}
}
]
}
)
How to get it as follows
HostedZoneId="ABDCEFGH",
ChangeBatch={
'Comment': 'Dns to ec2 instance',
'Changes': [
{
'Action': 'UPSERT',
'ResourceRecordSet': {
'Name': "ts-uat",
'Type': 'A',
'TTL': 120,
'ResourceRecords': [
{
'Value': 10.0.3.11
},
{
'Value': 10.0.2.22
},
]
}
}
]
}
)
I tried the following
def getListofIP(n):
return "{'value' : %s,}" % (n)
result = map(getListofIP, private_ip)
print(list(result))
output was
["{'value' : 10.0.3.11,}", "{'value' : 10.0.2.22,}"]
Thanks a lot #WillRichardson. it works for me
def getListofIP(n):
return {'Value' : '{}'.format(n)}
result = map(getListofIP, private_ip)
print(list(result))
Output
[{'Value': '10.0.3.11'}, {'Value': '10.0.2.22'}]

How to extract values from list and store it as dictionary(key-value pair)?

I need to extract 2 values from this list of dictionary and store it as a key-value pair.
Here I attached sample data..Where I need to extract "Name" and "Service" from this input and store it as a dictionary. Where "Name" is Key and corresponding "Service" is its value.
Input:
response = {
'Roles': [
{
'Path': '/',
'Name': 'Heera',
'Age': '25',
'Policy': 'Policy1',
'Start_Month': 'January',
'PolicyDocument':
{
'Date': '2012-10-17',
'Statement': [
{
'id': '',
'RoleStatus': 'New_Joinee',
'RoleType': {
'Service': 'Service1'
},
'Action': ''
}
]
},
'Duration': 3600
},
{
'Path': '/',
'Name': 'Prem',
'Age': '40',
'Policy': 'Policy2',
'Start_Month': 'April',
'PolicyDocument':
{
'Date': '2018-11-27',
'Statement': [
{
'id': '',
'RoleStatus': 'Senior',
'RoleType': {
'Service': ''
},
'Action': ''
}
]
},
'Duration': 2600
},
]
}
From this input, I need output as a dictionary type.
Output Format: { Name : Service }
Output:
{ "Heera":"Service1","Prem" : " "}
My try:
Role_name =[]
response = {#INPUT WHICH I SPECIFIED ABOVE#}
roles = response['Roles']
for role in roles:
Role_name.append(role['Name'])
print(Role_name)
I need to pair the name with its corresponding service. Any help would be really appreciable.
Thanks in advance.
You just have to write a long line which can reach till the key 'Service'.
And you a syntax error in line Start_Month': 'January') and 'Start_Month': 'April'). You can't have one unclosed brackets.
Fix it and run the following.
This is the code:
output_dict = {}
for r in response['Roles']:
output_dict[r["Name"]] = r['PolicyDocument']['Statement'][0]['RoleType']['Service']
print(output_dict)
Output:
{'Heera': 'Service1', 'Prem': ''}
You just have to do like this:
liste = []
for role in response['Roles']:
liste.append(
{
role['Name']:role['PolicyDocument']['Statement'][0]['RoleType']['Service'],
}
)
print(liste)
It seems your input data is structured kind of strange and I am not sure what the ) are doing next to the months since they make things invalid but here is a working script assuming you removed the parenthesis from your input.
response = {
'Roles': [
{
'Path': '/',
'Name': 'Heera',
'Age': '25',
'Policy': 'Policy1',
'Start_Month': 'January',
'PolicyDocument':
{
'Date': '2012-10-17',
'Statement': [
{
'id': '',
'RoleStatus': 'New_Joinee',
'RoleType': {
'Service': 'Service1'
},
'Action': ''
}
]
},
'Duration': 3600
},
{
'Path': '/',
'Name': 'Prem',
'Age': '40',
'Policy': 'Policy2',
'Start_Month': 'April',
'PolicyDocument':
{
'Date': '2018-11-27',
'Statement': [
{
'id': '',
'RoleStatus': 'Senior',
'RoleType': {
'Service': ''
},
'Action': ''
}
]
},
'Duration': 2600
},
]
}
output = {}
for i in response['Roles']:
output[i['Name']] = i['PolicyDocument']['Statement'][0]['RoleType']['Service']
print(output)
This should give you what you want in a variable called role_services:
role_services = {}
for role in response['Roles']:
for st in role['PolicyDocument']['Statement']:
role_services[role['Name']] = st['RoleType']['Service']
It will ensure you'll go through all of the statements within that data structure but be aware you'll overwrite key-value pairs as you traverse the response, if they exist in more than a single entry!
A reference on for loops which might be helpful, illustrates using if statements within them which can help you to extend this to check if items already exist!
Hope that helps

Updating complex JSON object in Python

I am grabbing sort of a complex MongoDB document with Python (v3.5) and I should update some values in it which are scattered all around the object and have no particular pattern in the structure and save it back to a different MongoDB collection. The object looks like this:
# after json.loads(mongo_db_document) my dict looks like this
notification = {
'_id': '570f934f45213b0d14b1256f',
'key': 'receipt',
'label': 'Delivery Receipt',
'version': '0.0.1',
'active': True,
'children': [
{
'key': 'started',
'label': 'Started',
'children': [
'date',
'time',
'offset'
]
},
{
'key': 'stop',
'label': 'Ended',
'children': [
'date',
'time',
'offset'
]
},
{
'label': '1. Particulars',
'template': 'formGroup',
'children': [
{
'children': [
{
'key': 'name',
'label': '2.1 Name',
'value': '********** THIS SHOULD BE UPDATED **********',
'readonly': 'true'
},
{
'key': 'ims_id',
'label': '2.2 IMS Number',
'value': '********** THIS SHOULD BE UPDATED **********',
'readonly': 'true'
}
]
},
{
'children': [
{
'key': 'type',
'readonly': '********** THIS SHOULD BE UPDATED **********',
'label': '2.3 Type',
'options': [
{
'label': 'Passenger',
'value': 'A37'
},
{
'label': 'Cargo',
'value': 'A35'
},
{
'label': 'Other',
'value': '********** THIS SHOULD BE UPDATED **********'
}
]
}
]
}
]
},
{
'template': 'formGroup',
'key': 'waste',
'label': '3. Waste',
'children': [
{
'label': 'Waste',
'children': [
{
'label': 'Plastics',
'key': 'A',
'inputType': 'number',
'inputAttributes': {
'min': 0
},
'value': '********** THIS SHOULD BE UPDATED **********'
},
{
'label': 'B. Oil',
'key': 'B',
'inputType': 'number',
'inputAttributes': {
'min': 0
},
'value': '********** THIS SHOULD BE UPDATED **********'
},
{
'label': 'C. Operational',
'key': 'C',
'inputType': 'number',
'inputAttributes': {
'min': 0
},
'value': '********** THIS SHOULD BE UPDATED **********'
}
]
}
]
},
{
'template': 'formRow',
'children': [
'empty',
'signature'
]
}
],
'filter': {
'timestamp_of_record': [
'date',
'time',
'offset'
]
}
}
My initial idea was to put placeholders (like $var_name) in places where I need to update values, and load the string with Python's string.Template, but that approach unfortunately breaks lots of stuff to other users of the same MongoDB document for some reason.
Is there a solution to simply modify this kind of object without "hardcoding" path to find the values I need to update?
There's this small script that I had written a couple years ago - I used it to find entries in some very long and unnerving JSONs. Admittedly it's not beautiful, but it might help in your case, perhaps?
You can find the script on Bitbucket, here (and here is the code).
Unfortunately it's not documented; at the time I wasn't really believing other people would use it, I guess.
Anyways, if you'd like to try it, save the script in your working directory and then use something like this:
from RecursiveSearch import Retriever
def alter_data(json_data, key, original, newval):
'''
Alter *all* values of said keys
'''
retr = Retriever(json_data)
for item_no, item in enumerate(retr.__track__(key)): # i.e. all 'value'
# Pick parent objects with a last element False in the __track__() result,
# indicating that `key` is either a dict key or a set element
if not item[-1]:
parent = retr.get_parent(key, item_no)
try:
if parent[key] == original:
parent[key] = newval
except TypeError:
# It's a set, this is not the key you're looking for
pass
if __name__ == '__main__':
alter_data(notification, key='value',
original = '********** THIS SHOULD BE UPDATED **********',
newval = '*UPDATED*')
Unfortunately as I said the script isn't well documented, so if you want to try it and need more info, I'll be glad to provide it.
Not sure if I understood correctly, but this will dynamically find all keys "value" and "readonly" and print out the paths to address the fields.
def findem(data, trail):
if isinstance(data, dict):
for k in data.keys():
if k in ('value', 'readonly'):
print("{}['{}']".format(trail, k))
else:
findem(data[k], "{}['{}']".format(trail, k))
elif isinstance(data, list):
for k in data:
findem(k, '{}[{}]'.format(trail, data.index(k)))
if __name__ == '__main__':
findem(notification, 'notification')
notification['children'][2]['children'][0]['children'][0]['readonly']
notification['children'][2]['children'][0]['children'][0]['value']
notification['children'][2]['children'][0]['children'][1]['readonly']
notification['children'][2]['children'][0]['children'][1]['value']
notification['children'][2]['children'][1]['children'][0]['readonly']
notification['children'][2]['children'][1]['children'][0]['options'][0]['value']
notification['children'][2]['children'][1]['children'][0]['options'][1]['value']
notification['children'][2]['children'][1]['children'][0]['options'][2]['value']
notification['children'][3]['children'][0]['children'][0]['value']
notification['children'][3]['children'][0]['children'][1]['value']
notification['children'][3]['children'][0]['children'][2]['value']
Add another list to the JSON object. Each item in that list would be a list of keys that lead to the values to be changed. An example for one such list is: ['children', 2, 'children', 'children', 0, 'value'].
Then, to access the value you could use a loop:
def change(json, path, newVal):
cur = json
for key in path[:-1]:
cur = cur[key]
cur[path[-1]] = newVal
path = notification['paths'][0]
#path, for example, could be ['children', 2, 'children', 'children', 0, 'value']
newVal = 'what ever you want'
change(notification, path, newVal)

Categories