I need to exactract a list of all Running IP's from AWS and along the list of IP i also want the tag-value & the region-name(or availability-zone). The list should look as follows
ex : [[('23.1.1.141', ' Production LB', 'us-east-1'), ('47.2.14.93', 'sung2-cloud-trial01-LBi-b89671', 'us-west-2'),................]]
The list consists of the following:
<'Running IP'>,<'Tag-value'>,<AvailabilityZone>
Using boto3 and I can achieve this using describe_instances(), so I tried using it as follows:
def gather_public_ip():
regions = ['us-west-2', 'eu-central-1', 'ap-southeast-1']
combined_list = [] ##This needs to be returned
for region in regions:
instance_information = {}
ip_dict = {}
client = boto3.client('ec2', aws_access_key_id=ACCESS_KEY, aws_secret_access_key=SECRET_KEY,
region_name=region, )
instance_dict = client.describe_instances().get('Reservations')
for instance in instance_dict:
.....
#WHAT CODE COMES HERE???
gather_public_ip()
So what exactly should i be looking into for this ? I am not sure how to do so, please help
NEW ANSWER RESPONSE:
Traceback (most recent call last):
File "C:/Users/nisingh/PycharmProjects/untitled1/more_specific.py", line 35, in <module>
gather_public_ip()
File "C:/Users/nisingh/PycharmProjects/untitled1/more_specific.py", line 27, in gather_public_ip
ipaddress = instance['PublicIpAddress'] # Which one do you need?
KeyError: 'PublicIpAddress'
{u'Monitoring': {u'State': 'disabled'}, u'PublicDnsName': '', u'RootDeviceType': 'ebs', u'State': {u'Code': 80, u'Name': 'stopped'}, u'EbsOptimized': False, u'LaunchTime': datetime.datetime(2015, 5, 2, 16, 34, 17, tzinfo=tzutc()), u'PrivateIpAddress': '10.11.32.10', u'ProductCodes': [], u'VpcId': 'vpc-8ae766ef', u'StateTransitionReason': 'User initiated (2016-01-29 22:20:18 GMT)', u'InstanceId': 'i-5e8ae8a8', u'ImageId': 'ami-5189a661', u'PrivateDnsName': 'ip-10-11-32-10.us-west-2.compute.internal', u'KeyName': 'pdx01-cump-050215', u'SecurityGroups': [{u'GroupName': 'pdx01-clv-jumpeng', u'GroupId': 'sg-50abbf35'}], u'ClientToken': 'inYin1430584456904', u'SubnetId': 'subnet-57e45b20', u'InstanceType': 't2.micro', u'NetworkInterfaces': [{u'Status': 'in-use', u'MacAddress': '06:91:7c:86:7c:d5', u'SourceDestCheck': True, u'VpcId': 'vpc-8ae766ef', u'Description': 'Primary network interface', u'NetworkInterfaceId': 'eni-7d566d0b', u'PrivateIpAddresses': [{u'Primary': True, u'PrivateIpAddress': '10.11.32.10'}], u'Attachment': {u'Status': 'attached', u'DeviceIndex': 0, u'DeleteOnTermination': True, u'AttachmentId': 'eni-attach-08d40929', u'AttachTime': datetime.datetime(2015, 5, 2, 16, 34, 17, tzinfo=tzutc())}, u'Groups': [{u'GroupName': 'pdx01-cloud-dev-jumpeng', u'GroupId': 'sg-50abbf35'}], u'SubnetId': 'subnet-57e45b20', u'OwnerId': '547316675291', u'PrivateIpAddress': '10.11.32.10'}], u'SourceDestCheck': True, u'Placement': {u'Tenancy': 'default', u'GroupName': '', u'AvailabilityZone': 'us-west-2b'}, u'Hypervisor': 'xen', u'BlockDeviceMappings': [{u'DeviceName': '/dev/sda1', u'Ebs': {u'Status': 'attached', u'DeleteOnTermination': True, u'VolumeId': 'vol-0990f21b', u'AttachTime': datetime.datetime(2015, 5, 2, 16, 34, 21, tzinfo=tzutc())}}], u'Architecture': 'x86_64', u'StateReason': {u'Message': 'Client.UserInitiatedShutdown: User initiated shutdown', u'Code': 'Client.UserInitiatedShutdown'}, u'RootDeviceName': '/dev/sda1', u'VirtualizationType': 'hvm', u'Tags': [{u'Value': 'pdx01-jump02-DEPRECATED', u'Key': 'Name'}, {u'Value': 'Cloud Eng Jump Box', u'Key': 'Description'}], u'AmiLaunchIndex': 0}
I think it is all about finding the information in the response and grabbing it;
def gather_public_ip():
regions = ['us-west-2', 'eu-central-1', 'ap-southeast-1']
combined_list = [] ##This needs to be returned
for region in regions:
instance_information = [] # I assume this is a list, not dict
ip_dict = {}
client = boto3.client('ec2', aws_access_key_id=ACCESS_KEY, aws_secret_access_key=SECRET_KEY,
region_name=region, )
instance_dict = client.describe_instances().get('Reservations')
for reservation in instance_dict:
for instance in reservation['Instances']: # This is rather not obvious
if instance[unicode('State')][unicode('Name')] == 'running':
try:
ipaddress = instance[unicode('PublicIpAddress')]
except:
break
tagValue = instance[unicode('Tags')][0][unicode('Value')] # 'Tags' is a list, took the first element, you might wanna switch this
zone = instances[unicode('Placement')][unicode('AvailabilityZone')]
info = ipaddress, tagValue, zone
instance_information.append(info)
combined_list.append(instance_information)
return combined_list
gather_public_ip()
I hope this is what you are looking for, and I also hope you do NOT copy this straight off, but look at how I extracted the info from the return dict in the docs.
Try ip_addr = instance_dict['Reservations'][0]['Instances'][0]['PublicIpAddress'] for IP addresses. describe_instaces() response is a dictionary whose value is an array which conatins multiple other dictionaries and so on....
Related
Any idea how to not include anything with None? I am trying to just pull in IP addresses at this point, but I don’t want to include empty elements.
My API Response
[{'name': '', 'serial': 'Q2KN-xxxx-438Z', 'mac': '0c:8d:db:c3:ad:c8', 'networkId': 'L_6575255xxx96096977', 'model': 'MX64', 'address': '', 'lat': 38.4180951010362, 'lng': -92.098531723022, 'notes': '', 'tags': '', 'wan1Ip': '47.134.13.195', 'wan2Ip': None}, {'name': '', 'serial': 'Q2PD-xxx-QQ9Y', 'mac': '0c:8d:db:dc:ed:f6', 'networkId': 'L_657525545596096977', 'model': 'MR33', 'address': '', 'lat': 38.4180951010362, 'lng': -92.098531723022, 'notes': '', 'tags': '', 'lanIp': '10.0.0.214'}]
Iterating through elements and selecting certain fields
response = requests.request("GET", url + id + '/devices', headers=headers)
data = response.json()
for item in data:
keys = [item.get(x) for x in ['wan1Ip', 'model', 'lanIp', 'wan2Ip']]
print(*keys, sep="\n", file=sys.stdout)
My output is:
47.134.13.195
MX64
None
None
None
MR33
10.0.0.214
None
My desired output is:
47.134.13.195
10.0.0.214
I’ve tried adding a re.findall for ip addresses, but not sure that’s going to work for me. I’ve also tried to add operators for not in None and several other things.
re.findall(“(?:[\d]{1,3}).(?:[\d]{1,3}).(?:[\d]{1,3}).(?:[\d]{1,3})?“,string2 )
Update
I've changed my line to
keys = [item.get(x) for x in ['wan1Ip', 'model', 'lanIp', 'wan2Ip', '{}']if x in item]
Obviously, I still have non IP addresses in my output, but I can select the elements that have IP addresses only. My main issue was None. I will also try some of the other suggestions.
Search each key ipv4 with regex(https://stackoverflow.com/a/5284410/6250402) and check key is None with item.get(x) or ''
import re
myRegex = re.compile(r'\b((25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)(\.|$)){4}\b')
# data here
for item in data:
keys = [item.get(x) for x in ['wan1Ip', 'model', 'lanIp', 'wan2Ip'] if myRegex.search(item.get(x) or '')]
print(*keys, sep="\n", file=sys.stdout)
Add a filter to your list comprehension to check to see if key x is in item.
for item in data:
keys = [item.get(x) for x in ['wan1Ip', 'model', 'lanIp', 'wan2Ip'] if x in item]
print(*keys, sep="\n", file=sys.stdout)
You could add this condition to the end of your list comprehension:
keys = [item.get(x) for x in ['wan1Ip', 'model', 'lanIp', 'wan2Ip'] if item.get(x) is not None]
The output will be what you want.
Could you try this:
>> data = response.json()
>> keys = [x.get(key) for key in ['wan1Ip', 'model', 'lanIp', 'wan2Ip'] for x in data if x.get(key)]
>> print(keys)
>> ['47.134.13.195', 'MX64', 'MR33', '10.0.0.214']
I have some JSON data that I'm pulling from a websocket:
while True:
result = ws.recv()
result = json.loads(result)
Here is Print(result):
{'type': 'ticker', 'sequence': 4779671311, 'product_id': 'BTC-USD', 'price': '15988.29000000', 'open_24h': '14566.71000000', 'volume_24h': '18276.75612545', 'low_24h': '15988.29000000', 'high_24h': '16102.00000000', 'volume_30d': '1018642.48337033', 'best_bid': '15988.28', 'best_ask': '15988.29', 'side': 'buy', 'time': '2018-01-05T15:38:21.568000Z', 'trade_id': 32155934, 'last_size': '0.02420000'}
Now I want to access the 'price' value.
print (result['price'])
This results with a KeyError:
File "C:/Users/Selzier/Documents/Python/temp.py", line 43, in <module>
print (result['price'])
KeyError: 'price'
However, if I perform a loop on the (results) data, then I can successfully print both i and result[i]
for i in result:
if i == "price":
print (i)
print (result[i])
Which will print the following data:
price
16091.00000000
Why do I get a 'KeyError' when calling:
result['price']
AND
result[0]
When I'm not inside of the 'for i in result' loop?
Create a guard in while True loop, like in for loop:
while True:
result = ws.recv()
result = json.loads(result)
if result and 'price' in result:
print(result['price'])
...
(read my comment)
I have a collection of an item like below in my mongoDB database:
{u'Keywords': [[u'european', 7], [u'bill', 5], [u'uk', 5], [u'years', 4], [u'brexit', 4]], u'Link': u'http://www.bbc.com/
news/uk-politics-39042876', u'date': datetime.datetime(2017, 2, 21, 22, 47, 7, 463000), u'_id': ObjectId('58acc36b3040a218bc62c6d3')}
.....
These come from a mongo DB query
mydb = client['BBCArticles']
##mydb.adminCommand({'setParameter': True, 'textSearchEnabled': True})
my_collection = mydb['Articles']
print 'Articles containing higher occurences of the keyword is sorted as follow:'
for doc in my_collection.find({"Keywords":{"$elemMatch" : {"$elemMatch": {"$in": [keyword.lower()]}}}}):
print doc
However, I want to print documents as follow:
doc1
Kewords: european,bill, uk
Link:"http://www.bbc.com/"
doc2
....
Since your collection looks like a list of dictionaries, it should be iterable and parseable using a for-loop. If indeed you want only a portion of the url and keywords, this should work:
# c = your_collection, a list of dictionaries
from urlparse import urlparse
for n in range(len(c)):
print 'doc{n}'.format(n=n+1)
for k, v in c[n].iteritems():
if k == 'Keywords':
print k+':', ', '.join([str(kw[0]) for kw in v[0:3]])
if k == 'Link':
parsed_uri = urlparse( v )
domain = '{uri.scheme}://{uri.netloc}/'.format(uri=parsed_uri)
print k+':', '"{0}"\n'.format(domain)
prints:
doc1
Keywords: european, bill, uk
Link: "http://www.bbc.com/"
Suppose I have this TSV file
It looks like:
status=200 protocol=http region_name=Podolsk datetime=2016-03-10 15:51:58 user_ip=0.120.81.243 user_agent=Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/48.0.2564.116 Safari/537.36 user_id=7885299833141807155 user_vhost=tindex.ru method=GET page=/search/
and I need to extract users' browsers and OS usage statistics. Here's the code that works just fine except that it is REALLY slow:
from ua_parser import user_agent_parser
import user_agents
dict0 = {}
def grep(filename, pattern):
for n,line in enumerate(open(filename)):
ua_start = line.find(pattern) + len(pattern)
ua_end = line.find('\t', ua_start)
ua = str(user_agents.parse(line[ua_start:ua_end]).browser.family)
try:
dict0[ua] += 1
except KeyError:
dict0[ua] = 1
grep('data/data.tsv', 'user_agent=')
It gives me a nice idea of statistics in a dictionary that look like this:
{'Android': 10890,
'Apache-HttpClient': 59,
'Avant': 21,
'BlackBerry WebKit': 16,
'CFNetwork': 301,
'Chrome': 20963,
'Chrome Mobile': 1442,
'Chrome Mobile iOS': 69,
'Chromium': 290,
'Dolfin': 34,
'Edge': 872,
'Edge Mobile': 12,
'Epiphany': 1,
'Firefox': 9757,
'Firefox Beta': 4,
'Firefox Mobile': 22,
'Firefox iOS': 1,
'Galeon': 1,
'Googlebot': 1,
'IE': 5399,
'IE Large Screen': 4,
'IE Mobile': 340,
'Iceweasel': 12,
'Iron': 7,
'Jasmine': 3,
'Mail.ru Chromium Browser': 649,
'Maxthon': 137,
'Midori': 1,
'Mobile Safari': 2778,
'Mobile Safari UI/WKWebView': 51,
'Nokia Browser': 14,
'Nokia OSS Browser': 1,
'Obigo': 2,
'Opera': 10652,
'Opera Coast': 2,
'Opera Mini': 675,
'Opera Mobile': 299,
'Opera Tablet': 25,
'Other': 13424,
'PhantomJS': 11,
'Pinterest': 1,
'Puffin': 25,
'Python Requests': 39,
'Python-urllib': 9,
'QQ Browser Mobile': 3,
'Safari': 255,
'Samsung Internet': 522,
'SeaMonkey': 1,
'Sogou Explorer': 2,
'Spider': 17,
'UC Browser': 528,
'Vivaldi': 7,
'WebKit Nightly': 104,
'Yandex Browser': 19969,
'YandexBot': 86,
'YandexDirect': 2,
'YandexMobileBot': 1,
'YandexSearch': 2601}
The component that is really slow here is the user_agents thing itself. I am not really attached to user_agents library or the way I store the results so if you have ANY ideas on how to improve processing speed you are welcome.
Actually I made a nice way that could make A HUGE speed improvement!
Basically we first add user_agent= lines to dictionary and count them. Then we parse keys to user_agents and count values!
from ua_parser import user_agent_parser
parsing_dict = {}
os_stats = {}
browser_stats = {}
target = 'tindex.ru'
def grep(filename, pattern):
def parse_ua(ua):
p = ua_parser.user_agent_parser.Parse(ua)
return [p.get('os').get('family'), p.get('user_agent').get('family')]
for n,line in enumerate(open(filename)):
if target in line:
ua_start = line.find(pattern) + len(pattern)
ua_end = line.find('\t', ua_start)
ua = line[ua_start:ua_end]
try:
parsing_dict[ua] += 1
except KeyError:
parsing_dict[ua] = 1
for key, value in parsing_dict.iteritems():
ua = parse_ua(key)
try:
os_stats[ua[0]] += value
browser_stats[ua[1]] += value
except KeyError:
os_stats[ua[0]] = value
browser_stats[ua[1]] = value
grep('data/data.tsv', 'user_agent=')
you can use the regex:
/user_agent=(.*) user_id/
I used the PyWurfl library, which provides UserAgents mapping with Device, OS and other such information.
It requires us to download the latest wurfl.xml file and then use wurfl2python.py file to convert the data to a python object. To improve performance, you can tweak this a little bit.
What you can do is, load the latest wurfl.xml data provided by the library (in conventional way the first time) and then convert this to say, csv format. Then load the csv file contents in memory (as a dictionary object). Then for my requirement, I used Spark Jobs to process the data by broadcasting this dictionary object across the nodes.
Here is my error when i'm going to create records.please advice me where is the missing point.?
File "/home/bellvantage/Documents/openerp-7.0/openerp-7/openerp/addons/bpl/bpl.py", line 394, in create
self.write(cr, uid, [id], {'name': name}, context)
File "/usr/lib/python2.7/dist-packages/psycopg2/extensions.py", line 129, in getquoted
pobjs = [adapt(o) for o in self._seq]
ProgrammingError: can't adapt type 'builtin_function_or_method'
2013-04-26 06:14:07,719 5739 INFO demo werkzeug: 127.0.0.1 - - [26/Apr/2013 06:14:07] "POST /web/dataset/call_kw HTTP/1.1" 200 -
relevant line in my code is like this
def create(self, cr, uid, values, context=None):
name = 'CheckRoll No : ' + str(values['checkroll_no']) + ' & Gang No : ' + str(values['gang_no'])
self.write(cr, uid, [id], {'name': name}, context)
return True
error comes and shows that my write method is incorrect.seems some parameter missing by me.
needs your advices to implement module
thanks..
EDITED
Now error like below
File "/home/bellvantage/Documents/openerp-7.0/openerp-7/openerp/addons/bpl/bpl.py", line 395, in create
res = super('bpl.work.offer',self).create(cr,uid,values,context=context)
TypeError: must be type, not str
2013-04-26 07:58:43,452 6947 INFO demo werkzeug: 127.0.0.1 - - [26/Apr/2013 07:58:43] "POST /web/dataset/call_kw HTTP/1.1" 200 -
here shows my fields in model class
'name': fields.char('Name', size=50),
'checkroll_no': fields.integer('Checkroll No'),
'gang_no': fields.integer('Gang'),
here shows values when i debug (at super.create() time),
dict: {'selected_tea_workers_line_ids': [[5, False, False], [0, False, {'worker_emp_no': '1000', 'worker_id':
1, 'tea_line_worker_id': False, 'is_selected': False}]], 'user_id': False, 'is_confirmed': False,
'date_of_offer': '2013-04-26', 'bpl_division_id': 1, 'work_type': 'tea', 'checkroll_no': 10, 'name':
'CheckRoll No : 10 & Gang No : 100', 'selected_rubber_workers_line_ids': [[5, False, False], [0, False,
{'worker_emp_no': '1001', 'worker_id': 2, 'rubber_line_worker_id': False, 'is_selected': False}]],
'work_update_id': False, 'field_no': 15, 'selected_sundry_workers_line_ids': [], 'payment_type':
'normal_work', 'gang_no': 100, 'total_workers': 0, 'no_of_workers': 0, 'norm': 15,
'selected_other_workers_line_ids': [], 'bpl_estate_id': 1, 'select_by': 'by_name'}
now its worked. thanks AnomA
Last Edited
def create(self, cr, uid, values, context=None):
name = 'CheckRoll No = ' + str(values['checkroll_no']) + ' & Gang No = ' + str(values['gang_no'])
values.update({'name': name})
return super(bpl_work_offer,self).create(cr,uid,values,context=context)
Actually you are doing wrong. You are trying to write to a record that is not created.Also I dont think you need to use write in this case.
class bpl_abc(osv.osv)
_inherit = 'bpl.work.offer'
def create(self, cr, uid, values, context=None):
name = 'CheckRoll No : ' + str(values['checkroll_no']) + ' & Gang No : ' + str(values['gang_no'])
values.update({'name': name})
res = super('bpl_abc',self).create(cr,uid,values,context=context)
return res
bpl_abc()
Please keep in mind that create function in openerp always return an id of the newly created record.
May be because 'write' method is the built-in method
EX:
with open("a.txt","r+") as f:
f.write("sometext")
Don't use self.write as write is the built-in method on 'file' object