We Keep Coding

How to add the value from list of tuples

I am extracting from the log file and print using the below code
for line in data:
g = re.findall(r'([\d.]+).*?(GET|POST|PUT|DELETE)', line)
print (g)
[('1.1.1.1', 'PUT')]
[('2.2.2.2', 'GET')]
[('1.1.1.1', 'PUT')]
[('2.2.2.2', 'POST')]
How to add to the output
output
1.1.1.1: PUT = 2
2.2.2.2: GET = 1,POST=1

You could use a dictionary to count:
# initialize the count dict
count_dict= dict()
for line in data:
g = re.findall(r'([\d.]+).*?(GET|POST|PUT|DELETE)', line)
for tup in g:
# get the counts for tuple tup if we don't have it yet
# use 0 (second argument to .get)
num= count_dict.get(tup, 0)
# increase the count and write it back
count_dict[tup]= num+1
# now iterate over the key (tuple) - value (counts)-pairs
# and print the result
for tup, count in count_dict.items():
print(tup, count)
Ok, I have to admit this doesn't give the exact output, you want, but from this you can do in a similar manner:
out_dict= dict()
for (comma_string, request_type), count in count_dict.items():
out_str= out_dict.get(comma_string, '')
sep='' if out_str == '' else ', '
out_str= f'{out_str}{sep}{request_type} = {count}'
out_dict[comma_string]= out_str
for tup, out_str in out_dict.items():
print(tup, out_str)
From your data that outputs:
1.1.1.1 PUT = 2
2.2.2.2 GET = 1, POST = 1

I would look towards Counter.
from collections import Counter
results = []
for line in data:
g = re.findall(r'([\d.]+).*?(GET|POST|PUT|DELETE)', line)
results.append(g[0])
ip_list = set(result[0] for result in results)
for ip in ip_list:
print(ip, Counter(result[1] for result in results if result[0] == ip ))

You can use collection.defaultdict
Ex:
from collections import defaultdict
result = defaultdict(list)
for line in data:
for ip, method in re.findall(r'([\d.]+).*?(GET|POST|PUT|DELETE)', line):
result[ip].append(method)
for k, v in result.items():
temp = ""
for i in set(v):
temp += " {} = {}".format(i, v.count(i))
print("{}{}".format(k, temp))

from collections import Counter
x = [[('1.1.1.1', 'PUT')],[('2.2.2.2', 'GET')],[('1.1.1.1', 'PUT')],[('2.2.2.2', 'POST')]]
# step 1: convert x into a dict.
m = {}
for i in x:
a, b = i[0]
if a not in m.keys():
m[a] = [b]
else:
x = m[a]
x.append(b)
m[a] = x
print('new dict is {}'.format(m))
# step 2 count frequency
m_values = list(m.values())
yy = []
for i in m_values:
x = []
k = list(Counter(i).keys())
v = list(Counter(i).values())
for i in range(len(k)):
x.append(k[i] + '=' + str(v[i]))
yy.append(x)
# step 3, update the value of the dict
m_keys = list(m.keys())
n = len(m_keys)
for i in range(n):
m[m_keys[i]] = yy[i]
print("final dict is{}".format(m))
Output is
new dict is {'1.1.1.1': ['PUT', 'PUT'], '2.2.2.2': ['GET', 'POST']}
final dict is{'1.1.1.1': ['PUT=2'], '2.2.2.2': ['GET=1', 'POST=1']}

Without dependencies and using a dict for counting, in a very basic way. Given the data_set:
data_set = [[('1.1.1.1', 'PUT')],
[('2.2.2.2', 'GET')],
[('2.2.2.2', 'POST')],
[('1.1.1.1', 'PUT')]]
Initialize the variables (manually, just few verbs) then iterate over the data:
counter = {'PUT': 0, 'GET': 0, 'POST': 0, 'DELETE': 0}
res = {}
for data in data_set:
ip, verb = data[0]
if not ip in res:
res[ip] = counter
else:
res[ip][verb] += 1
print(res)
#=> {'1.1.1.1': {'PUT': 1, 'GET': 0, 'POST': 1, 'DELETE': 0}, '2.2.2.2': {'PUT': 1, 'GET': 0, 'POST': 1, 'DELETE': 0}}
It's required to format the output to better fits your needs.

Dot notation to Json in python

I receive data from the Loggly service in dot notation, but to put data back in, it must be in JSON.
Hence, I need to convert:
{'json.message.status.time':50, 'json.message.code.response':80, 'json.time':100}
Into:
{'message': {'code': {'response': 80}, 'status': {'time': 50}}, 'time': 100}
I have put together a function to do so, but I wonder if there is a more direct and simpler way to accomplish the same result.
def dot_to_json(a):
# Create root for JSON tree structure
resp = {}
for k,v in a.items():
# eliminate json. (if metric comes from another type, it will keep its root)
k = re.sub(r'\bjson.\b','',k)
if '.' in k:
# Field has a dot
r = resp
s = ''
k2 = k.split('.')
l = len(k2)
count = 0
t = {}
for f in k2:
count += 1
if f not in resp.keys():
r[f]={}
r = r[f]
if count < l:
s += "['" + f + "']"
else:
s = "resp%s" % s
t = eval(s)
# Assign value to the last branch
t[f] = v
else:
r2 = resp
if k not in resp.keys():
r2[k] = {}
r2[k] = v
return resp

You can turn the path into dictionary access with:
def dot_to_json(a):
output = {}
for key, value in a.iteritems():
path = key.split('.')
if path[0] == 'json':
path = path[1:]
target = reduce(lambda d, k: d.setdefault(k, {}), path[:-1], output)
target[path[-1]] = value
return output
This takes the key as a path, ignoring the first json part. With reduce() you can walk the elements of path (except for the last one) and fetch the nested dictionary with it.
Essentially you start at output and for each element in path fetch the value and use that value as the input for the next iteration. Here dict.setdefault() is used to default to a new empty dictionary each time a key doesn't yet exist. For a path ['foo', 'bar', 'baz'] this comes down to the call output.setdefault('foo', {}).setdefault('bar', {}).setdefault('baz', {}), only more compact and supporting arbitrary length paths.
The innermost dictionary is then used to set the value with the last element of the path as the key.
Demo:
>>> def dot_to_json(a):
... output = {}
... for key, value in a.iteritems():
... path = key.split('.')[1:] # ignore the json. prefix
... target = reduce(lambda d, k: d.setdefault(k, {}), path[:-1], output)
... target[path[-1]] = value
... return output
...
>>> dot_to_json({'json.message.status.time':50, 'json.message.code.response':80, 'json.time':100}))
{'message': {'status': {'time': 50}, 'code': {'response': 80}}, 'time': 100}

Aggregating key value pair in python

I have a question related to python code.
I need to aggregate if the key = kv1, how can I do that?
input='num=123-456-7890&kv=1&kv2=12&kv3=0'
result={}
for pair in input.split('&'):
(key,value) = pair.split('=')
if key in 'kv1':
print value
result[key] += int(value)
print result['kv1']
Thanks a lot!!

I'm assuming you meant key == 'kv1' and also the kv within input was meant to be kv1 and that result is an empty dict that doesn't need result[key] += int(value) just result[key] = int(value)
input = 'num=123-456-7890&kv1=1&kv2=12&kv3=0'
keys = {k: v for k, v in [i.split('=') for i in input.split('&')]}
print keys # {'num': '123-456-7890', 'kv2': '12', 'kv1': '1', 'kv3': '0'}
result = {}
for key, value in keys.items():
if key == 'kv1':
# if you need to increase result['kv1']
_value = result[key] + int(value) if key in result else int(value)
result[key] = _value
# if you need to set result['kv1']
result[key] = int(value)
print result # {'kv1': 1}
Assuming you have multiple lines with data like:
num=123-456-7890&kv1=2&kv2=12&kv3=0
num=123-456-7891&kv1=1&kv2=12&kv3=0
num=123-456-7892&kv1=4&kv2=12&kv3=0
Reading line-by-line in a file:
def get_key(data, key):
keys = {k: v for k, v in [i.split('=') for i in data.split('&')]}
for k, v in keys.items():
if k == key: return int(v)
return None
results = []
for line in [line.strip() for line in open('filename', 'r')]:
value = get_key(line, 'kv1')
if value:
results.append({'kv1': value})
print results # could be [{'kv1': 2}, {'kv1': 1}, {'kv1': 4}]
Or just one string:
with open('filename', 'r') as f: data = f.read()
keys = {k: v for k, v in [i.split('=') for i in data.split('&')]}
result = {}
for key, value in keys.items():
if key == 'kv1':
result[key] = int(value)
Console i/o:
c:\nathan\python\bnutils>python
Python 2.7.5 (default, May 15 2013, 22:44:16) [MSC v.1500 64 bit (AMD64)] on win32
Type "help", "copyright", "credits" or "license" for more information.
>>> def get_key(data, key):
... keys = {k: v for k, v in [i.split('=') for i in data.split('&')]}
... for k, v in keys.items():
... if k == key: return int(v)
... return None
...
>>> results = []
>>> for line in [line.strip() for line in open('test.txt', 'r')]:
... value = get_key(line, 'kv1')
... if value:
... results.append({'kv1': value})
...
>>> print results
[{'kv1': 2}, {'kv1': 1}, {'kv1': 4}]
>>>
test.txt:
num=123-456-7890&kv1=2&kv2=12&kv3=0
num=123-456-7891&kv1=1&kv2=12&kv3=0
num=123-456-7892&kv1=4&kv2=12&kv3=0

import urlparse
urlparse.parse_qs(input)
results in: {'num': ['123-456-7890'], 'kv2': ['12'], 'kv': ['1'], 'kv3': ['0']}
The keys are aggregated for you.

You could do it this way, so basically just add an extra if else block dealing with the empty case for key
input='num=123-456-7890&kv=1&kv2=12&kv3=0'
result={}
for pair in input.split('&'):
temp = pair.split('=')
key = temp[0]
value = [1]
if key in 'kv1':
if key in p:
print value //do you really want to output this?
result[key] += int(value)
else:
print value //do you really want to output this?
result[key] = int(value)
print result['kv1']

Python - Convert JSON key/values into key/value where value is an array

I have a JSON file with numerous entries like this:
{
"area1": "California",
"area2": "Sierra Eastside",
"area3": "Bishop Area",
"area4": "Volcanic Tablelands (Happy/Sad Boulders)",
"area5": "Fish Slough Boulders",
"grade": "V6 ",
"route": "The Orgasm",
"type1": "Boulder",
"type2": "NONE",
"type3": "NONE",
"type4": "NONE",
},
I want to take the area and type entries and turn them into arrays:
{
"area": ["California","Sierra Eastside","Bishop Area","Volcanic Tablelands (Happy/Sad Boulders)","Fish Slough Boulders"]
"grade": "V6 ",
"route": "The Orgasm",
"type": ["Boulder","NONE","NONE","NONE"]
},
I have this code which almost works:
json_data=open('../json/routes_test.json')
datas = json.load(json_data)
datas_arrays = []
area_keys = ['area1','area2','area3','area4','area5']
type_keys = ['type1','type2','type3','type4']
for data in datas:
areaArray = []
typeArray = []
deleteArray = []
for k, v in data.iteritems():
for area_key in area_keys:
if (k == area_key):
areaArray.append(v)
deleteArray.append(k)
for type_key in type_keys:
if (k == type_key):
typeArray.append(v)
deleteArray.append(k)
for k in deleteArray:
del data[k]
data['area'] = areaArray
data['type'] = typeArray
datas_arrays.append(data)
print datas_arrays
print "********"
out = json.dumps(datas_arrays, sort_keys=True,indent=4, separators=(',', ': '))
print out
f_out= open('../json/toues_test_intoarrays.json', 'w')
f_out.write(out)
f_out.close()
The problem is that the area array is all out of order and the type array is backwards, which I can't have. I find it strange that one is unordered and one is ordered but backwards. To me it seems like the iteration should assure they're placed in order.

Python dictionaries have an arbitrary ordering, they are not sorted. You want to use your prebuilt lists of keys instead:
with open('../json/routes_test.json') as json_data:
datas = json.load(json_data)
area_keys = ['area1','area2','area3','area4','area5']
type_keys = ['type1','type2','type3','type4']
for data in datas:
data['area'] = [data[k] for k in area_keys]
data['type'] = [data[k] for k in type_keys]
for k in area_keys + type_keys:
del data[k]
out = json.dumps(datas, sort_keys=True, indent=4, separators=(',', ': '))
print out
with open('../json/toues_test_intoarrays.json', 'w') as f_out:
f_out.write(out)
which changes the dictionaries in-place.
You could even determine the area and type keys from each entry:
for data in datas:
keys = sorted(data.keys())
area_keys = [k for k in keys if k.startswith('area')]
data['area'] = [data[k] for k in area_keys]
type_keys = [k for k in keys if k.startswith('type')]
data['type'] = [data[k] for k in type_keys]
for k in area_keys + type_keys:
del data[k]
and omit the list literals with the 'area1', 'area2' etc. hardcoded lists altogether.

Iterate the keys in order.
for k, v in sorted(data.iteritems()):
This will fail once you get past 9, but it will do for now.

how to skip a value while printing from dict, if the value is blank

I have a dict
x4={'c;1': 'c4;;c6', 'b;1': 'a2;b2;c2;d2', 'b;0': 'A1;B1;C1;D1', 'a;1': 'a1;b1;c1;d1', 'a;0': 'A;B;C;D', 'c;0': 'c1;c2;c3'}
i am using this code:
for k,v in x4.iteritems():
a = k.split(";")
b = v.split(";")
if a[1] is not '0':
val = x4[a[0]+';0']
values = val.split(";")
for i in range(len(values)):
if values[i]=='' is True:
b[i]=''
else:
print '<%s>%s<%s>' % (values[i],b[i],values[i])
output its printing is
<c1>c4<c1>
<c2><c2>
<c3>c6<c3>
<A1>a2<A1>
<B1>b2<B1>
<C1>c2<C1>
<D1>d2<D1>
<A>a1<A>
<B>b1<B>
<C>c1<C>
<D>d1<D>
But What i want is
<c1>c4<c1>
<c3>c6<c3>
<A1>a2<A1>
<B1>b2<B1>
<C1>c2<C1>
<D1>d2<D1>
<A>a1<A>
<B>b1<B>
<C>c1<C>
<D>d1<D>
The 'c2' should not be printed.. I am using this code to print grouped data. pls help me

x4={'c;1': 'c4;;c6', 'b;1': 'a2;b2;c2;d2', 'b;0': 'A1;B1;C1;D1', 'a;1': 'a1;b1;c1;d1', 'a;0': 'A;B;C;D', 'c;0': 'c1;c2;c3'}
for k,v in x4.iteritems():
a = k.split(";")
b = v.split(";")
if a[1] is not '0':
val = x4[a[0]+';0']
values = val.split(";")
for i in range(len(values)):
if values[i] is '':
b[i]=''
else:
if b[i]:
print '<%s>%s<%s>' % (values[i],b[i],values[i])
output::
<A1>a2<A1>
<B1>b2<B1>
<C1>c2<C1>
<D1>d2<D1>
<c1>c4<c1>
<c3>c6<c3>
<A>a1<A>
<B>b1<B>
<C>c1<C>
<D>d1<D>