We Keep Coding

creating a set from multiple lists

I want to count the variables that occur in an AST (e.g., a+b+a should return {a,b}). Is there a one liner that I can write for each of the AST cases? I tried:
def count(e):
if isinstance(e, Add): # a + b
# XXX can I write the following as a one liner?
vs = set()
for v in s.args:
vs.update(count(v))
return vs
elif isinstance(e, Variable):
return e
...

To answer your actual question, you should probably be tree-walking for Name nodes.
This answer includes a variation of ast.NodeVisitor that passes the full path down to the node to the visitor function, since we need to be read some of the context to know whether we're in an Attribute access, or Storeing or loading.
There may be some corner cases I didn't think of yet, naturally, and attribute access (e.a, z.w) is currently fully ignored.
import ast
import collections
code = """
a = b + c + e.a
q = a + b + c
z.w = b + c
print(a)
"""
class NodeVisitorWithPath:
def visit(self, path):
node = path[-1]
method = "visit_" + node.__class__.__name__
visitor = getattr(self, method, self.generic_visit)
return visitor(path)
def generic_visit(self, path):
node = path[-1]
for field, value in ast.iter_fields(node):
if isinstance(value, list):
for item in value:
if isinstance(item, ast.AST):
self.visit(path + [item])
elif isinstance(value, ast.AST):
self.visit(path + [value])
class NameWalker(NodeVisitorWithPath):
def __init__(self):
self.events = collections.Counter()
def visit_Name(self, path):
if len(path) >= 1 and isinstance(
path[-2], ast.Attribute
): # Attribute access, ignore
return
event = (
"Assign"
if len(path) >= 1 and isinstance(path[-2], ast.Assign)
else "Load"
)
node = path[-1]
self.events[f"{event} {node.id}"] += 1
super().generic_visit(path)
tree = ast.parse(code, "x", "exec")
nw = NameWalker()
nw.visit([tree])
print(dict(nw.events))
outputs
{'Assign a': 1, 'Load b': 3, 'Load c': 3, 'Assign q': 1, 'Load a': 2, 'Load print': 1}

Extracting all path from a multi-level dictionary

I have a dictionary like this:
dirDict = {"DIR1" : {
"DIR11" : {
"DIR111" : "Maki111",
"DIR112" : "Maki112"
},
"DIR12" : "Maki12",
"DIR13" : {
"DIR131" : "Maki131"
}
}
}
Imagine this like a folder structure. And I would like to get similar as os.walk would do with a folder structure. Something like this:
["DIR1/DIR11/DIR111/Maki111",
"DIR1/DIR11/DIR112/Maki112",
"DIR1/DIR12/Maki12",
"DIR1/DIR13/DIR131/Maki131"]
So it is basically all the path for the dictionary values. I tried it many ways with recursive functions but I got lost.
Here is my latest trial:
def walk(input_dict, path_string = "", result = ""):
for key, value in input_dict.items():
if isinstance(value, dict):
path_string += "/" + key
print "==== DICT ====", "\nkey: ", key, "\nvalue: ", value, "\n\t\tpath_string: ", path_string
result = walk(value, path_string)
print "\t\t\t\tresulting: ", result
elif isinstance(value, str):
print "==== NOT DICT ===="
path_string += "/" + value
print "\t\tpath_string: ", path_string, "\nvalue: ", value
return path_string
else:
path_string = "/" + key
result += "\n" + result
return result

Using Python 3:
dirDict = {"DIR1" : {
"DIR11" : {
"DIR111" : "Maki111",
"DIR112" : "Maki112"
},
"DIR12" : "Maki12",
"DIR13" : {
"DIR131" : "Maki131"
}
}
}
def recurse(d, prefix=None, sep='/'):
if prefix is None:
prefix = []
for key, value in d.items():
if isinstance(value, dict):
yield from recurse(value, prefix + [key])
else:
yield sep.join(prefix + [key, value])
print(list(recurse(dirDict)))
Output:
['DIR1/DIR13/DIR131/Maki131', 'DIR1/DIR11/DIR111/Maki111', 'DIR1/DIR11/DIR112/Maki112', 'DIR1/DIR12/Maki12']

def walk(d, path):
paths = []
if len(d) == 0:
return path
for k, v in d.iteritems():
child_path = path + k + '/'
if isinstance(v, basestring):
paths.append(child_path + v)
else:
paths.extend(walk(v, child_path))
return paths

THe walk function I posted at https://gist.github.com/nvie/f304caf3b4f1ca4c3884#gistcomment-1597937 can be used as a helper for your problem:
def walk(obj, parent_first=True):
# Top down?
if parent_first:
yield (), obj
# For nested objects, the key is the path component.
if isinstance(obj, dict):
children = obj.items()
# For nested lists, the position is the path component.
elif isinstance(obj, (list, tuple)):
children = enumerate(obj)
# Scalar values have no children.
else:
children = []
# Recurse into children
for key, value in children:
for child_path, child in walk(value, parent_first):
yield (key,) + child_path, child
# Bottom up?
if not parent_first:
yield (), obj
Your problem can be approached using something like this:
for path, value in walk(obj):
if isinstance(value, str): # leaf node
path_with_value = path + (value,)
print("/".join(path_with_value))

A compact solution with a list comprehension:
def f(v):
if isinstance(v, dict):
return dict_to_list(v)
elif isinstance(v, list):
return v
else:
return [v]
def dict_to_list(d):
return ['{}/{}'.format(k, i) for k, v in d.items() for i in f(v)]
lst = dict_to_list(dirDict)
lst.sort()
print('\n'.join(lst))

Parsing JSON to find value for key

I am trying to parse json to find the value of a desired key. I am doing so recursively. If there is another, fast or more efficient way to do so, I am open
example json:
{
"data_version":"5",
"application":{
"platform":"iPhone",
"os":"iPhone OS",
"locale":"en_US",
"app_version":"unknown",
"mobile":{
"device":"iPhone",
"carrier":"Verizon",
}
},
"event_header":{
"accept_language":"en-us",
"topic_name":"mobile-clickstream",
"server_timestamp":1416958459572,
"version":"1.0"
},
"session":{
"properties":{
}
},
"event":{
"timestamp":1416958459185,
"properties":{
"event_sequence_number":97
}
}
}
here is what I have so far
def json_scan(json_obj, key):
result = None
for element in json_obj:
if str(element) == key:
result = json_obj[element]
else:
if type(json_obj[element]) == DictType:
json_scan(json_obj[element], key)
elif type(json_obj[element]) == ListType:
json_scan(element, key)
return result
expected output:
>>> json_scan(json_obj, "timestamp")
1416958459185
As I go through the debugger, I am able to find the the desired value but the line result = None resets result to None and at the end of the method, the value I get is None. I'm not sure how to fix this. I tried removing the line but I get error because result is not preset to a value.

Using json library in order to parse the json file (some commas should be deleted) and using native dict types :
def json_scan(json_obj, key):
d = json.loads(json_obj)
def _(dictobj, lookup):
if lookup in dictobj.keys():
return dictobj[lookup]
else:
for sub_dictobj in [d for d in dictobj.values() if type(d) == DictType]:
result = _(sub_dictobj, lookup)
if result:
return result
return None
return _(d, key)
A more complete version :
def json_scan(json_obj, key):
d = json.loads(json_obj)
def _(dictobj, lookup):
if lookup in dictobj.keys():
return dictobj[lookup]
else:
for sub_dictobj in [d for d in dictobj.values() if type(d) == DictType]:
result = _(sub_dictobj, lookup)
if result:
return result
# if objects in dictobj.values() are lists, go through them
for listobject in [l for l in dictobj.values() if type(d) == list]:
for sub_dictobj in [d for d in listobject if type(d) == DictType]:
result = _(sub_dictobj, lookup)
if result:
return result
return None
return _(d, key)
EDIT (2015/04/25):
After looking # PyCon 2015 videos, I came across dict_digger :
http://jtushman.github.io/blog/2013/11/06/dict-digger/
https://github.com/jtushman/dict_digger
It comes with tests...

You should return result from inside your if statement. So, your code would be:
def json_scan(json_obj, key):
for element in json_obj:
if str(element) == key:
result = json_obj[element]
return result
else:
if type(json_obj[element]) == DictType:
json_scan(json_obj[element], key)
elif type(json_obj[element]) == ListType:
json_scan(element, key)
return None
That way if you find the result, it'll return it immediately instead of resetting it to None. If it doesn't find it, it'll still return None at the end.

The problem is that you don't assign the recursive calls to result:
def json_scan(json_obj, key):
result = None
for element in json_obj:
if str(element) == key:
result = json_obj[element]
else:
if type(json_obj[element]) == DictType:
result = json_scan(json_obj[element], key)
elif type(json_obj[element]) == ListType:
result = json_scan(element, key)
return result
Another problem is that your scan doesn't work for lists - json_obj[element] is only going to work for dicts - but since your data doesn't have lists, its working for now. You should remove list processing completely (unless you really have lists, then the algorithm needs to change).

JSON dumps custom formatting

I'd like to dump a Python dictionary into a JSON file with a particular custom format. For example, the following dictionary my_dict,
'text_lines': [{"line1"}, {"line2"}]
dumped with
f.write(json.dumps(my_dict, sort_keys=True, indent=2))
looks like this
"text_lines": [
{
"line1"
},
{
"line2"
}
]
while I prefer that it looks like this
"text_lines":
[
{"line1"},
{"line2"}
]
Similarly, I want the following
"location": [
22,
-8
]
to look like this
"location": [22, -8]
(that is, more like a coordinate, which it is).
I know that this is a cosmetic issue, but it's important to me to preserve this formatting for easier hand editing of the file.
Any way of doing this kind of customisation? An explained example would be great (the docs did not get me very far).

I have used the example provided by Tim Ludwinski and adapted it to my preference:
class CompactJSONEncoder(json.JSONEncoder):
"""A JSON Encoder that puts small lists on single lines."""
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.indentation_level = 0
def encode(self, o):
"""Encode JSON object *o* with respect to single line lists."""
if isinstance(o, (list, tuple)):
if self._is_single_line_list(o):
return "[" + ", ".join(json.dumps(el) for el in o) + "]"
else:
self.indentation_level += 1
output = [self.indent_str + self.encode(el) for el in o]
self.indentation_level -= 1
return "[\n" + ",\n".join(output) + "\n" + self.indent_str + "]"
elif isinstance(o, dict):
self.indentation_level += 1
output = [self.indent_str + f"{json.dumps(k)}: {self.encode(v)}" for k, v in o.items()]
self.indentation_level -= 1
return "{\n" + ",\n".join(output) + "\n" + self.indent_str + "}"
else:
return json.dumps(o)
def _is_single_line_list(self, o):
if isinstance(o, (list, tuple)):
return not any(isinstance(el, (list, tuple, dict)) for el in o)\
and len(o) <= 2\
and len(str(o)) - 2 <= 60
#property
def indent_str(self) -> str:
return " " * self.indentation_level * self.indent
def iterencode(self, o, **kwargs):
"""Required to also work with `json.dump`."""
return self.encode(o)
Also see the version I have in use.

Here's something that I hacked together. Not very pretty but it seems to work. You could probably handle simple dictionaries in a similar way.
class MyJSONEncoder(json.JSONEncoder):
def __init__(self, *args, **kwargs):
super(MyJSONEncoder, self).__init__(*args, **kwargs)
self.current_indent = 0
self.current_indent_str = ""
def encode(self, o):
#Special Processing for lists
if isinstance(o, (list, tuple)):
primitives_only = True
for item in o:
if isinstance(item, (list, tuple, dict)):
primitives_only = False
break
output = []
if primitives_only:
for item in o:
output.append(json.dumps(item))
return "[ " + ", ".join(output) + " ]"
else:
self.current_indent += self.indent
self.current_indent_str = "".join( [ " " for x in range(self.current_indent) ])
for item in o:
output.append(self.current_indent_str + self.encode(item))
self.current_indent -= self.indent
self.current_indent_str = "".join( [ " " for x in range(self.current_indent) ])
return "[\n" + ",\n".join(output) + "\n" + self.current_indent_str + "]"
elif isinstance(o, dict):
output = []
self.current_indent += self.indent
self.current_indent_str = "".join( [ " " for x in range(self.current_indent) ])
for key, value in o.items():
output.append(self.current_indent_str + json.dumps(key) + ": " + self.encode(value))
self.current_indent -= self.indent
self.current_indent_str = "".join( [ " " for x in range(self.current_indent) ])
return "{\n" + ",\n".join(output) + "\n" + self.current_indent_str + "}"
else:
return json.dumps(o)
NOTE: It's pretty much unnecessary in this code to be inheriting from JSONEncoder.

You will need to create a subclass of the json.JSONEncoder class and override the methods
for each type of value so they write the format you need. You may end up re-implementing
most of them, depending on what your formatting needs are.
http://docs.python.org/2/library/json.html has an example for extending
the JSONEncoder.

urlencode a multidimensional dictionary in python

How can I get a URL-encoded version of a multidimensional dictionary in Python? Unfortunately, urllib.urlencode() only works in a single dimension. I would need a version capable of recursively encoding the dictionary.
For example, if I have the following dictionary:
{'a': 'b', 'c': {'d': 'e'}}
I want to obtain the following string:
a=b&c[d]=e

OK people. I implemented it myself:
import urllib
def recursive_urlencode(d):
"""URL-encode a multidimensional dictionary.
>>> data = {'a': 'b&c', 'd': {'e': {'f&g': 'h*i'}}, 'j': 'k'}
>>> recursive_urlencode(data)
u'a=b%26c&j=k&d[e][f%26g]=h%2Ai'
"""
def recursion(d, base=[]):
pairs = []
for key, value in d.items():
new_base = base + [key]
if hasattr(value, 'values'):
pairs += recursion(value, new_base)
else:
new_pair = None
if len(new_base) > 1:
first = urllib.quote(new_base.pop(0))
rest = map(lambda x: urllib.quote(x), new_base)
new_pair = "%s[%s]=%s" % (first, ']['.join(rest), urllib.quote(unicode(value)))
else:
new_pair = "%s=%s" % (urllib.quote(unicode(key)), urllib.quote(unicode(value)))
pairs.append(new_pair)
return pairs
return '&'.join(recursion(d))
if __name__ == "__main__":
import doctest
doctest.testmod()
Still, I'd be interested to know if there's a better way to do this. I can't believe Python's standard library doesn't implement this.

Something like this?
a = {'a': 'b', 'c': {'d': 'e'}}
url = urllib.urlencode([('%s[%s]'%(k,v.keys()[0]), v.values()[0] ) if type(v)==dict else (k,v) for k,v in a.iteritems()])
url = 'a=b&c%5Bd%5D=e'

Based on the code of #malaney, I think that the code below emulates the PHP function http_build_query() quite well.
#!/usr/bin/env python3
import urllib.parse
def http_build_query(data):
parents = list()
pairs = dict()
def renderKey(parents):
depth, outStr = 0, ''
for x in parents:
s = "[%s]" if depth > 0 or isinstance(x, int) else "%s"
outStr += s % str(x)
depth += 1
return outStr
def r_urlencode(data):
if isinstance(data, list) or isinstance(data, tuple):
for i in range(len(data)):
parents.append(i)
r_urlencode(data[i])
parents.pop()
elif isinstance(data, dict):
for key, value in data.items():
parents.append(key)
r_urlencode(value)
parents.pop()
else:
pairs[renderKey(parents)] = str(data)
return pairs
return urllib.parse.urlencode(r_urlencode(data))
if __name__ == '__main__':
payload = {
'action': 'add',
'controller': 'invoice',
'code': 'debtor',
'InvoiceLines': [
{'PriceExcl': 150, 'Description': 'Setupfee'},
{'PriceExcl':49.99, 'Description':'Subscription'}
],
'date': '2016-08-01',
'key': 'Yikes&ampersand'
}
print(http_build_query(payload))
payload2 = [
'item1',
'item2'
]
print(http_build_query(payload2))

I think the code below may be what you want
import urllib.parse
def url_encoder(params):
g_encode_params = {}
def _encode_params(params, p_key=None):
encode_params = {}
if isinstance(params, dict):
for key in params:
encode_key = '{}[{}]'.format(p_key,key)
encode_params[encode_key] = params[key]
elif isinstance(params, (list, tuple)):
for offset,value in enumerate(params):
encode_key = '{}[{}]'.format(p_key, offset)
encode_params[encode_key] = value
else:
g_encode_params[p_key] = params
for key in encode_params:
value = encode_params[key]
_encode_params(value, key)
if isinstance(params, dict):
for key in params:
_encode_params(params[key], key)
return urllib.parse.urlencode(g_encode_params)
if __name__ == '__main__':
params = {'name': 'interface_name', 'interfaces': [{'interface': 'inter1'}, {'interface': 'inter2'}]}
print(url_encoder(params))
the output is
interfaces%5B1%5D%5Binterface%5D=inter2&name=interface_name&interfaces%5B0%5D%5Binterface%5D=inter1
which is look like
interfaces[1][interface]=inter2&name=interface_name&interfaces[0][interface]=inter1
PS: you may want use OrderDict to replace dict above

The above solution only works for arrays with depth < 2. The code below will properly urlencode a multidimensional array of any depth.
#!/usr/bin/env python
import sys
import urllib
def recursive_urlencode(data):
def r_urlencode(data, parent=None, pairs=None):
if pairs is None:
pairs = {}
if parent is None:
parents = []
else:
parents = parent
for key, value in data.items():
if hasattr(value, 'values'):
parents.append(key)
r_urlencode(value, parents, pairs)
parents.pop()
else:
pairs[renderKey(parents + [key])] = renderVal(value)
return pairs
return urllib.urlencode(r_urlencode(data))
def renderKey(parents):
depth, outStr = 0, ''
for x in parents:
str = "[%s]" if depth > 0 else "%s"
outStr += str % renderVal(x)
depth += 1
return outStr
def renderVal(val):
return urllib.quote(unicode(val))
def main():
print recursive_urlencode(payload)
if __name__ == '__main__':
sys.exit(main())

The function get_encoded_url_params() takes a dict as argument and returns url encoded form of the dict.
def get_encoded_url_params(d):
"""URL-encode a nested dictionary.
:param d = dict
:returns url encoded string with dict key-value pairs as query parameters
e.g.
if d = { "addr":{ "country": "US", "line": ["a","b"] },
"routing_number": "011100915", "token": "asdf"
}
:returns 'addr[country]=US&addr[line][0]=a&addr[line][1]=b&routing_number=011100915&token=asdf'
or 'addr%5Bcountry%5D=US&addr%5Bline%5D%5B0%5D=a&addr%5Bline%5D%5B1%5D=b&routing_number=011100915&token=asdf'
(which is url encoded form of the former using quote_plus())
"""
def get_pairs(value, base):
if isinstance(value, dict):
return get_dict_pairs(value, base)
elif isinstance(value, list):
return get_list_pairs(value, base)
else:
return [base + '=' + str(value)]
# use quote_plus() to get url encoded string
# return [quote_plus(base) + '=' + quote_plus(str(value))]
def get_list_pairs(li, base):
pairs = []
for idx, value in enumerate(li):
new_base = base + '[' + str(idx) + ']'
pairs += get_pairs(value, new_base)
return pairs
def get_dict_pairs(d, base=''):
pairs = []
for key, value in d.items():
new_base = key if base == '' else base + '[' + key + ']'
pairs += get_pairs(value, new_base)
return pairs
return '&'.join(get_dict_pairs(d))

what about json.dumps and json.loads?
d = {'a': 'b', 'c': {'d': 'e'}}
s = json.dumps(d) # s: '{"a": "b", "c": {"d": "e"}}'
json.loads(s) # -> d

what about this simplified version:
def _clean(value):
return urllib.quote(unicode(value))
'&'.join([ v for val in [[ "%s[%s]=%s"%(k,ik, _(iv))
for ik, iv in v.items()] if type(v)==dict else ["%s=%s"%(k,_(v))]
for k,v in data.items() ]
for v in val ])
I agree is not readable, maybe flattening the list can be better done with itertools.chain instead of another list comprehension.
This only goes 1 level deeper, yours can go N levels deeper if you would add some logic to manage N numbers of "[%s]" depending on the level, but I guess is not that necesary

If you want to convert python dict/list/nested to PHP Array like urlencoded string.
In python, most of the data type you want to convert to urlencoded maybe: dict list tuple nested of them, Like
a = [1, 2]
print(recursive_urlencode(a))
# 0=1&1=2
a2 = (1, '2')
print(recursive_urlencode(a2))
# 0=1&1=2
b = {'a': 11, 'b': 'foo'}
print(recursive_urlencode(b))
# a=11&b=foo
c = {'a': 11, 'b': [1, 2]}
print(recursive_urlencode(c))
# a=11&b[0]=1&b[1]=2
d = [1, {'a': 11, 'b': 22}]
print(recursive_urlencode(d))
# 0=1&1[a]=11&1[b]=22
e = {'a': 11, 'b': [1, {'c': 123}, [3, 'foo']]}
print(recursive_urlencode(e))
# a=11&b[0]=1&b[1][c]=123&b[2][0]=3&b[2][1]=foo
https://github.com/Viky-zhang/to_php_post_arr
P.S. some code from: https://stackoverflow.com/a/4014164/2752670