We Keep Coding

Modular Command-line Application in python

I have already written a simple python cmd program using the cmd module. However, I would like it to be modular. For Example, there would be a folder named script and any .py that contains a command would add the command to the application. How would I go about that?
NOTE:
I have figured out how to find and load modules that are within a folder using importlib.

First, you must understand how the cmd module works. I would not go into it here but the gist is command is entered, then split up into the actual command itself and its arguments. The command is then searched in the implementation of the cmd module using getattr(). The resulting function returned is then executed. Raises an error if the function(attribute) is not found.
Add the list of modules added to a list.
lst.append(importlib.importlib.import_module(<module path here>))
When cmd is finding a command, modify the code to get it to run through the list of imported modules and see if the function/command exists in that module. If so, execute it.
rough code to find func
def findfunc(funcname):
for module in lst:
if hasattr(<module class where func is stored>, "<funcname>"):
return getattr(<module class where func is stored>,"<funcname>")
However, I will leave my implementation here. It also includes the capability to reload modules(limited to a certain extent) and private commands that do not show up in help or autocomplete(I am using py prompt toolkit). A fair bit of warning, no docs or help in understanding this code as it is only meant for my personal use only. Meaning don't directly copy it and also It was a modification to the python cmd module.
import importlib
import os
import string
import sys
import time
from prompt_toolkit import PromptSession
from prompt_toolkit.completion import WordCompleter
from prompt_toolkit.styles import Style
from internal.logger import logger # my logger Implementation, ignore it
__all__ = ["cli2"]
PROMPT = '>>> '
IDENTCHARS = string.ascii_letters + string.digits + '_'
style = Style.from_dict(
{
"completion-menu.completion": "bg:#008888 #ffffff",
"completion-menu.completion.current": "bg:#00aaaa #000000",
"scrollbar.background": "bg:#88aaaa",
"scrollbar.button": "bg:#222222",
}
)
def parse(arg):
return arg.split()
class Interpreter:
identchars = IDENTCHARS
ruler = '='
lastcmd = ''
doc_leader = ""
doc_header = "Available Commands (type help <topic> to get documentation for <topic> ):"
misc_header = "Miscellaneous help topics:"
nohelp = "*** No help on %s"
use_rawinput = 1
modulelst = []
internalcmdlst = ["help", "exit", "reload", "listmodules", "listvar"]
def __init__(self, intro=None, prompt=PROMPT, stdin=None, stdout=None):
"""Instantiate a line-oriented interpreter framework.
The optional arguments stdin and stdout
specify alternate input and output file objects; if not specified,
sys.stdin and sys.stdout are used.
"""
self.prompt = prompt
self.intro = intro
if stdin is not None:
self.stdin = stdin
else:
self.stdin = sys.stdin
if stdout is not None:
self.stdout = stdout
else:
self.stdout = sys.stdout
self.cmdqueue = []
def cmdloop(self):
self.preloop()
try:
if self.intro:
self.stdout.write(str(self.intro) + "\n")
stop = None
while not stop:
if self.cmdqueue:
line = self.cmdqueue.pop(0)
else:
line = self.session.prompt(self.prompt)
line = self.precmd(line)
stop = self.onecmd(line)
stop = self.postcmd(stop, line)
self.postloop()
finally:
pass
def precmd(self, line):
"""Hook method executed just before the command line is
interpreted, but after the input prompt is generated and issued.
"""
# Parse command to argument
return self.parseline(line)
def postcmd(self, stop, line):
"""Hook method executed just after a command dispatch is finished."""
return stop
def preloop(self):
"""Hook method executed once when the cmdloop() method is called."""
self.autocommands = [item[3:] for item in self.get_names() if (item[-5:] != '__pvt') and (item[:3] == 'do_')]
self.session = PromptSession(completer=WordCompleter(self.autocommands, ignore_case=False))
def postloop(self):
"""Hook method executed once when the cmdloop() method is about to
return.
"""
self.autocommands, self.session = None, None
def parseline(self, line):
"""Parse the line into a command name and a string containing
the arguments. Returns a tuple containing (command, args, line).
'command' and 'args' may be None if the line couldn't be parsed.
"""
line = line.strip()
if not line:
return None, None, line
elif line[0] == '?':
line = 'help ' + line[1:]
elif line[0] == '!':
if hasattr(self, 'do_shell'):
line = 'shell ' + line[1:]
else:
return None, None, line
i, n = 0, len(line)
while i < n and line[i] in self.identchars: i = i + 1
cmd, arg = line[:i], line[i:].strip()
return cmd, arg, line
def onecmd(self, linecommand):
"""Interpret the argument as though it had been typed in response
to the prompt.
This may be overridden, but should not normally need to be;
see the precmd() and postcmd() methods for useful execution hooks.
The return value is a flag indicating whether interpretation of
commands by the interpreter should stop.
"""
cmd, arg, line = linecommand
if not line:
return self.emptyline()
if cmd is None:
return self.default(line)
self.lastcmd = line
if line == 'EOF':
self.lastcmd = ''
if cmd == '':
return self.default(line)
if cmd in self.internalcmdlst:
return self.getfunc(cmd)(arg)
else:
try:
terminalclass, func = self.getfunc(cmd)
except AttributeError:
return self.default(line)
return func(terminalclass, arg)
def emptyline(self):
"""Called when an empty line is entered in response to the prompt.
If this method is not overridden, it repeats the last nonempty
command entered.
"""
if self.lastcmd:
return self.onecmd(self.lastcmd)
def default(self, line):
"""Called on an input line when the command prefix is not recognized.
If this method is not overridden, it prints an error message and
returns.
"""
self.stdout.write('*** Unknown syntax: %s\n' % line.split(" ")[0])
def get_names(self):
# This method used to pull in base class attributes
# at a time dir() didn't do it yet.
lst = [*dir(self.__class__)]
for modulejs in self.modulelst:
terminalclass = getattr(getattr(modulejs["module"], 'cli_helper'), "Terminal")
lst += dir(terminalclass)
return lst
def print_topics(self, header, cmds, maxcol):
if cmds:
self.stdout.write("%s\n" % str(header))
if self.ruler:
self.stdout.write("%s\n" % str(self.ruler * len(header)))
self.columnize(cmds, maxcol - 1)
self.stdout.write("\n")
def columnize(self, lst, displaywidth=80):
"""Display a list of strings as a compact set of columns.
Each column is only as wide as necessary.
Columns are separated by two spaces (one was not legible enough).
"""
if not lst:
self.stdout.write("<empty>\n")
return
nonstrings = [i for i in range(len(lst))
if not isinstance(lst[i], str)]
if nonstrings:
raise TypeError("list[i] not a string for i in %s"
% ", ".join(map(str, nonstrings)))
size = len(lst)
if size == 1:
self.stdout.write('%s\n' % str(lst[0]))
return
# Try every row count from 1 upwards
for nrows in range(1, len(lst)):
ncols = (size + nrows - 1) // nrows
colwidths = []
totwidth = -2
for col in range(ncols):
colwidth = 0
for row in range(nrows):
i = row + nrows * col
if i >= size:
break
x = lst[i]
colwidth = max(colwidth, len(x))
colwidths.append(colwidth)
totwidth += colwidth + 2
if totwidth > displaywidth:
break
if totwidth <= displaywidth:
break
else:
nrows = len(lst)
ncols = 1
colwidths = [0]
for row in range(nrows):
texts = []
for col in range(ncols):
i = row + nrows * col
if i >= size:
x = ""
else:
x = lst[i]
texts.append(x)
while texts and not texts[-1]:
del texts[-1]
for col in range(len(texts)):
texts[col] = texts[col].ljust(colwidths[col])
self.stdout.write("%s\n" % str(" ".join(texts)))
def addmodulesfromdirectory(self, directory):
try:
directory_contents = os.listdir(directory)
for item in directory_contents:
self.addmodule(os.path.join(directory, item), "".join([directory[2:], ".", item]))
except OSError as e:
print("Something Happened\nPlease try again :)")
logger.error("OSErrot, {}".format(e))
except ImportError as e:
print("Something Went Wrong while importing modules\nPlease try again :)")
logger.error("ImportError, {}".format(e))
except Exception as e:
print("Oops, my fault\nPlease try again :)")
logger.error("Exception, {}".format(e))
finally:
logger.info("Done importing cli modules")
def addmodule(self, pathdir, importdir):
if os.path.isdir(pathdir):
importedmodule = importlib.import_module(importdir)
modulejs = {
"module": importedmodule,
"name": importedmodule.__name__,
"importdir": importdir,
"pathdir": pathdir
}
self.modulelst.append(modulejs)
def removemodule(self, modulename):
if modulename in [x["name"] for x in self.modulelst]:
self.modulelst.remove(self.getmodulejs(modulename))
del sys.modules[modulename]
else:
raise ModuleNotFoundError
def replacemodule(self, modulename):
module = self.getmodulejs(modulename)
self.removemodule(modulename)
self.addmodule(module["pathdir"], module["importdir"])
def getmodulejs(self, modulename):
for i in self.modulelst:
if i['name'] == modulename:
return i
#staticmethod
def getmoduledep(modulename):
unsortedscriptsysmodules = [module for module in sys.modules if (("script" in module) and ("script" != module))]
sortedlst = []
for scriptmodules in unsortedscriptsysmodules:
mod = sys.modules[scriptmodules]
if not (hasattr(mod, "__path__") and getattr(mod, '__file__', None) is None) and (
scriptmodules != "script.{}".format(modulename)):
sortedlst.append(sys.modules[scriptmodules])
return sortedlst
def reloadmoduledep(self, modulename):
for dep in self.getmoduledep(modulename):
try:
importlib.reload(dep)
except ModuleNotFoundError as e:
print(e)
def getfunc(self, command):
if "do_" in command:
command = command[3:]
if command in self.internalcmdlst:
return getattr(self, "_".join(['do', command]))
else:
for modulejs in self.modulelst:
terminalclass = getattr(getattr(modulejs["module"], 'cli_helper'), "Terminal")
if hasattr(terminalclass, "_".join(['do', command])):
return terminalclass, getattr(terminalclass, "_".join(['do', command]))
raise AttributeError
def do_help(self, arg):
"""
List available commands with "help" or detailed help with "help cmd".
"""
if arg:
# XXX check arg syntax
try:
func = self.getfunc('help_' + arg)
except AttributeError:
try:
if arg[-5:] == '__pvt':
raise AttributeError
doc = self.getfunc('do_' + arg).__doc__
if doc:
self.stdout.write("%s\n" % str(doc))
return
except AttributeError:
pass
self.stdout.write("%s\n" % str(self.nohelp % (arg,)))
return
func()
else:
names = self.get_names()
cmds_doc = []
funchelp = {}
for name in names:
if name[:5] == 'help_':
funchelp[name[5:]] = 1
names.sort()
# There can be duplicates if routines overridden
prevname = ''
for name in names:
if name[:3] == 'do_' and name[-5:] != '__pvt':
if name == prevname:
continue
prevname = name
cmd = name[3:]
if cmd in funchelp:
cmds_doc.append(cmd)
del funchelp[cmd]
elif self.getfunc(name).__doc__:
cmds_doc.append(cmd)
self.stdout.write("%s\n" % str(self.doc_leader))
self.print_topics(self.doc_header, cmds_doc, 80)
self.print_topics(self.misc_header, list(funchelp.keys()), 80)
def do_exit(self, arg):
"""
Exit
"""
if arg:
print(self.prompt + "No arguments please")
print("Exiting")
time.sleep(1)
return True
def do_reload(self, arg):
if arg == "":
print("No Arguments found")
return
arg = parse(arg)
if arg[0] == "all":
localmodulelst = self.modulelst.copy()
self.modulelst = []
for i in localmodulelst:
print("Reloading", ".".join(i["name"].split(".")[1:]))
importlib.invalidate_caches()
self.reloadmoduledep(i["name"])
self.addmodule(i["pathdir"], i["importdir"])
self.autocommands = [item[3:] for item in self.get_names() if
(item[-5:] != '__pvt') and (item[:3] == 'do_')]
self.session = PromptSession(completer=WordCompleter(self.autocommands, ignore_case=False))
else:
print("Only argument \'all\' is accepted")
def do_listmodules(self, arg):
arg = parse(arg)
if arg[0] == "sys":
for i in sys.modules:
print(i)
elif len(arg) != 0:
print("No Argument Please")
else:
print("Listing all imported Modules")
for i in self.modulelst:
print(".".join(i["module"].__name__.split(".")[1:]))

Python - Formatting strings

I have the input file :
sun vehicle
one number
two number
reduce command
one speed
five speed
zero speed
speed command
kmh command
I used the following code:
from collections import OrderedDict
output = OrderedDict()
with open('final') as in_file:
for line in in_file:
columns = line.split(' ')
if len(columns) >= 2:
word,tag = line.strip().split()
if output.has_key(tag) == False:
output[tag] = [];
output[tag].append(word)
else:
print ""
for k, v in output.items():
print '<{}> {} </{}>'.format(k, ' '.join(v), k)
output = OrderedDict()
I am getting the output as:
<vehicle> sun </vehicle>
<number> one two </number>
<command> reduce speed kmh </command>
<speed> one five zero </speed>
But my expected output should be:
<vehicle> sun </vehicle>
<number> one two </number>
<command> reduce
<speed> one five zero </speed>
speed kmh </command>
Can someone help me in solving this?

It looks like the output you want to achieve is underspecified!
You presumably want the code to "know in advance" that speed is a part of command, before you get to the line speed command.
To do what you want, you will need a recursive function.
How about
for k, v in output.items():
print expandElements(k, v,output)
and somewhere you define
def expandElements(k,v, dic):
out = '<' +k + '>'
for i in v:
# check each item of v for matches in dic.
# if no match, then out=out+i
# otherwise expand using a recursive call of expandElements()
# and out=out+expandElements
out = out + '<' +k + '>'

It looks like you want some kind of tree structure for your output?
You are printing out with print '<{}> {} </{}>'.format(k, ' '.join(v), k) so all of your output is going to have the form of '<{}> {} </{}>'.
If you want to nest things you are going to need a nested structure to represent them.

For recursivly parsing the input file I would make a class representing the tag. Each tag can have its children. Every children is first a string added manually with tag.children.append("value") or by calling tag.add_value(tag.name, "value").
class Tag:
def __init__(self, name, parent=None):
self.name = name
self.children = []
self.has_root = True
self.parent = parent
def __str__(self):
""" compose string for this tag (recursivly) """
if not self.children:
return self.name
children_str = ' '.join([str(child) for child in self.children])
if not self.parent:
return children_str
return '<%s>%s</%s>' % (self.name, children_str, self.name)
#classmethod
def from_file(cls, file):
""" create root tag from file """
obj = cls('root')
columns = []
with open(file) as in_file:
for line in in_file:
value, tag = line.strip().split(' ')
obj.add_tag(tag, value)
return obj
def search_tag(self, tag):
""" search for a tag in the children """
if self.name == tag:
return self
for i, c in enumerate(self.children):
if isinstance(c, Tag) and c.name == tag:
return c
elif isinstance(c, str):
if c.strip() == tag.strip():
self.children[i] = Tag(tag, self)
return self.children[i]
else:
result = c.search_tag(tag)
if result:
return result
def add_tag(self, tag, value):
"""
add a value, tag pair to the children
Firstly this searches if the value is an child. If this is the
case it moves the children to the new location
Afterwards it searches the tag in the children. When found
the value is added to this tag. If not a new tag object
is created and added to this Tag. The flag has_root
is set to False so the element can be moved later.
"""
value_tag = self.search_tag(value)
if value_tag and not value_tag.has_root:
print("Found value: %s" % value)
if value_tag.parent:
i = value_tag.parent.children.index(value_tag)
value = value_tag.parent.children.pop(i)
value.has_root = True
else:
print("not %s" % value)
found = self.search_tag(tag)
if found:
found.children.append(value)
else:
# no root
tag_obj = Tag(tag, self)
self.children.append(tag_obj)
tag_obj.add_tag(tag, value)
tag_obj.has_root = False
tags = Tag.from_file('final')
print(tags)
I know in this example the speed-Tag is not added twice. I hope that's ok.
Sorry for the long code.

Python Code List Defined

This is part of my python code. I keep having a problem with my lists and it says it is not defined. Can you help or explain to me how to fix that?
import maya.cmds as cmds
def changeXtransformVal(percentage=1.0, myList = myList):
"""
Changes the value of each transform in the scene by a percentange.
Parameters:
percentage - Percentange to change each transform's value. Default value is 1.
Returns:
Nothing.
"""
# The ls command is the list command. It is used to list various nodes
# in the current scene. You can also use it to list selected nodes.
transformInScene = cmds.ls(type='transform')
found = False
for thisTransform in transformInScene:
if thisTransform not in ['front','persp','side','top']:
found = True
break
else:
found = False
if found == False:
sphere1 = cmds.polySphere()[0]
cmds.xform(sphere1, t = (0.5, 0.5, 0.5))
transformInScene = cmds.ls(type='transform')
# If there are no transforms in the scene, there is no point running this script
if not transformInScene:
raise RuntimeError, 'There are no transforms in the scene!'
badAttrs = list('front','persp','side','top')
# Loop through each transform
for thisTransform in transformInScene:
if thisTransform not in ['front','persp','side','top']:
allAttrs = cmds.listAttr(thisTransform, keyable=True, scalar=True)
allAttrs = [ i for i in badAttrs if i != "visibility" ]
print allAttrs
for attr in myList:
if attr in allAttrs:
currentVal = cmds.getAttr( thisTransform + "." + attr )
newVal = currentVal * percentage
cmds.setAttr(thisTransform + "." + attr, newval)
print "Changed %s. %s from %s to %s" % (thisTransform,attr,currentVal,newVal)
else:
badAttrs.append(attr)
if badAttrs:
print "These attributes %s are not valid" % str()
myList = ("translateX", "translateY", "translateZ", "scaleX" )
changeXtransformVal(percentage=2.0, myList = myList)

You are giving your function default parameters:
def changeXtransformVal(percentage=1.0, myList = myList):
These are evaluated when the function object is created, but at that time you have no myList object yet. You don't have to give these parameters default values, and you don't appear to use the values you do give.
If you don't need default values, use:
def changeXtransformVal(percentage, myList):
If you do need a default value for percentage, move the parameter to the end:
def changeXtransformVal(myList, percentage=1.0):
and adjust your function call accordingly:
changeXtransformVal(myList, percentage=2.0)
Note that you don't have to name parameters either; positional parameters work by position, and in this case percentage= is optional too.

What is the pythonic way to implement a parse tree for custom format?

I have a project that has a non-standard file format something like:
var foo = 5
load 'filename.txt'
var bar = 6
list baz = [1, 2, 3, 4]
And I want to parse this into a data structure much like BeautifulSoup does. But this format isn't supported by BeautifulSoup. What is the pythonic way to build a parse tree so that I can modify the values and re-write it out? In the end I would like to do something like:
data = parse_file('file.txt')
data.foo = data.foo * 2
data.write_file('file_new.txt')

Here is a solution using pyparsing... it works in your case. Beware that i'm not an expert therefore depending on your standards the code could be ugly... cheers
class ConfigFile (dict):
"""
Configuration file data
"""
def __init__ (self, filename):
"""
Parses config file.
"""
from pyparsing import Suppress, Word, alphas, alphanums, nums, \
delimitedList, restOfLine, printables, ZeroOrMore, Group, \
Combine
equal = Suppress ("=")
lbrack = Suppress ("[")
rbrack = Suppress ("]")
delim = Suppress ("'")
string = Word (printables, excludeChars = "'")
identifier = Word (alphas, alphanums + '_')
integer = Word (nums).setParseAction (lambda t: int (t[0]))
real = Combine( Word(nums) + '.' + Word(nums) ).setParseAction (lambda t: float(t[0]))
value = real | integer
var_kwd = Suppress ("var")
load_kwd = Suppress ("load")
list_kwd = Suppress ("list")
var_stm = Group (var_kwd + identifier + equal + value +
restOfLine.suppress ()).setParseAction (
lambda tok: tok[0].insert(len(tok[0]), 0))
load_stm = Group (load_kwd + delim + string + delim +
restOfLine.suppress ()).setParseAction (
lambda tok: tok[0].insert(len(tok[0]), 1))
list_stm = Group (list_kwd + identifier + equal + lbrack +
Group ( delimitedList (value, ",") ) +
rbrack + restOfLine.suppress ()).setParseAction (
lambda tok: tok[0].insert(len(tok[0]), 2))
cnf_file = ZeroOrMore (var_stm | load_stm | list_stm)
lines = cnf_file.parseFile (filename)
self._lines = []
for line in lines:
self._lines.append ((line[-1], line[0]))
if line[-1] != 1: dict.__setitem__(self, line[0], line[1])
self.__initialized = True
# after initialisation, setting attributes is the same as setting an item
def __getattr__ (self, key):
try:
return dict.__getitem__ (self, key)
except KeyError:
return None
def __setattr__ (self, key, value):
"""Maps attributes to values. Only if we are initialised"""
# this test allows attributes to be set in the __init__ method
if not self.__dict__.has_key ('_ConfigFile__initialized'):
return dict.__setattr__(self, key, value)
# any normal attributes are handled normally
elif self.__dict__.has_key (key):
dict.__setattr__(self, key, value)
# takes care of including new 'load' statements
elif key == 'load':
if not isinstance (value, str):
raise ValueError, "Invalid data type"
self._lines.append ((1, value))
# this is called when setting new attributes after __init__
else:
if not isinstance (value, int) and \
not isinstance (value, float) and \
not isinstance (value, list):
raise ValueError, "Invalid data type"
if dict.has_key (self, key):
if type(dict.__getitem__(self, key)) != type (value):
raise ValueError, "Cannot modify data type."
elif not isinstance (value, list): self._lines.append ((0, key))
else: self._lines.append ((2, key))
dict.__setitem__(self, key, value)
def Write (self, filename):
"""
Write config file.
"""
fid = open (filename, 'w')
for d in self._lines:
if d[0] == 0: fid.write ("var %s = %s\n" % (d[1], str(dict.__getitem__(self, d[1]))))
elif d[0] == 1: fid.write ("file '%s'\n" % (d[1]))
else: fid.write ("list %s = %s\n" % (d[1], str(dict.__getitem__(self, d[1]))))
if __name__ == "__main__":
input="""var foo = 5
load 'filename.txt'
var bar = 6
list baz = [1, 2, 3, 4]"""
file ("test.txt", 'w').write (input)
config = ConfigFile ("test.txt")
# Modify existent items
config.foo = config.foo * 2
# Add new items
config.foo2 = [4,5,6,7]
config.foo3 = 12.3456
config.load = 'filenameX.txt'
config.load = 'filenameXX.txt'
config.Write ("test_new.txt")
EDIT
I have modified the class to use
__getitem__, __setitem__
methods to mimic the 'access to member' syntax to parsed items as required by our poster. Enjoy!
PS
Overloading of the
__setitem__
method should be done with care to avoid interferences between setting of 'normal' attributes (class members) and the parsed items (that are accesses like attributes). The code is now fixed to avoid these problems. See the following reference http://code.activestate.com/recipes/389916/ for more info. It was funny to discover this!

What you have is a custom language you need to parse.
Use one of the many existing parsing libraries for Python. I personally recommend PLY. Alternatively, Pyparsing is also good and widely used & supported.
If your language is relatively simple, you can also implement a hand-written parser. Here is an example

refactor this dictionary-to-xml converter in python

It's a small thing, really: I have this function that converts dict objects to xml.
Here's the function:
def dictToXml(d):
from xml.sax.saxutils import escape
def unicodify(o):
if o is None:
return u'';
return unicode(o)
lines = []
def addDict(node, offset):
for name, value in node.iteritems():
if isinstance(value, dict):
lines.append(offset + u"<%s>" % name)
addDict(value, offset + u" " * 4)
lines.append(offset + u"</%s>" % name)
elif isinstance(value, list):
for item in value:
if isinstance(item, dict):
lines.append(offset + u"<%s>" % name)
addDict(item, offset + u" " * 4)
lines.append(offset + u"</%s>" % name)
else:
lines.append(offset + u"<%s>%s</%s>" % (name, escape(unicodify(item)), name))
else:
lines.append(offset + u"<%s>%s</%s>" % (name, escape(unicodify(value)), name))
addDict(d, u"")
lines.append(u"")
return u"\n".join(lines)
For example, it converts this dictionary
{ 'site': { 'name': 'stackoverflow', 'blogger': [ 'Jeff', 'Joel' ] } }
to:
<site>
<name>stackoverflow</name>
<blogger>jeff</blogger>
<blogger>joel</blogger>
</site>
It works, but the addDict function looks a little too repetitive. I'm sure there's a way to refactor it into 3 co-recursive functions named addDict, addList and addElse, but my brain is stuck. Any help?
Also, any way to get rid of the offset + thing in every line would be nice.
NOTE: I chose these semantics because I'm trying to match the behavior of the json-to-xml converter in org.json, which I use in a different part of my project. If you got to this page just looking for a dictionary to xml converter, there are some really good options in some of the answers. (Especially pyfo).

>>> from pyfo import pyfo
>>> d = ('site', { 'name': 'stackoverflow', 'blogger': [ 'Jeff', 'Joel' ] } )
>>> result = pyfo(d, pretty=True, prolog=True, encoding='ascii')
>>> print result.encode('ascii', 'xmlcharrefreplace')
<?xml version="1.0" encoding="ascii"?>
<site>
<blogger>
Jeff
Joel
</blogger>
<name>stackoverflow</name>
</site>
To install pyfo:
$ easy_install pyfo

I noticed you have commonality in adding items. Using this commonality I would refactor adding an item to a separate function.
def addItem(item, name, offset):
if isinstance(item, dict):
lines.append(offset + u"<%s>" % name)
addDict(item, offset + u" " * 4)
lines.append(offset + u"</%s>" % name)
else:
lines.append(offset + u"<%s>%s</%s>" % (name, escape(unicodify(item)), name))
def addList(value,name, offset):
for item in value:
addItem(item, name, offset)
def addDict(node, offset):
for name, value in node.iteritems():
if isinstance(value, list):
addList(value, name, offset)
else:
addItem(value, name, offset)
Advisory warning: this code is not tested or written by anybody who actually uses Python.

To get rid of repeated "offset+":
offset = 0
def addLine(str):
lines.append(u" " * (offset * 4) + str
then
...
addLine(u"<%s>" % name)
offset = offset + 1
addDict(value)
offset = offset - 1
addLine(u"</%s>" % name)
Don't have access to an interpreter here, so take this with a grain of salt :(

Your original code produce malformed XML and can produce the same XML for two different dictionaries (is not injective, speaking mathematically).
For example, if you have a list as a value of the only key in a dictionary:
d = { 'list': [1,2,3] }
I expect that your code would produce
<list>1</list><list>2</list><list>3</list>
and there is no root element. Any XML should have one and only one root element.
Then given the XML produced by your code, it is impossible to say if this XML
<tag>1</tag>
was produced from { 'tag': 1 } or from { 'tag': [1] }.
So, I suggest
always start from the root element
represent lists with either two special tags (e.g. <list/> and <item/>) or mark them as such in attributes
Then, after decisions about these conceptual shortcomings we can generate correct and unambiguous XML. I chose to use attributes to markup lists, and used ElementTree to construct the XML tree automatically. Also, recursion helps (add_value_to_xml is called recursively):
from xml.etree.ElementTree import Element, SubElement, tostring
def is_scalar(v):
return isinstance(v,basestring) or isinstance(v,float) \
or isinstance(v,int) or isinstance(v,bool)
def add_value_to_xml(root,v):
if type(v) == type({}):
for k,kv in v.iteritems():
vx = SubElement(root,unicode(k))
vx = add_value_to_xml(vx,kv)
elif type(v) == list:
root.set('type','list')
for e in v:
li = SubElement(root,root.tag)
li = add_value_to_xml(li,e)
li.set('type','item')
elif is_scalar(v):
root.text = unicode(v)
else:
raise Exception("add_value_to_xml: unsuppoted type (%s)"%type(v))
return root
def dict_to_xml(d,root='dict'):
x = Element(root)
x = add_value_to_xml(x,d)
return x
d = { 'float': 5194.177, 'str': 'eggs', 'int': 42,
'list': [1,2], 'dict': { 'recursion': True } }
x = dict_to_xml(d)
print tostring(x)
The result of the conversion of the test dict is:
<dict><int>42</int><dict><recursion>True</recursion></dict><float>5194.177</float><list type="list"><list type="item">1</list><list type="item">2</list></list><str>eggs</str></dict>

Here is my short sketch for a solution:
have a general addSomething() function that dispatches based on the type of the value to addDict(), addList() or addElse(). Those functions recursively call addSomething() again.
Basically you are factoring out the parts in the if clause and add a recursive call.

Here's what I find helpful when working with XML. Actually create the XML node structure first, then render this into text second.
This separates two unrelated concerns.
How do I transform my Python structure into an XML object model?
How to I format that XML object model?
It's hard when you put these two things together into one function. If, on the other hand, you separate them, then you have two things. First, you have a considerably simpler function to "walk" your Python structure and return an XML node. Your XML Nodes can be rendered into text with some preferred encoding and formatting rules applied.
from xml.sax.saxutils import escape
class Node( object ):
def __init__( self, name, *children ):
self.name= name
self.children= children
def toXml( self, indent ):
if len(self.children) == 0:
return u"%s<%s/>" % ( indent*4*u' ', self.name )
elif len(self.children) == 1:
child= self.children[0].toXml(0)
return u"%s<%s>%s</%s>" % ( indent*4*u' ', self.name, child, self.name )
else:
items = [ u"%s<%s>" % ( indent*4*u' ', self.name ) ]
items.extend( [ c.toXml(indent+1) for c in self.children ] )
items.append( u"%s</%s>" % ( indent*4*u' ', self.name ) )
return u"\n".join( items )
class Text( Node ):
def __init__( self, value ):
self.value= value
def toXml( self, indent ):
def unicodify(o):
if o is None:
return u'';
return unicode(o)
return "%s%s" % ( indent*4*u' ', escape( unicodify(self.value) ), )
def dictToXml(d):
def dictToNodeList(node):
nodes= []
for name, value in node.iteritems():
if isinstance(value, dict):
n= Node( name, *dictToNodeList( value ) )
nodes.append( n )
elif isinstance(value, list):
for item in value:
if isinstance(item, dict):
n= Node( name, *dictToNodeList( value ) )
nodes.append( n )
else:
n= Node( name, Text( item ) )
nodes.append( n )
else:
n= Node( name, Text( value ) )
nodes.append( n )
return nodes
return u"\n".join( [ n.toXml(0) for n in dictToNodeList(d) ] )