I'm looking for an XML to dictionary parser using ElementTree, I already found some but they are excluding the attributes, and in my case I have a lot of attributes.
34
votes
10 Answers
30
votes
def etree_to_dict(t):
d = {t.tag : map(etree_to_dict, t.iterchildren())}
d.update(('@' + k, v) for k, v in t.attrib.iteritems())
d['text'] = t.text
return d
Call as
tree = etree.parse("some_file.xml")
etree_to_dict(tree.getroot())
This works as long as you don't actually have an attribute text; if you do, then change the third line in the function body to use a different key. Also, you can't handle mixed content with this.
(Tested on LXML.)
50
votes
The following XML-to-Python-dict snippet parses entities as well as attributes following this XML-to-JSON "specification":
from collections import defaultdict
def etree_to_dict(t):
d = {t.tag: {} if t.attrib else None}
children = list(t)
if children:
dd = defaultdict(list)
for dc in map(etree_to_dict, children):
for k, v in dc.items():
dd[k].append(v)
d = {t.tag: {k: v[0] if len(v) == 1 else v
for k, v in dd.items()}}
if t.attrib:
d[t.tag].update(('@' + k, v)
for k, v in t.attrib.items())
if t.text:
text = t.text.strip()
if children or t.attrib:
if text:
d[t.tag]['#text'] = text
else:
d[t.tag] = text
return d
It is used:
from xml.etree import cElementTree as ET
e = ET.XML('''
<root>
<e />
<e>text</e>
<e name="value" />
<e name="value">text</e>
<e> <a>text</a> <b>text</b> </e>
<e> <a>text</a> <a>text</a> </e>
<e> text <a>text</a> </e>
</root>
''')
from pprint import pprint
d = etree_to_dict(e)
pprint(d)
The output of this example (as per above-linked "specification") should be:
{'root': {'e': [None,
'text',
{'@name': 'value'},
{'#text': 'text', '@name': 'value'},
{'a': 'text', 'b': 'text'},
{'a': ['text', 'text']},
{'#text': 'text', 'a': 'text'}]}}
Not necessarily pretty, but it is unambiguous, and simpler XML inputs result in simpler JSON. :)
Update
If you want to do the reverse, emit an XML string from a JSON/dict, you can use:
try:
basestring
except NameError: # python3
basestring = str
def dict_to_etree(d):
def _to_etree(d, root):
if not d:
pass
elif isinstance(d, str):
root.text = d
elif isinstance(d, dict):
for k,v in d.items():
assert isinstance(k, str)
if k.startswith('#'):
assert k == '#text' and isinstance(v, str)
root.text = v
elif k.startswith('@'):
assert isinstance(v, str)
root.set(k[1:], v)
elif isinstance(v, list):
for e in v:
_to_etree(e, ET.SubElement(root, k))
else:
_to_etree(v, ET.SubElement(root, k))
else:
assert d == 'invalid type', (type(d), d)
assert isinstance(d, dict) and len(d) == 1
tag, body = next(iter(d.items()))
node = ET.Element(tag)
_to_etree(body, node)
return node
print(ET.tostring(dict_to_etree(d)))
3
votes
1
votes
Here is a simple data structure in xml (save as file.xml):
<?xml version="1.0" encoding="UTF-8"?>
<Data>
<Person>
<First>John</First>
<Last>Smith</Last>
</Person>
<Person>
<First>Jane</First>
<Last>Doe</Last>
</Person>
</Data>
Here is the code to create a list of dictionary objects from it.
from lxml import etree
tree = etree.parse('file.xml')
root = tree.getroot()
datadict = []
for item in root:
d = {}
for elem in item:
d[elem.tag]=elem.text
datadict.append(d)
datadict now contains:
[{'First': 'John', 'Last': 'Smith'},{'First': 'Jane', 'Last': 'Doe'}]
and can be accessed like so:
datadict[0]['First']
'John'
datadict[1]['Last']
'Doe'
1
votes
You can use this snippet that directly converts it from xml to dictionary
import xml.etree.ElementTree as ET
xml = ('<xml>' +
'<first_name>Dean Christian</first_name>' +
'<middle_name>Christian</middle_name>' +
'<last_name>Armada</last_name>' +
'</xml>')
root = ET.fromstring(xml)
x = {x.tag: root.find(x.tag).text for x in root._children}
# returns {'first_name': 'Dean Christian', 'last_name': 'Armada', 'middle_name': 'Christian'}
1
votes
For transforming XML from/to python dictionaries, xmltodict has worked great for me:
import xmltodict
xml = '''
<root>
<e />
<e>text</e>
<e name="value" />
<e name="value">text</e>
<e> <a>text</a> <b>text</b> </e>
<e> <a>text</a> <a>text</a> </e>
<e> text <a>text</a> </e>
</root>
'''
xdict = xmltodict.parse(xml)
xdict will now look like
OrderedDict([('root',
OrderedDict([('e',
[None,
'text',
OrderedDict([('@name', 'value')]),
OrderedDict([('@name', 'value'),
('#text', 'text')]),
OrderedDict([('a', 'text'), ('b', 'text')]),
OrderedDict([('a', ['text', 'text'])]),
OrderedDict([('a', 'text'),
('#text', 'text')])])]))])
If your XML data is not in raw string/bytes form but in some ElementTree object, you just need to print it out as a string and use xmldict.parse again. For instance, if you are using lxml to process the XML documents, then
from lxml import etree
e = etree.XML(xml)
xmltodict.parse(etree.tostring(e))
will produce the same dictionary as above.
1
votes
The lxml documentation brings an example of how to map an XML tree into a dict of dicts:
def recursive_dict(element):
return element.tag, dict(map(recursive_dict, element)) or element.text
Note that this beautiful quick-and-dirty converter expects children to have unique tag names and will silently overwrite any data that was contained in preceding siblings with the same name. For any real-world application of xml-to-dict conversion, you would better write your own, longer version of this.
You could create a custom dictionary to deal with preceding siblings with the same name being overwritten:
from collections import UserDict, namedtuple
from lxml.etree import QName
class XmlDict(UserDict):
"""Custom dict to avoid preceding siblings with the same name being overwritten."""
__ROOTELM = namedtuple('RootElm', ['tag', 'node'])
def __setitem__(self, key, value):
if key in self:
if type(self.data[key]) is list:
self.data[key].append(value)
else:
self.data[key] = [self.data[key], value]
else:
self.data[key] = value
@staticmethod
def xml2dict(element):
"""Converts an ElementTree Element to a dictionary."""
elm = XmlDict.__ROOTELM(
tag=QName(element).localname,
node=XmlDict(map(XmlDict.xml2dict, element)) or element.text,
)
return elm
Usage
from lxml import etree
from pprint import pprint
xml_f = b"""<?xml version="1.0" encoding="UTF-8"?>
<Data>
<Person>
<First>John</First>
<Last>Smith</Last>
</Person>
<Person>
<First>Jane</First>
<Last>Doe</Last>
</Person>
</Data>"""
elm = etree.fromstring(xml_f)
d = XmlDict.xml2dict(elm)
Output
In [3]: pprint(d)
RootElm(tag='Data', node={'Person': [{'First': 'John', 'Last': 'Smith'}, {'First': 'Jane', 'Last': 'Doe'}]})
In [4]: pprint(d.node)
{'Person': [{'First': 'John', 'Last': 'Smith'},
{'First': 'Jane', 'Last': 'Doe'}]}
0
votes
from lxml import etree, objectify
def formatXML(parent):
"""
Recursive operation which returns a tree formated
as dicts and lists.
Decision to add a list is to find the 'List' word
in the actual parent tag.
"""
ret = {}
if parent.items(): ret.update(dict(parent.items()))
if parent.text: ret['__content__'] = parent.text
if ('List' in parent.tag):
ret['__list__'] = []
for element in parent:
ret['__list__'].append(formatXML(element))
else:
for element in parent:
ret[element.tag] = formatXML(element)
return ret
0
votes
Building on @larsmans, if the resulting keys contain xml namespace info, you can remove that before writing to the dict. Set a variable xmlns equal to the namespace and strip its value out.
xmlns = '{http://foo.namespaceinfo.com}'
def etree_to_dict(t):
if xmlns in t.tag:
t.tag = t.tag.lstrip(xmlns)
if d = {t.tag : map(etree_to_dict, t.iterchildren())}
d.update(('@' + k, v) for k, v in t.attrib.iteritems())
d['text'] = t.text
return d
