I'm looking for an XML to dictionary parser using ElementTree, I already found some but they are excluding the attributes, and in my case I have a lot of attributes.
The following XML-to-Python-dict snippet parses entities as well as attributes following this XML-to-JSON "specification":
from collections import defaultdict
def etree_to_dict(t):
d = {t.tag: {} if t.attrib else None}
children = list(t)
if children:
dd = defaultdict(list)
for dc in map(etree_to_dict, children):
for k, v in dc.items():
dd[k].append(v)
d = {t.tag: {k: v[0] if len(v) == 1 else v
for k, v in dd.items()}}
if t.attrib:
d[t.tag].update(('#' + k, v)
for k, v in t.attrib.items())
if t.text:
text = t.text.strip()
if children or t.attrib:
if text:
d[t.tag]['#text'] = text
else:
d[t.tag] = text
return d
It is used:
from xml.etree import cElementTree as ET
e = ET.XML('''
<root>
<e />
<e>text</e>
<e name="value" />
<e name="value">text</e>
<e> <a>text</a> <b>text</b> </e>
<e> <a>text</a> <a>text</a> </e>
<e> text <a>text</a> </e>
</root>
''')
from pprint import pprint
d = etree_to_dict(e)
pprint(d)
The output of this example (as per above-linked "specification") should be:
{'root': {'e': [None,
'text',
{'#name': 'value'},
{'#text': 'text', '#name': 'value'},
{'a': 'text', 'b': 'text'},
{'a': ['text', 'text']},
{'#text': 'text', 'a': 'text'}]}}
Not necessarily pretty, but it is unambiguous, and simpler XML inputs result in simpler JSON. :)
Update
If you want to do the reverse, emit an XML string from a JSON/dict, you can use:
try:
basestring
except NameError: # python3
basestring = str
def dict_to_etree(d):
def _to_etree(d, root):
if not d:
pass
elif isinstance(d, str):
root.text = d
elif isinstance(d, dict):
for k,v in d.items():
assert isinstance(k, str)
if k.startswith('#'):
assert k == '#text' and isinstance(v, str)
root.text = v
elif k.startswith('#'):
assert isinstance(v, str)
root.set(k[1:], v)
elif isinstance(v, list):
for e in v:
_to_etree(e, ET.SubElement(root, k))
else:
_to_etree(v, ET.SubElement(root, k))
else:
assert d == 'invalid type', (type(d), d)
assert isinstance(d, dict) and len(d) == 1
tag, body = next(iter(d.items()))
node = ET.Element(tag)
_to_etree(body, node)
return node
print(ET.tostring(dict_to_etree(d)))
def etree_to_dict(t):
d = {t.tag : map(etree_to_dict, t.iterchildren())}
d.update(('#' + k, v) for k, v in t.attrib.iteritems())
d['text'] = t.text
return d
Call as
tree = etree.parse("some_file.xml")
etree_to_dict(tree.getroot())
This works as long as you don't actually have an attribute text; if you do, then change the third line in the function body to use a different key. Also, you can't handle mixed content with this.
(Tested on LXML.)
For transforming XML from/to python dictionaries, xmltodict has worked great for me:
import xmltodict
xml = '''
<root>
<e />
<e>text</e>
<e name="value" />
<e name="value">text</e>
<e> <a>text</a> <b>text</b> </e>
<e> <a>text</a> <a>text</a> </e>
<e> text <a>text</a> </e>
</root>
'''
xdict = xmltodict.parse(xml)
xdict will now look like
OrderedDict([('root',
OrderedDict([('e',
[None,
'text',
OrderedDict([('#name', 'value')]),
OrderedDict([('#name', 'value'),
('#text', 'text')]),
OrderedDict([('a', 'text'), ('b', 'text')]),
OrderedDict([('a', ['text', 'text'])]),
OrderedDict([('a', 'text'),
('#text', 'text')])])]))])
If your XML data is not in raw string/bytes form but in some ElementTree object, you just need to print it out as a string and use xmldict.parse again. For instance, if you are using lxml to process the XML documents, then
from lxml import etree
e = etree.XML(xml)
xmltodict.parse(etree.tostring(e))
will produce the same dictionary as above.
Based on #larsmans, if you don't need attributes, this will give you a tighter dictionary --
def etree_to_dict(t):
return {t.tag : map(etree_to_dict, t.iterchildren()) or t.text}
Several answers already, but here's one compact solution that maps attributes, text value and children using dict-comprehension:
def etree_to_dict(t):
if type(t) is ET.ElementTree: return etree_to_dict(t.getroot())
return {
**t.attrib,
'text': t.text,
**{e.tag: etree_to_dict(e) for e in t}
}
The lxml documentation brings an example of how to map an XML tree into a dict of dicts:
def recursive_dict(element):
return element.tag, dict(map(recursive_dict, element)) or element.text
Note that this beautiful quick-and-dirty converter expects children to have unique tag names and will silently overwrite any data that was contained in preceding siblings with the same name. For any real-world application of xml-to-dict conversion, you would better write your own, longer version of this.
You could create a custom dictionary to deal with preceding siblings with the same name being overwritten:
from collections import UserDict, namedtuple
from lxml.etree import QName
class XmlDict(UserDict):
"""Custom dict to avoid preceding siblings with the same name being overwritten."""
__ROOTELM = namedtuple('RootElm', ['tag', 'node'])
def __setitem__(self, key, value):
if key in self:
if type(self.data[key]) is list:
self.data[key].append(value)
else:
self.data[key] = [self.data[key], value]
else:
self.data[key] = value
#staticmethod
def xml2dict(element):
"""Converts an ElementTree Element to a dictionary."""
elm = XmlDict.__ROOTELM(
tag=QName(element).localname,
node=XmlDict(map(XmlDict.xml2dict, element)) or element.text,
)
return elm
Usage
from lxml import etree
from pprint import pprint
xml_f = b"""<?xml version="1.0" encoding="UTF-8"?>
<Data>
<Person>
<First>John</First>
<Last>Smith</Last>
</Person>
<Person>
<First>Jane</First>
<Last>Doe</Last>
</Person>
</Data>"""
elm = etree.fromstring(xml_f)
d = XmlDict.xml2dict(elm)
Output
In [3]: pprint(d)
RootElm(tag='Data', node={'Person': [{'First': 'John', 'Last': 'Smith'}, {'First': 'Jane', 'Last': 'Doe'}]})
In [4]: pprint(d.node)
{'Person': [{'First': 'John', 'Last': 'Smith'},
{'First': 'Jane', 'Last': 'Doe'}]}
Here is a simple data structure in xml (save as file.xml):
<?xml version="1.0" encoding="UTF-8"?>
<Data>
<Person>
<First>John</First>
<Last>Smith</Last>
</Person>
<Person>
<First>Jane</First>
<Last>Doe</Last>
</Person>
</Data>
Here is the code to create a list of dictionary objects from it.
from lxml import etree
tree = etree.parse('file.xml')
root = tree.getroot()
datadict = []
for item in root:
d = {}
for elem in item:
d[elem.tag]=elem.text
datadict.append(d)
datadict now contains:
[{'First': 'John', 'Last': 'Smith'},{'First': 'Jane', 'Last': 'Doe'}]
and can be accessed like so:
datadict[0]['First']
'John'
datadict[1]['Last']
'Doe'
You can use this snippet that directly converts it from xml to dictionary
import xml.etree.ElementTree as ET
xml = ('<xml>' +
'<first_name>Dean Christian</first_name>' +
'<middle_name>Christian</middle_name>' +
'<last_name>Armada</last_name>' +
'</xml>')
root = ET.fromstring(xml)
x = {x.tag: root.find(x.tag).text for x in root._children}
# returns {'first_name': 'Dean Christian', 'last_name': 'Armada', 'middle_name': 'Christian'}
enhanced the accepted answer with python3 and use json list when all children have the same tag. Also provided an option whether to wrap the dict with root tag or not.
from collections import OrderedDict
from typing import Union
from xml.etree.ElementTree import ElementTree, Element
def etree_to_dict(root: Union[ElementTree, Element], include_root_tag=False):
root = root.getroot() if isinstance(root, ElementTree) else root
result = OrderedDict()
if len(root) > 1 and len({child.tag for child in root}) == 1:
result[next(iter(root)).tag] = [etree_to_dict(child) for child in root]
else:
for child in root:
result[child.tag] = etree_to_dict(child) if len(list(child)) > 0 else (child.text or "")
result.update(('#' + k, v) for k, v in root.attrib.items())
return {root.tag: result} if include_root_tag else result
d = etree_to_dict(etree.ElementTree.parse('data.xml'), True)
from lxml import etree, objectify
def formatXML(parent):
"""
Recursive operation which returns a tree formated
as dicts and lists.
Decision to add a list is to find the 'List' word
in the actual parent tag.
"""
ret = {}
if parent.items(): ret.update(dict(parent.items()))
if parent.text: ret['__content__'] = parent.text
if ('List' in parent.tag):
ret['__list__'] = []
for element in parent:
ret['__list__'].append(formatXML(element))
else:
for element in parent:
ret[element.tag] = formatXML(element)
return ret
Building on #larsmans, if the resulting keys contain xml namespace info, you can remove that before writing to the dict. Set a variable xmlns equal to the namespace and strip its value out.
xmlns = '{http://foo.namespaceinfo.com}'
def etree_to_dict(t):
if xmlns in t.tag:
t.tag = t.tag.lstrip(xmlns)
if d = {t.tag : map(etree_to_dict, t.iterchildren())}
d.update(('#' + k, v) for k, v in t.attrib.iteritems())
d['text'] = t.text
return d
If you have a schema, the xmlschema package already implements multiple XML-to-dict converters that honor the schema and attribute types. Quoting the following from the docs
Available converters
The library includes some converters. The default converter
xmlschema.XMLSchemaConverter is the base class of other converter
types. Each derived converter type implements a well know convention,
related to the conversion from XML to JSON data format:
xmlschema.ParkerConverter: Parker convention
xmlschema.BadgerFishConverter: BadgerFish convention
xmlschema.AbderaConverter: Apache Abdera project convention
xmlschema.JsonMLConverter: JsonML (JSON Mark-up Language) convention
Documentation of these different conventions is available here: http://wiki.open311.org/JSON_and_XML_Conversion/
Usage of the converters is straightforward, e.g.:
from xmlschema import ParkerConverter, XMLSchema, to_dict
xml = '...'
schema = XMLSchema('...')
to_dict(xml, schema=schema, converter=ParkerConverter)
Related
There is a jsx file with contents
<import name="abcd" color="green" age="25" />
<View color={dsdssd}>
<IBG
color={[color.imagecolor, color.image125]}
imageStyle={[styles.imageStyle, styles.image125]}
source={{ uri: contents.aimeecard }} >
<View color={styles.titleContainer}>
<Text color={[{green: 45}, styles.mainTileText]}</Text>
<View color={[abcde.text]} />
</View>
</View>
I need to fetch the details of first line using python script:
Expected output
name="abcd"
color="green"
age="25"
Also the path of jsx file is passed through list
ex: [abcd/file1.jsx , dcef/file2.jsx]
Python code tried for fetching jsx file through the list
for file in jsx_path:
data = md.parse("file")
print( file.firstChild.tagName )
Values are not fetched and getting error.
Can anyone help me in resolving this?
Assuming jsx_path is the list containing all the paths to the jsx files, you can iterate over each and use a context manager to avoid closing explicitly the files like so:
data = ""
for file in jsx_path:
with open(file) as f:
data += f.readline()[8:-4] + "\n"
print(data) # name="abcd" color="green" age="25"
Following your comment, if you want to output it as a dict, you can tweak the previous code:
import re
data = []
for file in jsx_path:
with open(file) as f:
data.append(re.split('\W+|=', f.readline()[8:-4]))
data_dict = []
for d in data:
data_dict.append({key:value for (key, value) in zip(d[::2], d[1::2])})
print(data_dict) # {'name': 'abcd', 'color': 'green', 'age': '25'}
Note that this is a hack. I only read the JSX file sequentially because your use case is simple enough to do so. You can also use a dedicated parser by extending the stlib class HTMLParser:
from html.parser import HTMLParser
class JSXImportParser(HTMLParser):
def handle_starttag(self, tag, attrs):
if tag == "import":
self._import_attrs = {key:value for (key, value) in attrs}
#property
def import_attrs(self):
return self._import_attrs
parser = JSXImportParser()
data = []
for file in jsx_path:
with open(file) as f:
parser.feed(f.read())
data.append(parser.import_attrs)
print(data) # [{'name': 'abcd', 'color': 'green', 'age': '25'}]
Note that this only extracts the details of the last import tag in each file, you can alter this behavior by tweaking the _import_attrs class attribute.
Edit: Following your additional comment about the requirement to use an XML parser library, the same thing can be achieved using ElementTree by sampling the file to extract only what's interesting for you (the import tag):
import xml.etree.ElementTree as ET
data = []
for file in jsx_path:
with open(file) as f:
import_statement = ET.XML(f.readline())
data.append(import_statement.attrib)
print(data) # [{'name': 'abcd', 'color': 'green', 'age': '25'}]
Of course this only works if the import statement is on the first line, if it's not the case, you'll have to locate it first before calling ET.XML.
Hello I am making a requests call to return order data from a online store. My issue is that once I have passed my data to a root variable the method iter is not returning the correct results. e.g. Display multiple tags of the same name rather than one and not showing the data within the tag.
I thought this was due to the XML not being correctly formatted so I formatted it by saving it to a file using pretty_print but that hasn't fixed the error.
How do I fix this? - Thanks in advance
Code:
import requests, xml.etree.ElementTree as ET, lxml.etree as etree
url="http://publicapi.ekmpowershop24.com/v1.1/publicapi.asmx"
headers = {'content-type': 'application/soap+xml'}
body = """<?xml version="1.0" encoding="utf-8"?>
<soap12:Envelope xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:xsd="http://www.w3.org/2001/XMLSchema" xmlns:soap12="http://www.w3.org/2003/05/soap-envelope">
<soap12:Body>
<GetOrders xmlns="http://publicapi.ekmpowershop.com/">
<GetOrdersRequest>
<APIKey>my_api_key</APIKey>
<FromDate>01/07/2018</FromDate>
<ToDate>04/07/2018</ToDate>
</GetOrdersRequest>
</GetOrders>
</soap12:Body>
</soap12:Envelope>"""
#send request to ekm
r = requests.post(url,data=body,headers=headers)
#save output to file
file = open("C:/Users/Mark/Desktop/test.xml", "w")
file.write(r.text)
file.close()
#take the file and format the xml
x = etree.parse("C:/Users/Mark/Desktop/test.xml")
newString = etree.tostring(x, pretty_print=True)
file = open("C:/Users/Mark/Desktop/test.xml", "w")
file.write(newString.decode('utf-8'))
file.close()
#parse the file to get the roots
tree = ET.parse("C:/Users/Mark/Desktop/test.xml")
root = tree.getroot()
#access elements names in the data
for child in root.iter('*'):
print(child.tag)
#show orders elements attributes
tree = ET.parse("C:/Users/Mark/Desktop/test.xml")
root = tree.getroot()
for order in root.iter('{http://publicapi.ekmpowershop.com/}Order'):
out = {}
for child in order:
if child.tag in ('OrderID'):
out[child.tag] = child.text
print(out)
Elements output:
{http://publicapi.ekmpowershop.com/}Orders
{http://publicapi.ekmpowershop.com/}Order
{http://publicapi.ekmpowershop.com/}OrderID
{http://publicapi.ekmpowershop.com/}OrderNumber
{http://publicapi.ekmpowershop.com/}CustomerID
{http://publicapi.ekmpowershop.com/}CustomerUserID
{http://publicapi.ekmpowershop.com/}Order
{http://publicapi.ekmpowershop.com/}OrderID
{http://publicapi.ekmpowershop.com/}OrderNumber
{http://publicapi.ekmpowershop.com/}CustomerID
{http://publicapi.ekmpowershop.com/}CustomerUserID
Orders Output:
{http://publicapi.ekmpowershop.com/}Order {}
{http://publicapi.ekmpowershop.com/}Order {}
XML Structure after formating:
<soap:Envelope xmlns:soap="http://www.w3.org/2003/05/soap-envelope" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:xsd="http://www.w3.org/2001/XMLSchema">
<soap:Body>
<GetOrdersResponse xmlns="http://publicapi.ekmpowershop.com/">
<GetOrdersResult>
<Status>Success</Status>
<Errors/>
<Date>2018-07-10T13:47:00.1682029+01:00</Date>
<TotalOrders>10</TotalOrders>
<TotalCost>100</TotalCost>
<Orders>
<Order>
<OrderID>100</OrderID>
<OrderNumber>102/040718/67</OrderNumber>
<CustomerID>6910</CustomerID>
<CustomerUserID>204</CustomerUserID>
<FirstName>TestFirst</FirstName>
<LastName>TestLast</LastName>
<CompanyName>Test Company</CompanyName>
<EmailAddress>test#Test.com</EmailAddress>
<OrderStatus>Dispatched</OrderStatus>
<OrderStatusColour>#00CC00</OrderStatusColour>
<TotalCost>85.8</TotalCost>
<OrderDate>10/07/2018 14:30:43</OrderDate>
<OrderDateISO>2018-07-10T14:30:43</OrderDateISO>
<AbandonedOrder>false</AbandonedOrder>
<EkmStatus>SUCCESS</EkmStatus>
</Order>
</Orders>
<Currency>GBP</Currency>
</GetOrdersResult>
</GetOrdersResponse>
</soap:Body>
</soap:Envelope>
You need to consider the namespace when checking for tags.
>>> # Include the namespace part of the tag in the tag values that we check.
>>> tags = ('{http://publicapi.ekmpowershop.com/}OrderID', '{http://publicapi.ekmpowershop.com/}OrderNumber')
>>> for order in root.iter('{http://publicapi.ekmpowershop.com/}Order'):
... out = {}
... for child in order:
... if child.tag in tags:
... out[child.tag] = child.text
... print(out)
...
{'{http://publicapi.ekmpowershop.com/}OrderID': '100', '{http://publicapi.ekmpowershop.com/}OrderNumber': '102/040718/67'}
If you don't want the namespace prefixes in the output, you can strip them by only including that part of the tag after the } character.
>>> for order in root.iter('{http://publicapi.ekmpowershop.com/}Order'):
... out = {}
... for child in order:
... if child.tag in tags:
... out[child.tag[child.tag.index('}')+1:]] = child.text
... print(out)
...
{'OrderID': '100', 'OrderNumber': '102/040718/67'}
If I've got an XML file like this:
<root
xmlns:a="http://example.com/a"
xmlns:b="http://example.com/b"
xmlns:c="http://example.com/c"
xmlns="http://example.com/base">
...
</root>
How can I get a list of the namespace definitions (ie, the xmlns:a="…", etc)?
Using:
import xml.etree.ElementTree as ET
tree = ET.parse('foo.xml')
root = tree.getroot()
print root.attrib()
Shows an empty attribute dictionary.
Via #mzjn, in the comments, here's how to do it with stock ElementTree: https://stackoverflow.com/a/42372404/407651 :
import xml.etree.ElementTree as ET
my_namespaces = dict([
node for (_, node) in ET.iterparse('file.xml', events=['start-ns'])
])
You might find it easier to use lxml.
from lxml import etree
xml_data = '<root xmlns:a="http://example.com/a" xmlns:b="http://example.com/b" xmlns:c="http://example.com/c" xmlns="http://example.com/base"></root>'
root_node = etree.fromstring(xml_data)
print root_node.nsmap
This outputs
{None: 'http://example.com/base',
'a': 'http://example.com/a',
'b': 'http://example.com/b',
'c': 'http://example.com/c'}
I have model with several fields that look like this:
class XMLData(models.Model):
name = models.CharField()
description = models.CharField()
price = models.CharField()
and xml data that wrapped in string, xml data look like this:
<Root>
<Header>
<information>info</information>
</Header>
<Main>
<Product>
<Name>name1</Name>
<Description>description1</Description>
<Price>1</Price>
</Product>
<Product>
<Name>name2</Name>
<Description>description2</Description>
<Price>2</Price>
</Product>
</Main>
</Root>
My question is: should i replace children nodes Product to the parent node and should i rename tags Name, Description, Price to name, description, price?
I tried to deserialize using this code:
for product in serializers.deserialize("xml", xmldata):
savedata = XMLData(product)
savedata.save()
so I hoped that will rise some errors and i would understand what to do next, but there was no errors and xml data didn't save to database.
Hope you understand my problem and thank you for your answer.
It seems you are only interested in storing the product information. In that case Product would be a better model name than XMLData.
Django serializers won't help since your data is not in the correct format. However you can deserialize with lxml. Code taken from: Converting xml to dictionary using ElementTree
from xml.etree import cElementTree as ET
from collections import defaultdict
def etree_to_dict(t):
d = {t.tag: {} if t.attrib else None}
children = list(t)
if children:
dd = defaultdict(list)
for dc in map(etree_to_dict, children):
for k, v in dc.iteritems():
dd[k].append(v)
d = {t.tag: {k:v[0] if len(v) == 1 else v for k, v in dd.iteritems()}}
if t.attrib:
d[t.tag].update(('#' + k, v) for k, v in t.attrib.iteritems())
if t.text:
text = t.text.strip()
if children or t.attrib:
if text:
d[t.tag]['#text'] = text
else:
d[t.tag] = text
return d
e = ET.XML('''
<Root>
<Header>
<information>info</information>
</Header>
<Main>
<Product>
<Name>name1</Name>
<Description>description1</Description>
<Price>1</Price>
</Product>
<Product>
<Name>name2</Name>
<Description>description2</Description>
<Price>2</Price>
</Product>
</Main>
</Root>
''')
from pprint import pprint
d = etree_to_dict(e)
pprint(d)
Now, instead of pretty printing, we can directly store the products:
d = etree_to_dict(e)
products = d['Root']['Main']['Product']
for p in products:
product = Product()
p.name = p['Name']
p.description = p['Description']
p.price = p['Price']
p.save()
That should store the products to the database.
I'm working with Elementtree to parse an XML file (Nessus data). Ive identified the item.attrib which looks to be a dictionary with a 'name': 'IPaddress'. I'd like to add this data into a dictionary, or if I can access just the ipaddress into a list. How can I access the value for name only? Ive tried using variations on item[0]/[1]/.attrib/text/ but still no luck.
Current Code
import elementtree.ElementTree as ET
def getDetails(nessus_file):
host_list = []
host_dict = {}
try:
tree = ET.parse(nessus_file)
doc = tree.getroot()
reporthost = doc.getiterator('ReportHost')
for child in doc:
if child.tag == 'Report':
for item in child:
if item.tag == 'ReportHost':
print item.attrib
except Exception as e:
print e
exit()
getDetails('file.nessus')
Example Output From Current Code
{'name': '172.121.26.80'}
{'name': '172.121.26.42'}
{'name': '172.121.26.41'}
{'name': '172.121.26.21'}
{'name': '172.121.26.15'}
{'name': '172.121.26.14'}
Use item.get('name'). See https://docs.python.org/2/library/xml.etree.elementtree.html#xml.etree.ElementTree.Element.get for details.