[PATCH 0/3] Add xml_to_dict parser
From: Royce Lv <lvroyce@linux.vnet.ibm.com> In kimchi we are having a increasing need of parse xml to dict. Bingbu proposed a very useful tool to fullfil this function. Resend it so that storage targets parse can use this. Royce Lv (3): xml parser: Add a parser to xmlutils xml parser: pep8 clean for xmlutils.py xml parser: Add testcase for xml_to_dict Makefile.am | 1 + src/kimchi/exception.py | 4 ++++ src/kimchi/xmlutils.py | 56 +++++++++++++++++++++++++++++++++++++++---- tests/test_xmlutil.py | 64 +++++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 121 insertions(+), 4 deletions(-) create mode 100644 tests/test_xmlutil.py -- 1.8.1.2
From: Royce Lv <lvroyce@linux.vnet.ibm.com> Add an xml parser to xmlutils.py to deal with the increasing need in kimchi to converse an xml to dict. Signed-off-by: Bing Bu Cao <mars@linux.vnet.ibm.com> Signed-off-by: Royce Lv <lvroyce@linux.vnet.ibm.com> --- src/kimchi/exception.py | 4 ++++ src/kimchi/xmlutils.py | 54 ++++++++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 55 insertions(+), 3 deletions(-) diff --git a/src/kimchi/exception.py b/src/kimchi/exception.py index bff0a18..623e6b3 100644 --- a/src/kimchi/exception.py +++ b/src/kimchi/exception.py @@ -43,3 +43,7 @@ class InvalidOperation(Exception): class IsoFormatError(Exception): pass + + +class ParseError(Exception): + pass diff --git a/src/kimchi/xmlutils.py b/src/kimchi/xmlutils.py index 51ff0ec..f94db30 100644 --- a/src/kimchi/xmlutils.py +++ b/src/kimchi/xmlutils.py @@ -23,7 +23,11 @@ import libxml2 -from xml.etree import ElementTree +from collections import defaultdict +from xml.etree import ElementTree as ET + + +from kimchi.exception import ParseError def xpath_get_text(xml, expr): @@ -36,7 +40,51 @@ def xpath_get_text(xml, expr): def xml_item_update(xml, xpath, value): - root = ElementTree.fromstring(xml) + root = ET.fromstring(xml) item = root.find(xpath) item.text = value - return ElementTree.tostring(root, encoding="utf-8") + return ET.tostring(root, encoding="utf-8") + + +def xml_to_dict(xml, fromstring=True): + try: + if fromstring: + t = ET.fromstring(xml) + else: + t = ET.parse(xml).getroot() + except ET.ParseError: + raise ParseError("XML parse failed, invalid XML file") + return _etree_to_dict(t) + + +def _etree_to_dict(t): + # Convert an etree object to dict + d = {t.tag: {} if t.attrib else None} + children = list(t) + td = {} + if children: + dd = defaultdict(list) + #if has subelement, iteration as it's a new element + for dc in map(_etree_to_dict, children): + for k, v in dc.iteritems(): + dd[k].append(v) + for k, v in dd.iteritems(): + if len(v) == 1: + va = v[0] + else: + va = v + td.update({k: va}) + d[t.tag] = td + + #prepend '@' to attr and '#' to value + if t.attrib: + d[t.tag].update((k, v) for k, v in t.attrib.iteritems()) + if t.text: + text = t.text.strip() + if children or t.attrib: + if text: + d[t.tag]['#value'] = text + else: + d[t.tag] = text + + return d -- 1.8.1.2
On 01/09/2014 06:32 PM, lvroyce@linux.vnet.ibm.com wrote:
From: Royce Lv <lvroyce@linux.vnet.ibm.com>
Add an xml parser to xmlutils.py to deal with the increasing need in kimchi to converse an xml to dict. I have suggested to adopt this patch several times. I need xml2dic. But seems this patch is not welcome. So Adam filed a issue about "generate and manipulate XML documents" 4 months ago. https://github.com/kimchi-project/kimchi/issues/124 Also we has give up libvirt-gobject.
and now we can get a python third module on some distros. on rhel6.5 and my F20 $ sudo yum search python-xmltodict python-xmltodict.noarch : Makes working with XML feel like you are working with JSON we need xml2dict and dict2xml. discuss with Mark about "generate and manipulate XML documents". we think maybe lxml is good for us. $ sudo yum search python-lxml python-lxml-docs.noarch : Documentation for python-lxml python-lxml.x86_64 : ElementTree-like Python bindings for libxml2 and libxslt you can consider lxml.
Signed-off-by: Bing Bu Cao <mars@linux.vnet.ibm.com> Signed-off-by: Royce Lv <lvroyce@linux.vnet.ibm.com> --- src/kimchi/exception.py | 4 ++++ src/kimchi/xmlutils.py | 54 ++++++++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 55 insertions(+), 3 deletions(-)
diff --git a/src/kimchi/exception.py b/src/kimchi/exception.py index bff0a18..623e6b3 100644 --- a/src/kimchi/exception.py +++ b/src/kimchi/exception.py @@ -43,3 +43,7 @@ class InvalidOperation(Exception):
class IsoFormatError(Exception): pass + + +class ParseError(Exception): + pass diff --git a/src/kimchi/xmlutils.py b/src/kimchi/xmlutils.py index 51ff0ec..f94db30 100644 --- a/src/kimchi/xmlutils.py +++ b/src/kimchi/xmlutils.py @@ -23,7 +23,11 @@ import libxml2
-from xml.etree import ElementTree +from collections import defaultdict +from xml.etree import ElementTree as ET + + +from kimchi.exception import ParseError
def xpath_get_text(xml, expr): @@ -36,7 +40,51 @@ def xpath_get_text(xml, expr):
def xml_item_update(xml, xpath, value): - root = ElementTree.fromstring(xml) + root = ET.fromstring(xml) item = root.find(xpath) item.text = value - return ElementTree.tostring(root, encoding="utf-8") + return ET.tostring(root, encoding="utf-8") + + +def xml_to_dict(xml, fromstring=True): + try: + if fromstring: + t = ET.fromstring(xml) + else: + t = ET.parse(xml).getroot() + except ET.ParseError: + raise ParseError("XML parse failed, invalid XML file") + return _etree_to_dict(t) + + +def _etree_to_dict(t): + # Convert an etree object to dict + d = {t.tag: {} if t.attrib else None} + children = list(t) + td = {} + if children: + dd = defaultdict(list) + #if has subelement, iteration as it's a new element + for dc in map(_etree_to_dict, children): + for k, v in dc.iteritems(): + dd[k].append(v) + for k, v in dd.iteritems(): + if len(v) == 1: + va = v[0] + else: + va = v + td.update({k: va}) + d[t.tag] = td + + #prepend '@' to attr and '#' to value + if t.attrib: + d[t.tag].update((k, v) for k, v in t.attrib.iteritems()) + if t.text: + text = t.text.strip() + if children or t.attrib: + if text: + d[t.tag]['#value'] = text + else: + d[t.tag] = text + + return d
-- Thanks and best regards! Sheldon Feng(冯少合)<shaohef@linux.vnet.ibm.com> IBM Linux Technology Center
From: Royce Lv <lvroyce@linux.vnet.ibm.com> Clean pep8 and add xmlutils.py to clean list. Signed-off-by: Royce Lv <lvroyce@linux.vnet.ibm.com> --- Makefile.am | 1 + src/kimchi/xmlutils.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/Makefile.am b/Makefile.am index 04ad696..f39c8a0 100644 --- a/Makefile.am +++ b/Makefile.am @@ -56,6 +56,7 @@ PEP8_WHITELIST = \ src/kimchi/rollbackcontext.py \ src/kimchi/root.py \ src/kimchi/server.py \ + src/kimchi/xmlutils.py \ tests/test_mockmodel.py \ tests/test_model.py \ tests/test_plugin.py \ diff --git a/src/kimchi/xmlutils.py b/src/kimchi/xmlutils.py index f94db30..4686113 100644 --- a/src/kimchi/xmlutils.py +++ b/src/kimchi/xmlutils.py @@ -33,7 +33,7 @@ from kimchi.exception import ParseError def xpath_get_text(xml, expr): doc = libxml2.parseDoc(xml) res = doc.xpathEval(expr) - ret = [None if x.children == None else x.children.content for x in res] + ret = [None if x.children is None else x.children.content for x in res] doc.freeDoc() return ret -- 1.8.1.2
From: Royce Lv <lvroyce@linux.vnet.ibm.com> Add testcase for xml_to_dict to validate string, xml file, and invalid string. Signed-off-by: Royce Lv <lvroyce@linux.vnet.ibm.com> --- tests/test_xmlutil.py | 64 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 tests/test_xmlutil.py diff --git a/tests/test_xmlutil.py b/tests/test_xmlutil.py new file mode 100644 index 0000000..0a93b54 --- /dev/null +++ b/tests/test_xmlutil.py @@ -0,0 +1,64 @@ +# +# Project Kimchi +# +# Copyright IBM, Corp. 2014 +# +# Authors: +# Royce Lv <lvroyce@linux.vnet.ibm.com> +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +import unittest + + +from kimchi.xmlutils import xml_to_dict +from kimchi.exception import ParseError + + +XML_STR = """ +<sources> + <source> + <host name='10.66.90.100'/> + <device path='iqn.2001-05.com.equallogic:0-8a0906-12a1f7d03'/> + </source> + <source> + <host name='10.66.90.100'/> + <device path='iqn.2001-05.com.equallogic:0-8a0906-9951f7d02'/> + </source> + <source> + <host name='10.66.90.100'/> + <device path='iqn.2001-05.com.equallogic:0-8a0906-6eb1f7d01'/> + </source> +</sources> +""" + +ILLEGAL_STR = """ +<sources> +<sources/> +""" + + +class XMLParseTests(unittest.TestCase): + def test_parse_xml_string(self): + res = xml_to_dict(XML_STR) + dic = {'sources': + {'source': [ + {'device': {'path': 'iqn.2001-05.com.equallogic:0-8a0906-12a1f7d03'}, 'host': {'name': '10.66.90.100'}}, + {'device': {'path': 'iqn.2001-05.com.equallogic:0-8a0906-9951f7d02'}, 'host': {'name': '10.66.90.100'}}, + {'device': {'path': 'iqn.2001-05.com.equallogic:0-8a0906-6eb1f7d01'}, 'host': {'name': '10.66.90.100'}}]}} + self.assertEquals(res, dic) + + def test_parse_illegal(self): + self.assertRaises(ParseError, xml_to_dict, ILLEGAL_STR) -- 1.8.1.2
participants (2)
-
lvroyce＠linux.vnet.ibm.com -
Sheldon