Many fixes to the treewalkers and added preliminary treewalkers tests (for now using the parser tests and serializing the treewalkers result tokens to the appropriate parser-tests format; all tests pass !!!)

tbroyer · tbroyer · commit 917094a64019 · 2007-05-26T23:13:47.000Z
--HG--
extra : convert_revision : svn%3Aacbfec75-9323-0410-a652-858a13e371e0/trunk%40619
diff --git a/src/treebuilders/etree.py b/src/treebuilders/etree.py
@@ -244,6 +244,6 @@ def getDocument(self):
                 return self.document._element.find("html")
         
         def getFragment(self):
-            return ElementTree.ElementTree(_base.TreeBuilder.getFragment(self)._element)
+            return _base.TreeBuilder.getFragment(self)._element
         
     return locals()
diff --git a/src/treewalkers/_base.py b/src/treewalkers/_base.py
@@ -1,3 +1,6 @@
+import gettext
+_ = gettext.gettext
+
 from constants import voidElements, spaceCharacters
 spaceCharacters = u"".join(spaceCharacters)
 
@@ -9,7 +12,7 @@ def walkChildren(self, node):
         raise NodeImplementedError
 
     def error(self, msg):
-        yield {"type": "SerializeError", "data": msg}
+        return {"type": "SerializeError", "data": msg}
 
     def normalizeAttrs(self, attrs):
         if not attrs:
@@ -18,14 +21,14 @@ def normalizeAttrs(self, attrs):
             attrs = attrs.items()
         return attrs
 
-    def element(self, name, attrs, hasChildren):
+    def element(self, node, name, attrs, hasChildren):
         if name in voidElements:
             for token in self.emptyTag(name, attrs, hasChildren):
                 yield token
         else:
             yield self.startTag(name, attrs)
             if hasChildren:
-                for token in self.serializeChildren(node):
+                for token in self.walkChildren(node):
                     yield token
             yield self.endTag(name)
 
@@ -47,14 +50,13 @@ def text(self, data):
         left = data[:len(data)-len(middle)]
         if left:
             yield {"type": "SpaceCharacters", "data": left}
+        data = middle
+        middle = data.rstrip(spaceCharacters)
+        right = data[len(middle):]
         if middle:
-            data = middle
-            middle = data.rstrip(spaceCharacters)
-            right = data[len(data)-len(middle):]
-            if middle:
-                yield {"type": "Characters", "data": middle}
-            if right:
-                yield {"type": "SpaceCharacters", "data": right}
+            yield {"type": "Characters", "data": middle}
+        if right:
+            yield {"type": "SpaceCharacters", "data": right}
 
     def comment(self, data):
         return {"type": "Comment", "data": data}
diff --git a/src/treewalkers/dom.py b/src/treewalkers/dom.py
@@ -7,29 +7,29 @@
 
 class TreeWalker(_base.TreeWalker):
     def walk(self, node):
-        if node.nodeType in (Node.DOCUMENT_NODE, Node.DOCUMENT_FRAGMENT_NODE):
-            for token in self.walkChildren(node):
-                yield token
-
-        elif node.nodeType == Node.DOCUMENT_TYPE_NODE:
+        if node.nodeType == Node.DOCUMENT_TYPE_NODE:
             yield self.doctype(node.nodeName)
 
         elif node.nodeType in (Node.TEXT_NODE, Node.CDATA_SECTION_NODE):
             for token in self.text(node.nodeValue):
                 yield token
 
         elif node.nodeType == Node.ELEMENT_NODE:
-            for token in self.element(node.nodeName, \
+            for token in self.element(node, node.nodeName, \
               node.attributes.items(), node.childNodes):
                 yield token
 
         elif node.nodeType == Node.COMMENT_NODE:
             yield self.comment(node.nodeValue)
 
+        elif node.nodeType in (Node.DOCUMENT_NODE, Node.DOCUMENT_FRAGMENT_NODE):
+            for token in self.walkChildren(node):
+                yield token
+
         else:
             yield self.unknown(node.nodeType)
 
     def walkChildren(self, node):
         for childNode in node.childNodes:
-            for token in self.walk(node):
+            for token in self.walk(childNode):
                 yield token
diff --git a/src/treewalkers/etree.py b/src/treewalkers/etree.py
@@ -5,6 +5,7 @@
 import copy
 
 import _base
+from constants import voidElements
 
 moduleCache = {}
 
@@ -24,28 +25,33 @@ def getETreeBuilder(ElementTreeImplementation):
 
     class TreeWalker(_base.TreeWalker):
         def walk(self, node):
-            if type(element) == type(ElementTree.ElementTree):
-                element = element.getroot()
+            if not(hasattr(node, "tag")):
+                node = node.getroot()
 
             if node.tag in ("<DOCUMENT_ROOT>", "<DOCUMENT_FRAGMENT>"):
                 for token in self.walkChildren(node):
                     yield token
 
             elif node.tag == "<!DOCTYPE>":
                 yield self.doctype(node.text)
-                if node.tail:
-                    for token in self.text(node.tail):
-                        yield token
 
             elif type(node.tag) == type(ElementTree.Comment):
                 yield self.comment(node.text)
-                if node.tail:
-                    for token in self.text(node.tail):
-                        yield token
 
             else:
                 #This is assumed to be an ordinary element
-                for token in self.element(node):
+                if node.tag in voidElements:
+                    for token in self.emptyTag(node.tag, \
+                      node.attrib.items(), len(node) or node.text):
+                        yield token
+                else:
+                    yield self.startTag(node.tag, node.attrib.items())
+                    for token in self.walkChildren(node):
+                        yield token
+                    yield self.endTag(node.tag)
+
+            if node.tail:
+                for token in self.text(node.tail):
                     yield token
 
         def walkChildren(self, node):
@@ -55,8 +61,5 @@ def walkChildren(self, node):
             for childNode in node.getchildren():
                 for token in self.walk(childNode):
                     yield token
-            if node.tail:
-                for token in self.text(node.tail):
-                    yield token
 
     return locals()
diff --git a/src/treewalkers/simpletree.py b/src/treewalkers/simpletree.py
@@ -7,7 +7,7 @@ class TreeWalker(_base.TreeWalker):
     def walk(self, node):
         # testing node.type allows us not to import treebuilders.simpletree
         if node.type in (1, 2): # Document or DocumentFragment
-            for token in self.serializeChildren(node):
+            for token in self.walkChildren(node):
                 yield token
 
         elif node.type == 3: # DocumentType
@@ -18,7 +18,7 @@ def walk(self, node):
                 yield token
 
         elif node.type == 5: # Element
-            for token in self.element(node.name, \
+            for token in self.element(node, node.name, \
               node.attributes.items(), node.childNodes):
                 yield token
 
diff --git a/src/treewalkers/soup.py b/src/treewalkers/soup.py
@@ -22,7 +22,8 @@ def walk(self, node):
                 yield token
 
         elif isinstance(node, Tag): # Element
-            for token in self.element(node):
+            for token in self.element(node, node.name, \
+              node.attrs.items(), node.contents):
                 yield token
 
         else:
diff --git a/tests/test_parser.py b/tests/test_parser.py
@@ -60,8 +60,6 @@
 except ImportError:
     pass
 
-sys.stdout.write('Testing trees '+ " ".join(treeTypes.keys()) + "\n")
-
 #Run the parse error checks
 checkParseErrors = False
 
@@ -141,6 +139,7 @@ def runParserTest(self, innerHTML, input, expected, errors, treeClass):
             self.assertEquals(len(p.errors), len(errors), errorMsg2)
 
 def test_parser():
+    sys.stdout.write('Testing tree builders '+ " ".join(treeTypes.keys()) + "\n")
     for name, cls in treeTypes.iteritems():
         for filename in glob.glob('tree-construction/*.dat'):
             f = open(filename)
diff --git a/tests/test_serializer.py b/tests/test_serializer.py
@@ -92,6 +92,13 @@ def buildTestSuite():
     return unittest.TestLoader().loadTestsFromTestCase(TestCase)
 
 def main():
+    # the following is temporary while the unit tests for parse errors are
+    # still in flux
+    if '-p' in sys.argv: # suppress check for serialize errors
+        sys.argv.remove('-p')
+        global checkSerializeErrors
+        checkSerializeErrors = False
+       
     buildTestSuite()
     unittest.main()
 
diff --git a/tests/test_treewalkers.py b/tests/test_treewalkers.py
