inline attribute value quoted states; fix test_tokenizer.py to throw a parse error for end tags with attributes

annevk · annevk · commit 83ddb4cd46ad · 2007-01-12T19:57:09.000Z
--HG--
extra : convert_revision : svn%3Aacbfec75-9323-0410-a652-858a13e371e0/trunk%40463
diff --git a/src/tokenizer.py b/src/tokenizer.py
@@ -257,19 +257,6 @@ def emitCurrentToken(self):
         self.tokenQueue.append(self.currentToken)
         self.state = self.states["data"]
 
-    def attributeValueQuotedStateHandler(self, quoteType):
-        data = self.stream.char()
-        if data == quoteType:
-            self.state = self.states["beforeAttributeName"]
-        elif data == u"&":
-            self.processEntityInAttribute()
-        elif data == EOF:
-            self.tokenQueue.append({"type": "ParseError", "data":
-              _("Unexpected end of file in attribute value.")})
-            self.emitCurrentToken()
-        else:
-            self.currentToken["data"][-1][1] += data + self.stream.charsUntil(\
-              (quoteType, u"&"))
 
     # Below are the various tokenizer states worked out.
 
@@ -567,14 +554,33 @@ def beforeAttributeValueState(self):
         return True
 
     def attributeValueDoubleQuotedState(self):
-        # AT We could also let self.attributeValueQuotedStateHandler always
-        # return true and then return that directly here. Not sure what is
-        # faster or better...
-        self.attributeValueQuotedStateHandler(u"\"")
+        data = self.stream.char()
+        if data == "\"":
+            self.state = self.states["beforeAttributeName"]
+        elif data == u"&":
+            self.processEntityInAttribute()
+        elif data == EOF:
+            self.tokenQueue.append({"type": "ParseError", "data":
+              _("Unexpected end of file in attribute value (\").")})
+            self.emitCurrentToken()
+        else:
+            self.currentToken["data"][-1][1] += data +\
+              self.stream.charsUntil(("\"", u"&"))
         return True
 
     def attributeValueSingleQuotedState(self):
-        self.attributeValueQuotedStateHandler(u"'")
+        data = self.stream.char()
+        if data == "'":
+            self.state = self.states["beforeAttributeName"]
+        elif data == u"&":
+            self.processEntityInAttribute()
+        elif data == EOF:
+            self.tokenQueue.append({"type": "ParseError", "data":
+              _("Unexpected end of file in attribute value (').")})
+            self.emitCurrentToken()
+        else:
+            self.currentToken["data"][-1][1] += data +\
+              self.stream.charsUntil(("'", u"&"))
         return True
 
     def attributeValueUnQuotedState(self):
diff --git a/tests/test_parser.py b/tests/test_parser.py
@@ -21,7 +21,7 @@
 
 #Run the parse error checks
 #XXX - ideally want this to be a command line argument
-checkParseErrors = True
+checkParseErrors = False
 
 def parseTestcase(testString):
     testString = testString.split("\n")
diff --git a/tests/test_tokenizer.py b/tests/test_tokenizer.py
@@ -46,6 +46,8 @@ def processEmptyTag(self, token):
         self.outputTokens.append([u"StartTag", token["name"], token["data"]])
 
     def processEndTag(self, token):
+        if token["data"]:
+            self.processParseError(None)
         self.outputTokens.append([u"EndTag", token["name"]])
 
     def processComment(self, token):
