Several changes related to character encoding; convert utf-16 to utf-8 if found in pre-parse algorithm, allow chardet to be switched off, start implementing reparsing if <meta> found during actual parse (not yet complete)

jgraham · jgraham · commit d05f439ca084 · 2007-10-16T20:49:50.000Z
--HG--
extra : convert_revision : svn%3Aacbfec75-9323-0410-a652-858a13e371e0/trunk%401056
diff --git a/src/html5lib/html5parser.py b/src/html5lib/html5parser.py
@@ -78,14 +78,15 @@ def __init__(self, strict = False, tree=simpletree.TreeBuilder,
         }
 
     def _parse(self, stream, innerHTML=False, container="div",
-               encoding=None, **kwargs):
+               encoding=None, parseMeta=True, useChardet=True, **kwargs):
         
         self.tree.reset()
         self.firstStartTag = False
         self.errors = []
 
         self.tokenizer = self.tokenizer_class(stream, encoding=encoding,
-                                              parseMeta=not innerHTML, **kwargs)
+                                              parseMeta=parseMeta,
+                                              useChardet=useChardet, **kwargs)
 
         if innerHTML:
             self.innerHTML = container.lower()
@@ -131,7 +132,7 @@ def _parse(self, stream, innerHTML=False, container="div",
         # When the loop finishes it's EOF
         self.phase.processEOF()
 
-    def parse(self, stream, encoding=None):
+    def parse(self, stream, encoding=None, parseMeta=True, useChardet=True):
         """Parse a HTML document into a well-formed tree
 
         stream - a filelike object or string containing the HTML to be parsed
@@ -144,7 +145,8 @@ def parse(self, stream, encoding=None):
         self._parse(stream, innerHTML=False, encoding=encoding)
         return self.tree.getDocument()
     
-    def parseFragment(self, stream, container="div", encoding=None):
+    def parseFragment(self, stream, container="div", encoding=None,
+                      parseMeta=False, useChardet=True):
         """Parse a HTML fragment into a well-formed tree fragment
         
         container - name of the element we're setting the innerHTML property
diff --git a/src/html5lib/inputstream.py b/src/html5lib/inputstream.py
@@ -38,7 +38,7 @@ def __init__(self, source, encoding=None, parseMeta=True, chardet=True):
         # List of where new lines occur
         self.newLines = [0]
 
-        self.charEncoding = encoding
+        self.charEncoding = (encoding, "certian")
 
         # Raw Stream - for unicode objects this will encode to utf-8 and set
         #              self.charEncoding as appropriate
@@ -54,11 +54,11 @@ def __init__(self, source, encoding=None, parseMeta=True, chardet=True):
         self.defaultEncoding = "windows-1252"
         
         #Detect encoding iff no explicit "transport level" encoding is supplied
-        if self.charEncoding is None or not isValidEncoding(self.charEncoding):
+        if self.charEncoding[0] is None or not isValidEncoding(self.charEncoding[0]):
             self.charEncoding = self.detectEncoding(parseMeta, chardet)
 
-        self.dataStream = codecs.getreader(self.charEncoding)(self.rawStream,
-                                                              'replace')
+        self.dataStream = codecs.getreader(self.charEncoding[0])(self.rawStream,
+                                                              '  replace')
 
         self.queue = deque([])
         self.readChars = []
@@ -92,12 +92,15 @@ def detectEncoding(self, parseMeta=True, chardet=True):
         #First look for a BOM
         #This will also read past the BOM if present
         encoding = self.detectBOM()
+        confidence = "certain"
         #If there is no BOM need to look for meta elements with encoding 
         #information
         if encoding is None and parseMeta:
             encoding = self.detectEncodingMeta()
+            confidence = "tentative"
         #Guess with chardet, if avaliable
         if encoding is None and chardet:
+            confidence = "tentative"
             try:
                 from chardet.universaldetector import UniversalDetector
                 buffers = []
@@ -115,6 +118,7 @@ def detectEncoding(self, parseMeta=True, chardet=True):
                 pass
         # If all else fails use the default encoding
         if encoding is None:
+            confidence="tentative"
             encoding = self.defaultEncoding
         
         #Substitute for equivalent encodings:
@@ -123,7 +127,7 @@ def detectEncoding(self, parseMeta=True, chardet=True):
         if encoding.lower() in encodingSub:
             encoding = encodingSub[encoding.lower()]
 
-        return encoding
+        return encoding, confidence
 
     def detectBOM(self):
         """Attempts to detect at BOM at the start of the stream. If
@@ -200,7 +204,8 @@ def detectEncodingMeta(self):
         buffer = self.rawStream.read(self.numBytesMeta)
         parser = EncodingParser(buffer)
         self.seek(buffer, 0)
-        return parser.getEncoding()
+        encoding = parser.getEncoding()
+        return encoding
 
     def updatePosition(self):
         #Remove EOF from readChars, if present
@@ -414,7 +419,12 @@ def getEncoding(self):
             if not keepParsing:
                 break
         if self.encoding is not None:
-            self.encoding = self.encoding.strip()
+            self.encoding = self.encoding.strip()        
+            #Spec violation that complies with hsivonen + mjs
+            if self.encoding.upper() in ("UTF-16", "UTF-16BE", "UTF-16LE",
+                                         "UTF-32", "UTF-32BE", "UTF-32LE"):
+                self.encoding = "utf-8"
+        
         return self.encoding
 
     def handleComment(self):
@@ -531,7 +541,7 @@ def getAttribute(self):
                 #11.5
                 else:
                     attrValue.extend(self.data.currentByte)
-        elif self.data.currentByte in (">", '<'):
+        elif self.data.currentByte in (">", "<"):
                 return "".join(attrName), ""
         elif self.data.currentByte in asciiUppercase:
             attrValue.extend(self.data.currentByte.lower())
@@ -540,7 +550,7 @@ def getAttribute(self):
         while True:
             self.data.position +=1
             if self.data.currentByte in (
-                list(spaceCharacters) + [">", '<']):
+                list(spaceCharacters) + [">", "<"]):
                 return "".join(attrName), "".join(attrValue)
             elif self.data.currentByte in asciiUppercase:
                 attrValue.extend(self.data.currentByte.lower())
diff --git a/src/html5lib/sanitizer.py b/src/html5lib/sanitizer.py
@@ -188,11 +188,11 @@ def sanitize_css(self, style):
         return ' '.join(clean)
 
 class HTMLSanitizer(HTMLTokenizer, HTMLSanitizerMixin):
-    def __init__(self, stream, encoding=None, parseMeta=True,
+    def __init__(self, stream, encoding=None, parseMeta=True, useChardet=True,
                  lowercaseElementName=False, lowercaseAttrName=False):
         #Change case matching defaults as we only output lowercase html anyway
         #This solution doesn't seem ideal...
-        HTMLTokenizer.__init__(self, stream, encoding, parseMeta,
+        HTMLTokenizer.__init__(self, stream, encoding, parseMeta, useChardet,
                                lowercaseElementName, lowercaseAttrName)
 
     def __iter__(self):
diff --git a/src/html5lib/tokenizer.py b/src/html5lib/tokenizer.py
@@ -30,9 +30,9 @@ class HTMLTokenizer(object):
 
     # XXX need to fix documentation
 
-    def __init__(self, stream, encoding=None, parseMeta=True,
+    def __init__(self, stream, encoding=None, parseMeta=True, useChardet=True,
                  lowercaseElementName=True, lowercaseAttrName=True,):
-        self.stream = HTMLInputStream(stream, encoding, parseMeta)
+        self.stream = HTMLInputStream(stream, encoding, parseMeta, useChardet)
         
         #Perform case conversions?
         self.lowercaseElementName = lowercaseElementName
diff --git a/tests/test_encoding.py b/tests/test_encoding.py
@@ -16,7 +16,7 @@ def buildTestSuite():
         for idx, test in enumerate(tests):
             def encodingTest(self, data=test['data'], encoding=test['encoding']):
                 stream = inputstream.HTMLInputStream(data,chardet=False)
-                self.assertEquals(encoding.lower(), stream.charEncoding)
+                self.assertEquals(encoding.lower(), stream.charEncoding[0])
             setattr(Html5EncodingTestCase, 'test_%s_%d' % (test_name, idx+1),
                 encodingTest)
 
@@ -25,7 +25,7 @@ def encodingTest(self, data=test['data'], encoding=test['encoding']):
         def test_chardet(self):
             data = open(os.path.join(test_dir, "encoding" , "chardet", "test_big5.txt")).read()
             encoding = inputstream.HTMLInputStream(data).charEncoding
-            assert encoding.lower() == "big5"
+            assert encoding[0].lower() == "big5"
         setattr(Html5EncodingTestCase, 'test_chardet', test_chardet)
     except ImportError:
         print "chardet not found, skipping chardet tests"
diff --git a/tests/test_stream.py b/tests/test_stream.py
@@ -7,7 +7,7 @@ class HTMLInputStreamTest(unittest.TestCase):
 
     def test_char_ascii(self):
         stream = HTMLInputStream("'", encoding='ascii')
-        self.assertEquals(stream.charEncoding, 'ascii')
+        self.assertEquals(stream.charEncoding[0], 'ascii')
         self.assertEquals(stream.char(), "'")
 
     def test_char_null(self):
@@ -16,24 +16,24 @@ def test_char_null(self):
 
     def test_char_utf8(self):
         stream = HTMLInputStream(u'\u2018'.encode('utf-8'), encoding='utf-8')
-        self.assertEquals(stream.charEncoding, 'utf-8')
+        self.assertEquals(stream.charEncoding[0], 'utf-8')
         self.assertEquals(stream.char(), u'\u2018')
 
     def test_char_win1252(self):
         stream = HTMLInputStream(u"\xa9\xf1\u2019".encode('windows-1252'))
-        self.assertEquals(stream.charEncoding, 'windows-1252')
+        self.assertEquals(stream.charEncoding[0], 'windows-1252')
         self.assertEquals(stream.char(), u"\xa9")
         self.assertEquals(stream.char(), u"\xf1")
         self.assertEquals(stream.char(), u"\u2019")
 
     def test_bom(self):
         stream = HTMLInputStream(codecs.BOM_UTF8 + "'")
-        self.assertEquals(stream.charEncoding, 'utf-8')
+        self.assertEquals(stream.charEncoding[0], 'utf-8')
         self.assertEquals(stream.char(), "'")
 
     def test_utf_16(self):
         stream = HTMLInputStream((' '*1025).encode('utf-16'))
-        self.assert_(stream.charEncoding in ['utf-16-le','utf-16-be'])
+        self.assert_(stream.charEncoding[0] in ['utf-16-le','utf-16-be'], stream.charEncoding)
         self.assertEquals(len(stream.charsUntil(' ',True)),1025)
 
     def test_newlines(self):
