getcode2git
diff --git a/‎lib/request/basic.py‎
Lines changed: 10 additions & 0 deletions b/‎lib/request/basic.py‎
Lines changed: 10 additions & 0 deletions
@@ -17,6 +17,7 @@
 from lib.core.common import extractRegexResult
 from lib.core.common import getPublicTypeMembers
 from lib.core.common import getUnicode
+from lib.core.common import randomStr
 from lib.core.common import readInput
 from lib.core.common import resetCookieJar
 from lib.core.common import singleTimeLogMessage
@@ -206,6 +207,15 @@ def checkCharEncoding(encoding, warn=True):
             singleTimeLogMessage(warnMsg, logging.WARN, encoding)
         encoding = None
 
+    if encoding:
+        try:
+            unicode(randomStr(), encoding)
+        except:
+            if warn:
+                warnMsg = "invalid web page charset '%s'" % encoding
+                singleTimeLogMessage(warnMsg, logging.WARN, encoding)
+            encoding = None
+
     return encoding
 
 def getHeuristicCharEncoding(page):