- Fix segfault with invalid coding.

nnorwitz · nnorwitz · commit 40d378141663 · 2005-10-02T01:48:49.000Z
- SF Bug #772896, unknown encoding results in MemoryError, which is not helpful

I will only backport the segfault fix.  I'll let Anthony decide if he wants
the other changes backported.  I will do the backport if asked.
diff --git a/Lib/test/bad_coding.py b/Lib/test/bad_coding.py
@@ -0,0 +1 @@
+# -*- coding: uft-8 -*-
diff --git a/Lib/test/test_coding.py b/Lib/test/test_coding.py
@@ -0,0 +1,21 @@
+
+import test.test_support, unittest
+import os
+
+class CodingTest(unittest.TestCase):
+    def test_bad_coding(self):
+        module_name = 'bad_coding'
+        self.assertRaises(SyntaxError, __import__, 'test.' + module_name)
+
+        path = os.path.dirname(__file__)
+        filename = os.path.join(path, module_name + '.py')
+        fp = open(filename)
+        text = fp.read()
+        fp.close()
+        self.assertRaises(SyntaxError, compile, text, filename, 'exec')
+
+def test_main():
+    test.test_support.run_unittest(CodingTest)
+
+if __name__ == "__main__":
+    test_main()
diff --git a/Misc/NEWS b/Misc/NEWS
@@ -12,6 +12,10 @@ What's New in Python 2.5 alpha 1?
 Core and builtins
 -----------------
 
+- Fix segfault with invalid coding.
+
+- SF bug #772896: unknown encoding results in MemoryError.
+
 - All iterators now have a Boolean value of true.  Formerly, some iterators
   supported a __len__() method which evaluated to False when the iterator
   was empty.
diff --git a/Parser/parsetok.c b/Parser/parsetok.c
@@ -42,7 +42,7 @@ PyParser_ParseStringFlagsFilename(const char *s, const char *filename,
 	initerr(err_ret, filename);
 
 	if ((tok = PyTokenizer_FromString(s)) == NULL) {
-		err_ret->error = E_NOMEM;
+		err_ret->error = PyErr_Occurred() ? E_DECODE : E_NOMEM;
 		return NULL;
 	}
 
diff --git a/Parser/pgenmain.c b/Parser/pgenmain.c
@@ -116,6 +116,13 @@ getgrammar(char *filename)
 	return g;
 }
 
+/* Can't happen in pgen */
+PyObject*
+PyErr_Occurred()
+{
+	return 0;
+}
+
 void
 Py_FatalError(const char *msg)
 {
diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c
@@ -603,8 +603,11 @@ decode_str(const char *str, struct tok_state *tok)
 	if (tok->enc != NULL) {
 		assert(utf8 == NULL);
 		utf8 = translate_into_utf8(str, tok->enc);
-		if (utf8 == NULL)
+		if (utf8 == NULL) {
+			PyErr_Format(PyExc_SyntaxError,
+				"unknown encoding: %s", tok->enc);
 			return NULL;
+		}
 		str = PyString_AsString(utf8);
 	}
 #endif
diff --git a/Python/pythonrun.c b/Python/pythonrun.c
@@ -1487,7 +1487,7 @@ err_input(perrdetail *err)
 			msg = "unknown decode error";
 		Py_DECREF(type);
 		Py_DECREF(value);
-		Py_DECREF(tb);
+		Py_XDECREF(tb);
 		break;
 	}
 	case E_LINECONT:

Original file line number	Diff line number	Diff line change
`@@ -42,7 +42,7 @@ PyParser_ParseStringFlagsFilename(const char s, const char filename,`
`42`	`42`	`initerr(err_ret, filename);`
`43`	`43`
`44`	`44`	`if ((tok = PyTokenizer_FromString(s)) == NULL) {`
`45`		`- err_ret->error = E_NOMEM;`
	`45`	`+ err_ret->error = PyErr_Occurred() ? E_DECODE : E_NOMEM;`
`46`	`46`	`return NULL;`
`47`	`47`	`}`
`48`	`48`
Original file line number	Diff line number	Diff line change
`@@ -116,6 +116,13 @@ getgrammar(char *filename)`
`116`	`116`	`return g;`
`117`	`117`	`}`
`118`	`118`
	`119`	`+/* Can't happen in pgen */`
	`120`	`+PyObject*`
	`121`	`+PyErr_Occurred()`
	`122`	`+{`
	`123`	`+ return 0;`
	`124`	`+}`
	`125`	`+`
`119`	`126`	`void`
`120`	`127`	`Py_FatalError(const char *msg)`
`121`	`128`	`{`
Original file line number	Diff line number	Diff line change
`@@ -1487,7 +1487,7 @@ err_input(perrdetail *err)`
`1487`	`1487`	`msg = "unknown decode error";`
`1488`	`1488`	`Py_DECREF(type);`
`1489`	`1489`	`Py_DECREF(value);`
`1490`		`- Py_DECREF(tb);`
	`1490`	`+ Py_XDECREF(tb);`
`1491`	`1491`	`break;`
`1492`	`1492`	`}`
`1493`	`1493`	`case E_LINECONT:`