python · vstinner · Dec 17, 2020 · serhiy-storchaka · Dec 18, 2020 · vstinner
@@ -29,7 +29,7 @@ Py_DEPRECATED(3.2) PyAPI_FUNC(const char *) PyModule_GetFilename(PyObject *);
 PyAPI_FUNC(PyObject *) PyModule_GetFilenameObject(PyObject *);
 #ifndef Py_LIMITED_API
 PyAPI_FUNC(void) _PyModule_Clear(PyObject *);
-PyAPI_FUNC(void) _PyModule_ClearDict(PyObject *);
+PyAPI_FUNC(void) _PyModule_ClearDict(PyObject *, PyObject *);
 PyAPI_FUNC(int) _PyModuleSpec_IsInitializing(PyObject *);
 #endif
 PyAPI_FUNC(struct PyModuleDef*) PyModule_GetDef(PyObject*);

diff --git a/Lib/test/test_module.py b/Lib/test/test_module.py
@@ -266,14 +266,15 @@ def test_module_repr_source(self):
                          '{!r} does not end with {!r}'.format(r, ends_with))
 
     def test_module_finalization_at_shutdown(self):
-        # Module globals and builtins should still be available during shutdown
+        # Most module globals and builtins should still be available during
+        # shutdown.
         rc, out, err = assert_python_ok("-c", "from test import final_a")
         self.assertFalse(err)
         lines = out.splitlines()
         self.assertEqual(set(lines), {
             b"x = a",
             b"x = b",
-            b"final_a.x = a",
+            b"final_a.x = None",
             b"final_b.x = b",
             b"len = len",
             b"shutil.rmtree = rmtree"})

diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py
@@ -901,8 +901,8 @@ def __del__(self, sys=sys):
         self.assertIn(b'sys.flags', out[0])
         self.assertIn(b'sys.float_info', out[1])
 
-    def test_sys_ignores_cleaning_up_user_data(self):
-        code = """if 1:
+    def test_sys_cleaning_up_user_data(self):
+        code = textwrap.dedent("""
             import struct, sys
 
             class C:
@@ -912,11 +912,11 @@ def __del__(self):
                     self.pack('I', -42)
 
             sys.x = C()
-            """
+        """)
         rc, stdout, stderr = assert_python_ok('-c', code)
         self.assertEqual(rc, 0)
         self.assertEqual(stdout.rstrip(), b"")
-        self.assertEqual(stderr.rstrip(), b"")
+        self.assertIn(b'Exception ignored in: <function C.__del__ at ', stderr)
 
     @unittest.skipUnless(hasattr(sys, 'getandroidapilevel'),
                          'need sys.getandroidapilevel()')

diff --git a/Lib/test/test_threading.py b/Lib/test/test_threading.py
@@ -615,18 +615,23 @@ def test_main_thread_during_shutdown(self):
         code = """if 1:
             import gc, threading
 
-            main_thread = threading.current_thread()
-            assert main_thread is threading.main_thread()  # sanity check
-
             class RefCycle:
                 def __init__(self):
                     self.cycle = self
 
+                    self.current_thread = threading.current_thread
+                    self.main_thread = threading.main_thread
+                    self.enumerate = threading.enumerate
+
+                    self.thread = self.current_thread()
+                    assert self.thread is threading.main_thread()  # sanity check
+
+
                 def __del__(self):
                     print("GC:",
-                          threading.current_thread() is main_thread,
-                          threading.main_thread() is main_thread,
-                          threading.enumerate() == [main_thread])
+                          self.current_thread() is self.thread,
+                          self.main_thread() is self.thread,
+                          self.enumerate() == [self.thread])
 
             RefCycle()
             gc.collect()  # sanity check

diff --git a/Misc/NEWS.d/next/Core and Builtins/2020-12-17-22-18-44.bpo-42671.M_xDGc.rst b/Misc/NEWS.d/next/Core and Builtins/2020-12-17-22-18-44.bpo-42671.M_xDGc.rst
@@ -0,0 +1,8 @@
+Make the Python finalization more deterministic:
+
+* First, clear the ``__main__`` module.
+* Then, clear modules from the most recently imported to the least
+  recently imported: ``reversed(sys.modules)``.
+* :mod:`builtins` and :mod:`sys` modules are always cleared last.
+* Module attributes are set to None from the most recently defined to the
+  least recently defined: ``reversed(module.__dict__)``.
@@ -559,65 +559,88 @@ PyModule_GetState(PyObject* m)
 void
 _PyModule_Clear(PyObject *m)
 {
-    PyObject *d = ((PyModuleObject *)m)->md_dict;
-    if (d != NULL)
-        _PyModule_ClearDict(d);
+    PyModuleObject *module = (PyModuleObject *)m;
+    _PyModule_ClearDict(module->md_name, module->md_dict);
 }
 
 void
-_PyModule_ClearDict(PyObject *d)
+_PyModule_ClearDict(PyObject *module_name, PyObject *dict)
 {
+    int verbose = _Py_GetConfig()->verbose;
+    if (verbose) {
+        PySys_FormatStderr("# cleanup[3] wiping %S module\n", module_name);
+    }
+
+    // If the module has no dict: there is nothing to do.
+    if (dict == NULL) {
+        return;
+    }
+
     /* To make the execution order of destructors for global
        objects a bit more predictable, we first zap all objects
        whose name starts with a single underscore, before we clear
        the entire dictionary.  We zap them by replacing them with
        None, rather than deleting them from the dictionary, to
        avoid rehashing the dictionary (to some extent). */
 
-    Py_ssize_t pos;
-    PyObject *key, *value;
-
-    int verbose = _Py_GetConfig()->verbose;
+    for (int step=1; step <= 2; step++) {
+        PyObject *reversed = PyObject_CallOneArg((PyObject*)&PyReversed_Type, dict);
+        if (reversed == NULL) {
+            PyErr_WriteUnraisable(NULL);
+            return;
+        }
+        PyObject *keys = PyObject_CallOneArg((PyObject*)&PyList_Type, reversed);
+        Py_DECREF(reversed);
+        if (keys == NULL) {
+            PyErr_WriteUnraisable(NULL);
+            return;
+        }
+        PyObject *iter = PyObject_GetIter(keys);
+        Py_DECREF(keys);
+        if (iter == NULL) {
+            PyErr_WriteUnraisable(NULL);
+            return;
+        }
 
-    /* First, clear only names starting with a single underscore */
-    pos = 0;
-    while (PyDict_Next(d, &pos, &key, &value)) {
-        if (value != Py_None && PyUnicode_Check(key)) {
-            if (PyUnicode_READ_CHAR(key, 0) == '_' &&
-                PyUnicode_READ_CHAR(key, 1) != '_') {
-                if (verbose > 1) {
-                    const char *s = PyUnicode_AsUTF8(key);
-                    if (s != NULL)
-                        PySys_WriteStderr("#   clear[1] %s\n", s);
-                    else
-                        PyErr_Clear();
-                }
-                if (PyDict_SetItem(d, key, Py_None) != 0) {
-                    PyErr_WriteUnraisable(NULL);
-                }
+        /* First, clear only names starting with a single underscore */
+        PyObject *key;
+        while ((key = PyIter_Next(iter))) {
+            assert(!PyErr_Occurred());
+            PyObject *value = PyObject_GetItem(dict, key);
+            Py_XDECREF(value);  // only the value pointer is useful
+            if (value == Py_None) {
+                continue;
+            }
+            if (value == NULL) {
+                // ignore error
+                PyErr_Clear();
             }
-        }
-    }
 
-    /* Next, clear all names except for __builtins__ */
-    pos = 0;
-    while (PyDict_Next(d, &pos, &key, &value)) {
-        if (value != Py_None && PyUnicode_Check(key)) {
-            if (PyUnicode_READ_CHAR(key, 0) != '_' ||
-                !_PyUnicode_EqualToASCIIString(key, "__builtins__"))
-            {
-                if (verbose > 1) {
-                    const char *s = PyUnicode_AsUTF8(key);
-                    if (s != NULL)
-                        PySys_WriteStderr("#   clear[2] %s\n", s);
-                    else
-                        PyErr_Clear();
+            if (PyUnicode_Check(key)) {
+                if (step == 1) {
+                    if (PyUnicode_READ_CHAR(key, 0) != '_' ||
+                        PyUnicode_READ_CHAR(key, 1) == '_') {
+                        continue;
+                    }
                 }
-                if (PyDict_SetItem(d, key, Py_None) != 0) {
-                    PyErr_WriteUnraisable(NULL);
+                else {
+                    /* Step 2: clear all names except for __builtins__ */
+                    if (_PyUnicode_EqualToASCIIString(key, "__builtins__")) {
+                        continue;
+                    }
                 }
             }
+            if (verbose > 1) {
+                PySys_FormatStderr("#   clear[%i] %S.%S\n",
+                                   step, module_name, key);
+            }
+            assert(!PyErr_Occurred());
+            if (PyDict_SetItem(dict, key, Py_None) != 0) {
+                PyErr_WriteUnraisable(NULL);
+            }
+            Py_DECREF(key);
         }
+        Py_DECREF(iter);
     }
 
     /* Note: we leave __builtins__ in place, so that destructors