python · hnarkaytis · Apr 21, 2020 · Apr 21, 2020 · Apr 21, 2020 · Apr 21, 2020
diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
@@ -74,6 +74,8 @@ Include/pytime.h              @pganssle @abalkin
 
 # Parser/Pgen
 /Parser/pgen/                 @pablogsal
+/Parser/pegen/                @pablogsal
+/Tools/peg_generator/         @pablogsal
 
 # SQLite 3
 **/*sqlite*                   @berkerpeksag

diff --git a/.travis.yml b/.travis.yml
@@ -1,5 +1,5 @@
 language: c
-dist: xenial
+dist: bionic
 
 # To cache doc-building dependencies and C compiler output.
 cache:
@@ -157,7 +157,9 @@ install:
 before_script:
   # -Og is much faster than -O0
   - CFLAGS="${CFLAGS} -Og" ./configure --with-pydebug
-  - make -j4 regen-all
+  - eval "$(pyenv init -)"
+  - pyenv global 3.8
+  - PYTHON_FOR_REGEN=python3.8 make -j4 regen-all
   - changes=`git status --porcelain`
   - |
       # Check for changes in regenerated files

diff --git a/Doc/c-api/contextvars.rst b/Doc/c-api/contextvars.rst
@@ -101,11 +101,6 @@ Context object management functions:
    current context for the current thread.  Returns ``0`` on success,
    and ``-1`` on error.
 
-.. c:function:: int PyContext_ClearFreeList()
-
-   Clear the context variable free list. Return the total number of
-   freed items.  This function always succeeds.
-
 
 Context variable functions:
 

diff --git a/Doc/c-api/dict.rst b/Doc/c-api/dict.rst
@@ -232,10 +232,3 @@ Dictionary Objects
           for key, value in seq2:
               if override or key not in a:
                   a[key] = value
-
-
-.. c:function:: int PyDict_ClearFreeList()
-
-   Clear the free list. Return the total number of freed items.
-
-   .. versionadded:: 3.3
diff --git a/Doc/c-api/float.rst b/Doc/c-api/float.rst
@@ -76,8 +76,3 @@ Floating Point Objects
 .. c:function:: double PyFloat_GetMin()
 
    Return the minimum normalized positive float *DBL_MIN* as C :c:type:`double`.
-
-.. c:function:: int PyFloat_ClearFreeList()
-
-   Clear the float free list. Return the number of items that could not
-   be freed.
diff --git a/Doc/c-api/init.rst b/Doc/c-api/init.rst
@@ -1074,8 +1074,10 @@ All of the following functions must be called after :c:func:`Py_Initialize`.
 
 .. c:function:: PyFrameObject* PyThreadState_GetFrame(PyThreadState *tstate)
 
-   Get the current frame of the Python thread state *tstate*. It can be
-   ``NULL`` if no frame is currently executing.
+   Get the current frame of the Python thread state *tstate*.
+
+   Return a strong reference. Return ``NULL`` if no frame is currently
+   executing.
 
    See also :c:func:`PyEval_GetFrame`.
 

diff --git a/Doc/c-api/init_config.rst b/Doc/c-api/init_config.rst
@@ -686,6 +686,16 @@ PyConfig
 
       :data:`sys._xoptions`.
 
+   .. c:member:: int _use_peg_parser
+
+      Enable PEG parser? Default: 1.
+
+      Set to 0 by :option:`-X oldparser <-X>` and :envvar:`PYTHONOLDPARSER`.
+
+      See also :pep:`617`.
+
+      .. deprecated-removed:: 3.9 3.10
+
 If ``parse_argv`` is non-zero, ``argv`` arguments are parsed the same
 way the regular Python parses command line arguments, and Python
 arguments are stripped from ``argv``: see :ref:`Command Line Arguments

diff --git a/Doc/c-api/intro.rst b/Doc/c-api/intro.rst
@@ -187,6 +187,39 @@ complete listing.
    .. versionchanged:: 3.8
       MSVC support was added.
 
+.. c:macro:: PyDoc_STRVAR(name, str)
+
+   Creates a variable with name ``name`` that can be used in docstrings.
+   If Python is built without docstrings, the value will be empty.
+
+   Use :c:macro:`PyDoc_STRVAR` for docstrings to support building
+   Python without docstrings, as specified in :pep:`7`.
+
+   Example::
+
+      PyDoc_STRVAR(pop_doc, "Remove and return the rightmost element.");
+
+      static PyMethodDef deque_methods[] = {
+          // ...
+          {"pop", (PyCFunction)deque_pop, METH_NOARGS, pop_doc},
+          // ...
+      }
+
+.. c:macro:: PyDoc_STR(str)
+
+   Creates a docstring for the given input string or an empty string
+   if docstrings are disabled.
+
+   Use :c:macro:`PyDoc_STR` in specifying docstrings to support
+   building Python without docstrings, as specified in :pep:`7`.
+
+   Example::
+
+      static PyMethodDef pysqlite_row_methods[] = {
+          {"keys", (PyCFunction)pysqlite_row_keys, METH_NOARGS,
+              PyDoc_STR("Returns the keys of the row.")},
+          {NULL, NULL}
+      };
 
 .. _api-objects:
 

diff --git a/Doc/c-api/list.rst b/Doc/c-api/list.rst
@@ -142,10 +142,3 @@ List Objects
 
    Return a new tuple object containing the contents of *list*; equivalent to
    ``tuple(list)``.
-
-
-.. c:function:: int PyList_ClearFreeList()
-
-   Clear the free list. Return the total number of freed items.
-
-   .. versionadded:: 3.3
diff --git a/Doc/c-api/method.rst b/Doc/c-api/method.rst
@@ -92,9 +92,3 @@ no longer available.
 .. c:function:: PyObject* PyMethod_GET_SELF(PyObject *meth)
 
    Macro version of :c:func:`PyMethod_Self` which avoids error checking.
-
-
-.. c:function:: int PyMethod_ClearFreeList()
-
-   Clear the free list. Return the total number of freed items.
-
diff --git a/Doc/c-api/module.rst b/Doc/c-api/module.rst
@@ -153,7 +153,7 @@ or request "multi-phase initialization" by returning the definition struct itsel
    .. c:member:: const char *m_doc
 
       Docstring for the module; usually a docstring variable created with
-      :c:func:`PyDoc_STRVAR` is used.
+      :c:macro:`PyDoc_STRVAR` is used.
 
    .. c:member:: Py_ssize_t m_size
 

diff --git a/Doc/c-api/reflection.rst b/Doc/c-api/reflection.rst
@@ -31,10 +31,34 @@ Reflection
    See also :c:func:`PyThreadState_GetFrame`.
 
 
+.. c:function:: int PyFrame_GetBack(PyFrameObject *frame)
+
+   Get the *frame* next outer frame.
+
+   Return a strong reference, or ``NULL`` if *frame* has no outer frame.
+
+   *frame* must not be ``NULL``.
+
+   .. versionadded:: 3.9
+
+
+.. c:function:: int PyFrame_GetCode(PyFrameObject *frame)
+
+   Get the *frame* code.
+
+   Return a strong reference.
+
+   *frame* must not be ``NULL``. The result (frame code) cannot be ``NULL``.
+
+   .. versionadded:: 3.9
+
+
 .. c:function:: int PyFrame_GetLineNumber(PyFrameObject *frame)
 
    Return the line number that *frame* is currently executing.
 
+   *frame* must not be ``NULL``.
+
 
 .. c:function:: const char* PyEval_GetFuncName(PyObject *func)
 

diff --git a/Doc/c-api/set.rst b/Doc/c-api/set.rst
@@ -157,10 +157,3 @@ subtypes but not for instances of :class:`frozenset` or its subtypes.
 .. c:function:: int PySet_Clear(PyObject *set)
 
    Empty an existing set of all elements.
-
-
-.. c:function:: int PySet_ClearFreeList()
-
-   Clear the free list. Return the total number of freed items.
-
-   .. versionadded:: 3.3
diff --git a/Doc/c-api/tuple.rst b/Doc/c-api/tuple.rst
@@ -111,11 +111,6 @@ Tuple Objects
    raises :exc:`MemoryError` or :exc:`SystemError`.
 
 
-.. c:function:: int PyTuple_ClearFreeList()
-
-   Clear the free list. Return the total number of freed items.
-
-
 Struct Sequence Objects
 -----------------------
 

diff --git a/Doc/extending/extending.rst b/Doc/extending/extending.rst
@@ -395,18 +395,26 @@ optionally followed by an import of the module::
        }
 
        /* Add a built-in module, before Py_Initialize */
-       PyImport_AppendInittab("spam", PyInit_spam);
+       if (PyImport_AppendInittab("spam", PyInit_spam) == -1) {
+           fprintf(stderr, "Error: could not extend in-built modules table\n");
+           exit(1);
+       }
 
        /* Pass argv[0] to the Python interpreter */
        Py_SetProgramName(program);
 
-       /* Initialize the Python interpreter.  Required. */
+       /* Initialize the Python interpreter.  Required.
+          If this step fails, it will be a fatal error. */
        Py_Initialize();
 
        /* Optionally import the module; alternatively,
           import can be deferred until the embedded script
           imports it. */
-       PyImport_ImportModule("spam");
+       pmodule = PyImport_ImportModule("spam");
+       if (!pmodule) {
+           PyErr_Print();
+           fprintf(stderr, "Error: could not import module 'spam'\n");
+       }
 
        ...
 

diff --git a/Doc/faq/programming.rst b/Doc/faq/programming.rst
@@ -851,10 +851,11 @@ For integers, use the built-in :func:`int` type constructor, e.g. ``int('144')
 e.g. ``float('144') == 144.0``.
 
 By default, these interpret the number as decimal, so that ``int('0144') ==
-144`` and ``int('0x144')`` raises :exc:`ValueError`. ``int(string, base)`` takes
-the base to convert from as a second optional argument, so ``int('0x144', 16) ==
-324``.  If the base is specified as 0, the number is interpreted using Python's
-rules: a leading '0o' indicates octal, and '0x' indicates a hex number.
+144`` holds true, and ``int('0x144')`` raises :exc:`ValueError`. ``int(string,
+base)`` takes the base to convert from as a second optional argument, so ``int(
+'0x144', 16) == 324``.  If the base is specified as 0, the number is interpreted
+using Python's rules: a leading '0o' indicates octal, and '0x' indicates a hex
+number.
 
 Do not use the built-in function :func:`eval` if all you need is to convert
 strings to numbers.  :func:`eval` will be significantly slower and it presents a
@@ -1494,8 +1495,8 @@ to uppercase::
 
 Here the ``UpperOut`` class redefines the ``write()`` method to convert the
 argument string to uppercase before calling the underlying
-``self.__outfile.write()`` method.  All other methods are delegated to the
-underlying ``self.__outfile`` object.  The delegation is accomplished via the
+``self._outfile.write()`` method.  All other methods are delegated to the
+underlying ``self._outfile`` object.  The delegation is accomplished via the
 ``__getattr__`` method; consult :ref:`the language reference <attribute-access>`
 for more information about controlling attribute access.
 

diff --git a/Doc/library/2to3.rst b/Doc/library/2to3.rst
@@ -9,9 +9,7 @@
 of *fixers* to transform it into valid Python 3.x code.  The standard library
 contains a rich set of fixers that will handle almost all code.  2to3 supporting
 library :mod:`lib2to3` is, however, a flexible and generic library, so it is
-possible to write your own fixers for 2to3.  :mod:`lib2to3` could also be
-adapted to custom applications in which Python code needs to be edited
-automatically.
+possible to write your own fixers for 2to3.
 
 
 .. _2to3-using:
@@ -466,9 +464,17 @@ and off individually.  They are described here in more detail.
 
 --------------
 
+.. deprecated:: 3.10
+   Python 3.9 will switch to a PEG parser (see :pep:`617`), and Python 3.10 may
+   include new language syntax that is not parsable by lib2to3's LL(1) parser.
+   The ``lib2to3`` module may be removed from the standard library in a future
+   Python version. Consider third-party alternatives such as `LibCST`_ or
+   `parso`_.
+
 .. note::
 
    The :mod:`lib2to3` API should be considered unstable and may change
    drastically in the future.
 
-.. XXX What is the public interface anyway?
+.. _LibCST: https://libcst.readthedocs.io/
+.. _parso: https://parso.readthedocs.io/
diff --git a/Doc/library/enum.rst b/Doc/library/enum.rst
@@ -273,6 +273,10 @@ overridden::
     the next :class:`int` in sequence with the last :class:`int` provided, but
     the way it does this is an implementation detail and may change.
 
+.. note::
+
+    The :meth:`_generate_next_value_` method must be defined before any members.
+
 Iteration
 ---------
 

diff --git a/Doc/library/imaplib.rst b/Doc/library/imaplib.rst
@@ -582,6 +582,15 @@ An :class:`IMAP4` instance has the following methods:
 
    Unsubscribe from old mailbox.
 
+.. method:: IMAP4.unselect()
+
+   :meth:`imaplib.IMAP4.unselect` frees server's resources associated with the
+   selected mailbox and returns the server to the authenticated
+   state. This command performs the same actions as :meth:`imaplib.IMAP4.close`, except
+   that no messages are permanently removed from the currently
+   selected mailbox.
+
+   .. versionadded:: 3.9
 
 .. method:: IMAP4.xatom(name[, ...])
 

diff --git a/Doc/library/multiprocessing.rst b/Doc/library/multiprocessing.rst
@@ -878,6 +878,16 @@ For an example of the usage of queues for interprocess communication see
 
    It is a simplified :class:`Queue` type, very close to a locked :class:`Pipe`.
 
+   .. method:: close()
+
+      Close the queue: release internal resources.
+
+      A queue must not be used anymore after it is closed. For example,
+      :meth:`get`, :meth:`put` and :meth:`empty` methods must no longer be
+      called.
+
+      .. versionadded:: 3.9
+
    .. method:: empty()
 
       Return ``True`` if the queue is empty, ``False`` otherwise.

diff --git a/Doc/library/queue.rst b/Doc/library/queue.rst
@@ -190,32 +190,28 @@ fully processed by daemon consumer threads.
 
 Example of how to wait for enqueued tasks to be completed::
 
+    import threading, queue
+
+    q = queue.Queue()
+
     def worker():
         while True:
             item = q.get()
-            if item is None:
-                break
-            do_work(item)
+            print(f'Working on {item}')
+            print(f'Finished {item}')
             q.task_done()
 
-    q = queue.Queue()
-    threads = []
-    for i in range(num_worker_threads):
-        t = threading.Thread(target=worker)
-        t.start()
-        threads.append(t)
+    # turn-on the worker thread
+    threading.Thread(target=worker, daemon=True).start()
 
-    for item in source():
+    # send thirty task requests to the worker
+    for item in range(30):
         q.put(item)
+    print('All task requests sent\n', end='')
 
     # block until all tasks are done
     q.join()
-
-    # stop workers
-    for i in range(num_worker_threads):
-        q.put(None)
-    for t in threads:
-        t.join()
+    print('All work completed')
 
 
 SimpleQueue Objects