diff --git a/.gitattributes b/.gitattributes index b8189f12ded0f4..f4d65dfd1dfbd3 100644 --- a/.gitattributes +++ b/.gitattributes @@ -34,6 +34,9 @@ Lib/test/xmltestdata/* noeol Lib/venv/scripts/common/activate text eol=lf Lib/venv/scripts/posix/* text eol=lf +# Prevent GitHub's web conflict editor from converting LF to CRLF +*.rst text eol=lf + # CRLF files [attr]dos text eol=crlf diff --git a/.github/workflows/mypy.yml b/.github/workflows/mypy.yml index e5a5b3939e58e3..7f6571ef954576 100644 --- a/.github/workflows/mypy.yml +++ b/.github/workflows/mypy.yml @@ -19,6 +19,7 @@ on: - "Tools/build/consts_getter.py" - "Tools/build/deepfreeze.py" - "Tools/build/generate-build-details.py" + - "Tools/build/generate_levenshtein_examples.py" - "Tools/build/generate_sbom.py" - "Tools/build/generate_stdlib_module_names.py" - "Tools/build/mypy.ini" diff --git a/.github/workflows/reusable-san.yml b/.github/workflows/reusable-san.yml index 9d4f412cfcf6f7..33f6f0ef455fe0 100644 --- a/.github/workflows/reusable-san.yml +++ b/.github/workflows/reusable-san.yml @@ -40,17 +40,15 @@ jobs: # Install clang wget https://apt.llvm.org/llvm.sh chmod +x llvm.sh + sudo ./llvm.sh 20 + sudo update-alternatives --install /usr/bin/clang clang /usr/bin/clang-20 100 + sudo update-alternatives --set clang /usr/bin/clang-20 + sudo update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-20 100 + sudo update-alternatives --set clang++ /usr/bin/clang++-20 if [ "${SANITIZER}" = "TSan" ]; then - sudo ./llvm.sh 17 # gh-121946: llvm-18 package is temporarily broken - sudo update-alternatives --install /usr/bin/clang clang /usr/bin/clang-17 100 - sudo update-alternatives --set clang /usr/bin/clang-17 - sudo update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-17 100 - sudo update-alternatives --set clang++ /usr/bin/clang++-17 # Reduce ASLR to avoid TSan crashing sudo sysctl -w vm.mmap_rnd_bits=28 - else - sudo ./llvm.sh 20 fi - name: Sanitizer option setup diff --git a/Doc/c-api/allocation.rst b/Doc/c-api/allocation.rst index 59044d2d88cc16..09c9ed3ca54cf6 100644 --- a/Doc/c-api/allocation.rst +++ b/Doc/c-api/allocation.rst @@ -2,7 +2,7 @@ .. _allocating-objects: -Allocating Objects on the Heap +Allocating objects on the heap ============================== @@ -153,10 +153,12 @@ Allocating Objects on the Heap To allocate and create extension modules. -Deprecated aliases -^^^^^^^^^^^^^^^^^^ +Soft-deprecated aliases +^^^^^^^^^^^^^^^^^^^^^^^ -These are :term:`soft deprecated` aliases to existing functions and macros. +.. soft-deprecated:: 3.15 + +These are aliases to existing functions and macros. They exist solely for backwards compatibility. @@ -164,7 +166,7 @@ They exist solely for backwards compatibility. :widths: auto :header-rows: 1 - * * Deprecated alias + * * Soft-deprecated alias * Function * * .. c:macro:: PyObject_NEW(type, typeobj) * :c:macro:`PyObject_New` diff --git a/Doc/c-api/buffer.rst b/Doc/c-api/buffer.rst index fe950196297a24..dc3e0f37c36c5b 100644 --- a/Doc/c-api/buffer.rst +++ b/Doc/c-api/buffer.rst @@ -258,7 +258,9 @@ readonly, format .. c:macro:: PyBUF_WRITEABLE - This is a :term:`soft deprecated` alias to :c:macro:`PyBUF_WRITABLE`. + This is an alias to :c:macro:`PyBUF_WRITABLE`. + + .. soft-deprecated:: 3.13 .. c:macro:: PyBUF_FORMAT diff --git a/Doc/c-api/bytes.rst b/Doc/c-api/bytes.rst index d1fde1baf71a45..f56bcd6333a37d 100644 --- a/Doc/c-api/bytes.rst +++ b/Doc/c-api/bytes.rst @@ -47,9 +47,9 @@ called with a non-bytes parameter. *len* on success, and ``NULL`` on failure. If *v* is ``NULL``, the contents of the bytes object are uninitialized. - .. deprecated:: 3.15 - ``PyBytes_FromStringAndSize(NULL, len)`` is :term:`soft deprecated`, - use the :c:type:`PyBytesWriter` API instead. + .. soft-deprecated:: 3.15 + Use the :c:type:`PyBytesWriter` API instead of + ``PyBytes_FromStringAndSize(NULL, len)``. .. c:function:: PyObject* PyBytes_FromFormat(const char *format, ...) @@ -238,9 +238,8 @@ called with a non-bytes parameter. *\*bytes* is set to ``NULL``, :exc:`MemoryError` is set, and ``-1`` is returned. - .. deprecated:: 3.15 - The function is :term:`soft deprecated`, - use the :c:type:`PyBytesWriter` API instead. + .. soft-deprecated:: 3.15 + Use the :c:type:`PyBytesWriter` API instead. .. c:function:: PyObject *PyBytes_Repr(PyObject *bytes, int smartquotes) diff --git a/Doc/c-api/code.rst b/Doc/c-api/code.rst index be2c85ec97489e..57b77f92a7d2e6 100644 --- a/Doc/c-api/code.rst +++ b/Doc/c-api/code.rst @@ -212,7 +212,7 @@ bound into a function. .. c:function:: PyObject *PyCode_Optimize(PyObject *code, PyObject *consts, PyObject *names, PyObject *lnotab_obj) - This is a :term:`soft deprecated` function that does nothing. + This is a function that does nothing. Prior to Python 3.10, this function would perform basic optimizations to a code object. @@ -220,6 +220,8 @@ bound into a function. .. versionchanged:: 3.10 This function now does nothing. + .. soft-deprecated:: 3.13 + .. _c_codeobject_flags: diff --git a/Doc/c-api/concrete.rst b/Doc/c-api/concrete.rst index 1746fe95eaaca9..3f38411a52de6b 100644 --- a/Doc/c-api/concrete.rst +++ b/Doc/c-api/concrete.rst @@ -112,6 +112,7 @@ Other Objects picklebuffer.rst weakref.rst capsule.rst + sentinel.rst frame.rst gen.rst coro.rst diff --git a/Doc/c-api/descriptor.rst b/Doc/c-api/descriptor.rst index b913e24b3c7cc8..539c4610ce4f5b 100644 --- a/Doc/c-api/descriptor.rst +++ b/Doc/c-api/descriptor.rst @@ -140,7 +140,7 @@ found in the dictionary of type objects. .. c:macro:: PyDescr_COMMON - This is a :term:`soft deprecated` macro including the common fields for a + This is a macro including the common fields for a descriptor object. This was included in Python's C API by mistake; do not use it in extensions. @@ -148,6 +148,8 @@ found in the dictionary of type objects. descriptor protocol (:c:member:`~PyTypeObject.tp_descr_get` and :c:member:`~PyTypeObject.tp_descr_set`). + .. soft-deprecated:: 3.15 + Built-in descriptors ^^^^^^^^^^^^^^^^^^^^ diff --git a/Doc/c-api/exceptions.rst b/Doc/c-api/exceptions.rst index 8ecd7c62517104..7a07818b7b4d1a 100644 --- a/Doc/c-api/exceptions.rst +++ b/Doc/c-api/exceptions.rst @@ -818,7 +818,7 @@ Exception Classes .. c:macro:: PyException_HEAD - This is a :term:`soft deprecated` macro including the base fields for an + This is a macro including the base fields for an exception object. This was included in Python's C API by mistake and is not designed for use @@ -826,6 +826,8 @@ Exception Classes :c:func:`PyErr_NewException` or otherwise create a class inheriting from :c:data:`PyExc_BaseException`. + .. soft-deprecated:: 3.15 + Exception Objects ================= diff --git a/Doc/c-api/extension-modules.rst b/Doc/c-api/extension-modules.rst index 92b531665e135d..7bc04970b19503 100644 --- a/Doc/c-api/extension-modules.rst +++ b/Doc/c-api/extension-modules.rst @@ -191,10 +191,10 @@ the :c:data:`Py_mod_multiple_interpreters` slot. ``PyInit`` function ................... -.. deprecated:: 3.15 +.. soft-deprecated:: 3.15 - This functionality is :term:`soft deprecated`. - It will not get new features, but there are no plans to remove it. + This functionality will not get new features, + but there are no plans to remove it. Instead of :c:func:`PyModExport_modulename`, an extension module can define an older-style :dfn:`initialization function` with the signature: @@ -272,10 +272,9 @@ For example, a module called ``spam`` would be defined like this:: Legacy single-phase initialization ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -.. deprecated:: 3.15 +.. soft-deprecated:: 3.15 - Single-phase initialization is :term:`soft deprecated`. - It is a legacy mechanism to initialize extension + Single-phase initialization is a legacy mechanism to initialize extension modules, with known drawbacks and design flaws. Extension module authors are encouraged to use multi-phase initialization instead. diff --git a/Doc/c-api/file.rst b/Doc/c-api/file.rst index d89072ab24e241..dcafefdc045872 100644 --- a/Doc/c-api/file.rst +++ b/Doc/c-api/file.rst @@ -2,7 +2,7 @@ .. _fileobjects: -File Objects +File objects ------------ .. index:: pair: object; file @@ -136,11 +136,12 @@ the :mod:`io` APIs instead. failure; the appropriate exception will be set. -Deprecated API -^^^^^^^^^^^^^^ +Soft-deprecated API +^^^^^^^^^^^^^^^^^^^ +.. soft-deprecated:: 3.15 -These are :term:`soft deprecated` APIs that were included in Python's C API +These are APIs that were included in Python's C API by mistake. They are documented solely for completeness; use other ``PyFile*`` APIs instead. diff --git a/Doc/c-api/float.rst b/Doc/c-api/float.rst index 929b56bd8e8203..a12ad11abb107d 100644 --- a/Doc/c-api/float.rst +++ b/Doc/c-api/float.rst @@ -86,8 +86,7 @@ Floating-Point Objects It is equivalent to the :c:macro:`!INFINITY` macro from the C11 standard ```` header. - .. deprecated:: 3.15 - The macro is :term:`soft deprecated`. + .. soft-deprecated:: 3.15 .. c:macro:: Py_NAN @@ -103,8 +102,7 @@ Floating-Point Objects Equivalent to :c:macro:`!INFINITY`. - .. deprecated:: 3.14 - The macro is :term:`soft deprecated`. + .. soft-deprecated:: 3.14 .. c:macro:: Py_MATH_E @@ -161,8 +159,8 @@ Floating-Point Objects that is, it is normal, subnormal or zero, but not infinite or NaN. Return ``0`` otherwise. - .. deprecated:: 3.14 - The macro is :term:`soft deprecated`. Use :c:macro:`!isfinite` instead. + .. soft-deprecated:: 3.14 + Use :c:macro:`!isfinite` instead. .. c:macro:: Py_IS_INFINITY(X) @@ -170,8 +168,8 @@ Floating-Point Objects Return ``1`` if the given floating-point number *X* is positive or negative infinity. Return ``0`` otherwise. - .. deprecated:: 3.14 - The macro is :term:`soft deprecated`. Use :c:macro:`!isinf` instead. + .. soft-deprecated:: 3.14 + Use :c:macro:`!isinf` instead. .. c:macro:: Py_IS_NAN(X) @@ -179,8 +177,8 @@ Floating-Point Objects Return ``1`` if the given floating-point number *X* is a not-a-number (NaN) value. Return ``0`` otherwise. - .. deprecated:: 3.14 - The macro is :term:`soft deprecated`. Use :c:macro:`!isnan` instead. + .. soft-deprecated:: 3.14 + Use :c:macro:`!isnan` instead. Pack and Unpack functions diff --git a/Doc/c-api/frame.rst b/Doc/c-api/frame.rst index 967cfc727655ec..4159ff6e5965fb 100644 --- a/Doc/c-api/frame.rst +++ b/Doc/c-api/frame.rst @@ -1,6 +1,6 @@ .. highlight:: c -Frame Objects +Frame objects ------------- .. c:type:: PyFrameObject @@ -147,7 +147,7 @@ See also :ref:`Reflection `. Return the line number that *frame* is currently executing. -Frame Locals Proxies +Frame locals proxies ^^^^^^^^^^^^^^^^^^^^ .. versionadded:: 3.13 @@ -169,7 +169,7 @@ See :pep:`667` for more information. Return non-zero if *obj* is a frame :func:`locals` proxy. -Legacy Local Variable APIs +Legacy local variable APIs ^^^^^^^^^^^^^^^^^^^^^^^^^^ These APIs are :term:`soft deprecated`. As of Python 3.13, they do nothing. @@ -178,40 +178,34 @@ They exist solely for backwards compatibility. .. c:function:: void PyFrame_LocalsToFast(PyFrameObject *f, int clear) - This function is :term:`soft deprecated` and does nothing. - Prior to Python 3.13, this function would copy the :attr:`~frame.f_locals` attribute of *f* to the internal "fast" array of local variables, allowing changes in frame objects to be visible to the interpreter. If *clear* was true, this function would process variables that were unset in the locals dictionary. - .. versionchanged:: 3.13 + .. soft-deprecated:: 3.13 This function now does nothing. .. c:function:: void PyFrame_FastToLocals(PyFrameObject *f) - This function is :term:`soft deprecated` and does nothing. - Prior to Python 3.13, this function would copy the internal "fast" array of local variables (which is used by the interpreter) to the :attr:`~frame.f_locals` attribute of *f*, allowing changes in local variables to be visible to frame objects. - .. versionchanged:: 3.13 + .. soft-deprecated:: 3.13 This function now does nothing. .. c:function:: int PyFrame_FastToLocalsWithError(PyFrameObject *f) - This function is :term:`soft deprecated` and does nothing. - Prior to Python 3.13, this function was similar to :c:func:`PyFrame_FastToLocals`, but would return ``0`` on success, and ``-1`` with an exception set on failure. - .. versionchanged:: 3.13 + .. soft-deprecated:: 3.13 This function now does nothing. @@ -219,7 +213,7 @@ They exist solely for backwards compatibility. :pep:`667` -Internal Frames +Internal frames ^^^^^^^^^^^^^^^ Unless using :pep:`523`, you will not need this. @@ -249,5 +243,3 @@ Unless using :pep:`523`, you will not need this. Return the currently executing line number, or -1 if there is no line number. .. versionadded:: 3.12 - - diff --git a/Doc/c-api/gen.rst b/Doc/c-api/gen.rst index 74db49a6814800..ed121726b89620 100644 --- a/Doc/c-api/gen.rst +++ b/Doc/c-api/gen.rst @@ -90,7 +90,9 @@ Deprecated API .. c:macro:: PyAsyncGenASend_CheckExact(op) - This is a :term:`soft deprecated` API that was included in Python's C API + This is an API that was included in Python's C API by mistake. It is solely here for completeness; do not use this API. + + .. soft-deprecated:: 3.14 diff --git a/Doc/c-api/intro.rst b/Doc/c-api/intro.rst index 2a22a023bdafb4..500f2818e2e40a 100644 --- a/Doc/c-api/intro.rst +++ b/Doc/c-api/intro.rst @@ -536,16 +536,14 @@ have been standardized in C11 (or previous standards). Use the standard ``alignas`` specifier rather than this macro. - .. deprecated:: 3.15 - The macro is :term:`soft deprecated`. + .. soft-deprecated:: 3.15 .. c:macro:: PY_FORMAT_SIZE_T The :c:func:`printf` formatting modifier for :c:type:`size_t`. Use ``"z"`` directly instead. - .. deprecated:: 3.15 - The macro is :term:`soft deprecated`. + .. soft-deprecated:: 3.15 .. c:macro:: Py_LL(number) Py_ULL(number) @@ -558,8 +556,7 @@ have been standardized in C11 (or previous standards). Consider using the C99 standard suffixes ``LL`` and ``LLU`` directly. - .. deprecated:: 3.15 - The macro is :term:`soft deprecated`. + .. soft-deprecated:: 3.15 .. c:macro:: PY_LONG_LONG PY_INT32_T @@ -572,8 +569,7 @@ have been standardized in C11 (or previous standards). respectively. Historically, these types needed compiler-specific extensions. - .. deprecated:: 3.15 - These macros are :term:`soft deprecated`. + .. soft-deprecated:: 3.15 .. c:macro:: PY_LLONG_MIN PY_LLONG_MAX @@ -587,16 +583,14 @@ have been standardized in C11 (or previous standards). The required header, ````, :ref:`is included ` in ``Python.h``. - .. deprecated:: 3.15 - These macros are :term:`soft deprecated`. + .. soft-deprecated:: 3.15 .. c:macro:: Py_MEMCPY(dest, src, n) - This is a :term:`soft deprecated` alias to :c:func:`!memcpy`. - Use :c:func:`!memcpy` directly instead. + This is an alias to :c:func:`!memcpy`. - .. deprecated:: 3.14 - The macro is :term:`soft deprecated`. + .. soft-deprecated:: 3.14 + Use :c:func:`!memcpy` directly instead. .. c:macro:: Py_UNICODE_SIZE @@ -606,29 +600,25 @@ have been standardized in C11 (or previous standards). The required header for the latter, ````, :ref:`is included ` in ``Python.h``. - .. deprecated:: 3.15 - The macro is :term:`soft deprecated`. + .. soft-deprecated:: 3.15 .. c:macro:: Py_UNICODE_WIDE Defined if ``wchar_t`` can hold a Unicode character (UCS-4). Use ``sizeof(wchar_t) >= 4`` instead - .. deprecated:: 3.15 - The macro is :term:`soft deprecated`. + .. soft-deprecated:: 3.15 .. c:macro:: Py_VA_COPY - This is a :term:`soft deprecated` alias to the C99-standard ``va_copy`` - function. + This is an alias to the C99-standard ``va_copy`` function. Historically, this would use a compiler-specific method to copy a ``va_list``. .. versionchanged:: 3.6 This is now an alias to ``va_copy``. - .. deprecated:: 3.15 - The macro is :term:`soft deprecated`. + .. soft-deprecated:: 3.15 .. _api-objects: diff --git a/Doc/c-api/long.rst b/Doc/c-api/long.rst index 790ec8da109ba8..60e3ae4a064e72 100644 --- a/Doc/c-api/long.rst +++ b/Doc/c-api/long.rst @@ -197,12 +197,10 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. .. c:function:: long PyLong_AS_LONG(PyObject *obj) - A :term:`soft deprecated` alias. Exactly equivalent to the preferred ``PyLong_AsLong``. In particular, it can fail with :exc:`OverflowError` or another exception. - .. deprecated:: 3.14 - The function is soft deprecated. + .. soft-deprecated:: 3.14 .. c:function:: int PyLong_AsInt(PyObject *obj) diff --git a/Doc/c-api/module.rst b/Doc/c-api/module.rst index a66a1bfd7f8489..b67ca671a2a118 100644 --- a/Doc/c-api/module.rst +++ b/Doc/c-api/module.rst @@ -965,9 +965,7 @@ or code that creates modules dynamically. // PyModule_AddObject() stole a reference to obj: // Py_XDECREF(obj) is not needed here. - .. deprecated:: 3.13 - - :c:func:`PyModule_AddObject` is :term:`soft deprecated`. + .. soft-deprecated:: 3.13 .. c:function:: int PyModule_AddIntConstant(PyObject *module, const char *name, long value) diff --git a/Doc/c-api/monitoring.rst b/Doc/c-api/monitoring.rst index b0227c2f4faf15..4bfcb86abf58ed 100644 --- a/Doc/c-api/monitoring.rst +++ b/Doc/c-api/monitoring.rst @@ -205,6 +205,4 @@ would typically correspond to a Python function. .. versionadded:: 3.13 - .. deprecated:: 3.14 - - This function is :term:`soft deprecated`. + .. soft-deprecated:: 3.14 diff --git a/Doc/c-api/sentinel.rst b/Doc/c-api/sentinel.rst new file mode 100644 index 00000000000000..710ded56e2a6db --- /dev/null +++ b/Doc/c-api/sentinel.rst @@ -0,0 +1,35 @@ +.. highlight:: c + +.. _sentinelobjects: + +Sentinel objects +---------------- + +.. c:var:: PyTypeObject PySentinel_Type + + This instance of :c:type:`PyTypeObject` represents the Python + :class:`sentinel` type. This is the same object as :class:`sentinel`. + + .. versionadded:: next + +.. c:function:: int PySentinel_Check(PyObject *o) + + Return true if *o* is a :class:`sentinel` object. The :class:`sentinel` type + does not allow subclasses, so this check is exact. + + .. versionadded:: next + +.. c:function:: PyObject* PySentinel_New(const char *name, const char *module_name) + + Return a new :class:`sentinel` object with :attr:`~sentinel.__name__` set to + *name* and :attr:`~sentinel.__module__` set to *module_name*. + *name* must not be ``NULL``. If *module_name* is ``NULL``, :attr:`~sentinel.__module__` + is set to ``None``. + Return ``NULL`` with an exception set on failure. + + For pickling to work, *module_name* must be the name of an importable + module, and the sentinel must be accessible from that module under a + path matching *name*. Pickle treats *name* as a global variable name + in *module_name* (see :meth:`object.__reduce__`). + + .. versionadded:: next diff --git a/Doc/c-api/sequence.rst b/Doc/c-api/sequence.rst index df5bf6b64a93a0..6bae8f25ad75d1 100644 --- a/Doc/c-api/sequence.rst +++ b/Doc/c-api/sequence.rst @@ -109,9 +109,8 @@ Sequence Protocol Alias for :c:func:`PySequence_Contains`. - .. deprecated:: 3.14 - The function is :term:`soft deprecated` and should no longer be used to - write new code. + .. soft-deprecated:: 3.14 + The function should no longer be used to write new code. .. c:function:: Py_ssize_t PySequence_Index(PyObject *o, PyObject *value) diff --git a/Doc/c-api/set.rst b/Doc/c-api/set.rst index 53febd0c4c116c..db537aff2e6ce5 100644 --- a/Doc/c-api/set.rst +++ b/Doc/c-api/set.rst @@ -201,7 +201,7 @@ Deprecated API .. c:macro:: PySet_MINSIZE - A :term:`soft deprecated` constant representing the size of an internal + A constant representing the size of an internal preallocated table inside :c:type:`PySetObject` instances. This is documented solely for completeness, as there are no guarantees @@ -211,3 +211,5 @@ Deprecated API :c:macro:`!PySet_MINSIZE` can be replaced with a small constant like ``8``. If looking for the size of a set, use :c:func:`PySet_Size` instead. + + .. soft-deprecated:: 3.14 diff --git a/Doc/c-api/typeobj.rst b/Doc/c-api/typeobj.rst index c3960d6ff87ec8..d3d8239365f9bf 100644 --- a/Doc/c-api/typeobj.rst +++ b/Doc/c-api/typeobj.rst @@ -1391,8 +1391,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. versionchanged:: 3.9 - Renamed to the current name, without the leading underscore. - The old provisional name is :term:`soft deprecated`. + Renamed to the current name, without the leading underscore. + The old provisional name is :term:`soft deprecated`. .. versionchanged:: 3.12 @@ -1501,11 +1501,13 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:macro:: Py_TPFLAGS_HAVE_VERSION_TAG - This is a :term:`soft deprecated` macro that does nothing. + This macro does nothing. Historically, this would indicate that the :c:member:`~PyTypeObject.tp_version_tag` field was available and initialized. + .. soft-deprecated:: 3.13 + .. c:macro:: Py_TPFLAGS_INLINE_VALUES diff --git a/Doc/data/refcounts.dat b/Doc/data/refcounts.dat index 2a6e6b963134bb..663b79e45eec17 100644 --- a/Doc/data/refcounts.dat +++ b/Doc/data/refcounts.dat @@ -2037,6 +2037,10 @@ PySeqIter_Check:PyObject *:op:0: PySeqIter_New:PyObject*::+1: PySeqIter_New:PyObject*:seq:0: +PySentinel_New:PyObject*::+1: +PySentinel_New:const char*:name:: +PySentinel_New:const char*:module_name:: + PySequence_Check:int::: PySequence_Check:PyObject*:o:0: diff --git a/Doc/deprecations/pending-removal-in-3.17.rst b/Doc/deprecations/pending-removal-in-3.17.rst index ea9fb93ddd8c84..952ffad64356d9 100644 --- a/Doc/deprecations/pending-removal-in-3.17.rst +++ b/Doc/deprecations/pending-removal-in-3.17.rst @@ -35,7 +35,7 @@ Pending removal in Python 3.17 - Passing non-ascii *encoding* names to :func:`encodings.normalize_encoding` is deprecated and scheduled for removal in Python 3.17. - (Contributed by Stan Ulbrych in :gh:`136702`) + (Contributed by Stan Ulbrych in :gh:`136702`.) * :mod:`typing`: diff --git a/Doc/howto/enum.rst b/Doc/howto/enum.rst index 5260c2ca4add47..2fe5814bb04a73 100644 --- a/Doc/howto/enum.rst +++ b/Doc/howto/enum.rst @@ -371,7 +371,7 @@ Equality comparisons are defined though:: >>> Color.BLUE == Color.BLUE True -Comparisons against non-enumeration values will always compare not equal +Equality comparisons against non-enumeration values will always return ``False`` (again, :class:`IntEnum` was explicitly designed to behave differently, see below):: diff --git a/Doc/howto/free-threading-extensions.rst b/Doc/howto/free-threading-extensions.rst index 2f089a3d89680a..b21ed1c8f37be1 100644 --- a/Doc/howto/free-threading-extensions.rst +++ b/Doc/howto/free-threading-extensions.rst @@ -416,11 +416,9 @@ C API extensions need to be built specifically for the free-threaded build. The wheels, shared libraries, and binaries are indicated by a ``t`` suffix. * `pypa/manylinux `_ supports the - free-threaded build, with the ``t`` suffix, such as ``python3.13t``. -* `pypa/cibuildwheel `_ supports the - free-threaded build on Python 3.13 and 3.14. On Python 3.14, free-threaded - wheels will be built by default. On Python 3.13, you will need to set - `CIBW_ENABLE to cpython-freethreading `_. + free-threaded build, with the ``t`` suffix, such as ``python3.14t``. +* `pypa/cibuildwheel `_ supports + building wheels for the free-threaded build of Python 3.14 and newer. Limited C API and Stable ABI ............................ diff --git a/Doc/library/asyncio-dev.rst b/Doc/library/asyncio-dev.rst index f3409bcd2df648..713b40d746680a 100644 --- a/Doc/library/asyncio-dev.rst +++ b/Doc/library/asyncio-dev.rst @@ -304,7 +304,7 @@ generator can occur in an unexpected order:: try: yield 2 finally: - await asyncio.sleep(0.1) # immitate some async work + await asyncio.sleep(0.1) # imitate some async work work_done = True diff --git a/Doc/library/asyncio-task.rst b/Doc/library/asyncio-task.rst index 4e60eee44290af..f0fe91b363d95e 100644 --- a/Doc/library/asyncio-task.rst +++ b/Doc/library/asyncio-task.rst @@ -355,6 +355,34 @@ and reliable way to wait for all tasks in the group to finish. Passes on all *kwargs* to :meth:`loop.create_task` + .. method:: cancel() + + Cancel the task group. This is a non-exceptional, early exit of the + task group's lifetime -- useful once the group's goal has been met or + its services no longer needed. + + :meth:`~asyncio.Task.cancel` will be called on any tasks in the group that + aren't yet done, as well as the parent (body) of the group. The task group + context manager will exit *without* :exc:`asyncio.CancelledError` being raised. + + If :meth:`cancel` is called before entering the task group, the group will be + cancelled upon entry. This is useful for patterns where one piece of + code passes an unused :class:`asyncio.TaskGroup` instance to another in order to have + the ability to cancel anything run within the group. + + :meth:`cancel` is idempotent and may be called after the task group has + already exited. + + Some ways to use :meth:`cancel`: + + * call it from the task group body based on some condition or event + * pass the task group instance to child tasks via :meth:`create_task`, allowing a child + task to conditionally cancel the entire entire group + * pass the task group instance or bound :meth:`cancel` method to some other task *before* + opening the task group, allowing remote cancellation + + .. versionadded:: next + Example:: async def main(): @@ -366,7 +394,8 @@ Example:: The ``async with`` statement will wait for all tasks in the group to finish. While waiting, new tasks may still be added to the group (for example, by passing ``tg`` into one of the coroutines -and calling ``tg.create_task()`` in that coroutine). +and calling ``tg.create_task()`` in that coroutine). There is also opportunity +to short-circuit the entire task group with ``tg.cancel()``, based on some condition. Once the last task has finished and the ``async with`` block is exited, no new tasks may be added to the group. @@ -427,53 +456,6 @@ reported by :meth:`asyncio.Task.cancelling`. Improved handling of simultaneous internal and external cancellations and correct preservation of cancellation counts. -Terminating a task group ------------------------- - -While terminating a task group is not natively supported by the standard -library, termination can be achieved by adding an exception-raising task -to the task group and ignoring the raised exception: - -.. code-block:: python - - import asyncio - from asyncio import TaskGroup - - class TerminateTaskGroup(Exception): - """Exception raised to terminate a task group.""" - - async def force_terminate_task_group(): - """Used to force termination of a task group.""" - raise TerminateTaskGroup() - - async def job(task_id, sleep_time): - print(f'Task {task_id}: start') - await asyncio.sleep(sleep_time) - print(f'Task {task_id}: done') - - async def main(): - try: - async with TaskGroup() as group: - # spawn some tasks - group.create_task(job(1, 0.5)) - group.create_task(job(2, 1.5)) - # sleep for 1 second - await asyncio.sleep(1) - # add an exception-raising task to force the group to terminate - group.create_task(force_terminate_task_group()) - except* TerminateTaskGroup: - pass - - asyncio.run(main()) - -Expected output: - -.. code-block:: text - - Task 1: start - Task 2: start - Task 1: done - Sleeping ======== diff --git a/Doc/library/calendar.rst b/Doc/library/calendar.rst index 54cafaf4fe47d8..1c8f25e96dcd6c 100644 --- a/Doc/library/calendar.rst +++ b/Doc/library/calendar.rst @@ -54,13 +54,13 @@ interpreted as prescribed by the ISO 8601 standard. Year 0 is 1 BC, year -1 is .. method:: setfirstweekday(firstweekday) - Set the first weekday to *firstweekday*, passed as an :class:`int` (0--6) + Set the first weekday to *firstweekday*, passed as an :class:`int` (0--6). Identical to setting the :attr:`~Calendar.firstweekday` property. .. method:: iterweekdays() - Return an iterator for the week day numbers that will be used for one + Return an iterator for the weekday numbers that will be used for one week. The first value from the iterator will be the same as the value of the :attr:`~Calendar.firstweekday` property. @@ -86,7 +86,7 @@ interpreted as prescribed by the ISO 8601 standard. Year 0 is 1 BC, year -1 is Return an iterator for the month *month* in the year *year* similar to :meth:`itermonthdates`, but not restricted by the :class:`datetime.date` range. Days returned will be tuples consisting of a day of the month - number and a week day number. + number and a weekday number. .. method:: itermonthdays3(year, month) @@ -408,7 +408,7 @@ For simple text calendars this module provides the following functions. .. function:: monthrange(year, month) - Returns weekday of first day of the month and number of days in month, for the + Returns weekday of first day of the month and number of days in month, for the specified *year* and *month*. @@ -446,7 +446,7 @@ For simple text calendars this module provides the following functions. An unrelated but handy function that takes a time tuple such as returned by the :func:`~time.gmtime` function in the :mod:`time` module, and returns the corresponding Unix timestamp value, assuming an epoch of 1970, and the POSIX - encoding. In fact, :func:`time.gmtime` and :func:`timegm` are each others' + encoding. In fact, :func:`time.gmtime` and :func:`timegm` are each other's inverse. @@ -580,9 +580,14 @@ The :mod:`!calendar` module defines the following exceptions: .. exception:: IllegalMonthError(month) - A subclass of :exc:`ValueError`, + A subclass of :exc:`ValueError` and :exc:`IndexError`, raised when the given month number is outside of the range 1-12 (inclusive). + .. versionchanged:: 3.12 + :exc:`IllegalMonthError` is now also a subclass of + :exc:`ValueError`. New code should avoid catching + :exc:`IndexError`. + .. attribute:: month The invalid month number. diff --git a/Doc/library/collections.rst b/Doc/library/collections.rst index cb9300f072b9e7..e42bdc06be09ff 100644 --- a/Doc/library/collections.rst +++ b/Doc/library/collections.rst @@ -326,7 +326,7 @@ For example:: .. versionadded:: 3.10 The usual dictionary methods are available for :class:`Counter` objects - except for two which work differently for counters. + except for these two which work differently for counters: .. method:: fromkeys(iterable) diff --git a/Doc/library/ctypes.rst b/Doc/library/ctypes.rst index 571975d4674397..b8d615565a590c 100644 --- a/Doc/library/ctypes.rst +++ b/Doc/library/ctypes.rst @@ -1735,7 +1735,7 @@ If wrapping a shared library with :mod:`!ctypes`, consider determining the shared library name at development time, and hardcoding it into the wrapper module instead of using :func:`!find_library` to locate the library at runtime. -Also consider addding a configuration option or environment variable to let +Also consider adding a configuration option or environment variable to let users select a library to use, and then perhaps use :func:`!find_library` as a default or fallback. @@ -1756,11 +1756,10 @@ as a default or fallback. (or by) Python. It is recommended to only use this function as a default or fallback, - .. deprecated:: 3.15 + .. soft-deprecated:: 3.15 - This function is :term:`soft deprecated`. - It is kept for use in cases where it works, but not expected to be - updated for additional platforms and configurations. + This function is kept for use in cases where it works, but not expected to + be updated for additional platforms and configurations. On Linux, :func:`!find_library` tries to run external programs (``/sbin/ldconfig``, ``gcc``, ``objdump`` and ``ld``) to find the @@ -3191,8 +3190,8 @@ Arrays and pointers Equivalent to ``type * length``, where *type* is a :mod:`!ctypes` data type and *length* an integer. - This function is :term:`soft deprecated` in favor of multiplication. - There are no plans to remove it. + .. soft-deprecated:: 3.14 + In favor of multiplication. .. class:: _Pointer diff --git a/Doc/library/dataclasses.rst b/Doc/library/dataclasses.rst index fd8e0c0bea1cb1..0bce3e5b762b8b 100644 --- a/Doc/library/dataclasses.rst +++ b/Doc/library/dataclasses.rst @@ -371,8 +371,8 @@ Module contents Converts the dataclass *obj* to a dict (by using the factory function *dict_factory*). Each dataclass is converted to a dict of its fields, as ``name: value`` pairs. dataclasses, dicts, - lists, and tuples are recursed into. Other objects are copied with - :func:`copy.deepcopy`. + frozendicts, lists, and tuples are recursed into. Other objects are copied + with :func:`copy.deepcopy`. Example of using :func:`!asdict` on nested dataclasses:: @@ -402,8 +402,8 @@ Module contents Converts the dataclass *obj* to a tuple (by using the factory function *tuple_factory*). Each dataclass is converted - to a tuple of its field values. dataclasses, dicts, lists, and - tuples are recursed into. Other objects are copied with + to a tuple of its field values. dataclasses, dicts, frozendicts, lists, + and tuples are recursed into. Other objects are copied with :func:`copy.deepcopy`. Continuing from the previous example:: diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index 119141d2e6daf3..aa99d198e436d5 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -19,13 +19,13 @@ are always available. They are listed here in alphabetical order. | | :func:`ascii` | | :func:`filter` | | :func:`map` | | **S** | | | | | :func:`float` | | :func:`max` | | |func-set|_ | | | **B** | | :func:`format` | | |func-memoryview|_ | | :func:`setattr` | -| | :func:`bin` | | |func-frozenset|_ | | :func:`min` | | :func:`slice` | -| | :func:`bool` | | | | | | :func:`sorted` | -| | :func:`breakpoint` | | **G** | | **N** | | :func:`staticmethod` | -| | |func-bytearray|_ | | :func:`getattr` | | :func:`next` | | |func-str|_ | -| | |func-bytes|_ | | :func:`globals` | | | | :func:`sum` | -| | | | | | **O** | | :func:`super` | -| | **C** | | **H** | | :func:`object` | | | +| | :func:`bin` | | |func-frozenset|_ | | :func:`min` | | :func:`sentinel` | +| | :func:`bool` | | | | | | :func:`slice` | +| | :func:`breakpoint` | | **G** | | **N** | | :func:`sorted` | +| | |func-bytearray|_ | | :func:`getattr` | | :func:`next` | | :func:`staticmethod` | +| | |func-bytes|_ | | :func:`globals` | | | | |func-str|_ | +| | | | | | **O** | | :func:`sum` | +| | **C** | | **H** | | :func:`object` | | :func:`super` | | | :func:`callable` | | :func:`hasattr` | | :func:`oct` | | **T** | | | :func:`chr` | | :func:`hash` | | :func:`open` | | |func-tuple|_ | | | :func:`classmethod` | | :func:`help` | | :func:`ord` | | :func:`type` | @@ -1827,6 +1827,61 @@ are always available. They are listed here in alphabetical order. :func:`setattr`. +.. class:: sentinel(name, /) + + Return a new unique sentinel object. *name* must be a :class:`str`, and is + used as the returned object's representation:: + + >>> MISSING = sentinel("MISSING") + >>> MISSING + MISSING + + Sentinel objects are truthy and compare equal only to themselves. They are + intended to be compared with the :keyword:`is` operator. + + Shallow and deep copies of a sentinel object return the object itself. + + Sentinels are conventionally assigned to a variable with a matching name. + Sentinels defined in this way can be used in :term:`type hints `:: + + MISSING = sentinel("MISSING") + + def next_value(default: int | MISSING = MISSING): + ... + + Sentinel objects support the :ref:`| ` operator for use in type expressions. + + :mod:`Pickling ` is supported for sentinel objects that are + placed in the global scope of a module under a name matching the sentinel's + name, and for sentinels placed in class scopes with a name matching the + :term:`qualified name` of the sentinel. Other sentinels, such as those + defined in a function scope, are not picklable. The identity of the sentinel is preserved + after pickling:: + + import pickle + + PICKLABLE = sentinel("PICKLABLE") + + assert pickle.loads(pickle.dumps(PICKLABLE)) is PICKLABLE + + class Cls: + PICKLABLE = sentinel("Cls.PICKLABLE") + + assert pickle.loads(pickle.dumps(Cls.PICKLABLE)) is Cls.PICKLABLE + + Sentinel objects have the following attributes: + + .. attribute:: __name__ + + The sentinel's name. + + .. attribute:: __module__ + + The name of the module where the sentinel was created. + + .. versionadded:: next + + .. class:: slice(stop, /) slice(start, stop, step=None, /) diff --git a/Doc/library/importlib.rst b/Doc/library/importlib.rst index 785f6c614b4391..0b76020eacc1da 100644 --- a/Doc/library/importlib.rst +++ b/Doc/library/importlib.rst @@ -286,7 +286,7 @@ ABC hierarchy:: This method can potentially yield a very large number of objects, and it may carry out IO operations when computing these values. - Because of this, it will generaly be desirable to compute the result + Because of this, it will generally be desirable to compute the result values on-the-fly, as they are needed. As such, the returned object is only guaranteed to be an :class:`iterable `, instead of a :class:`list` or other @@ -340,7 +340,7 @@ ABC hierarchy:: This method can potentially yield a very large number of objects, and it may carry out IO operations when computing these values. - Because of this, it will generaly be desirable to compute the result + Because of this, it will generally be desirable to compute the result values on-the-fly, as they are needed. As such, the returned object is only guaranteed to be an :class:`iterable `, instead of a :class:`list` or other diff --git a/Doc/library/itertools.rst b/Doc/library/itertools.rst index 5a0ac60ab7d2ec..06f8bf2a8b6fa8 100644 --- a/Doc/library/itertools.rst +++ b/Doc/library/itertools.rst @@ -833,6 +833,7 @@ and :term:`generators ` which incur interpreter overhead. from collections import Counter, deque from contextlib import suppress from functools import reduce + from heapq import heappush, heappushpop, heappush_max, heappushpop_max from math import comb, isqrt, prod, sumprod from operator import getitem, is_not, itemgetter, mul, neg, truediv @@ -848,11 +849,6 @@ and :term:`generators ` which incur interpreter overhead. # prepend(1, [2, 3, 4]) → 1 2 3 4 return chain([value], iterable) - def running_mean(iterable): - "Yield the average of all values seen so far." - # running_mean([8.5, 9.5, 7.5, 6.5]) → 8.5 9.0 8.5 8.0 - return map(truediv, accumulate(iterable), count(1)) - def repeatfunc(function, times=None, *args): "Repeat calls to a function with specified arguments." if times is None: @@ -1150,6 +1146,49 @@ and :term:`generators ` which incur interpreter overhead. return n + # ==== Running statistics ==== + + def running_mean(iterable): + "Average of values seen so far." + # running_mean([37, 33, 38, 28]) → 37 35 36 34 + return map(truediv, accumulate(iterable), count(1)) + + def running_min(iterable): + "Smallest of values seen so far." + # running_min([37, 33, 38, 28]) → 37 33 33 28 + return accumulate(iterable, func=min) + + def running_max(iterable): + "Largest of values seen so far." + # running_max([37, 33, 38, 28]) → 37 37 38 38 + return accumulate(iterable, func=max) + + def running_median(iterable): + "Median of values seen so far." + # running_median([37, 33, 38, 28]) → 37 35 37 35 + read = iter(iterable).__next__ + lo = [] # max-heap + hi = [] # min-heap the same size as or one smaller than lo + with suppress(StopIteration): + while True: + heappush_max(lo, heappushpop(hi, read())) + yield lo[0] + heappush(hi, heappushpop_max(lo, read())) + yield (lo[0] + hi[0]) / 2 + + def running_statistics(iterable): + "Aggregate statistics for values seen so far." + # Generate tuples: (size, minimum, median, maximum, mean) + t0, t1, t2, t3 = tee(iterable, 4) + return zip( + count(1), + running_min(t0), + running_median(t1), + running_max(t2), + running_mean(t3), + ) + + .. doctest:: :hide: @@ -1226,10 +1265,6 @@ and :term:`generators ` which incur interpreter overhead. [(0, 'a'), (1, 'b'), (2, 'c')] - >>> list(running_mean([8.5, 9.5, 7.5, 6.5])) - [8.5, 9.0, 8.5, 8.0] - - >>> for _ in loops(5): ... print('hi') ... @@ -1789,6 +1824,28 @@ and :term:`generators ` which incur interpreter overhead. True + >>> list(running_mean([8.5, 9.5, 7.5, 6.5])) + [8.5, 9.0, 8.5, 8.0] + >>> list(running_mean([37, 33, 38, 28])) + [37.0, 35.0, 36.0, 34.0] + + + >>> list(running_min([37, 33, 38, 28])) + [37, 33, 33, 28] + + + >>> list(running_max([37, 33, 38, 28])) + [37, 37, 38, 38] + + + >>> list(running_median([37, 33, 38, 28])) + [37, 35.0, 37, 35.0] + + + >>> list(running_statistics([37, 33, 38, 28])) + [(1, 37, 37, 37, 37.0), (2, 33, 35.0, 37, 35.0), (3, 33, 37, 38, 36.0), (4, 28, 35.0, 38, 34.0)] + + .. testcode:: :hide: diff --git a/Doc/library/math.rst b/Doc/library/math.rst index 4a11aec15dfb73..9cc8c5d6886324 100644 --- a/Doc/library/math.rst +++ b/Doc/library/math.rst @@ -781,9 +781,8 @@ the following functions from the :mod:`math.integer` module: Floats with integral values (like ``5.0``) are no longer accepted in the :func:`factorial` function. -.. deprecated:: 3.15 - These aliases are :term:`soft deprecated` in favor of the - :mod:`math.integer` functions. +.. soft-deprecated:: 3.15 + Use the :mod:`math.integer` functions instead of these aliases. Constants diff --git a/Doc/library/mimetypes.rst b/Doc/library/mimetypes.rst index 1e599bde8bcddd..0facacd50fd389 100644 --- a/Doc/library/mimetypes.rst +++ b/Doc/library/mimetypes.rst @@ -54,8 +54,8 @@ the information :func:`init` sets up. .. versionchanged:: 3.8 Added support for *url* being a :term:`path-like object`. - .. deprecated:: 3.13 - Passing a file path instead of URL is :term:`soft deprecated`. + .. soft-deprecated:: 3.13 + Passing a file path instead of URL. Use :func:`guess_file_type` for this. diff --git a/Doc/library/multiprocessing.rst b/Doc/library/multiprocessing.rst index 63bc252e129dfb..3ceb5e717c4825 100644 --- a/Doc/library/multiprocessing.rst +++ b/Doc/library/multiprocessing.rst @@ -1336,12 +1336,12 @@ Connection objects are usually created using Note that multiple connection objects may be polled at once by using :func:`multiprocessing.connection.wait`. - .. method:: send_bytes(buffer[, offset[, size]]) + .. method:: send_bytes(buf[, offset[, size]]) Send byte data from a :term:`bytes-like object` as a complete message. - If *offset* is given then data is read from that position in *buffer*. If - *size* is given then that many bytes will be read from buffer. Very large + If *offset* is given then data is read from that position in *buf*. If + *size* is given then that many bytes will be read from *buf*. Very large buffers (approximately 32 MiB+, though it depends on the OS) may raise a :exc:`ValueError` exception @@ -1361,18 +1361,18 @@ Connection objects are usually created using alias of :exc:`OSError`. - .. method:: recv_bytes_into(buffer[, offset]) + .. method:: recv_bytes_into(buf[, offset]) - Read into *buffer* a complete message of byte data sent from the other end + Read into *buf* a complete message of byte data sent from the other end of the connection and return the number of bytes in the message. Blocks until there is something to receive. Raises :exc:`EOFError` if there is nothing left to receive and the other end was closed. - *buffer* must be a writable :term:`bytes-like object`. If + *buf* must be a writable :term:`bytes-like object`. If *offset* is given then the message will be written into the buffer from that position. Offset must be a non-negative integer less than the - length of *buffer* (in bytes). + length of *buf* (in bytes). If the buffer is too short then a :exc:`BufferTooShort` exception is raised and the complete message is available as ``e.args[0]`` where ``e`` diff --git a/Doc/library/os.rst b/Doc/library/os.rst index 7547967c6b32f0..d2534b3e974f36 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -5110,9 +5110,8 @@ written in Python, such as a mail server's external command delivery program. Use :class:`subprocess.Popen` or :func:`subprocess.run` to control options like encodings. - .. deprecated:: 3.14 - The function is :term:`soft deprecated` and should no longer be used to - write new code. The :mod:`subprocess` module is recommended instead. + .. soft-deprecated:: 3.14 + The :mod:`subprocess` module is recommended instead. .. function:: posix_spawn(path, argv, env, *, file_actions=None, \ @@ -5340,9 +5339,8 @@ written in Python, such as a mail server's external command delivery program. .. versionchanged:: 3.6 Accepts a :term:`path-like object`. - .. deprecated:: 3.14 - These functions are :term:`soft deprecated` and should no longer be used - to write new code. The :mod:`subprocess` module is recommended instead. + .. soft-deprecated:: 3.14 + The :mod:`subprocess` module is recommended instead. .. data:: P_NOWAIT diff --git a/Doc/library/re.rst b/Doc/library/re.rst index 7e0a00cba2f63f..a46fd42458158c 100644 --- a/Doc/library/re.rst +++ b/Doc/library/re.rst @@ -931,7 +931,6 @@ Functions .. function:: prefixmatch(pattern, string, flags=0) -.. function:: match(pattern, string, flags=0) If zero or more characters at the beginning of *string* match the regular expression *pattern*, return a corresponding :class:`~re.Match`. Return @@ -954,7 +953,11 @@ Functions :func:`~re.match`. Use that name when you need to retain compatibility with older Python versions. - .. deprecated:: 3.15 + .. versionadded:: 3.15 + +.. function:: match(pattern, string, flags=0) + + .. soft-deprecated:: 3.15 :func:`~re.match` has been :term:`soft deprecated` in favor of the alternate :func:`~re.prefixmatch` name of this API which is more explicitly descriptive. Use it to better @@ -1285,7 +1288,6 @@ Regular expression objects .. method:: Pattern.prefixmatch(string[, pos[, endpos]]) -.. method:: Pattern.match(string[, pos[, endpos]]) If zero or more characters at the *beginning* of *string* match this regular expression, return a corresponding :class:`~re.Match`. Return ``None`` if the @@ -1310,7 +1312,11 @@ Regular expression objects :meth:`~Pattern.match`. Use that name when you need to retain compatibility with older Python versions. - .. deprecated:: 3.15 + .. versionadded:: 3.15 + +.. method:: Pattern.match(string[, pos[, endpos]]) + + .. soft-deprecated:: 3.15 :meth:`~Pattern.match` has been :term:`soft deprecated` in favor of the alternate :meth:`~Pattern.prefixmatch` name of this API which is more explicitly descriptive. Use it to @@ -1794,8 +1800,8 @@ while new code should prefer :func:`!prefixmatch`. .. versionadded:: 3.15 :func:`!prefixmatch` -.. deprecated:: 3.15 - :func:`!match` is :term:`soft deprecated` +.. soft-deprecated:: 3.15 + :func:`!match` Making a phonebook ^^^^^^^^^^^^^^^^^^ @@ -1947,7 +1953,7 @@ successive matches:: class Token(NamedTuple): type: str - value: str + value: int | float | str line: int column: int diff --git a/Doc/library/sys.monitoring.rst b/Doc/library/sys.monitoring.rst index 16e6b1d6dc786b..7cca6f2bcdae91 100644 --- a/Doc/library/sys.monitoring.rst +++ b/Doc/library/sys.monitoring.rst @@ -180,8 +180,8 @@ Local events '''''''''''' Local events are associated with normal execution of the program and happen -at clearly defined locations. All local events can be disabled. -The local events are: +at clearly defined locations. All local events can be disabled +per location. The local events are: * :monitoring-event:`PY_START` * :monitoring-event:`PY_RESUME` @@ -205,6 +205,8 @@ Using :monitoring-event:`BRANCH_LEFT` and :monitoring-event:`BRANCH_RIGHT` events will give much better performance as they can be disabled independently. +.. _monitoring-ancillary-events: + Ancillary events '''''''''''''''' @@ -226,7 +228,7 @@ Other events '''''''''''' Other events are not necessarily tied to a specific location in the -program and cannot be individually disabled via :data:`DISABLE`. +program and cannot be individually disabled per location. The other events that can be monitored are: @@ -234,6 +236,12 @@ The other events that can be monitored are: * :monitoring-event:`PY_UNWIND` * :monitoring-event:`RAISE` * :monitoring-event:`EXCEPTION_HANDLED` +* :monitoring-event:`RERAISE` + +.. versionchanged:: 3.15 + Other events can now be turned on and disabled on a per code object + basis. Returning :data:`DISABLE` from a callback disables the event + for the entire code object (for the current tool). The STOP_ITERATION event @@ -247,8 +255,7 @@ raise an exception unless it would be visible to other code. To allow tools to monitor for real exceptions without slowing down generators and coroutines, the :monitoring-event:`STOP_ITERATION` event is provided. -:monitoring-event:`STOP_ITERATION` can be locally disabled, unlike -:monitoring-event:`RAISE`. +:monitoring-event:`STOP_ITERATION` can be locally disabled. Note that the :monitoring-event:`STOP_ITERATION` event and the :monitoring-event:`RAISE` event for a :exc:`StopIteration` exception are @@ -314,15 +321,14 @@ location by returning :data:`sys.monitoring.DISABLE` from a callback function. This does not change which events are set, or any other code locations for the same event. -Disabling events for specific locations is very important for high -performance monitoring. For example, a program can be run under a -debugger with no overhead if the debugger disables all monitoring -except for a few breakpoints. +:ref:`Other events ` can be disabled on a per code +object basis by returning :data:`sys.monitoring.DISABLE` from a callback +function. This disables the event for the entire code object (for the current +tool). -If :data:`DISABLE` is returned by a callback for a -:ref:`global event `, :exc:`ValueError` will be raised -by the interpreter in a non-specific location (that is, no traceback will be -provided). +Disabling events for specific locations is very important for high performance +monitoring. For example, a program can be run under a debugger with no overhead +if the debugger disables all monitoring except for a few breakpoints. .. function:: restart_events() -> None diff --git a/Doc/library/typing.rst b/Doc/library/typing.rst index 2ce868cf84da9d..1957cadcbb1592 100644 --- a/Doc/library/typing.rst +++ b/Doc/library/typing.rst @@ -1174,7 +1174,8 @@ These can be used as types in annotations. They all support subscription using or transforms parameters of another callable. Usage is in the form ``Concatenate[Arg1Type, Arg2Type, ..., ParamSpecVariable]``. ``Concatenate`` - is currently only valid when used as the first argument to a :ref:`Callable `. + is valid when used in :ref:`Callable ` type hints + and when instantiating user-defined generic classes with :class:`ParamSpec` parameters. The last parameter to ``Concatenate`` must be a :class:`ParamSpec` or ellipsis (``...``). @@ -1980,7 +1981,7 @@ without the dedicated syntax, as documented below. .. _typevartuple: -.. class:: TypeVarTuple(name, *, default=typing.NoDefault) +.. class:: TypeVarTuple(name, *, bound=None, covariant=False, contravariant=False, infer_variance=False, default=typing.NoDefault) Type variable tuple. A specialized form of :ref:`type variable ` that enables *variadic* generics. @@ -2090,6 +2091,24 @@ without the dedicated syntax, as documented below. The name of the type variable tuple. + .. attribute:: __covariant__ + + Whether the type variable tuple has been explicitly marked as covariant. + + .. versionadded:: 3.15 + + .. attribute:: __contravariant__ + + Whether the type variable tuple has been explicitly marked as contravariant. + + .. versionadded:: 3.15 + + .. attribute:: __infer_variance__ + + Whether the type variable tuple's variance should be inferred by type checkers. + + .. versionadded:: 3.15 + .. attribute:: __default__ The default value of the type variable tuple, or :data:`typing.NoDefault` if it @@ -2116,6 +2135,11 @@ without the dedicated syntax, as documented below. .. versionadded:: 3.13 + Type variable tuples created with ``covariant=True`` or + ``contravariant=True`` can be used to declare covariant or contravariant + generic types. The ``bound`` argument is also accepted, similar to + :class:`TypeVar`, but its actual semantics are yet to be decided. + .. versionadded:: 3.11 .. versionchanged:: 3.12 @@ -2127,6 +2151,11 @@ without the dedicated syntax, as documented below. Support for default values was added. + .. versionchanged:: 3.15 + + Added support for the ``bound``, ``covariant``, ``contravariant``, and + ``infer_variance`` parameters. + .. class:: ParamSpec(name, *, bound=None, covariant=False, contravariant=False, default=typing.NoDefault) Parameter specification variable. A specialized version of @@ -2196,6 +2225,20 @@ without the dedicated syntax, as documented below. The name of the parameter specification. + .. attribute:: __covariant__ + + Whether the parameter specification has been explicitly marked as covariant. + + .. attribute:: __contravariant__ + + Whether the parameter specification has been explicitly marked as contravariant. + + .. attribute:: __infer_variance__ + + Whether the parameter specification's variance should be inferred by type checkers. + + .. versionadded:: 3.12 + .. attribute:: __default__ The default value of the parameter specification, or :data:`typing.NoDefault` if it @@ -3358,6 +3401,36 @@ Functions and decorators .. versionadded:: 3.12 +.. decorator:: disjoint_base + + Decorator to mark a class as a disjoint base. + + Type checkers do not allow child classes of a disjoint base ``C`` to + inherit from other disjoint bases that are not parent or child classes of ``C``. + + For example:: + + @disjoint_base + class Disjoint1: pass + + @disjoint_base + class Disjoint2: pass + + class Disjoint3(Disjoint1, Disjoint2): pass # Type checker error + + Type checkers can use knowledge of disjoint bases to detect unreachable code + and determine when two types can overlap. + + The corresponding runtime concept is a solid base (see :ref:`multiple-inheritance`). + Classes that are solid bases at runtime can be marked with ``@disjoint_base`` in stub files. + Users may also mark other classes as disjoint bases to indicate to type checkers that + multiple inheritance with other disjoint bases should not be allowed. + + Note that the concept of a solid base is a CPython implementation + detail, and the exact set of standard library classes that are + disjoint bases at runtime may change in future versions of Python. + + .. versionadded:: next .. decorator:: type_check_only @@ -3380,13 +3453,13 @@ Functions and decorators Introspection helpers --------------------- -.. function:: get_type_hints(obj, globalns=None, localns=None, include_extras=False) +.. function:: get_type_hints(obj, globalns=None, localns=None, include_extras=False, *, format=Format.VALUE) Return a dictionary containing type hints for a function, method, module, class object, or other callable object. - This is often the same as ``obj.__annotations__``, but this function makes - the following changes to the annotations dictionary: + This is often the same as :func:`annotationlib.get_annotations`, but this + function makes the following changes to the annotations dictionary: * Forward references encoded as string literals or :class:`ForwardRef` objects are handled by evaluating them in *globalns*, *localns*, and @@ -3400,17 +3473,15 @@ Introspection helpers annotations from ``C``'s base classes with those on ``C`` directly. This is done by traversing :attr:`C.__mro__ ` and iteratively combining - ``__annotations__`` dictionaries. Annotations on classes appearing - earlier in the :term:`method resolution order` always take precedence over - annotations on classes appearing later in the method resolution order. + :term:`annotations ` of each base class. Annotations + on classes appearing earlier in the :term:`method resolution order` always + take precedence over annotations on classes appearing later in the method + resolution order. * The function recursively replaces all occurrences of ``Annotated[T, ...]``, ``Required[T]``, ``NotRequired[T]``, and ``ReadOnly[T]`` with ``T``, unless *include_extras* is set to ``True`` (see :class:`Annotated` for more information). - See also :func:`annotationlib.get_annotations`, a lower-level function that - returns annotations more directly. - .. caution:: This function may execute arbitrary code contained in annotations. @@ -3418,11 +3489,12 @@ Introspection helpers .. note:: - If any forward references in the annotations of *obj* are not resolvable - or are not valid Python code, this function will raise an exception - such as :exc:`NameError`. For example, this can happen with imported - :ref:`type aliases ` that include forward references, - or with names imported under :data:`if TYPE_CHECKING `. + If :attr:`Format.VALUE ` is used and any + forward references in the annotations of *obj* are not resolvable, a + :exc:`NameError` exception is raised. For example, this can happen + with names imported under :data:`if TYPE_CHECKING `. + More generally, any kind of exception can be raised if an annotation + contains invalid Python code. .. note:: @@ -3440,6 +3512,10 @@ Introspection helpers if a default value equal to ``None`` was set. Now the annotation is returned unchanged. + .. versionchanged:: 3.14 + Added the ``format`` parameter. See the documentation on + :func:`annotationlib.get_annotations` for more information. + .. versionchanged:: 3.14 Calling :func:`get_type_hints` on instances is no longer supported. Some instances were accepted in earlier versions as an undocumented diff --git a/Doc/reference/compound_stmts.rst b/Doc/reference/compound_stmts.rst index 0cf0a41bfb400c..72e1cad3bbd892 100644 --- a/Doc/reference/compound_stmts.rst +++ b/Doc/reference/compound_stmts.rst @@ -858,7 +858,7 @@ A literal pattern corresponds to most : | "None" : | "True" : | "False" - signed_number: ["-"] NUMBER + signed_number: ["+" | "-"] NUMBER The rule ``strings`` and the token ``NUMBER`` are defined in the :doc:`standard Python grammar <./grammar>`. Triple-quoted strings are diff --git a/Doc/tools/extensions/changes.py b/Doc/tools/extensions/changes.py index 8de5e7f78c6627..02dc51b3a76943 100644 --- a/Doc/tools/extensions/changes.py +++ b/Doc/tools/extensions/changes.py @@ -2,8 +2,10 @@ from __future__ import annotations -from typing import TYPE_CHECKING +import re +from docutils import nodes +from sphinx import addnodes from sphinx.domains.changeset import ( VersionChange, versionlabel_classes, @@ -11,6 +13,7 @@ ) from sphinx.locale import _ as sphinx_gettext +TYPE_CHECKING = False if TYPE_CHECKING: from docutils.nodes import Node from sphinx.application import Sphinx @@ -73,6 +76,76 @@ def run(self) -> list[Node]: versionlabel_classes[self.name] = "" +class SoftDeprecated(PyVersionChange): + """Directive for soft deprecations that auto-links to the glossary term. + + Usage:: + + .. soft-deprecated:: 3.15 + + Use :func:`new_thing` instead. + + Renders as: "Soft deprecated since version 3.15: Use new_thing() instead." + with "Soft deprecated" linking to the glossary definition. + """ + + _TERM_RE = re.compile(r":term:`([^`]+)`") + + def run(self) -> list[Node]: + versionlabels[self.name] = sphinx_gettext( + ":term:`Soft deprecated` since version %s" + ) + versionlabel_classes[self.name] = "soft-deprecated" + try: + result = super().run() + finally: + versionlabels[self.name] = "" + versionlabel_classes[self.name] = "" + + for node in result: + # Add "versionchanged" class so existing theme CSS applies + node["classes"] = node.get("classes", []) + ["versionchanged"] + # Replace the plain-text "Soft deprecated" with a glossary reference + for inline in node.findall(nodes.inline): + if "versionmodified" in inline.get("classes", []): + self._add_glossary_link(inline) + + return result + + @classmethod + def _add_glossary_link(cls, inline: nodes.inline) -> None: + """Replace :term:`soft deprecated` text with a cross-reference to the + 'Soft deprecated' glossary entry.""" + for child in inline.children: + if not isinstance(child, nodes.Text): + continue + + text = str(child) + match = cls._TERM_RE.search(text) + if match is None: + continue + + ref = addnodes.pending_xref( + "", + nodes.Text(match.group(1)), + refdomain="std", + reftype="term", + reftarget="soft deprecated", + refwarn=True, + ) + + start, end = match.span() + new_nodes: list[nodes.Node] = [] + if start > 0: + new_nodes.append(nodes.Text(text[:start])) + new_nodes.append(ref) + if end < len(text): + new_nodes.append(nodes.Text(text[end:])) + + child.parent.replace(child, new_nodes) + break + + def setup(app: Sphinx) -> ExtensionMetadata: # Override Sphinx's directives with support for 'next' app.add_directive("versionadded", PyVersionChange, override=True) @@ -83,6 +156,9 @@ def setup(app: Sphinx) -> ExtensionMetadata: # Register the ``.. deprecated-removed::`` directive app.add_directive("deprecated-removed", DeprecatedRemoved) + # Register the ``.. soft-deprecated::`` directive + app.add_directive("soft-deprecated", SoftDeprecated) + return { "version": "1.0", "parallel_read_safe": True, diff --git a/Doc/tools/removed-ids.txt b/Doc/tools/removed-ids.txt index f3cd8bf0ef5bb9..5e3ef2efe271fd 100644 --- a/Doc/tools/removed-ids.txt +++ b/Doc/tools/removed-ids.txt @@ -1 +1,7 @@ # HTML IDs excluded from the check-html-ids.py check. + +# Remove from here in 3.16 +c-api/allocation.html: deprecated-aliases +c-api/file.html: deprecated-api + +library/asyncio-task.html: terminating-a-task-group diff --git a/Doc/tools/templates/dummy.html b/Doc/tools/templates/dummy.html index 75f6607d8f3698..699e518801cbcd 100644 --- a/Doc/tools/templates/dummy.html +++ b/Doc/tools/templates/dummy.html @@ -29,6 +29,7 @@ {% trans %}Deprecated since version %s, will be removed in version %s{% endtrans %} {% trans %}Deprecated since version %s, removed in version %s{% endtrans %} +{% trans %}:term:`Soft deprecated` since version %s{% endtrans %} In docsbuild-scripts, when rewriting indexsidebar.html with actual versions: diff --git a/Doc/using/android.rst b/Doc/using/android.rst index 45345d045ddfd9..60a13569318141 100644 --- a/Doc/using/android.rst +++ b/Doc/using/android.rst @@ -30,7 +30,7 @@ Adding Python to an Android app Most app developers should use one of the following tools, which will provide a much easier experience: -* `Briefcase `__, from the BeeWare project +* `Briefcase `__, from the BeeWare project * `Buildozer `__, from the Kivy project * `Chaquopy `__ * `pyqtdeploy `__ diff --git a/Doc/whatsnew/3.15.rst b/Doc/whatsnew/3.15.rst index 7ea7c901eceb19..f23791146704b5 100644 --- a/Doc/whatsnew/3.15.rst +++ b/Doc/whatsnew/3.15.rst @@ -69,6 +69,8 @@ Summary -- Release highlights ` * :pep:`814`: :ref:`Add frozendict built-in type ` +* :pep:`661`: :ref:`Add sentinel built-in type + ` * :pep:`799`: :ref:`A dedicated profiling package for organizing Python profiling tools ` * :pep:`799`: :ref:`Tachyon: High frequency statistical sampling profiler @@ -80,6 +82,7 @@ Summary -- Release highlights * :pep:`728`: ``TypedDict`` with typed extra items * :pep:`747`: :ref:`Annotating type forms with TypeForm ` +* :pep:`800`: Disjoint bases in the type system * :pep:`782`: :ref:`A new PyBytesWriter C API to create a Python bytes object ` * :pep:`803`: :ref:`Stable ABI for Free-Threaded Builds ` @@ -141,7 +144,7 @@ In the case where loading a lazily imported module fails (for example, if the module does not exist), Python raises the exception at the point of first use rather than at import time. The associated traceback includes both the location where the name was accessed and the original import statement, -making it straightforward to diagnose & debug the failure. +making it straightforward to diagnose and debug the failure. For cases where you want to enable lazy loading globally without modifying source code, Python provides the :option:`-X lazy_imports <-X>` command-line @@ -234,6 +237,20 @@ to accept also other mapping types such as :class:`~types.MappingProxyType`. (Contributed by Victor Stinner and Donghee Na in :gh:`141510`.) +.. _whatsnew315-sentinel: + +:pep:`661`: Add sentinel built-in type +-------------------------------------- + +A new :class:`sentinel` type is added to the :mod:`builtins` module for +creating unique sentinel values with a concise representation. Sentinel +objects preserve identity when copied, support use in type expressions with +the ``|`` operator, and can be pickled when they are importable by module and +name. + +(PEP by Tal Einat; contributed by Jelle Zijlstra in :gh:`148829`.) + + .. _whatsnew315-profiling-package: :pep:`799`: A dedicated profiling package @@ -450,14 +467,36 @@ Improved error messages Running this code now produces a clearer suggestion: - .. code-block:: pycon + .. code-block:: pytb Traceback (most recent call last): - File "/home/pablogsal/github/python/main/lel.py", line 42, in - print(container.area) - ^^^^^^^^^^^^^^ + File "/home/pablogsal/github/python/main/lel.py", line 42, in + print(container.area) + ^^^^^^^^^^^^^^ AttributeError: 'Container' object has no attribute 'area'. Did you mean '.inner.area' instead of '.area'? +* The interpreter now tries to provide a suggestion when + :func:`delattr` fails due to a missing attribute. + When an attribute name that closely resembles an existing attribute is used, + the interpreter will suggest the correct attribute name in the error message. + For example: + + .. doctest:: + + >>> class A: + ... pass + >>> a = A() + >>> a.abcde = 1 + >>> del a.abcdf # doctest: +ELLIPSIS + Traceback (most recent call last): + ... + AttributeError: 'A' object has no attribute 'abcdf'. Did you mean: 'abcde'? + + (Contributed by Nikita Sobolev and Pranjal Prajapati in :gh:`136588`.) + +* Several error messages incorrectly using the term "argument" have been corrected. + (Contributed by Stan Ulbrych in :gh:`133382`.) + Other language changes ====================== @@ -489,28 +528,6 @@ Other language changes (Contributed by Adam Turner in :gh:`133711`; PEP 686 written by Inada Naoki.) -* Several error messages incorrectly using the term "argument" have been corrected. - (Contributed by Stan Ulbrych in :gh:`133382`.) - -* The interpreter now tries to provide a suggestion when - :func:`delattr` fails due to a missing attribute. - When an attribute name that closely resembles an existing attribute is used, - the interpreter will suggest the correct attribute name in the error message. - For example: - - .. doctest:: - - >>> class A: - ... pass - >>> a = A() - >>> a.abcde = 1 - >>> del a.abcdf # doctest: +ELLIPSIS - Traceback (most recent call last): - ... - AttributeError: 'A' object has no attribute 'abcdf'. Did you mean: 'abcde'? - - (Contributed by Nikita Sobolev and Pranjal Prajapati in :gh:`136588`.) - * Unraisable exceptions are now highlighted with color by default. This can be controlled by :ref:`environment variables `. (Contributed by Peter Bierma in :gh:`134170`.) @@ -644,6 +661,10 @@ Other language changes * Allow the *count* argument of :meth:`bytes.replace` to be a keyword. (Contributed by Stan Ulbrych in :gh:`147856`.) +* Unary plus is now accepted in :keyword:`match` literal patterns, mirroring + the existing support for unary minus. + (Contributed by Bartosz Sławecki in :gh:`145239`.) + New modules =========== @@ -707,7 +728,7 @@ base64 (Contributed by Serhiy Storchaka in :gh:`143214` and :gh:`146431`.) * Added the *ignorechars* parameter in :func:`~base64.b16decode`, - :func:`~base64.b32decode`, :func:`~base64.b32hexdecode`, + :func:`~base64.b32decode`, :func:`~base64.b32hexdecode`, :func:`~base64.b64decode`, :func:`~base64.b85decode`, and :func:`~base64.z85decode`. (Contributed by Serhiy Storchaka in :gh:`144001` and :gh:`146431`.) @@ -879,13 +900,13 @@ inspect json ---- -* Add the *array_hook* parameter to :func:`~json.load` and +* Add the *array_hook* parameter to :func:`~json.load` and :func:`~json.loads` functions: allow a callback for JSON literal array types to customize Python lists in the resulting decoded object. Passing combined :class:`frozendict` to *object_pairs_hook* param and :class:`tuple` to ``array_hook`` will yield a deeply nested immutable Python structure representing the JSON data. - (Contributed by Joao S. O. Bueno in :gh:`146440`) + (Contributed by Joao S. O. Bueno in :gh:`146440`.) locale @@ -913,23 +934,11 @@ math mimetypes --------- -* Add ``application/dicom`` MIME type for ``.dcm`` extension. - (Contributed by Benedikt Johannes in :gh:`144217`.) -* Add ``application/efi``. (Contributed by Charlie Lin in :gh:`145720`.) -* Add ``application/node`` MIME type for ``.cjs`` extension. - (Contributed by John Franey in :gh:`140937`.) -* Add ``application/toml``. (Contributed by Gil Forcada in :gh:`139959`.) -* Add ``application/sql`` and ``application/vnd.sqlite3``. - (Contributed by Charlie Lin in :gh:`145698`.) -* Add the following MIME types: - - - ``application/vnd.ms-cab-compressed`` for ``.cab`` extension - - ``application/vnd.ms-htmlhelp`` for ``.chm`` extension - - ``application/vnd.ms-officetheme`` for ``.thmx`` extension - - (Contributed by Charlie Lin in :gh:`145718`.) - -* Add ``image/jxl``. (Contributed by Foolbar in :gh:`144213`.) +* Add more MIME types. + (Contributed by Benedikt Johannes, Charlie Lin, Foolbar, Gil Forcada and + John Franey + in :gh:`144217`, :gh:`145720`, :gh:`140937`, :gh:`139959`, :gh:`145698`, + :gh:`145718` and :gh:`144213`.) * Rename ``application/x-texinfo`` to ``application/texinfo``. (Contributed by Charlie Lin in :gh:`140165`.) * Changed the MIME type for ``.ai`` files to ``application/pdf``. @@ -978,6 +987,20 @@ pickle (Contributed by Zackery Spytz and Serhiy Storchaka in :gh:`77188`.) +pprint +------ + +* Add an *expand* keyword argument for :func:`pprint.pprint`, + :func:`pprint.pformat`, :func:`pprint.pp`. If true, the output will be + formatted similar to pretty-printed :func:`json.dumps` when + *indent* is supplied. + (Contributed by Stefan Todoran, Semyon Moroz and Hugo van Kemenade in + :gh:`112632`.) + +* Add t-string support to :mod:`pprint`. + (Contributed by Loïc Simon and Hugo van Kemenade in :gh:`134551`.) + + re -- @@ -1099,7 +1122,7 @@ subprocess If none of these mechanisms are available, the function falls back to the traditional busy loop (non-blocking call and short sleeps). - (Contributed by Giampaolo Rodola in :gh:`83069`). + (Contributed by Giampaolo Rodola in :gh:`83069`.) symtable @@ -1116,6 +1139,19 @@ sys (Contributed by Klaus Zimmermann in :gh:`137476`.) +sys.monitoring +-------------- + +* The :ref:`other events ` + (:monitoring-event:`PY_THROW`, :monitoring-event:`PY_UNWIND`, + :monitoring-event:`RAISE`, :monitoring-event:`EXCEPTION_HANDLED`, and + :monitoring-event:`RERAISE`) can now be turned on and disabled on a per code + object basis. Returning :data:`~sys.monitoring.DISABLE` from a callback for + one of these events disables the event for the entire code object (for the + current tool), rather than raising :exc:`ValueError` as in prior versions. + (Contributed by Gabriele N. Tornetta in :gh:`146182`.) + + tarfile ------- @@ -1156,7 +1192,7 @@ timeit * Make the target time of :meth:`timeit.Timer.autorange` configurable and add ``--target-time`` option to the command-line interface. - (Contributed by Alessandro Cucci and Miikka Koskinen in :gh:`140283`.) + (Contributed by Alessandro Cucci and Miikka Koskinen in :gh:`80642`.) tkinter @@ -1201,10 +1237,7 @@ tomllib Previously an inline table had to be on a single line and couldn't end with a trailing comma. This is now relaxed so that the following is valid: - .. syntax highlighting needs TOML 1.1.0 support in Pygments, - see https://github.com/pygments/pygments/issues/3026 - - .. code-block:: text + .. code-block:: toml tbl = { key = "a string", @@ -1216,7 +1249,7 @@ tomllib - Add ``\xHH`` notation to basic strings for codepoints under 255, and the ``\e`` escape for the escape character: - .. code-block:: text + .. code-block:: toml null = "null byte: \x00; letter a: \x61" csi = "\e[" @@ -1224,7 +1257,7 @@ tomllib - Seconds in datetime and time values are now optional. The following are now valid: - .. code-block:: text + .. code-block:: toml dt = 2010-02-03 14:15 t = 14:15 @@ -1276,6 +1309,18 @@ typing as it was incorrectly inferred in runtime before. (Contributed by Nikita Sobolev in :gh:`137191`.) +* :pep:`800`: Add :deco:`typing.disjoint_base`, a new decorator marking a class + as a disjoint base. This is an advanced feature primarily intended to allow + type checkers to faithfully reflect the runtime semantics of types defined + as builtins or in compiled extensions. If a class ``C`` is a disjoint base, then + child classes of that class cannot inherit from other disjoint bases that are + not parent or child classes of ``C``. (Contributed by Jelle Zijlstra in :gh:`148639`.) + +* :class:`~typing.TypeVarTuple` now accepts ``bound``, ``covariant``, + ``contravariant``, and ``infer_variance`` keyword arguments, matching the + interface of :class:`~typing.TypeVar` and :class:`~typing.ParamSpec`. + ``bound`` semantics remain undefined in the specification. + unicodedata ----------- @@ -1399,7 +1444,7 @@ Optimizations ============= * ``mimalloc`` is now used as the default allocator for - for raw memory allocations such as via :c:func:`PyMem_RawMalloc` + raw memory allocations such as via :c:func:`PyMem_RawMalloc` for better performance on :term:`free-threaded builds `. (Contributed by Kumar Aditya in :gh:`144914`.) @@ -1418,7 +1463,7 @@ base64 & binascii * Implementation for Base32 has been rewritten in C. Encoding and decoding is now two orders of magnitude faster. - (Contributed by James Seo in :gh:`146192`) + (Contributed by James Seo in :gh:`146192`.) csv @@ -1512,7 +1557,7 @@ The JIT compiler's machine code generator now produces better machine code for x86-64 and AArch64 macOS and Linux targets. In general, users should experience lower memory usage for generated machine code and more efficient machine code versus 3.14. -(Contributed by Brandt Bucher in :gh:`136528` and :gh:`136528`. +(Contributed by Brandt Bucher in :gh:`136528` and :gh:`135905`. Implementation for AArch64 contributed by Mark Shannon in :gh:`139855`. Additional optimizations for AArch64 contributed by Mark Shannon and Diego Russo in :gh:`140683` and :gh:`142305`.) @@ -1535,6 +1580,14 @@ collections.abc deprecated since Python 3.12, and is scheduled for removal in Python 3.17. +ctypes +------ + +* Removed the undocumented function :func:`!ctypes.SetPointerType`, + which has been deprecated since Python 3.13. + (Contributed by Bénédikt Tran in :gh:`133866`.) + + datetime -------- @@ -1544,14 +1597,6 @@ datetime (Contributed by Stan Ulbrych and Gregory P. Smith in :gh:`70647`.) -ctypes ------- - -* Removed the undocumented function :func:`!ctypes.SetPointerType`, - which has been deprecated since Python 3.13. - (Contributed by Bénédikt Tran in :gh:`133866`.) - - glob ---- @@ -1594,20 +1639,6 @@ platform (Contributed by Alexey Makridenko in :gh:`133604`.) -pprint ------- - -* Add an *expand* keyword argument for :func:`pprint.pprint`, - :func:`pprint.pformat`, :func:`pprint.pp`. If true, the output will be - formatted similar to pretty-printed :func:`json.dumps` when - *indent* is supplied. - (Contributed by Stefan Todoran, Semyon Moroz and Hugo van Kemenade in - :gh:`112632`.) - -* Add t-string support to :mod:`pprint`. - (Contributed by Loïc Simon and Hugo van Kemenade in :gh:`134551`.) - - sre_* ----- @@ -1650,6 +1681,9 @@ typing or ``TD = TypedDict("TD", {})`` instead. (Contributed by Bénédikt Tran in :gh:`133823`.) +* Deprecated :func:`!typing.no_type_check_decorator` has been removed. + (Contributed by Nikita Sobolev in :gh:`133601`.) + wave ---- @@ -1765,8 +1799,6 @@ New deprecations :func:`issubclass`, but warnings were not previously emitted if it was merely imported or accessed from the :mod:`!typing` module. - * Deprecated :func:`!typing.no_type_check_decorator` has been removed. - (Contributed by Nikita Sobolev in :gh:`133601`.) * ``__version__`` @@ -2053,8 +2085,8 @@ Deprecated C APIs - :c:macro:`Py_ALIGNED`: Prefer ``alignas`` instead. - :c:macro:`PY_FORMAT_SIZE_T`: Use ``"z"`` directly. - - :c:macro:`Py_LL` & :c:macro:`Py_ULL`: - Use standard suffixes, ``LL`` & ``ULL``. + - :c:macro:`Py_LL` and :c:macro:`Py_ULL`: + Use standard suffixes, ``LL`` and ``ULL``. - :c:macro:`PY_LONG_LONG`, :c:macro:`PY_LLONG_MIN`, :c:macro:`PY_LLONG_MAX`, :c:macro:`PY_ULLONG_MAX`, :c:macro:`PY_INT32_T`, :c:macro:`PY_UINT32_T`, :c:macro:`PY_INT64_T`, :c:macro:`PY_UINT64_T`, :c:macro:`PY_SIZE_MAX`: diff --git a/Grammar/python.gram b/Grammar/python.gram index 3a91d426c36501..9bf3a67939fcf3 100644 --- a/Grammar/python.gram +++ b/Grammar/python.gram @@ -554,10 +554,12 @@ complex_number[expr_ty]: signed_number[expr_ty]: | NUMBER + | '+' number=NUMBER { number } | '-' number=NUMBER { _PyAST_UnaryOp(USub, number, EXTRA) } signed_real_number[expr_ty]: | real_number + | '+' real=real_number { real } | '-' real=real_number { _PyAST_UnaryOp(USub, real, EXTRA) } real_number[expr_ty]: @@ -565,6 +567,7 @@ real_number[expr_ty]: imaginary_number[expr_ty]: | imag=NUMBER { _PyPegen_ensure_imaginary(p, imag) } + | '+' imag=NUMBER { _PyPegen_ensure_imaginary(p, imag) } capture_pattern[pattern_ty]: | target=pattern_capture_target { _PyAST_MatchAs(NULL, target->v.Name.id, EXTRA) } diff --git a/Include/Python.h b/Include/Python.h index e6e5cab67e2045..1272e2464f91d1 100644 --- a/Include/Python.h +++ b/Include/Python.h @@ -9,10 +9,11 @@ // is not needed. -// Include Python header files -#include "patchlevel.h" -#include "pyconfig.h" -#include "pymacconfig.h" +// Include Python configuration headers +#include "patchlevel.h" // the Python version +#include "pyconfig.h" // information from configure +#include "pymacconfig.h" // overrides for pyconfig +#include "pyabi.h" // feature/ABI selection // Include standard header files @@ -46,13 +47,11 @@ # endif #endif -#if defined(Py_GIL_DISABLED) -# if defined(_MSC_VER) -# include // __readgsqword() -# endif - -# if defined(__MINGW32__) -# include // __readgsqword() +#if !defined(Py_LIMITED_API) +# if defined(Py_GIL_DISABLED) +# if defined(_MSC_VER) || defined(__MINGW32__) +# include // __readgsqword() +# endif # endif #endif // Py_GIL_DISABLED @@ -67,6 +66,7 @@ __pragma(warning(disable: 4201)) // Include Python header files #include "pyport.h" +#include "exports.h" #include "pymacro.h" #include "pymath.h" #include "pymem.h" @@ -117,6 +117,7 @@ __pragma(warning(disable: 4201)) #include "cpython/genobject.h" #include "descrobject.h" #include "genericaliasobject.h" +#include "sentinelobject.h" #include "warnings.h" #include "weakrefobject.h" #include "structseq.h" diff --git a/Include/cpython/monitoring.h b/Include/cpython/monitoring.h index 5094c8c23ae32b..fa6168d95cd210 100644 --- a/Include/cpython/monitoring.h +++ b/Include/cpython/monitoring.h @@ -24,16 +24,20 @@ extern "C" { #define PY_MONITORING_EVENT_STOP_ITERATION 10 #define PY_MONITORING_IS_INSTRUMENTED_EVENT(ev) \ - ((ev) < _PY_MONITORING_LOCAL_EVENTS) +((ev) <= PY_MONITORING_EVENT_STOP_ITERATION) -/* Other events, mainly exceptions */ +/* Other events, mainly exceptions. + * These can now be turned on and disabled on a per code object basis. */ -#define PY_MONITORING_EVENT_RAISE 11 +#define PY_MONITORING_EVENT_PY_UNWIND 11 #define PY_MONITORING_EVENT_EXCEPTION_HANDLED 12 -#define PY_MONITORING_EVENT_PY_UNWIND 13 +#define PY_MONITORING_EVENT_RAISE 13 #define PY_MONITORING_EVENT_PY_THROW 14 #define PY_MONITORING_EVENT_RERAISE 15 +#define _PY_MONITORING_IS_UNGROUPED_EVENT(ev) \ +((ev) < _PY_MONITORING_UNGROUPED_EVENTS) + /* Ancillary events */ diff --git a/Include/cpython/pystats.h b/Include/cpython/pystats.h index e473110eca7415..5d1f44988a6df1 100644 --- a/Include/cpython/pystats.h +++ b/Include/cpython/pystats.h @@ -144,6 +144,7 @@ typedef struct _optimization_stats { uint64_t unknown_callee; uint64_t trace_immediately_deopts; uint64_t executors_invalidated; + uint64_t fitness_terminated_traces; UOpStats opcode[PYSTATS_MAX_UOP_ID + 1]; uint64_t unsupported_opcode[256]; uint64_t trace_length_hist[_Py_UOP_HIST_SIZE]; diff --git a/Include/exports.h b/Include/exports.h index 97a674ec2403a4..a863ecb33078ab 100644 --- a/Include/exports.h +++ b/Include/exports.h @@ -36,7 +36,7 @@ #define Py_LOCAL_SYMBOL #endif /* module init functions outside the core must be exported */ - #if defined(Py_BUILD_CORE) + #if defined(_PyEXPORTS_CORE) #define _PyINIT_EXPORTED_SYMBOL Py_EXPORTED_SYMBOL #else #define _PyINIT_EXPORTED_SYMBOL __declspec(dllexport) @@ -64,13 +64,13 @@ /* only get special linkage if built as shared or platform is Cygwin */ #if defined(Py_ENABLE_SHARED) || defined(__CYGWIN__) # if defined(HAVE_DECLSPEC_DLL) -# if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) +# if defined(_PyEXPORTS_CORE) && !defined(_PyEXPORTS_CORE_MODULE) /* module init functions inside the core need no external linkage */ /* except for Cygwin to handle embedding */ # if !defined(__CYGWIN__) # define _PyINIT_FUNC_DECLSPEC # endif /* __CYGWIN__ */ -# else /* Py_BUILD_CORE */ +# else /* _PyEXPORTS_CORE */ /* Building an extension module, or an embedded situation */ /* public Python functions and data are imported */ /* Under Cygwin, auto-import functions to prevent compilation */ @@ -80,7 +80,7 @@ # define PyAPI_FUNC(RTYPE) Py_IMPORTED_SYMBOL RTYPE # endif /* !__CYGWIN__ */ # define PyAPI_DATA(RTYPE) extern Py_IMPORTED_SYMBOL RTYPE -# endif /* Py_BUILD_CORE */ +# endif /* _PyEXPORTS_CORE */ # endif /* HAVE_DECLSPEC_DLL */ #endif /* Py_ENABLE_SHARED */ diff --git a/Include/internal/pycore_blocks_output_buffer.h b/Include/internal/pycore_blocks_output_buffer.h index 016e7a18665859..322c1e93344ba3 100644 --- a/Include/internal/pycore_blocks_output_buffer.h +++ b/Include/internal/pycore_blocks_output_buffer.h @@ -242,9 +242,12 @@ static inline PyObject * _BlocksOutputBuffer_Finish(_BlocksOutputBuffer *buffer, const Py_ssize_t avail_out) { + PyObject *obj; assert(buffer->writer != NULL); - return PyBytesWriter_FinishWithSize(buffer->writer, - buffer->allocated - avail_out); + obj = PyBytesWriter_FinishWithSize(buffer->writer, + buffer->allocated - avail_out); + buffer->writer = NULL; + return obj; } /* Clean up the buffer when an error occurred. */ diff --git a/Include/internal/pycore_ceval.h b/Include/internal/pycore_ceval.h index 94a1f687b7b150..f9507fda1606db 100644 --- a/Include/internal/pycore_ceval.h +++ b/Include/internal/pycore_ceval.h @@ -121,18 +121,11 @@ _PyEval_EvalFrame(PyThreadState *tstate, _PyInterpreterFrame *frame, int throwfl } #ifdef _Py_TIER2 -#ifdef _Py_JIT -_Py_CODEUNIT *_Py_LazyJitShim( - struct _PyExecutorObject *current_executor, _PyInterpreterFrame *frame, - _PyStackRef *stack_pointer, PyThreadState *tstate -); -#else _Py_CODEUNIT *_PyTier2Interpreter( struct _PyExecutorObject *current_executor, _PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate ); #endif -#endif extern _PyJitEntryFuncPtr _Py_jit_entry; @@ -320,7 +313,7 @@ PyObject * _PyEval_ImportNameWithImport( PyAPI_FUNC(PyObject *)_PyEval_MatchClass(PyThreadState *tstate, PyObject *subject, PyObject *type, Py_ssize_t nargs, PyObject *kwargs); PyAPI_FUNC(PyObject *)_PyEval_MatchKeys(PyThreadState *tstate, PyObject *map, PyObject *keys); PyAPI_FUNC(void) _PyEval_MonitorRaise(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *instr); -PyAPI_FUNC(bool) _PyEval_NoToolsForUnwind(PyThreadState *tstate); +PyAPI_FUNC(bool) _PyEval_NoToolsForUnwind(PyThreadState *tstate, _PyInterpreterFrame *frame); PyAPI_FUNC(int) _PyEval_UnpackIterableStackRef(PyThreadState *tstate, PyObject *v, int argcnt, int argcntafter, _PyStackRef *sp); PyAPI_FUNC(void) _PyEval_FrameClearAndPop(PyThreadState *tstate, _PyInterpreterFrame *frame); PyAPI_FUNC(PyObject **) _PyObjectArray_FromStackRefArray(_PyStackRef *input, Py_ssize_t nargs, PyObject **scratch); diff --git a/Include/internal/pycore_genobject.h b/Include/internal/pycore_genobject.h index a3badb59cb771a..2c264c39ae9de0 100644 --- a/Include/internal/pycore_genobject.h +++ b/Include/internal/pycore_genobject.h @@ -33,6 +33,9 @@ PyAPI_FUNC(int) _PyGen_FetchStopIterationValue(PyObject **); PyAPI_FUNC(PyObject *)_PyCoro_GetAwaitableIter(PyObject *o); PyAPI_FUNC(PyObject *)_PyAsyncGenValueWrapperNew(PyThreadState *state, PyObject *); +// Exported for external JIT support +PyAPI_FUNC(PyObject *) _PyCoro_ComputeOrigin(int origin_depth, _PyInterpreterFrame *current_frame); + extern PyTypeObject _PyCoroWrapper_Type; extern PyTypeObject _PyAsyncGenWrappedValue_Type; extern PyTypeObject _PyAsyncGenAThrow_Type; diff --git a/Include/internal/pycore_instruments.h b/Include/internal/pycore_instruments.h index 1da8237e93f766..56b55e93a014cb 100644 --- a/Include/internal/pycore_instruments.h +++ b/Include/internal/pycore_instruments.h @@ -70,16 +70,15 @@ PyAPI_DATA(PyObject) _PyInstrumentation_DISABLE; /* Total tool ids available */ #define PY_MONITORING_TOOL_IDS 8 -/* Count of all local monitoring events */ -#define _PY_MONITORING_LOCAL_EVENTS 11 -/* Count of all "real" monitoring events (not derived from other events) */ +/* Count of all "real" monitoring events (not derived from other events). + * "Other" events can now be turned on/disabled per code object. */ #define _PY_MONITORING_UNGROUPED_EVENTS 16 /* Count of all monitoring events */ #define _PY_MONITORING_EVENTS 19 /* Tables of which tools are active for each monitored event. */ typedef struct _Py_LocalMonitors { - uint8_t tools[_PY_MONITORING_LOCAL_EVENTS]; + uint8_t tools[_PY_MONITORING_UNGROUPED_EVENTS]; } _Py_LocalMonitors; typedef struct _Py_GlobalMonitors { @@ -122,6 +121,15 @@ typedef struct _PyCoMonitoringData { extern int _Py_Instrumentation_GetLine(PyCodeObject *code, _PyCoLineInstrumentationData *line_data, int index); +static inline uint8_t +_PyCode_GetOriginalOpcode(_PyCoLineInstrumentationData *line_data, int index) +{ + return line_data->data[index*line_data->bytes_per_entry]; +} + +// Exported for external JIT support +PyAPI_FUNC(uint8_t) _PyCode_Deinstrument(uint8_t opcode); + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_interp_structs.h b/Include/internal/pycore_interp_structs.h index 2bfb84da36cbc8..01adadd1485189 100644 --- a/Include/internal/pycore_interp_structs.h +++ b/Include/internal/pycore_interp_structs.h @@ -449,6 +449,9 @@ typedef struct _PyOptimizationConfig { uint16_t side_exit_initial_value; uint16_t side_exit_initial_backoff; + // Trace fitness thresholds + uint16_t fitness_initial; + // Optimization flags bool specialization_enabled; bool uops_optimize_enabled; diff --git a/Include/internal/pycore_jit.h b/Include/internal/pycore_jit.h index 70bccce4166c18..b3cadcce8247d0 100644 --- a/Include/internal/pycore_jit.h +++ b/Include/internal/pycore_jit.h @@ -23,9 +23,13 @@ typedef _Py_CODEUNIT *(*jit_func)( _PyStackRef _tos_cache0, _PyStackRef _tos_cache1, _PyStackRef _tos_cache2 ); +_Py_CODEUNIT *_PyJIT( + _PyExecutorObject *executor, _PyInterpreterFrame *frame, + _PyStackRef *stack_pointer, PyThreadState *tstate +); + int _PyJIT_Compile(_PyExecutorObject *executor, const _PyUOpInstruction *trace, size_t length); void _PyJIT_Free(_PyExecutorObject *executor); -void _PyJIT_Fini(void); PyAPI_FUNC(int) _PyJIT_AddressInJitCode(PyInterpreterState *interp, uintptr_t addr); #endif // _Py_JIT diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h index a809a7e25526f6..7c2e0e95a80c3f 100644 --- a/Include/internal/pycore_optimizer.h +++ b/Include/internal/pycore_optimizer.h @@ -15,6 +15,50 @@ extern "C" { #include "pycore_optimizer_types.h" #include +/* Fitness controls how long a trace can grow. + * Starts at FITNESS_INITIAL, then decreases from per-bytecode buffer usage + * plus branch/frame heuristics. The trace stops when fitness drops below the + * current exit_quality. + * + * Design targets for the constants below: + * 1. Reaching the abstract frame-depth limit should drop fitness below + * EXIT_QUALITY_SPECIALIZABLE. + * 2. A backward edge should leave budget for roughly N_BACKWARD_SLACK more + * bytecodes, assuming AVG_SLOTS_PER_INSTRUCTION. + * 3. Roughly seven balanced branches should reduce fitness to + * EXIT_QUALITY_DEFAULT after per-slot costs. + * 4. A push followed by a matching return is net-zero on frame-specific + * fitness, excluding per-slot costs. + */ +#define MAX_TARGET_LENGTH (UOP_MAX_TRACE_LENGTH / 2) +#define OPTIMIZER_EFFECTIVENESS 2 +#define FITNESS_INITIAL (MAX_TARGET_LENGTH * OPTIMIZER_EFFECTIVENESS) + +/* Exit quality thresholds: trace stops when fitness < exit_quality. + * Higher = trace is more willing to stop here. */ +#define EXIT_QUALITY_CLOSE_LOOP (FITNESS_INITIAL - AVG_SLOTS_PER_INSTRUCTION*4) +#define EXIT_QUALITY_ENTER_EXECUTOR (FITNESS_INITIAL * 1 / 8) +#define EXIT_QUALITY_DEFAULT (FITNESS_INITIAL / 40) +#define EXIT_QUALITY_SPECIALIZABLE (FITNESS_INITIAL / 80) + +/* Estimated buffer slots per bytecode, used only to derive heuristics. + * Runtime charging uses trace-buffer capacity consumed for each bytecode. */ +#define AVG_SLOTS_PER_INSTRUCTION 6 + +/* Heuristic backward-edge exit quality: leave room for about 1 unroll and + * N_BACKWARD_SLACK more bytecodes before reaching EXIT_QUALITY_CLOSE_LOOP, + * based on AVG_SLOTS_PER_INSTRUCTION. */ +#define N_BACKWARD_SLACK 10 +#define EXIT_QUALITY_BACKWARD_EDGE (EXIT_QUALITY_CLOSE_LOOP / 2 - N_BACKWARD_SLACK * AVG_SLOTS_PER_INSTRUCTION) + +/* Penalty for a balanced branch. + * It is sized so repeated balanced branches can drive a trace toward + * EXIT_QUALITY_DEFAULT, while compute_branch_penalty() keeps any single branch + * from dominating the budget. + */ +#define FITNESS_BRANCH_BALANCED ((FITNESS_INITIAL - EXIT_QUALITY_DEFAULT - \ + (MAX_TARGET_LENGTH / 14 * AVG_SLOTS_PER_INSTRUCTION)) / (14)) + typedef struct _PyJitUopBuffer { _PyUOpInstruction *start; @@ -103,7 +147,8 @@ typedef struct _PyJitTracerPreviousState { } _PyJitTracerPreviousState; typedef struct _PyJitTracerTranslatorState { - int jump_backward_seen; + int32_t fitness; // Current trace fitness, starts high, decrements + int frame_depth; // Current inline depth (0 = root frame) } _PyJitTracerTranslatorState; typedef struct _PyJitTracerState { diff --git a/Include/internal/pycore_stackref.h b/Include/internal/pycore_stackref.h index 329045b5faaa22..ca4a7c216eda53 100644 --- a/Include/internal/pycore_stackref.h +++ b/Include/internal/pycore_stackref.h @@ -71,8 +71,10 @@ static const _PyStackRef PyStackRef_NULL = { .index = 0 }; static const _PyStackRef PyStackRef_ERROR = { .index = (1 << Py_TAGGED_SHIFT) }; #define PyStackRef_None ((_PyStackRef){ .index = (2 << Py_TAGGED_SHIFT) } ) -#define PyStackRef_False ((_PyStackRef){ .index = (3 << Py_TAGGED_SHIFT) }) -#define PyStackRef_True ((_PyStackRef){ .index = (4 << Py_TAGGED_SHIFT) }) +#define _Py_STACKREF_FALSE_INDEX (3 << Py_TAGGED_SHIFT) +#define _Py_STACKREF_TRUE_INDEX (4 << Py_TAGGED_SHIFT) +#define PyStackRef_False ((_PyStackRef){ .index = _Py_STACKREF_FALSE_INDEX }) +#define PyStackRef_True ((_PyStackRef){ .index = _Py_STACKREF_TRUE_INDEX }) #define INITIAL_STACKREF_INDEX (5 << Py_TAGGED_SHIFT) diff --git a/Include/patchlevel.h b/Include/patchlevel.h index 9f5c36230a7e45..974246f896e10b 100644 --- a/Include/patchlevel.h +++ b/Include/patchlevel.h @@ -61,32 +61,4 @@ #define PYTHON_ABI_VERSION 3 #define PYTHON_ABI_STRING "3" - -/* Stable ABI for free-threaded builds (introduced in PEP 803) - is enabled by one of: - - Py_TARGET_ABI3T, or - - Py_LIMITED_API and Py_GIL_DISABLED. - "Output" macros to be used internally: - - Py_LIMITED_API (defines the subset of API we expose) - - _Py_OPAQUE_PYOBJECT (additionally hides what's ABI-incompatible between - free-threaded & GIL) - (Don't use Py_TARGET_ABI3T directly: it's currently only used to set these - 2 macros. It's also available for users' convenience.) - */ -#if defined(Py_LIMITED_API) && defined(Py_GIL_DISABLED) \ - && !defined(Py_TARGET_ABI3T) -# define Py_TARGET_ABI3T Py_LIMITED_API -#endif -#if defined(Py_TARGET_ABI3T) -# define _Py_OPAQUE_PYOBJECT -# if !defined(Py_LIMITED_API) -# define Py_LIMITED_API Py_TARGET_ABI3T -# elif Py_LIMITED_API > Py_TARGET_ABI3T - // if both are defined, use the *lower* version, - // i.e. maximum compatibility -# undef Py_LIMITED_API -# define Py_LIMITED_API Py_TARGET_ABI3T -# endif -#endif - #endif //_Py_PATCHLEVEL_H diff --git a/Include/pyabi.h b/Include/pyabi.h new file mode 100644 index 00000000000000..8c4ae281a43faf --- /dev/null +++ b/Include/pyabi.h @@ -0,0 +1,121 @@ +/* Macros that restrict available definitions and select implementations + * to match an ABI stability promise: + * + * - internal API/ABI (may change at any time) -- Py_BUILD_CORE* + * - general CPython API/ABI (may change in 3.x.0) -- default + * - Stable ABI: abi3, abi3t (long-term stable) -- Py_LIMITED_API, + * Py_TARGET_ABI3T, _Py_OPAQUE_PYOBJECT + * - Free-threading (incompatible with non-free-threading builds) + * -- Py_GIL_DISABLED + */ + +#ifndef _Py_PYABI_H +#define _Py_PYABI_H + +/* Defines to build Python and its standard library: + * + * - Py_BUILD_CORE: Build Python core. Gives access to Python internals; should + * not be used by third-party modules. + * - Py_BUILD_CORE_BUILTIN: Build a Python stdlib module as a built-in module. + * - Py_BUILD_CORE_MODULE: Build a Python stdlib module as a dynamic library. + * + * Py_BUILD_CORE_BUILTIN and Py_BUILD_CORE_MODULE imply Py_BUILD_CORE. + * + * On Windows, Py_BUILD_CORE_MODULE exports "PyInit_xxx" symbol, whereas + * Py_BUILD_CORE_BUILTIN does not. + */ +#if defined(Py_BUILD_CORE_BUILTIN) && !defined(Py_BUILD_CORE) +# define Py_BUILD_CORE +#endif +#if defined(Py_BUILD_CORE_MODULE) && !defined(Py_BUILD_CORE) +# define Py_BUILD_CORE +#endif + +/* Check valid values for target ABI macros. + */ +#if defined(Py_LIMITED_API) && Py_LIMITED_API+0 < 3 + // Empty Py_LIMITED_API used to work; redefine to + // Python 3.2 to be explicit. +# undef Py_LIMITED_API +# define Py_LIMITED_API 0x03020000 +#endif +#if defined(Py_TARGET_ABI3T) && Py_TARGET_ABI3T+0 < 0x030f0000 +# error "Py_TARGET_ABI3T must be 0x030f0000 (3.15) or above" +#endif + +/* Stable ABI for free-threaded builds (abi3t, introduced in PEP 803) + * is enabled by one of: + * - Py_TARGET_ABI3T, or + * - Py_LIMITED_API and Py_GIL_DISABLED. + * + * These affect set the following, which Python.h should use internally: + * - Py_LIMITED_API (defines the subset of API we expose) + * - _Py_OPAQUE_PYOBJECT (additionally hides what's ABI-incompatible between + * free-threaded & GIL) + * + * (Don't use Py_TARGET_ABI3T directly. It's currently only used to set these + * 2 macros, and defined for users' convenience.) + */ +#if defined(Py_LIMITED_API) && defined(Py_GIL_DISABLED) \ + && !defined(Py_TARGET_ABI3T) +# define Py_TARGET_ABI3T Py_LIMITED_API +#endif +#if defined(Py_TARGET_ABI3T) +# define _Py_OPAQUE_PYOBJECT +# if !defined(Py_LIMITED_API) +# define Py_LIMITED_API Py_TARGET_ABI3T +# elif Py_LIMITED_API > Py_TARGET_ABI3T + // if both are defined, use the *lower* version, + // i.e. maximum compatibility +# undef Py_LIMITED_API +# define Py_LIMITED_API Py_TARGET_ABI3T +# endif +#else +# ifdef _Py_OPAQUE_PYOBJECT + // _Py_OPAQUE_PYOBJECT is a private macro; do not define it directly. +# error "Define Py_TARGET_ABI3T to target abi3t." +# endif +#endif + +#if defined(Py_TARGET_ABI3T) +# if !defined(Py_GIL_DISABLED) + // Define Py_GIL_DISABLED for users' needs. Users check this macro to see + // whether they need extra synchronization. +# define Py_GIL_DISABLED +# endif +# if defined(_Py_IS_TESTCEXT) + // When compiling for abi3t, contents of Python.h should not depend + // on Py_GIL_DISABLED. + // We ask GCC to error if it sees the macro from this point on. + // Since users are free to the macro, and there's no way to undo the + // poisoning at the end of Python.h, we only do this in a test module + // (test_cext). + // + // Clang's poisoning is stricter than GCC's: it looks in `#elif` + // expressions after matching `#if`s. We disable it for now. + // We also provide an undocumented, unsupported opt-out macro to help + // porting to other compilers. Consider reaching out if you use it. +# if defined(__GNUC__) && !defined(__clang__) && !defined(_Py_NO_GCC_POISON) +# undef Py_GIL_DISABLED +# pragma GCC poison Py_GIL_DISABLED +# endif +# endif +#endif + +/* The internal C API must not be used with the limited C API: make sure + * that Py_BUILD_CORE* macros are not defined in this case. + * But, keep the "original" values, under different names, for "exports.h" + */ +#ifdef Py_BUILD_CORE +# define _PyEXPORTS_CORE +#endif +#ifdef Py_BUILD_CORE_MODULE +# define _PyEXPORTS_CORE_MODULE +#endif +#ifdef Py_LIMITED_API +# undef Py_BUILD_CORE +# undef Py_BUILD_CORE_BUILTIN +# undef Py_BUILD_CORE_MODULE +#endif + +#endif // _Py_PYABI_H diff --git a/Include/pyport.h b/Include/pyport.h index 62cba4c1421f99..c975921beafb9e 100644 --- a/Include/pyport.h +++ b/Include/pyport.h @@ -58,34 +58,6 @@ #endif -/* Defines to build Python and its standard library: - * - * - Py_BUILD_CORE: Build Python core. Give access to Python internals, but - * should not be used by third-party modules. - * - Py_BUILD_CORE_BUILTIN: Build a Python stdlib module as a built-in module. - * - Py_BUILD_CORE_MODULE: Build a Python stdlib module as a dynamic library. - * - * Py_BUILD_CORE_BUILTIN and Py_BUILD_CORE_MODULE imply Py_BUILD_CORE. - * - * On Windows, Py_BUILD_CORE_MODULE exports "PyInit_xxx" symbol, whereas - * Py_BUILD_CORE_BUILTIN does not. - */ -#if defined(Py_BUILD_CORE_BUILTIN) && !defined(Py_BUILD_CORE) -# define Py_BUILD_CORE -#endif -#if defined(Py_BUILD_CORE_MODULE) && !defined(Py_BUILD_CORE) -# define Py_BUILD_CORE -#endif - -#if defined(Py_TARGET_ABI3T) -# if !defined(Py_GIL_DISABLED) -// Define Py_GIL_DISABLED for users' needs. This macro is used to enable -// locking needed in for free-threaded interpreters builds. -# define Py_GIL_DISABLED -# endif -#endif - - /************************************************************************** Symbols and macros to supply platform-independent interfaces to basic C language & library operations whose spellings vary across platforms. @@ -393,17 +365,6 @@ extern "C" { # define Py_NO_INLINE #endif -#include "exports.h" - -#ifdef Py_LIMITED_API - // The internal C API must not be used with the limited C API: make sure - // that Py_BUILD_CORE macro is not defined in this case. These 3 macros are - // used by exports.h, so only undefine them afterwards. -# undef Py_BUILD_CORE -# undef Py_BUILD_CORE_BUILTIN -# undef Py_BUILD_CORE_MODULE -#endif - /* limits.h constants that may be missing */ #ifndef INT_MAX diff --git a/Include/sentinelobject.h b/Include/sentinelobject.h new file mode 100644 index 00000000000000..9d8577767b7485 --- /dev/null +++ b/Include/sentinelobject.h @@ -0,0 +1,22 @@ +/* Sentinel object interface */ + +#ifndef Py_SENTINELOBJECT_H +#define Py_SENTINELOBJECT_H +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_LIMITED_API +PyAPI_DATA(PyTypeObject) PySentinel_Type; + +#define PySentinel_Check(op) Py_IS_TYPE((op), &PySentinel_Type) + +PyAPI_FUNC(PyObject *) PySentinel_New( + const char *name, + const char *module_name); +#endif + +#ifdef __cplusplus +} +#endif +#endif /* !Py_SENTINELOBJECT_H */ diff --git a/Lib/_colorize.py b/Lib/_colorize.py index 478f81894911e7..852ad38f08618e 100644 --- a/Lib/_colorize.py +++ b/Lib/_colorize.py @@ -1,3 +1,4 @@ +import builtins import os import sys @@ -202,25 +203,25 @@ class Difflib(ThemeSection): @dataclass(frozen=True, kw_only=True) class FancyCompleter(ThemeSection): # functions and methods - function: str = ANSIColors.BOLD_BLUE - builtin_function_or_method: str = ANSIColors.BOLD_BLUE - method: str = ANSIColors.BOLD_CYAN - method_wrapper: str = ANSIColors.BOLD_CYAN - wrapper_descriptor: str = ANSIColors.BOLD_CYAN - method_descriptor: str = ANSIColors.BOLD_CYAN + function: builtins.str = ANSIColors.BOLD_BLUE + builtin_function_or_method: builtins.str = ANSIColors.BOLD_BLUE + method: builtins.str = ANSIColors.BOLD_CYAN + method_wrapper: builtins.str = ANSIColors.BOLD_CYAN + wrapper_descriptor: builtins.str = ANSIColors.BOLD_CYAN + method_descriptor: builtins.str = ANSIColors.BOLD_CYAN # numbers - int: str = ANSIColors.BOLD_YELLOW - float: str = ANSIColors.BOLD_YELLOW - complex: str = ANSIColors.BOLD_YELLOW - bool: str = ANSIColors.BOLD_YELLOW + int: builtins.str = ANSIColors.BOLD_YELLOW + float: builtins.str = ANSIColors.BOLD_YELLOW + complex: builtins.str = ANSIColors.BOLD_YELLOW + bool: builtins.str = ANSIColors.BOLD_YELLOW # others - type: str = ANSIColors.BOLD_MAGENTA - module: str = ANSIColors.CYAN - NoneType: str = ANSIColors.GREY - bytes: str = ANSIColors.BOLD_GREEN - str: str = ANSIColors.BOLD_GREEN + type: builtins.str = ANSIColors.BOLD_MAGENTA + module: builtins.str = ANSIColors.CYAN + NoneType: builtins.str = ANSIColors.GREY + bytes: builtins.str = ANSIColors.BOLD_GREEN + str: builtins.str = ANSIColors.BOLD_GREEN @dataclass(frozen=True, kw_only=True) diff --git a/Lib/annotationlib.py b/Lib/annotationlib.py index 9fee2564114339..5c9a0812646f81 100644 --- a/Lib/annotationlib.py +++ b/Lib/annotationlib.py @@ -47,6 +47,7 @@ class Format(enum.IntEnum): "__cell__", "__owner__", "__stringifier_dict__", + "__resolved_str_cache__", ) @@ -94,6 +95,7 @@ def __init__( # value later. self.__code__ = None self.__ast_node__ = None + self.__resolved_str_cache__ = None def __init_subclass__(cls, /, *args, **kwds): raise TypeError("Cannot subclass ForwardRef") @@ -113,7 +115,7 @@ def evaluate( """ match format: case Format.STRING: - return self.__forward_arg__ + return self.__resolved_str__ case Format.VALUE: is_forwardref_format = False case Format.FORWARDREF: @@ -258,6 +260,24 @@ def __forward_arg__(self): "Attempted to access '__forward_arg__' on an uninitialized ForwardRef" ) + @property + def __resolved_str__(self): + # __forward_arg__ with any names from __extra_names__ replaced + # with the type_repr of the value they represent + if self.__resolved_str_cache__ is None: + resolved_str = self.__forward_arg__ + names = self.__extra_names__ + + if names: + visitor = _ExtraNameFixer(names) + ast_expr = ast.parse(resolved_str, mode="eval").body + node = visitor.visit(ast_expr) + resolved_str = ast.unparse(node) + + self.__resolved_str_cache__ = resolved_str + + return self.__resolved_str_cache__ + @property def __forward_code__(self): if self.__code__ is not None: @@ -321,7 +341,7 @@ def __repr__(self): extra.append(", is_class=True") if self.__owner__ is not None: extra.append(f", owner={self.__owner__!r}") - return f"ForwardRef({self.__forward_arg__!r}{''.join(extra)})" + return f"ForwardRef({self.__resolved_str__!r}{''.join(extra)})" _Template = type(t"") @@ -357,6 +377,7 @@ def __init__( self.__cell__ = cell self.__owner__ = owner self.__stringifier_dict__ = stringifier_dict + self.__resolved_str_cache__ = None # Needed for ForwardRef def __convert_to_ast(self, other): if isinstance(other, _Stringifier): @@ -1163,3 +1184,14 @@ def _get_dunder_annotations(obj): if not isinstance(ann, dict): raise ValueError(f"{obj!r}.__annotations__ is neither a dict nor None") return ann + + +class _ExtraNameFixer(ast.NodeTransformer): + """Fixer for __extra_names__ items in ForwardRef __repr__ and string evaluation""" + def __init__(self, extra_names): + self.extra_names = extra_names + + def visit_Name(self, node: ast.Name): + if (new_name := self.extra_names.get(node.id, _sentinel)) is not _sentinel: + node = ast.Name(id=type_repr(new_name)) + return node diff --git a/Lib/asyncio/taskgroups.py b/Lib/asyncio/taskgroups.py index 00e8f6d5d1a68b..45dfebc65904fc 100644 --- a/Lib/asyncio/taskgroups.py +++ b/Lib/asyncio/taskgroups.py @@ -37,6 +37,7 @@ def __init__(self): self._errors = [] self._base_error = None self._on_completed_fut = None + self._cancel_on_enter = False def __repr__(self): info = [''] @@ -63,6 +64,8 @@ async def __aenter__(self): raise RuntimeError( f'TaskGroup {self!r} cannot determine the parent task') self._entered = True + if self._cancel_on_enter: + self.cancel() return self @@ -178,6 +181,9 @@ async def _aexit(self, et, exc): finally: exc = None + # Suppress any remaining exception (exceptions deserving to be raised + # were raised above). + return True def create_task(self, coro, **kwargs): """Create a new task in this group and return it. @@ -278,3 +284,30 @@ def _on_task_done(self, task): self._abort() self._parent_cancel_requested = True self._parent_task.cancel() + + def cancel(self): + """Cancel the task group + + `cancel()` will be called on any tasks in the group that aren't yet + done, as well as the parent (body) of the group. This will cause the + task group context manager to exit *without* `asyncio.CancelledError` + being raised. + + If `cancel()` is called before entering the task group, the group will be + cancelled upon entry. This is useful for patterns where one piece of + code passes an unused TaskGroup instance to another in order to have + the ability to cancel anything run within the group. + + `cancel()` is idempotent and may be called after the task group has + already exited. + """ + if not self._entered: + self._cancel_on_enter = True + return + if self._exiting and not self._tasks: + return + if not self._aborting: + self._abort() + if self._parent_task and not self._parent_cancel_requested: + self._parent_cancel_requested = True + self._parent_task.cancel() diff --git a/Lib/dataclasses.py b/Lib/dataclasses.py index 0c7e01cb16b192..9d5bed6b96fc49 100644 --- a/Lib/dataclasses.py +++ b/Lib/dataclasses.py @@ -1496,7 +1496,8 @@ class C: If given, 'dict_factory' will be used instead of built-in dict. The function applies recursively to field values that are dataclass instances. This will also look into built-in containers: - tuples, lists, and dicts. Other objects are copied with 'copy.deepcopy()'. + tuples, lists, dicts, and frozendicts. Other objects are copied + with 'copy.deepcopy()'. """ if not _is_dataclass_instance(obj): raise TypeError("asdict() should be called on dataclass instances") @@ -1552,7 +1553,7 @@ def _asdict_inner(obj, dict_factory): return obj_type(*[_asdict_inner(v, dict_factory) for v in obj]) else: return obj_type(_asdict_inner(v, dict_factory) for v in obj) - elif issubclass(obj_type, dict): + elif issubclass(obj_type, (dict, frozendict)): if hasattr(obj_type, 'default_factory'): # obj is a defaultdict, which has a different constructor from # dict as it requires the default_factory as its first arg. @@ -1587,7 +1588,8 @@ class C: If given, 'tuple_factory' will be used instead of built-in tuple. The function applies recursively to field values that are dataclass instances. This will also look into built-in containers: - tuples, lists, and dicts. Other objects are copied with 'copy.deepcopy()'. + tuples, lists, dicts, and frozendicts. Other objects are copied + with 'copy.deepcopy()'. """ if not _is_dataclass_instance(obj): @@ -1616,7 +1618,7 @@ def _astuple_inner(obj, tuple_factory): # generator (which is not true for namedtuples, handled # above). return type(obj)(_astuple_inner(v, tuple_factory) for v in obj) - elif isinstance(obj, dict): + elif isinstance(obj, (dict, frozendict)): obj_type = type(obj) if hasattr(obj_type, 'default_factory'): # obj is a defaultdict, which has a different constructor from diff --git a/Lib/http/cookies.py b/Lib/http/cookies.py index 769541116993c4..660fec4f1be865 100644 --- a/Lib/http/cookies.py +++ b/Lib/http/cookies.py @@ -391,17 +391,21 @@ def __repr__(self): return '<%s: %s>' % (self.__class__.__name__, self.OutputString()) def js_output(self, attrs=None): + import base64 # Print javascript output_string = self.OutputString(attrs) if _has_control_character(output_string): raise CookieError("Control characters are not allowed in cookies") + # Base64-encode value to avoid template + # injection in cookie values. + output_encoded = base64.b64encode(output_string.encode('utf-8')).decode("ascii") return """ - """ % (output_string.replace('"', r'\"')) + """ % (output_encoded,) def OutputString(self, attrs=None): # Build up our result diff --git a/Lib/json/tool.py b/Lib/json/tool.py index e0b944b197d38b..e56a601c581ae5 100644 --- a/Lib/json/tool.py +++ b/Lib/json/tool.py @@ -89,7 +89,8 @@ def main(): infile = open(options.infile, encoding='utf-8') try: if options.json_lines: - objs = (json.loads(line) for line in infile) + lines = infile.readlines() + objs = (json.loads(line) for line in lines) else: objs = (json.load(infile),) finally: diff --git a/Lib/locale.py b/Lib/locale.py index e7382796905ebd..4ff6f8c0f0a775 100644 --- a/Lib/locale.py +++ b/Lib/locale.py @@ -1505,8 +1505,8 @@ def getpreferredencoding(do_setlocale=True): # This maps Windows language identifiers to locale strings. # # This list has been updated from -# http://msdn.microsoft.com/library/default.asp?url=/library/en-us/intl/nls_238z.asp -# to include every locale up to Windows Vista. +# https://learn.microsoft.com/en-us/openspecs/windows_protocols/ms-lcid/70feba9f-294e-491e-b6eb-56532684c37f +# to include every locale up to protocol revision 16.0 (2024-04-23). # # NOTE: this mapping is incomplete. If your language is missing, please # submit a bug report as detailed in the Python devguide at: @@ -1516,10 +1516,15 @@ def getpreferredencoding(do_setlocale=True): # windows_locale = { - 0x0436: "af_ZA", # Afrikaans - 0x041c: "sq_AL", # Albanian - 0x0484: "gsw_FR",# Alsatian - France + 0x0036: "af", # Afrikaans + 0x0436: "af_ZA", # Afrikaans - South Africa + 0x001c: "sq", # Albanian + 0x041c: "sq_AL", # Albanian - Albania + 0x0084: "gsw", # Alsatian + 0x0484: "gsw_FR", # Alsatian - France + 0x005e: "am", # Amharic 0x045e: "am_ET", # Amharic - Ethiopia + 0x0001: "ar", # Arabic 0x0401: "ar_SA", # Arabic - Saudi Arabia 0x0801: "ar_IQ", # Arabic - Iraq 0x0c01: "ar_EG", # Arabic - Egypt @@ -1533,39 +1538,72 @@ def getpreferredencoding(do_setlocale=True): 0x2c01: "ar_JO", # Arabic - Jordan 0x3001: "ar_LB", # Arabic - Lebanon 0x3401: "ar_KW", # Arabic - Kuwait - 0x3801: "ar_AE", # Arabic - United Arab Emirates + 0x3801: "ar_AE", # Arabic - U.A.E. 0x3c01: "ar_BH", # Arabic - Bahrain 0x4001: "ar_QA", # Arabic - Qatar - 0x042b: "hy_AM", # Armenian + 0x002b: "hy", # Armenian + 0x042b: "hy_AM", # Armenian - Armenia + 0x004d: "as", # Assamese 0x044d: "as_IN", # Assamese - India - 0x042c: "az_AZ", # Azeri - Latin - 0x082c: "az_AZ", # Azeri - Cyrillic - 0x046d: "ba_RU", # Bashkir - 0x042d: "eu_ES", # Basque - Russia - 0x0423: "be_BY", # Belarusian - 0x0445: "bn_IN", # Begali - 0x201a: "bs_BA", # Bosnian - Cyrillic - 0x141a: "bs_BA", # Bosnian - Latin + 0x002c: "az", # Azerbaijani (Latin) + 0x742c: "az", # Azerbaijani (Cyrillic) + 0x782c: "az", # Azerbaijani (Latin) + 0x042c: "az_AZ", # Azerbaijani (Latin) - Azerbaijan + 0x0045: "bn", # Bangla + 0x0445: "bn_IN", # Bangla - India + 0x0845: "bn_BD", # Bangla - Bangladesh + 0x006d: "ba", # Bashkir + 0x046d: "ba_RU", # Bashkir - Russia + 0x002d: "eu", # Basque + 0x042d: "eu_ES", # Basque - Spain + 0x0023: "be", # Belarusian + 0x0423: "be_BY", # Belarusian - Belarus + 0x641a: "bs", # Bosnian (Cyrillic) + 0x681a: "bs", # Bosnian (Latin) + 0x141a: "bs_BA", # Bosnian (Latin) - Bosnia and Herzegovina + 0x201a: "bs_BA", # Bosnian (Cyrillic) - Bosnia and Herzegovina + 0x781a: "bs", # Bosnian (Latin) + 0x007e: "br", # Breton 0x047e: "br_FR", # Breton - France - 0x0402: "bg_BG", # Bulgarian -# 0x0455: "my_MM", # Burmese - Not supported - 0x0403: "ca_ES", # Catalan - 0x0004: "zh_CHS",# Chinese - Simplified - 0x0404: "zh_TW", # Chinese - Taiwan - 0x0804: "zh_CN", # Chinese - PRC - 0x0c04: "zh_HK", # Chinese - Hong Kong S.A.R. - 0x1004: "zh_SG", # Chinese - Singapore - 0x1404: "zh_MO", # Chinese - Macao S.A.R. - 0x7c04: "zh_CHT",# Chinese - Traditional + 0x0002: "bg", # Bulgarian + 0x0402: "bg_BG", # Bulgarian - Bulgaria + 0x0055: "my", # Burmese + 0x0455: "my_MM", # Burmese - Myanmar + 0x0003: "ca", # Catalan + 0x0403: "ca_ES", # Catalan - Spain + 0x0803: "ca_ES", # Valencian - Spain + 0x0092: "ku", # Central Kurdish + 0x7c92: "ku", # Central Kurdish + 0x0492: "ku_IQ", # Central Kurdish - Iraq + 0x005c: "chr", # Cherokee + 0x7c5c: "chr", # Cherokee + 0x045c: "chr_US", # Cherokee - United States + 0x0004: "zh", # Chinese (Simplified) + 0x7804: "zh", # Chinese (Simplified) + 0x7c04: "zh", # Chinese (Traditional) + 0x0404: "zh_TW", # Chinese (Traditional) - Taiwan + 0x0804: "zh_CN", # Chinese (Simplified) - People's Republic of China + 0x0c04: "zh_HK", # Chinese (Traditional) - Hong Kong S.A.R. + 0x1004: "zh_SG", # Chinese (Simplified) - Singapore + 0x1404: "zh_MO", # Chinese (Traditional) - Macao S.A.R. + 0x0083: "co", # Corsican 0x0483: "co_FR", # Corsican - France - 0x041a: "hr_HR", # Croatian - 0x101a: "hr_BA", # Croatian - Bosnia - 0x0405: "cs_CZ", # Czech - 0x0406: "da_DK", # Danish - 0x048c: "gbz_AF",# Dari - Afghanistan - 0x0465: "div_MV",# Divehi - Maldives - 0x0413: "nl_NL", # Dutch - The Netherlands + 0x001a: "hr", # Croatian + 0x041a: "hr_HR", # Croatian - Croatia + 0x101a: "hr_BA", # Croatian (Latin) - Bosnia and Herzegovina + 0x0005: "cs", # Czech + 0x0405: "cs_CZ", # Czech - Czech Republic + 0x0006: "da", # Danish + 0x0406: "da_DK", # Danish - Denmark + 0x008c: "prs", # Dari + 0x048c: "prs_AF", # Dari - Afghanistan + 0x0065: "dv", # Divehi + 0x0465: "dv_MV", # Divehi - Maldives + 0x0013: "nl", # Dutch + 0x0413: "nl_NL", # Dutch - Netherlands 0x0813: "nl_BE", # Dutch - Belgium + 0x0c51: "dz_BT", # Dzongkha - Bhutan + 0x0009: "en", # English 0x0409: "en_US", # English - United States 0x0809: "en_GB", # English - United Kingdom 0x0c09: "en_AU", # English - Australia @@ -1573,122 +1611,248 @@ def getpreferredencoding(do_setlocale=True): 0x1409: "en_NZ", # English - New Zealand 0x1809: "en_IE", # English - Ireland 0x1c09: "en_ZA", # English - South Africa - 0x2009: "en_JA", # English - Jamaica - 0x2409: "en_CB", # English - Caribbean + 0x2009: "en_JM", # English - Jamaica 0x2809: "en_BZ", # English - Belize - 0x2c09: "en_TT", # English - Trinidad + 0x2c09: "en_TT", # English - Trinidad and Tobago 0x3009: "en_ZW", # English - Zimbabwe - 0x3409: "en_PH", # English - Philippines + 0x3409: "en_PH", # English - Republic of the Philippines + 0x3c09: "en_HK", # English - Hong Kong 0x4009: "en_IN", # English - India 0x4409: "en_MY", # English - Malaysia - 0x4809: "en_IN", # English - Singapore - 0x0425: "et_EE", # Estonian - 0x0438: "fo_FO", # Faroese - 0x0464: "fil_PH",# Filipino - 0x040b: "fi_FI", # Finnish + 0x4809: "en_SG", # English - Singapore + 0x4c09: "en_AE", # English - United Arab Emirates + 0x0025: "et", # Estonian + 0x0425: "et_EE", # Estonian - Estonia + 0x0038: "fo", # Faroese + 0x0438: "fo_FO", # Faroese - Faroe Islands + 0x0064: "fil", # Filipino + 0x0464: "fil_PH", # Filipino - Philippines + 0x000b: "fi", # Finnish + 0x040b: "fi_FI", # Finnish - Finland + 0x000c: "fr", # French 0x040c: "fr_FR", # French - France 0x080c: "fr_BE", # French - Belgium 0x0c0c: "fr_CA", # French - Canada 0x100c: "fr_CH", # French - Switzerland 0x140c: "fr_LU", # French - Luxembourg - 0x180c: "fr_MC", # French - Monaco + 0x180c: "fr_MC", # French - Principality of Monaco + 0x1c0c: "fr_029", # French - Caribbean + 0x200c: "fr_RE", # French - Reunion + 0x240c: "fr_CD", # French - Congo, DRC + 0x280c: "fr_SN", # French - Senegal + 0x2c0c: "fr_CM", # French - Cameroon + 0x300c: "fr_CI", # French - Côte d'Ivoire + 0x340c: "fr_ML", # French - Mali + 0x380c: "fr_MA", # French - Morocco + 0x3c0c: "fr_HT", # French - Haiti + 0x0062: "fy", # Frisian 0x0462: "fy_NL", # Frisian - Netherlands - 0x0456: "gl_ES", # Galician - 0x0437: "ka_GE", # Georgian + 0x0067: "ff", # Fulah + 0x7c67: "ff", # Fulah (Latin) + 0x0467: "ff_NG", + 0x0867: "ff_SN", # Fulah - Senegal + 0x0056: "gl", # Galician + 0x0456: "gl_ES", # Galician - Spain + 0x0037: "ka", # Georgian + 0x0437: "ka_GE", # Georgian - Georgia + 0x0007: "de", # German 0x0407: "de_DE", # German - Germany 0x0807: "de_CH", # German - Switzerland 0x0c07: "de_AT", # German - Austria 0x1007: "de_LU", # German - Luxembourg 0x1407: "de_LI", # German - Liechtenstein - 0x0408: "el_GR", # Greek + 0x0008: "el", # Greek + 0x0408: "el_GR", # Greek - Greece + 0x006f: "kl", # Greenlandic 0x046f: "kl_GL", # Greenlandic - Greenland - 0x0447: "gu_IN", # Gujarati - 0x0468: "ha_NG", # Hausa - Latin - 0x040d: "he_IL", # Hebrew - 0x0439: "hi_IN", # Hindi - 0x040e: "hu_HU", # Hungarian - 0x040f: "is_IS", # Icelandic - 0x0421: "id_ID", # Indonesian - 0x045d: "iu_CA", # Inuktitut - Syllabics - 0x085d: "iu_CA", # Inuktitut - Latin + 0x0074: "gn", # Guarani + 0x0474: "gn_PY", # Guarani - Paraguay + 0x0047: "gu", # Gujarati + 0x0447: "gu_IN", # Gujarati - India + 0x0068: "ha", # Hausa (Latin) + 0x7c68: "ha", # Hausa (Latin) + 0x0468: "ha_NG", # Hausa (Latin) - Nigeria + 0x0075: "haw", # Hawaiian + 0x0475: "haw_US", # Hawaiian - United States + 0x000d: "he", # Hebrew + 0x040d: "he_IL", # Hebrew - Israel + 0x0039: "hi", # Hindi + 0x0439: "hi_IN", # Hindi - India + 0x000e: "hu", # Hungarian + 0x040e: "hu_HU", # Hungarian - Hungary + 0x000f: "is", # Icelandic + 0x040f: "is_IS", # Icelandic - Iceland + 0x0070: "ig", # Igbo + 0x0470: "ig_NG", # Igbo - Nigeria + 0x0021: "id", # Indonesian + 0x0421: "id_ID", # Indonesian - Indonesia + 0x005d: "iu", # Inuktitut (Latin) + 0x785d: "iu", # Inuktitut (Syllabics) + 0x7c5d: "iu", # Inuktitut (Latin) + 0x045d: "iu_CA", # Inuktitut (Syllabics) - Canada + 0x085d: "iu_CA", # Inuktitut (Latin) - Canada + 0x003c: "ga", # Irish 0x083c: "ga_IE", # Irish - Ireland + 0x0010: "it", # Italian 0x0410: "it_IT", # Italian - Italy 0x0810: "it_CH", # Italian - Switzerland - 0x0411: "ja_JP", # Japanese + 0x0011: "ja", # Japanese + 0x0411: "ja_JP", # Japanese - Japan + 0x004b: "kn", # Kannada 0x044b: "kn_IN", # Kannada - India - 0x043f: "kk_KZ", # Kazakh - 0x0453: "kh_KH", # Khmer - Cambodia - 0x0486: "qut_GT",# K'iche - Guatemala + 0x0471: "kr_NG", # Kanuri (Latin) - Nigeria + 0x0060: "ks", # Kashmiri + 0x0460: "ks", # Kashmiri - Perso_Arabic + 0x0860: "ks_IN", # Kashmiri (Devanagari) - India + 0x003f: "kk", # Kazakh + 0x043f: "kk_KZ", # Kazakh - Kazakhstan + 0x0053: "km", # Khmer + 0x0453: "km_KH", # Khmer - Cambodia + 0x0087: "rw", # Kinyarwanda 0x0487: "rw_RW", # Kinyarwanda - Rwanda - 0x0457: "kok_IN",# Konkani - 0x0412: "ko_KR", # Korean - 0x0440: "ky_KG", # Kyrgyz - 0x0454: "lo_LA", # Lao - Lao PDR - 0x0426: "lv_LV", # Latvian - 0x0427: "lt_LT", # Lithuanian - 0x082e: "dsb_DE",# Lower Sorbian - Germany - 0x046e: "lb_LU", # Luxembourgish - 0x042f: "mk_MK", # FYROM Macedonian + 0x0041: "sw", # Kiswahili + 0x0441: "sw_KE", # Kiswahili - Kenya + 0x0057: "kok", # Konkani + 0x0457: "kok_IN", # Konkani - India + 0x0012: "ko", # Korean + 0x0412: "ko_KR", # Korean - Korea + 0x0040: "ky", # Kyrgyz + 0x0440: "ky_KG", # Kyrgyz - Kyrgyzstan + 0x0054: "lo", # Lao + 0x0454: "lo_LA", # Lao - Lao P.D.R. + 0x0476: "la_VA", # Latin - Vatican City + 0x0026: "lv", # Latvian + 0x0426: "lv_LV", # Latvian - Latvia + 0x0027: "lt", # Lithuanian + 0x0427: "lt_LT", # Lithuanian - Lithuania + 0x7c2e: "dsb", # Lower Sorbian + 0x082e: "dsb_DE", # Lower Sorbian - Germany + 0x006e: "lb", # Luxembourgish + 0x046e: "lb_LU", # Luxembourgish - Luxembourg + 0x002f: "mk", # Macedonian + 0x042f: "mk_MK", # Macedonian - North Macedonia + 0x003e: "ms", # Malay 0x043e: "ms_MY", # Malay - Malaysia 0x083e: "ms_BN", # Malay - Brunei Darussalam + 0x004c: "ml", # Malayalam 0x044c: "ml_IN", # Malayalam - India - 0x043a: "mt_MT", # Maltese - 0x0481: "mi_NZ", # Maori - 0x047a: "arn_CL",# Mapudungun - 0x044e: "mr_IN", # Marathi - 0x047c: "moh_CA",# Mohawk - Canada - 0x0450: "mn_MN", # Mongolian - Cyrillic - 0x0850: "mn_CN", # Mongolian - PRC - 0x0461: "ne_NP", # Nepali - 0x0414: "nb_NO", # Norwegian - Bokmal - 0x0814: "nn_NO", # Norwegian - Nynorsk + 0x003a: "mt", # Maltese + 0x043a: "mt_MT", # Maltese - Malta + 0x0081: "mi", # Maori + 0x0481: "mi_NZ", # Maori - New Zealand + 0x007a: "arn", # Mapudungun + 0x047a: "arn_CL", # Mapudungun - Chile + 0x004e: "mr", # Marathi + 0x044e: "mr_IN", # Marathi - India + 0x007c: "moh", # Mohawk + 0x047c: "moh_CA", # Mohawk - Canada + 0x0050: "mn", # Mongolian (Cyrillic) + 0x7850: "mn", # Mongolian (Cyrillic) + 0x7c50: "mn", # Mongolian (Traditional Mongolian) + 0x0450: "mn_MN", # Mongolian (Cyrillic) - Mongolia + 0x0c50: "mn_MN", # Mongolian (Traditional Mongolian) - Mongolia + 0x0061: "ne", # Nepali + 0x0461: "ne_NP", # Nepali - Nepal + 0x0861: "ne_IN", # Nepali - India + 0x0014: "no", # Norwegian (Bokmal) + 0x0414: "nb_NO", # Norwegian (Bokmal) - Norway + 0x0814: "nn_NO", # Norwegian (Nynorsk) - Norway + 0x7814: "nn", # Norwegian (Nynorsk) + 0x7c14: "nb", # Norwegian (Bokmal) + 0x0082: "oc", # Occitan 0x0482: "oc_FR", # Occitan - France - 0x0448: "or_IN", # Oriya - India + 0x0048: "or", # Odia + 0x0448: "or_IN", # Odia - India + 0x0072: "om", # Oromo + 0x0472: "om_ET", # Oromo - Ethiopia + 0x0063: "ps", # Pashto 0x0463: "ps_AF", # Pashto - Afghanistan - 0x0429: "fa_IR", # Persian - 0x0415: "pl_PL", # Polish + 0x0029: "fa", # Persian + 0x0429: "fa_IR", # Persian - Iran + 0x0015: "pl", # Polish + 0x0415: "pl_PL", # Polish - Poland + 0x0016: "pt", # Portuguese 0x0416: "pt_BR", # Portuguese - Brazil 0x0816: "pt_PT", # Portuguese - Portugal - 0x0446: "pa_IN", # Punjabi - 0x046b: "quz_BO",# Quechua (Bolivia) - 0x086b: "quz_EC",# Quechua (Ecuador) - 0x0c6b: "quz_PE",# Quechua (Peru) + 0x0046: "pa", # Punjabi + 0x7c46: "pa", # Punjabi + 0x0446: "pa_IN", # Punjabi - India + 0x0846: "pa_PK", # Punjabi - Islamic Republic of Pakistan + 0x006b: "quz", # Quechua + 0x046b: "quz_BO", # Quechua - Bolivia + 0x086b: "quz_EC", # Quechua - Ecuador + 0x0c6b: "quz_PE", # Quechua - Peru + 0x0018: "ro", # Romanian 0x0418: "ro_RO", # Romanian - Romania - 0x0417: "rm_CH", # Romansh - 0x0419: "ru_RU", # Russian - 0x243b: "smn_FI",# Sami Finland - 0x103b: "smj_NO",# Sami Norway - 0x143b: "smj_SE",# Sami Sweden - 0x043b: "se_NO", # Sami Northern Norway - 0x083b: "se_SE", # Sami Northern Sweden - 0x0c3b: "se_FI", # Sami Northern Finland - 0x203b: "sms_FI",# Sami Skolt - 0x183b: "sma_NO",# Sami Southern Norway - 0x1c3b: "sma_SE",# Sami Southern Sweden - 0x044f: "sa_IN", # Sanskrit - 0x0c1a: "sr_SP", # Serbian - Cyrillic - 0x1c1a: "sr_BA", # Serbian - Bosnia Cyrillic - 0x081a: "sr_SP", # Serbian - Latin - 0x181a: "sr_BA", # Serbian - Bosnia Latin + 0x0818: "ro_MD", # Romanian - Moldova + 0x0017: "rm", # Romansh + 0x0417: "rm_CH", # Romansh - Switzerland + 0x0019: "ru", # Russian + 0x0419: "ru_RU", # Russian - Russia + 0x0819: "ru_MD", # Russian - Moldova + 0x0085: "sah", # Sakha + 0x0485: "sah_RU", # Sakha - Russia + 0x003b: "se", # Sami (Northern) + 0x043b: "se_NO", # Sami (Northern) - Norway + 0x083b: "se_SE", # Sami (Northern) - Sweden + 0x0c3b: "se_FI", # Sami (Northern) - Finland + 0x7c3b: "smj", # Sami (Lule) + 0x103b: "smj_NO", # Sami (Lule) - Norway + 0x143b: "smj_SE", # Sami (Lule) - Sweden + 0x783b: "sma", # Sami (Southern) + 0x183b: "sma_NO", # Sami (Southern) - Norway + 0x1c3b: "sma_SE", # Sami (Southern) - Sweden + 0x743b: "sms", # Sami (Skolt) + 0x203b: "sms_FI", # Sami (Skolt) - Finland + 0x703b: "smn", # Sami (Inari) + 0x243b: "smn_FI", # Sami (Inari) - Finland + 0x004f: "sa", # Sanskrit + 0x044f: "sa_IN", # Sanskrit - India + 0x0091: "gd", # Scottish Gaelic + 0x0491: "gd_GB", # Scottish Gaelic - United Kingdom + 0x6c1a: "sr", # Serbian (Cyrillic) + 0x701a: "sr", # Serbian (Latin) + 0x7c1a: "sr", # Serbian (Latin) + 0x081a: "sr_CS", # Serbian (Latin) - Serbia and Montenegro (Former) + 0x0c1a: "sr_CS", # Serbian (Cyrillic) - Serbia and Montenegro (Former) + 0x181a: "sr_BA", # Serbian (Latin) - Bosnia and Herzegovina + 0x1c1a: "sr_BA", # Serbian (Cyrillic) - Bosnia and Herzegovina + 0x241a: "sr_RS", # Serbian (Latin) - Serbia + 0x281a: "sr_RS", # Serbian (Cyrillic) - Serbia + 0x2c1a: "sr_ME", # Serbian (Latin) - Montenegro + 0x301a: "sr_ME", # Serbian (Cyrillic) - Montenegro + 0x006c: "nso", # Sesotho sa Leboa + 0x046c: "nso_ZA", # Sesotho sa Leboa - South Africa + 0x0032: "tn", # Setswana + 0x0432: "tn_ZA", # Setswana - South Africa + 0x0832: "tn_BW", # Setswana - Botswana + 0x0059: "sd", # Sindhi + 0x7c59: "sd", # Sindhi + 0x0859: "sd_PK", # Sindhi - Islamic Republic of Pakistan + 0x005b: "si", # Sinhala 0x045b: "si_LK", # Sinhala - Sri Lanka - 0x046c: "ns_ZA", # Northern Sotho - 0x0432: "tn_ZA", # Setswana - Southern Africa - 0x041b: "sk_SK", # Slovak - 0x0424: "sl_SI", # Slovenian + 0x001b: "sk", # Slovak + 0x041b: "sk_SK", # Slovak - Slovakia + 0x0024: "sl", # Slovenian + 0x0424: "sl_SI", # Slovenian - Slovenia + 0x0477: "so_SO", # Somali - Somalia + 0x0030: "st", # Sotho + 0x0430: "st_ZA", # Sotho - South Africa + 0x000a: "es", # Spanish 0x040a: "es_ES", # Spanish - Spain 0x080a: "es_MX", # Spanish - Mexico - 0x0c0a: "es_ES", # Spanish - Spain (Modern) + 0x0c0a: "es_ES", # Spanish - Spain 0x100a: "es_GT", # Spanish - Guatemala 0x140a: "es_CR", # Spanish - Costa Rica 0x180a: "es_PA", # Spanish - Panama 0x1c0a: "es_DO", # Spanish - Dominican Republic - 0x200a: "es_VE", # Spanish - Venezuela + 0x200a: "es_VE", # Spanish - Bolivarian Republic of Venezuela 0x240a: "es_CO", # Spanish - Colombia 0x280a: "es_PE", # Spanish - Peru 0x2c0a: "es_AR", # Spanish - Argentina 0x300a: "es_EC", # Spanish - Ecuador 0x340a: "es_CL", # Spanish - Chile - 0x380a: "es_UR", # Spanish - Uruguay + 0x380a: "es_UY", # Spanish - Uruguay 0x3c0a: "es_PY", # Spanish - Paraguay 0x400a: "es_BO", # Spanish - Bolivia 0x440a: "es_SV", # Spanish - El Salvador @@ -1696,36 +1860,87 @@ def getpreferredencoding(do_setlocale=True): 0x4c0a: "es_NI", # Spanish - Nicaragua 0x500a: "es_PR", # Spanish - Puerto Rico 0x540a: "es_US", # Spanish - United States -# 0x0430: "", # Sutu - Not supported - 0x0441: "sw_KE", # Swahili + 0x5c0a: "es_CU", # Spanish - Cuba + 0x001d: "sv", # Swedish 0x041d: "sv_SE", # Swedish - Sweden 0x081d: "sv_FI", # Swedish - Finland - 0x045a: "syr_SY",# Syriac - 0x0428: "tg_TJ", # Tajik - Cyrillic - 0x085f: "tmz_DZ",# Tamazight - Latin - 0x0449: "ta_IN", # Tamil - 0x0444: "tt_RU", # Tatar - 0x044a: "te_IN", # Telugu - 0x041e: "th_TH", # Thai - 0x0851: "bo_BT", # Tibetan - Bhutan - 0x0451: "bo_CN", # Tibetan - PRC - 0x041f: "tr_TR", # Turkish - 0x0442: "tk_TM", # Turkmen - Cyrillic - 0x0480: "ug_CN", # Uighur - Arabic - 0x0422: "uk_UA", # Ukrainian - 0x042e: "wen_DE",# Upper Sorbian - Germany - 0x0420: "ur_PK", # Urdu + 0x005a: "syr", # Syriac + 0x045a: "syr_SY", # Syriac - Syria + 0x0028: "tg", # Tajik (Cyrillic) + 0x7c28: "tg", # Tajik (Cyrillic) + 0x0428: "tg_TJ", # Tajik (Cyrillic) - Tajikistan + 0x005f: "tzm", # Tamazight (Latin) + 0x785f: "tzm", + 0x7c5f: "tzm", # Tamazight (Latin) + 0x085f: "tzm_DZ", # Tamazight (Latin) - Algeria + 0x045f: "tzm_MA", # Central Atlas Tamazight (Arabic) - Morocco + 0x105f: "tzm_MA", + 0x0049: "ta", # Tamil + 0x0449: "ta_IN", # Tamil - India + 0x0849: "ta_LK", # Tamil - Sri Lanka + 0x0044: "tt", # Tatar + 0x0444: "tt_RU", # Tatar - Russia + 0x004a: "te", # Telugu + 0x044a: "te_IN", # Telugu - India + 0x001e: "th", # Thai + 0x041e: "th_TH", # Thai - Thailand + 0x0051: "bo", # Tibetan + 0x0451: "bo_CN", # Tibetan - People's Republic of China + 0x0073: "ti", # Tigrinya + 0x0473: "ti_ET", # Tigrinya - Ethiopia + 0x0873: "ti_ER", # Tigrinya - Eritrea + 0x0031: "ts", # Tsonga + 0x0431: "ts_ZA", # Tsonga - South Africa + 0x001f: "tr", # Turkish + 0x041f: "tr_TR", # Turkish - Turkey + 0x0042: "tk", # Turkmen + 0x0442: "tk_TM", # Turkmen - Turkmenistan + 0x0022: "uk", # Ukrainian + 0x0422: "uk_UA", # Ukrainian - Ukraine + 0x002e: "hsb", # Upper Sorbian + 0x042e: "hsb_DE", # Upper Sorbian - Germany + 0x0020: "ur", # Urdu + 0x0420: "ur_PK", # Urdu - Islamic Republic of Pakistan 0x0820: "ur_IN", # Urdu - India - 0x0443: "uz_UZ", # Uzbek - Latin - 0x0843: "uz_UZ", # Uzbek - Cyrillic - 0x042a: "vi_VN", # Vietnamese - 0x0452: "cy_GB", # Welsh + 0x0080: "ug", # Uyghur + 0x0480: "ug_CN", # Uyghur - People's Republic of China + 0x0043: "uz", # Uzbek (Latin) + 0x7843: "uz", # Uzbek (Cyrillic) + 0x7c43: "uz", # Uzbek (Latin) + 0x0443: "uz_UZ", # Uzbek (Latin) - Uzbekistan + 0x0033: "ve", # Venda + 0x0433: "ve_ZA", # Venda - South Africa + 0x002a: "vi", # Vietnamese + 0x042a: "vi_VN", # Vietnamese - Vietnam + 0x0052: "cy", # Welsh + 0x0452: "cy_GB", # Welsh - United Kingdom + 0x0088: "wo", # Wolof 0x0488: "wo_SN", # Wolof - Senegal + 0x0034: "xh", # Xhosa 0x0434: "xh_ZA", # Xhosa - South Africa - 0x0485: "sah_RU",# Yakut - Cyrillic - 0x0478: "ii_CN", # Yi - PRC + 0x0078: "ii", # Yi + 0x0478: "ii_CN", # Yi - People's Republic of China + 0x043d: "yi_001", # Yiddish - World + 0x006a: "yo", # Yoruba 0x046a: "yo_NG", # Yoruba - Nigeria - 0x0435: "zu_ZA", # Zulu + 0x0035: "zu", # Zulu + 0x0435: "zu_ZA", # Zulu - South Africa + 0x0086: "qut", + +# 0x0001007f: "x-IV-mathan", # math alphanumeric sorting + 0x00010407: "de_DE", + 0x0001040e: "hu_HU", + 0x00010437: "ka_GE", + 0x00020804: "zh_CN", + 0x00021004: "zh_SG", + 0x00021404: "zh_MO", + 0x00030404: "zh_TW", + 0x00040404: "zh_TW", + 0x00040411: "ja_JP", + 0x00040c04: "zh_HK", + 0x00041404: "zh_MO", + 0x00050804: "zh_CN", + 0x00051004: "zh_SG", } def _print_locale(): diff --git a/Lib/statistics.py b/Lib/statistics.py index e635b99f958e44..32fcf2313a815a 100644 --- a/Lib/statistics.py +++ b/Lib/statistics.py @@ -136,7 +136,7 @@ from fractions import Fraction from decimal import Decimal -from itertools import count, groupby, repeat +from itertools import compress, count, groupby, repeat from bisect import bisect_left, bisect_right from math import hypot, sqrt, fabs, exp, erfc, tau, log, fsum, sumprod from math import isfinite, isinf, pi, cos, sin, tan, cosh, asin, atan, acos @@ -195,9 +195,9 @@ def fmean(data, weights=None): n = len(data) except TypeError: # Handle iterators that do not define __len__(). - counter = count() - total = fsum(map(itemgetter(0), zip(data, counter))) - n = next(counter) + counter = count(1) + total = fsum(compress(data, counter)) + n = next(counter) - 1 else: total = fsum(data) diff --git a/Lib/test/.ruff.toml b/Lib/test/.ruff.toml index a960543f277935..dca74eb6e14bbd 100644 --- a/Lib/test/.ruff.toml +++ b/Lib/test/.ruff.toml @@ -18,6 +18,8 @@ extend-exclude = [ "test_lazy_import/__init__.py", "test_lazy_import/data/*.py", "test_lazy_import/data/**/*.py", + # Unary plus literal pattern is not yet supported by Ruff (GH-145239) + "test_patma.py", ] [lint] diff --git a/Lib/test/pickletester.py b/Lib/test/pickletester.py index 6366f12257f3b5..c2018c9785b9b3 100644 --- a/Lib/test/pickletester.py +++ b/Lib/test/pickletester.py @@ -3244,6 +3244,7 @@ def test_builtin_types(self): 'BuiltinImporter': (3, 3), 'str': (3, 4), # not interoperable with Python < 3.4 'frozendict': (3, 15), + 'sentinel': (3, 15), } for t in builtins.__dict__.values(): if isinstance(t, type) and not issubclass(t, BaseException): diff --git a/Lib/test/support/strace_helper.py b/Lib/test/support/strace_helper.py index cf95f7bdc7d2ca..bf15283d3027da 100644 --- a/Lib/test/support/strace_helper.py +++ b/Lib/test/support/strace_helper.py @@ -74,7 +74,7 @@ def sections(self): def _filter_memory_call(call): # mmap can operate on a fd or "MAP_ANONYMOUS" which gives a block of memory. # Ignore "MAP_ANONYMOUS + the "MAP_ANON" alias. - if call.syscall == "mmap" and "MAP_ANON" in call.args[3]: + if call.syscall in ("mmap", "mmap2") and "MAP_ANON" in call.args[3]: return True if call.syscall in ("munmap", "mprotect"): diff --git a/Lib/test/test__colorize.py b/Lib/test/test__colorize.py index 67e0595943d356..0353ff7530b92a 100644 --- a/Lib/test/test__colorize.py +++ b/Lib/test/test__colorize.py @@ -5,6 +5,7 @@ import unittest import unittest.mock import _colorize +from test.support import cpython_only, import_helper from test.support.os_helper import EnvironmentVarGuard @@ -22,6 +23,15 @@ def supports_virtual_terminal(): return contextlib.nullcontext() +class TestImportTime(unittest.TestCase): + + @cpython_only + def test_lazy_import(self): + import_helper.ensure_lazy_imports( + "_colorize", {"copy", "re"} + ) + + class TestTheme(unittest.TestCase): def test_attributes(self): diff --git a/Lib/test/test_annotationlib.py b/Lib/test/test_annotationlib.py index 50cf8fcb6b4ed6..77f2a77882fce2 100644 --- a/Lib/test/test_annotationlib.py +++ b/Lib/test/test_annotationlib.py @@ -1961,6 +1961,15 @@ def test_forward_repr(self): "typing.List[ForwardRef('int', owner='class')]", ) + def test_forward_repr_extra_names(self): + def f(a: undefined | str): ... + + annos = get_annotations(f, format=Format.FORWARDREF) + + self.assertRegex( + repr(annos['a']), r"ForwardRef\('undefined \| str'.*\)" + ) + def test_forward_recursion_actually(self): def namespace1(): a = ForwardRef("A") @@ -2037,6 +2046,17 @@ def test_evaluate_string_format(self): fr = ForwardRef("set[Any]") self.assertEqual(fr.evaluate(format=Format.STRING), "set[Any]") + def test_evaluate_string_format_extra_names(self): + # Test that internal extra_names are replaced when evaluating as strings + def f(a: unknown | str | int | list[str] | tuple[int, ...]): ... + + fr = get_annotations(f, format=Format.FORWARDREF)['a'] + # Test the cache is not populated before access + self.assertIsNone(fr.__resolved_str_cache__) + + self.assertEqual(fr.evaluate(format=Format.STRING), "unknown | str | int | list[str] | tuple[int, ...]") + self.assertEqual(fr.__resolved_str_cache__, "unknown | str | int | list[str] | tuple[int, ...]") + def test_evaluate_forwardref_format(self): fr = ForwardRef("undef") evaluated = fr.evaluate(format=Format.FORWARDREF) diff --git a/Lib/test/test_asyncio/test_sock_lowlevel.py b/Lib/test/test_asyncio/test_sock_lowlevel.py index df4ec7948975f6..f32dcd589e2de2 100644 --- a/Lib/test/test_asyncio/test_sock_lowlevel.py +++ b/Lib/test/test_asyncio/test_sock_lowlevel.py @@ -427,6 +427,27 @@ def test_recvfrom_into(self): self.loop.run_until_complete( self._basetest_datagram_recvfrom_into(server_address)) + async def _basetest_datagram_recvfrom_into_wrong_size(self, server_address): + # Call sock_sendto() with a size larger than the buffer + with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as sock: + sock.setblocking(False) + + buf = bytearray(5000) + data = b'\x01' * 4096 + wrong_size = len(buf) + 1 + await self.loop.sock_sendto(sock, data, server_address) + with self.assertRaises(ValueError): + await self.loop.sock_recvfrom_into( + sock, buf, wrong_size) + + size, addr = await self.loop.sock_recvfrom_into(sock, buf) + self.assertEqual(buf[:size], data) + + def test_recvfrom_into_wrong_size(self): + with test_utils.run_udp_echo_server() as server_address: + self.loop.run_until_complete( + self._basetest_datagram_recvfrom_into_wrong_size(server_address)) + async def _basetest_datagram_sendto_blocking(self, server_address): # Sad path, sock.sendto() raises BlockingIOError # This involves patching sock.sendto() to raise BlockingIOError but diff --git a/Lib/test/test_asyncio/test_taskgroups.py b/Lib/test/test_asyncio/test_taskgroups.py index 91f6b03b4597a5..8925884b9dcf73 100644 --- a/Lib/test/test_asyncio/test_taskgroups.py +++ b/Lib/test/test_asyncio/test_taskgroups.py @@ -1102,6 +1102,131 @@ async def throw_error(): # cancellation happens here and error is more understandable await asyncio.sleep(0) + async def test_taskgroup_cancel_children(self): + # (asserting that TimeoutError is not raised) + async with asyncio.timeout(1): + async with asyncio.TaskGroup() as tg: + tg.create_task(asyncio.sleep(10)) + tg.create_task(asyncio.sleep(10)) + await asyncio.sleep(0) + tg.cancel() + + async def test_taskgroup_cancel_body(self): + count = 0 + async with asyncio.TaskGroup() as tg: + tg.cancel() + count += 1 + await asyncio.sleep(0) + count += 1 + self.assertEqual(count, 1) + + async def test_taskgroup_cancel_idempotent(self): + count = 0 + async with asyncio.TaskGroup() as tg: + tg.cancel() + tg.cancel() + count += 1 + await asyncio.sleep(0) + count += 1 + self.assertEqual(count, 1) + + async def test_taskgroup_cancel_after_exit(self): + async with asyncio.TaskGroup() as tg: + await asyncio.sleep(0) + # (asserting that exception is not raised) + tg.cancel() + + async def test_taskgroup_cancel_before_enter(self): + tg = asyncio.TaskGroup() + tg.cancel() + count = 0 + async with tg: + count += 1 + await asyncio.sleep(0) + count += 1 + self.assertEqual(count, 1) + + async def test_taskgroup_cancel_before_create_task(self): + async with asyncio.TaskGroup() as tg: + tg.cancel() + # TODO: This behavior is not ideal. We'd rather have no exception + # raised, and the child task run until the first await. + with self.assertRaises(RuntimeError): + tg.create_task(asyncio.sleep(1)) + + async def test_taskgroup_cancel_before_exception(self): + async def raise_exc(parent_tg: asyncio.TaskGroup): + parent_tg.cancel() + raise RuntimeError + + with self.assertRaises(ExceptionGroup): + async with asyncio.TaskGroup() as tg: + tg.create_task(raise_exc(tg)) + await asyncio.sleep(1) + + async def test_taskgroup_cancel_after_exception(self): + async def raise_exc(parent_tg: asyncio.TaskGroup): + try: + raise RuntimeError + finally: + parent_tg.cancel() + + with self.assertRaises(ExceptionGroup): + async with asyncio.TaskGroup() as tg: + tg.create_task(raise_exc(tg)) + await asyncio.sleep(1) + + async def test_taskgroup_body_cancel_before_exception(self): + with self.assertRaises(ExceptionGroup): + async with asyncio.TaskGroup() as tg: + tg.cancel() + raise RuntimeError + + async def test_taskgroup_body_cancel_after_exception(self): + with self.assertRaises(ExceptionGroup): + async with asyncio.TaskGroup() as tg: + try: + raise RuntimeError + finally: + tg.cancel() + + async def test_taskgroup_cancel_one_winner(self): + async def race(*fns): + outcome = None + async def run(fn): + nonlocal outcome + outcome = await fn() + tg.cancel() + + async with asyncio.TaskGroup() as tg: + for fn in fns: + tg.create_task(run(fn)) + return outcome + + event = asyncio.Event() + record = [] + async def fn_1(): + record.append("1 started") + await event.wait() + record.append("1 finished") + return 1 + + async def fn_2(): + record.append("2 started") + await event.wait() + record.append("2 finished") + return 2 + + async def fn_3(): + record.append("3 started") + event.set() + await asyncio.sleep(10) + record.append("3 finished") + return 3 + + self.assertEqual(await race(fn_1, fn_2, fn_3), 1) + self.assertListEqual(record, ["1 started", "2 started", "3 started", "1 finished"]) + class TestTaskGroup(BaseTestTaskGroup, unittest.IsolatedAsyncioTestCase): loop_factory = asyncio.EventLoop diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py index 844656eb0e2c2e..e323742665234c 100644 --- a/Lib/test/test_builtin.py +++ b/Lib/test/test_builtin.py @@ -4,6 +4,7 @@ import builtins import collections import contextlib +import copy import decimal import fractions import gc @@ -21,6 +22,7 @@ import typing import unittest import warnings +import weakref from contextlib import ExitStack from functools import partial from inspect import CO_COROUTINE @@ -52,6 +54,10 @@ # used as proof of globals being used A_GLOBAL_VALUE = 123 +A_SENTINEL = sentinel("A_SENTINEL") + +class SentinelContainer: + CLASS_SENTINEL = sentinel("SentinelContainer.CLASS_SENTINEL") class Squares: @@ -1903,6 +1909,98 @@ class C: __repr__ = None self.assertRaises(TypeError, repr, C()) + def test_sentinel(self): + missing = sentinel("MISSING") + other = sentinel("MISSING") + + self.assertIsInstance(missing, sentinel) + self.assertIs(type(missing), sentinel) + self.assertEqual(missing.__name__, "MISSING") + self.assertEqual(missing.__module__, __name__) + self.assertIsNot(missing, other) + self.assertEqual(repr(missing), "MISSING") + self.assertTrue(missing) + self.assertIs(copy.copy(missing), missing) + self.assertIs(copy.deepcopy(missing), missing) + self.assertEqual(missing, missing) + self.assertNotEqual(missing, other) + self.assertRaises(TypeError, sentinel) + self.assertRaises(TypeError, sentinel, "MISSING", "EXTRA") + self.assertRaises(TypeError, sentinel, name="MISSING") + with self.assertRaisesRegex(TypeError, "must be str"): + sentinel(1) + self.assertTrue(sentinel.__flags__ & support._TPFLAGS_IMMUTABLETYPE) + self.assertTrue(sentinel.__flags__ & support._TPFLAGS_HAVE_GC) + self.assertFalse(sentinel.__flags__ & support._TPFLAGS_BASETYPE) + with self.assertRaises(TypeError): + class SubSentinel(sentinel): + pass + with self.assertRaises(TypeError): + sentinel.attribute = "value" + with self.assertRaises(AttributeError): + missing.__name__ = "CHANGED" + with self.assertRaises(AttributeError): + missing.__module__ = "changed" + with self.assertRaises(AttributeError): + del missing.__name__ + with self.assertRaises(AttributeError): + del missing.__module__ + + def test_sentinel_pickle(self): + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(protocol=proto): + self.assertIs( + pickle.loads(pickle.dumps(A_SENTINEL, protocol=proto)), + A_SENTINEL) + self.assertIs( + pickle.loads(pickle.dumps( + SentinelContainer.CLASS_SENTINEL, protocol=proto)), + SentinelContainer.CLASS_SENTINEL) + + missing = sentinel("MISSING") + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(protocol=proto): + with self.assertRaises(pickle.PicklingError): + pickle.dumps(missing, protocol=proto) + + def test_sentinel_str_subclass_name_cycle(self): + class Name(str): + pass + + name = Name("MISSING") + missing = sentinel(name) + self.assertIs(missing.__name__, name) + self.assertTrue(gc.is_tracked(missing)) + + name.missing = missing + ref = weakref.ref(name) + del name, missing + support.gc_collect() + self.assertIsNone(ref()) + + def test_sentinel_union(self): + missing = sentinel("MISSING") + + self.assertIsInstance(missing | int, typing.Union) + self.assertEqual((missing | int).__args__, (missing, int)) + self.assertIsInstance(int | missing, typing.Union) + self.assertEqual((int | missing).__args__, (int, missing)) + self.assertIs(missing | missing, missing) + self.assertEqual(repr(int | missing), "int | MISSING") + self.assertIsInstance(missing | None, typing.Union) + self.assertEqual((missing | None).__args__, (missing, type(None))) + self.assertIsInstance(None | missing, typing.Union) + self.assertEqual((None | missing).__args__, (type(None), missing)) + self.assertIsInstance(missing | list[int], typing.Union) + self.assertEqual((missing | list[int]).__args__, (missing, list[int])) + self.assertIsInstance(missing | (int | str), typing.Union) + self.assertEqual((missing | (int | str)).__args__, (missing, int, str)) + + with self.assertRaises(TypeError): + missing | 1 + with self.assertRaises(TypeError): + 1 | missing + def test_round(self): self.assertEqual(round(0.0), 0.0) self.assertEqual(type(round(0.0)), int) diff --git a/Lib/test/test_calendar.py b/Lib/test/test_calendar.py index fe9a59d335b6b0..79f0ebb78ffe9c 100644 --- a/Lib/test/test_calendar.py +++ b/Lib/test/test_calendar.py @@ -495,12 +495,17 @@ def test_formatmonth(self): calendar.TextCalendar().formatmonth(0, 2), result_0_02_text ) + def test_formatmonth_with_invalid_month(self): with self.assertRaises(calendar.IllegalMonthError): calendar.TextCalendar().formatmonth(2017, 13) with self.assertRaises(calendar.IllegalMonthError): calendar.TextCalendar().formatmonth(2017, -1) + def test_illegal_month_error_bases(self): + self.assertIsSubclass(calendar.IllegalMonthError, ValueError) + self.assertIsSubclass(calendar.IllegalMonthError, IndexError) + def test_formatmonthname_with_year(self): self.assertEqual( calendar.HTMLCalendar().formatmonthname(2004, 1, withyear=True), diff --git a/Lib/test/test_capi/test_object.py b/Lib/test/test_capi/test_object.py index 67572ab1ba268d..635deaa73f7efa 100644 --- a/Lib/test/test_capi/test_object.py +++ b/Lib/test/test_capi/test_object.py @@ -1,5 +1,6 @@ import enum import os +import pickle import sys import textwrap import unittest @@ -63,6 +64,27 @@ def test_get_constant_borrowed(self): self.check_get_constant(_testlimitedcapi.get_constant_borrowed) +class SentinelTest(unittest.TestCase): + + def test_pysentinel_new(self): + marker = _testcapi.pysentinel_new("CAPI_SENTINEL", __name__) + self.assertIs(type(marker), sentinel) + self.assertTrue(_testcapi.pysentinel_check(marker)) + self.assertFalse(_testcapi.pysentinel_check(object())) + self.assertEqual(marker.__name__, "CAPI_SENTINEL") + self.assertEqual(marker.__module__, __name__) + self.assertEqual(repr(marker), "CAPI_SENTINEL") + + no_module = _testcapi.pysentinel_new("NO_MODULE") + self.assertIs(type(no_module), sentinel) + self.assertEqual(no_module.__name__, "NO_MODULE") + self.assertIs(no_module.__module__, None) + + globals()["CAPI_SENTINEL"] = marker + self.addCleanup(globals().pop, "CAPI_SENTINEL", None) + self.assertIs(pickle.loads(pickle.dumps(marker)), marker) + + class PrintTest(unittest.TestCase): def testPyObjectPrintObject(self): diff --git a/Lib/test/test_capi/test_opt.py b/Lib/test/test_capi/test_opt.py index 59266b000ed4df..39075fc64cf02b 100644 --- a/Lib/test/test_capi/test_opt.py +++ b/Lib/test/test_capi/test_opt.py @@ -1427,9 +1427,13 @@ def testfunc(n): for _ in gen(n): pass testfunc(TIER2_THRESHOLD * 2) + # The generator may be inlined into testfunc's trace, + # so check whichever executor contains _YIELD_VALUE. gen_ex = get_first_executor(gen) - self.assertIsNotNone(gen_ex) - uops = get_opnames(gen_ex) + testfunc_ex = get_first_executor(testfunc) + ex = gen_ex or testfunc_ex + self.assertIsNotNone(ex) + uops = get_opnames(ex) self.assertNotIn("_MAKE_HEAP_SAFE", uops) self.assertIn("_YIELD_VALUE", uops) diff --git a/Lib/test/test_cext/setup.py b/Lib/test/test_cext/setup.py index 7262a110d83415..25fe50df603883 100644 --- a/Lib/test/test_cext/setup.py +++ b/Lib/test/test_cext/setup.py @@ -18,6 +18,11 @@ # The purpose of test_cext extension is to check that building a C # extension using the Python C API does not emit C compiler warnings. '-Werror', + # Enable extra checks for header files, which: + # - need to be enabled somewhere inside Python headers (rather than + # before including Python.h) + # - should not be checked for user code + '-D_Py_IS_TESTCEXT', ] # C compiler flags for GCC and clang diff --git a/Lib/test/test_ctypes/test_byteswap.py b/Lib/test/test_ctypes/test_byteswap.py index f14e1aa32e17ab..6a1bae14773d27 100644 --- a/Lib/test/test_ctypes/test_byteswap.py +++ b/Lib/test/test_ctypes/test_byteswap.py @@ -1,4 +1,5 @@ import binascii +import ctypes import math import struct import sys @@ -165,6 +166,48 @@ def test_endian_double(self): self.assertEqual(s.value, math.pi) self.assertEqual(bin(struct.pack(">d", math.pi)), bin(s)) + @unittest.skipUnless(hasattr(ctypes, 'c_float_complex'), "No complex types") + def test_endian_float_complex(self): + c_float_complex = ctypes.c_float_complex + if sys.byteorder == "little": + self.assertIs(c_float_complex.__ctype_le__, c_float_complex) + self.assertIs(c_float_complex.__ctype_be__.__ctype_le__, + c_float_complex) + else: + self.assertIs(c_float_complex.__ctype_be__, c_float_complex) + self.assertIs(c_float_complex.__ctype_le__.__ctype_be__, + c_float_complex) + s = c_float_complex(math.pi+1j) + self.assertEqual(bin(struct.pack("F", math.pi+1j)), bin(s)) + self.assertAlmostEqual(s.value, math.pi+1j, places=6) + s = c_float_complex.__ctype_le__(math.pi+1j) + self.assertAlmostEqual(s.value, math.pi+1j, places=6) + self.assertEqual(bin(struct.pack("F", math.pi+1j)), bin(s)) + + @unittest.skipUnless(hasattr(ctypes, 'c_double_complex'), "No complex types") + def test_endian_double_complex(self): + c_double_complex = ctypes.c_double_complex + if sys.byteorder == "little": + self.assertIs(c_double_complex.__ctype_le__, c_double_complex) + self.assertIs(c_double_complex.__ctype_be__.__ctype_le__, + c_double_complex) + else: + self.assertIs(c_double_complex.__ctype_be__, c_double_complex) + self.assertIs(c_double_complex.__ctype_le__.__ctype_be__, + c_double_complex) + s = c_double_complex(math.pi+1j) + self.assertEqual(bin(struct.pack("D", math.pi+1j)), bin(s)) + self.assertAlmostEqual(s.value, math.pi+1j, places=6) + s = c_double_complex.__ctype_le__(math.pi+1j) + self.assertAlmostEqual(s.value, math.pi+1j, places=6) + self.assertEqual(bin(struct.pack("D", math.pi+1j)), bin(s)) + def test_endian_other(self): self.assertIs(c_byte.__ctype_le__, c_byte) self.assertIs(c_byte.__ctype_be__, c_byte) diff --git a/Lib/test/test_dataclasses/__init__.py b/Lib/test/test_dataclasses/__init__.py index b44b1da0336d09..e0cfe3df3e6357 100644 --- a/Lib/test/test_dataclasses/__init__.py +++ b/Lib/test/test_dataclasses/__init__.py @@ -1693,17 +1693,24 @@ class GroupTuple: class GroupDict: id: int users: Dict[str, User] + @dataclass + class GroupFrozenDict: + id: int + users: frozendict[str, User] a = User('Alice', 1) b = User('Bob', 2) gl = GroupList(0, [a, b]) gt = GroupTuple(0, (a, b)) gd = GroupDict(0, {'first': a, 'second': b}) + gfd = GroupFrozenDict(0, frozendict({'first': a, 'second': b})) self.assertEqual(asdict(gl), {'id': 0, 'users': [{'name': 'Alice', 'id': 1}, {'name': 'Bob', 'id': 2}]}) self.assertEqual(asdict(gt), {'id': 0, 'users': ({'name': 'Alice', 'id': 1}, {'name': 'Bob', 'id': 2})}) - self.assertEqual(asdict(gd), {'id': 0, 'users': {'first': {'name': 'Alice', 'id': 1}, - 'second': {'name': 'Bob', 'id': 2}}}) + expected_dict = {'id': 0, 'users': {'first': {'name': 'Alice', 'id': 1}, + 'second': {'name': 'Bob', 'id': 2}}} + self.assertEqual(asdict(gd), expected_dict) + self.assertEqual(asdict(gfd), expected_dict) def test_helper_asdict_builtin_object_containers(self): @dataclass @@ -1884,14 +1891,21 @@ class GroupTuple: class GroupDict: id: int users: Dict[str, User] + @dataclass + class GroupFrozenDict: + id: int + users: frozendict[str, User] a = User('Alice', 1) b = User('Bob', 2) gl = GroupList(0, [a, b]) gt = GroupTuple(0, (a, b)) gd = GroupDict(0, {'first': a, 'second': b}) + gfd = GroupFrozenDict(0, frozendict({'first': a, 'second': b})) self.assertEqual(astuple(gl), (0, [('Alice', 1), ('Bob', 2)])) self.assertEqual(astuple(gt), (0, (('Alice', 1), ('Bob', 2)))) - self.assertEqual(astuple(gd), (0, {'first': ('Alice', 1), 'second': ('Bob', 2)})) + d = {'first': ('Alice', 1), 'second': ('Bob', 2)} + self.assertEqual(astuple(gd), (0, d)) + self.assertEqual(astuple(gfd), (0, frozendict(d))) def test_helper_astuple_builtin_object_containers(self): @dataclass diff --git a/Lib/test/test_free_threading/test_str.py b/Lib/test/test_free_threading/test_str.py index 9a1ce3620ac4b2..11e04009956db1 100644 --- a/Lib/test/test_free_threading/test_str.py +++ b/Lib/test/test_free_threading/test_str.py @@ -1,7 +1,9 @@ +import sys +import threading import unittest from itertools import cycle -from threading import Event, Thread +from threading import Barrier, Event, Thread from unittest import TestCase from test.support import threading_helper @@ -69,6 +71,24 @@ def reader_func(): for reader in readers: reader.join() + def test_intern_unowned_string(self): + # Test interning strings owned by various threads. + strings = [f"intern_race_owner_{i}" for i in range(50)] + + NUM_THREADS = 5 + b = Barrier(NUM_THREADS) + + def interner(): + tid = threading.get_ident() + for i in range(20): + strings.append(f"intern_{tid}_{i}") + b.wait() + for s in strings: + r = sys.intern(s) + self.assertTrue(sys._is_interned(r)) + + threading_helper.run_concurrently(interner, nthreads=NUM_THREADS) + def test_maketrans_dict_concurrent_modification(self): for _ in range(5): d = {2000: 'a'} diff --git a/Lib/test/test_http_cookies.py b/Lib/test/test_http_cookies.py index e2c7551c0b3341..cfcbc17bd6df80 100644 --- a/Lib/test/test_http_cookies.py +++ b/Lib/test/test_http_cookies.py @@ -1,5 +1,5 @@ # Simple test suite for http/cookies.py - +import base64 import copy import unittest import doctest @@ -175,17 +175,19 @@ def test_load(self): self.assertEqual(C.output(['path']), 'Set-Cookie: Customer="WILE_E_COYOTE"; Path=/acme') - self.assertEqual(C.js_output(), r""" + cookie_encoded = base64.b64encode(b'Customer="WILE_E_COYOTE"; Path=/acme; Version=1').decode('ascii') + self.assertEqual(C.js_output(), fr""" """) - self.assertEqual(C.js_output(['path']), r""" + cookie_encoded = base64.b64encode(b'Customer="WILE_E_COYOTE"; Path=/acme').decode('ascii') + self.assertEqual(C.js_output(['path']), fr""" """) @@ -290,17 +292,19 @@ def test_quoted_meta(self): self.assertEqual(C.output(['path']), 'Set-Cookie: Customer="WILE_E_COYOTE"; Path=/acme') - self.assertEqual(C.js_output(), r""" + expected_encoded_cookie = base64.b64encode(b'Customer=\"WILE_E_COYOTE\"; Path=/acme; Version=1').decode('ascii') + self.assertEqual(C.js_output(), fr""" """) - self.assertEqual(C.js_output(['path']), r""" + expected_encoded_cookie = base64.b64encode(b'Customer=\"WILE_E_COYOTE\"; Path=/acme').decode('ascii') + self.assertEqual(C.js_output(['path']), fr""" """) @@ -391,13 +395,16 @@ def test_setter(self): self.assertEqual( M.output(), "Set-Cookie: %s=%s; Path=/foo" % (i, "%s_coded_val" % i)) + expected_encoded_cookie = base64.b64encode( + ("%s=%s; Path=/foo" % (i, "%s_coded_val" % i)).encode("ascii") + ).decode('ascii') expected_js_output = """ - """ % (i, "%s_coded_val" % i) + """ % (expected_encoded_cookie,) self.assertEqual(M.js_output(), expected_js_output) for i in ["foo bar", "foo@bar"]: # Try some illegal characters diff --git a/Lib/test/test_json/json_lines.jsonl b/Lib/test/test_json/json_lines.jsonl new file mode 100644 index 00000000000000..d2f292111956f3 --- /dev/null +++ b/Lib/test/test_json/json_lines.jsonl @@ -0,0 +1,2 @@ +{"ingredients":["frog", "water", "chocolate", "glucose"]} +{"ingredients":["chocolate","steel bolts"]} diff --git a/Lib/test/test_json/test_tool.py b/Lib/test/test_json/test_tool.py index 7b5d217a21558c..0a96b318b15b1c 100644 --- a/Lib/test/test_json/test_tool.py +++ b/Lib/test/test_json/test_tool.py @@ -1,4 +1,5 @@ import errno +import pathlib import os import sys import textwrap @@ -157,6 +158,14 @@ def test_jsonlines(self): self.assertEqual(process.stdout, self.jsonlines_expect) self.assertEqual(process.stderr, '') + @force_not_colorized + def test_jsonlines_from_file(self): + jsonl = pathlib.Path(__file__).parent / 'json_lines.jsonl' + args = sys.executable, '-m', self.module, '--json-lines', jsonl + process = subprocess.run(args, capture_output=True, text=True, check=True) + self.assertEqual(process.stdout, self.jsonlines_expect) + self.assertEqual(process.stderr, '') + def test_help_flag(self): rc, out, err = assert_python_ok('-m', self.module, '-h', PYTHON_COLORS='0') diff --git a/Lib/test/test_marshal.py b/Lib/test/test_marshal.py index 78db4219e2997c..9c4d91c456dc5d 100644 --- a/Lib/test/test_marshal.py +++ b/Lib/test/test_marshal.py @@ -317,6 +317,138 @@ def test_recursion_limit(self): last.append([0]) self.assertRaises(ValueError, marshal.dumps, head) + def test_reference_loop_list(self): + a = [] + a.append(a) + for v in range(3): + self.assertRaises(ValueError, marshal.dumps, a, v) + for v in range(3, marshal.version + 1): + d = marshal.dumps(a, v) + b = marshal.loads(d) + self.assertIsInstance(b, list) + self.assertIs(b[0], b) + + def test_reference_loop_dict(self): + a = {} + a[None] = a + for v in range(3): + self.assertRaises(ValueError, marshal.dumps, a, v) + for v in range(3, marshal.version + 1): + d = marshal.dumps(a, v) + b = marshal.loads(d) + self.assertIsInstance(b, dict) + self.assertIs(b[None], b) + + def test_reference_loop_tuple(self): + a = ([],) + a[0].append(a) + for v in range(3): + self.assertRaises(ValueError, marshal.dumps, a, v) + for v in range(3, marshal.version + 1): + d = marshal.dumps(a, v) + b = marshal.loads(d) + self.assertIsInstance(b, tuple) + self.assertIsInstance(b[0], list) + self.assertIs(b[0][0], b) + + def test_reference_loop_code(self): + def f(): + return 1234.5 + code = f.__code__ + a = [] + code = code.replace(co_consts=code.co_consts + (a,)) + # This test creates a reference loop which leads to reference leaks, + # so we need to break the loop manually. See gh-148722. + self.addCleanup(a.clear) + a.append(code) + for v in range(marshal.version + 1): + self.assertRaises(ValueError, marshal.dumps, code, v) + + def test_reference_loop_slice(self): + a = slice([], None) + a.start.append(a) + for v in range(marshal.version + 1): + self.assertRaises(ValueError, marshal.dumps, a, v) + + a = slice(None, []) + a.stop.append(a) + for v in range(marshal.version + 1): + self.assertRaises(ValueError, marshal.dumps, a, v) + + a = slice(None, None, []) + a.step.append(a) + for v in range(marshal.version + 1): + self.assertRaises(ValueError, marshal.dumps, a, v) + + def test_reference_loop_frozendict(self): + a = frozendict({None: []}) + a[None].append(a) + for v in range(marshal.version + 1): + self.assertRaises(ValueError, marshal.dumps, a, v) + + def test_loads_reference_loop_list(self): + data = b'\xdb\x01\x00\x00\x00r\x00\x00\x00\x00' # [] + a = marshal.loads(data) + self.assertIsInstance(a, list) + self.assertIs(a[0], a) + + def test_loads_reference_loop_dict(self): + data = b'\xfbNr\x00\x00\x00\x000' # {None: } + a = marshal.loads(data) + self.assertIsInstance(a, dict) + self.assertIs(a[None], a) + + def test_loads_abnormal_reference_loops(self): + # Indirect self-references of tuples. + data = b'\xa8\x01\x00\x00\x00[\x01\x00\x00\x00r\x00\x00\x00\x00' # ([],) + a = marshal.loads(data) + self.assertIsInstance(a, tuple) + self.assertIsInstance(a[0], list) + self.assertIs(a[0][0], a) + + data = b'\xa8\x01\x00\x00\x00{Nr\x00\x00\x00\x000' # ({None: },) + a = marshal.loads(data) + self.assertIsInstance(a, tuple) + self.assertIsInstance(a[0], dict) + self.assertIs(a[0][None], a) + + # Direct self-reference which cannot be created in Python. + # This creates a reference loop which cannot be collected. + if False: + data = b'\xa8\x01\x00\x00\x00r\x00\x00\x00\x00' # (,) + a = marshal.loads(data) + self.assertIsInstance(a, tuple) + self.assertIs(a[0], a) + + # Direct self-references which cannot be created in Python + # because of unhashability. + data = b'\xfbr\x00\x00\x00\x00N0' # {: None} + self.assertRaises(TypeError, marshal.loads, data) + data = b'\xbc\x01\x00\x00\x00r\x00\x00\x00\x00' # {} + self.assertRaises(TypeError, marshal.loads, data) + + for data in [ + # Indirect self-references of immutable objects. + b'\xba[\x01\x00\x00\x00r\x00\x00\x00\x00NN', # slice([], None) + b'\xbaN[\x01\x00\x00\x00r\x00\x00\x00\x00N', # slice(None, []) + b'\xbaNN[\x01\x00\x00\x00r\x00\x00\x00\x00', # slice(None, None, []) + b'\xba{Nr\x00\x00\x00\x000NN', # slice({None: }, None) + b'\xbaN{Nr\x00\x00\x00\x000N', # slice(None, {None: }) + b'\xbaNN{Nr\x00\x00\x00\x000', # slice(None, None, {None: }) + b'\xfdN[\x01\x00\x00\x00r\x00\x00\x00\x000', # frozendict({None: []}) + b'\xfdN{Nr\x00\x00\x00\x0000', # frozendict({None: {None: }) + + # Direct self-references which cannot be created in Python. + b'\xbe\x01\x00\x00\x00r\x00\x00\x00\x00', # frozenset({}) + b'\xfdNr\x00\x00\x00\x000', # frozendict({None: }) + b'\xfdr\x00\x00\x00\x00N0', # frozendict({: None}) + b'\xbar\x00\x00\x00\x00NN', # slice(, None) + b'\xbaNr\x00\x00\x00\x00N', # slice(None, ) + b'\xbaNNr\x00\x00\x00\x00', # slice(None, None, ) + ]: + with self.subTest(data=data): + self.assertRaises(ValueError, marshal.loads, data) + def test_exact_type_match(self): # Former bug: # >>> class Int(int): pass diff --git a/Lib/test/test_monitoring.py b/Lib/test/test_monitoring.py index bc7af6e15380a4..b8861d09e1564b 100644 --- a/Lib/test/test_monitoring.py +++ b/Lib/test/test_monitoring.py @@ -196,13 +196,10 @@ def test_c_return_count(self): (E.BRANCH, "branch"), ] -EXCEPT_EVENTS = [ +SIMPLE_EVENTS = INSTRUMENTED_EVENTS + [ (E.RAISE, "raise"), - (E.PY_UNWIND, "unwind"), (E.EXCEPTION_HANDLED, "exception_handled"), -] - -SIMPLE_EVENTS = INSTRUMENTED_EVENTS + EXCEPT_EVENTS + [ + (E.PY_UNWIND, "unwind"), (E.C_RAISE, "c_raise"), (E.C_RETURN, "c_return"), ] @@ -738,18 +735,6 @@ def test_disable_legal_events(self): sys.monitoring.register_callback(TEST_TOOL, event, None) - def test_disable_illegal_events(self): - for event, name in EXCEPT_EVENTS: - try: - counter = CounterWithDisable() - counter.disable = True - sys.monitoring.register_callback(TEST_TOOL, event, counter) - sys.monitoring.set_events(TEST_TOOL, event) - with self.assertRaises(ValueError): - self.raise_handle_reraise() - finally: - sys.monitoring.set_events(TEST_TOOL, 0) - sys.monitoring.register_callback(TEST_TOOL, event, None) class ExceptionRecorder: @@ -1481,8 +1466,334 @@ def func3(): ('line', 'func3', 6)]) def test_set_non_local_event(self): + # C_RETURN/C_RAISE are ancillary (derived) events — not settable as local with self.assertRaises(ValueError): - sys.monitoring.set_local_events(TEST_TOOL, just_call.__code__, E.RAISE) + sys.monitoring.set_local_events(TEST_TOOL, just_call.__code__, E.C_RETURN) + + def test_local_reraise(self): + """RERAISE fires as a local event only for the instrumented code object.""" + + def foo(): + try: + raise RuntimeError("test") + except RuntimeError: + raise + + def bar(): + try: + raise RuntimeError("test") + except RuntimeError: + raise + + events = set() + + def callback(code, offset, exc): + events.add(code.co_name) + + try: + sys.monitoring.register_callback(TEST_TOOL, E.RERAISE, callback) + sys.monitoring.set_local_events(TEST_TOOL, foo.__code__, E.RERAISE) + try: + foo() + except RuntimeError: + pass + try: + bar() # should NOT trigger the callback + except RuntimeError: + pass + self.assertEqual(events, {'foo'}) + finally: + sys.monitoring.set_local_events(TEST_TOOL, foo.__code__, 0) + sys.monitoring.register_callback(TEST_TOOL, E.RERAISE, None) + + def test_local_reraise_disable(self): + """Returning DISABLE from a RERAISE callback disables it for that code object.""" + + call_count = 0 + + def foo(): + try: + raise RuntimeError("test") + except RuntimeError: + raise + + def callback(code, offset, exc): + nonlocal call_count + call_count += 1 + return sys.monitoring.DISABLE + + try: + sys.monitoring.register_callback(TEST_TOOL, E.RERAISE, callback) + sys.monitoring.set_local_events(TEST_TOOL, foo.__code__, E.RERAISE) + try: + foo() + except RuntimeError: + pass + self.assertEqual(call_count, 1) + try: + foo() + except RuntimeError: + pass + self.assertEqual(call_count, 1) # not fired again — disabled + finally: + sys.monitoring.set_local_events(TEST_TOOL, foo.__code__, 0) + sys.monitoring.register_callback(TEST_TOOL, E.RERAISE, None) + + def test_local_py_throw(self): + """PY_THROW fires as a local event only for the instrumented code object.""" + + def gen_foo(): + yield 1 + yield 2 + + def gen_bar(): + yield 1 + yield 2 + + events = [] + + def callback(code, offset, exc): + events.append(code.co_name) + + try: + sys.monitoring.register_callback(TEST_TOOL, E.PY_THROW, callback) + sys.monitoring.set_local_events(TEST_TOOL, gen_foo.__code__, E.PY_THROW) + + g = gen_foo() + next(g) + try: + g.throw(RuntimeError("test")) + except RuntimeError: + pass + + h = gen_bar() + next(h) + try: + h.throw(RuntimeError("test")) # should NOT trigger the callback + except RuntimeError: + pass + + self.assertEqual(events, ['gen_foo']) + finally: + sys.monitoring.set_local_events(TEST_TOOL, gen_foo.__code__, 0) + sys.monitoring.register_callback(TEST_TOOL, E.PY_THROW, None) + + def test_local_py_throw_disable(self): + """Returning DISABLE from a PY_THROW callback disables it for that code object.""" + + call_count = 0 + + def gen_foo(): + yield 1 + yield 2 + + def callback(code, offset, exc): + nonlocal call_count + call_count += 1 + return sys.monitoring.DISABLE + + try: + sys.monitoring.register_callback(TEST_TOOL, E.PY_THROW, callback) + sys.monitoring.set_local_events(TEST_TOOL, gen_foo.__code__, E.PY_THROW) + + g = gen_foo() + next(g) + try: + g.throw(RuntimeError("test")) + except RuntimeError: + pass + self.assertEqual(call_count, 1) + + g2 = gen_foo() + next(g2) + try: + g2.throw(RuntimeError("test")) + except RuntimeError: + pass + self.assertEqual(call_count, 1) # not fired again — disabled + finally: + sys.monitoring.set_local_events(TEST_TOOL, gen_foo.__code__, 0) + sys.monitoring.register_callback(TEST_TOOL, E.PY_THROW, None) + + def test_local_raise(self): + """RAISE fires as a local event only for the instrumented code object.""" + + def foo(): + try: + raise RuntimeError("test") + except RuntimeError: + pass + + def bar(): + try: + raise RuntimeError("test") + except RuntimeError: + pass + + events = [] + + def callback(code, offset, exc): + events.append(code.co_name) + + try: + sys.monitoring.register_callback(TEST_TOOL, E.RAISE, callback) + sys.monitoring.set_local_events(TEST_TOOL, foo.__code__, E.RAISE) + foo() + bar() # should NOT trigger the callback + self.assertEqual(events, ['foo']) + finally: + sys.monitoring.set_local_events(TEST_TOOL, foo.__code__, 0) + sys.monitoring.register_callback(TEST_TOOL, E.RAISE, None) + + def test_local_raise_disable(self): + """Returning DISABLE from a RAISE callback disables it for that code object.""" + + call_count = 0 + + def foo(): + try: + raise RuntimeError("test") + except RuntimeError: + pass + + def callback(code, offset, exc): + nonlocal call_count + call_count += 1 + return sys.monitoring.DISABLE + + try: + sys.monitoring.register_callback(TEST_TOOL, E.RAISE, callback) + sys.monitoring.set_local_events(TEST_TOOL, foo.__code__, E.RAISE) + foo() + self.assertEqual(call_count, 1) + foo() + self.assertEqual(call_count, 1) # not fired again — disabled + finally: + sys.monitoring.set_local_events(TEST_TOOL, foo.__code__, 0) + sys.monitoring.register_callback(TEST_TOOL, E.RAISE, None) + + def test_local_exception_handled(self): + """EXCEPTION_HANDLED fires as a local event only for the instrumented code object.""" + + def foo(): + try: + raise RuntimeError("test") + except RuntimeError: + pass + + def bar(): + try: + raise RuntimeError("test") + except RuntimeError: + pass + + events = [] + + def callback(code, offset, exc): + events.append(code.co_name) + + try: + sys.monitoring.register_callback(TEST_TOOL, E.EXCEPTION_HANDLED, callback) + sys.monitoring.set_local_events(TEST_TOOL, foo.__code__, E.EXCEPTION_HANDLED) + foo() + bar() # should NOT trigger the callback + self.assertEqual(events, ['foo']) + finally: + sys.monitoring.set_local_events(TEST_TOOL, foo.__code__, 0) + sys.monitoring.register_callback(TEST_TOOL, E.EXCEPTION_HANDLED, None) + + def test_local_exception_handled_disable(self): + """Returning DISABLE from an EXCEPTION_HANDLED callback disables it for that code object.""" + + call_count = 0 + + def foo(): + try: + raise RuntimeError("test") + except RuntimeError: + pass + + def callback(code, offset, exc): + nonlocal call_count + call_count += 1 + return sys.monitoring.DISABLE + + try: + sys.monitoring.register_callback(TEST_TOOL, E.EXCEPTION_HANDLED, callback) + sys.monitoring.set_local_events(TEST_TOOL, foo.__code__, E.EXCEPTION_HANDLED) + foo() + self.assertEqual(call_count, 1) + foo() + self.assertEqual(call_count, 1) # not fired again — disabled + finally: + sys.monitoring.set_local_events(TEST_TOOL, foo.__code__, 0) + sys.monitoring.register_callback(TEST_TOOL, E.EXCEPTION_HANDLED, None) + + def test_local_py_unwind(self): + """PY_UNWIND fires as a local event only for the instrumented code object.""" + + def foo(): + raise RuntimeError("test") + + def bar(): + raise RuntimeError("test") + + events = [] + + def callback(code, offset, exc): + events.append(code.co_name) + + try: + sys.monitoring.register_callback(TEST_TOOL, E.PY_UNWIND, callback) + sys.monitoring.set_local_events(TEST_TOOL, foo.__code__, E.PY_UNWIND) + + try: + foo() + except RuntimeError: + pass + + try: + bar() # should NOT trigger the callback + except RuntimeError: + pass + + self.assertEqual(events, ['foo']) + finally: + sys.monitoring.set_local_events(TEST_TOOL, foo.__code__, 0) + sys.monitoring.register_callback(TEST_TOOL, E.PY_UNWIND, None) + + def test_local_py_unwind_disable(self): + """Returning DISABLE from a PY_UNWIND callback disables it for that code object.""" + + call_count = 0 + + def foo(): + raise RuntimeError("test") + + def callback(code, offset, exc): + nonlocal call_count + call_count += 1 + return sys.monitoring.DISABLE + + try: + sys.monitoring.register_callback(TEST_TOOL, E.PY_UNWIND, callback) + sys.monitoring.set_local_events(TEST_TOOL, foo.__code__, E.PY_UNWIND) + + try: + foo() + except RuntimeError: + pass + self.assertEqual(call_count, 1) # fired once + + try: + foo() + except RuntimeError: + pass + self.assertEqual(call_count, 1) # not fired again — disabled by DISABLE return + + finally: + sys.monitoring.set_local_events(TEST_TOOL, foo.__code__, 0) + sys.monitoring.register_callback(TEST_TOOL, E.PY_UNWIND, None) def line_from_offset(code, offset): for start, end, line in code.co_lines(): diff --git a/Lib/test/test_patma.py b/Lib/test/test_patma.py index 5d0857b059ea23..29cce4ee6d271f 100644 --- a/Lib/test/test_patma.py +++ b/Lib/test/test_patma.py @@ -2762,6 +2762,96 @@ def test_patma_255(self): self.assertEqual(y, 1) self.assertIs(z, x) + def test_patma_256(self): + x = 0 + match x: + case +0: + y = 0 + self.assertEqual(x, 0) + self.assertEqual(y, 0) + + def test_patma_257(self): + x = 0 + match x: + case +0.0: + y = 0 + self.assertEqual(x, 0) + self.assertEqual(y, 0) + + def test_patma_258(self): + x = 0 + match x: + case +0j: + y = 0 + self.assertEqual(x, 0) + self.assertEqual(y, 0) + + def test_patma_259(self): + x = 0 + match x: + case +0.0j: + y = 0 + self.assertEqual(x, 0) + self.assertEqual(y, 0) + + def test_patma_260(self): + x = 1 + match x: + case +1: + y = 0 + self.assertEqual(x, 1) + self.assertEqual(y, 0) + + def test_patma_261(self): + x = 1.5 + match x: + case +1.5: + y = 0 + self.assertEqual(x, 1.5) + self.assertEqual(y, 0) + + def test_patma_262(self): + x = 1j + match x: + case +1j: + y = 0 + self.assertEqual(x, 1j) + self.assertEqual(y, 0) + + def test_patma_263(self): + x = 1.5j + match x: + case +1.5j: + y = 0 + self.assertEqual(x, 1.5j) + self.assertEqual(y, 0) + + def test_patma_264(self): + x = 0.25 + 1.75j + match x: + case +0.25 + 1.75j: + y = 0 + self.assertEqual(x, 0.25 + 1.75j) + self.assertEqual(y, 0) + + def test_patma_265(self): + x = 0.25 - 1.75j + match x: + case 0.25 - +1.75j: + y = 0 + self.assertEqual(x, 0.25 - 1.75j) + self.assertEqual(y, 0) + + def test_patma_266(self): + x = 0 + match x: + case +1e1000: + y = 0 + case 0: + y = 1 + self.assertEqual(x, 0) + self.assertEqual(y, 1) + def test_patma_runtime_checkable_protocol(self): # Runtime-checkable protocol from typing import Protocol, runtime_checkable diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index 9c0172f6ba7f23..6c3d67fb6b7383 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -29,7 +29,7 @@ from typing import assert_type, cast, runtime_checkable from typing import get_type_hints from typing import get_origin, get_args, get_protocol_members -from typing import override +from typing import override, disjoint_base from typing import is_typeddict, is_protocol from typing import reveal_type from typing import dataclass_transform @@ -780,7 +780,7 @@ def test_typevartuple_none(self): self.assertIs(U_None.__default__, None) self.assertIs(U_None.has_default(), True) - class X[**Ts]: ... + class X[*Ts]: ... Ts, = X.__type_params__ self.assertIs(Ts.__default__, NoDefault) self.assertIs(Ts.has_default(), False) @@ -1288,6 +1288,57 @@ def test_cannot_call_instance(self): with self.assertRaises(TypeError): Ts() + def test_default_variance(self): + Ts = TypeVarTuple('Ts') + self.assertIs(Ts.__covariant__, False) + self.assertIs(Ts.__contravariant__, False) + self.assertIs(Ts.__infer_variance__, False) + self.assertIsNone(Ts.__bound__) + + def test_covariant(self): + Ts_co = TypeVarTuple('Ts_co', covariant=True) + self.assertIs(Ts_co.__covariant__, True) + self.assertIs(Ts_co.__contravariant__, False) + self.assertIs(Ts_co.__infer_variance__, False) + + def test_contravariant(self): + Ts_contra = TypeVarTuple('Ts_contra', contravariant=True) + self.assertIs(Ts_contra.__covariant__, False) + self.assertIs(Ts_contra.__contravariant__, True) + self.assertIs(Ts_contra.__infer_variance__, False) + + def test_infer_variance(self): + Ts = TypeVarTuple('Ts', infer_variance=True) + self.assertIs(Ts.__covariant__, False) + self.assertIs(Ts.__contravariant__, False) + self.assertIs(Ts.__infer_variance__, True) + + def test_bound(self): + Ts_bound = TypeVarTuple('Ts_bound', bound=int) + self.assertIs(Ts_bound.__bound__, int) + Ts_no_bound = TypeVarTuple('Ts_no_bound') + self.assertIsNone(Ts_no_bound.__bound__) + + def test_no_bivariant(self): + with self.assertRaises(ValueError): + TypeVarTuple('Ts', covariant=True, contravariant=True) + + def test_cannot_combine_explicit_and_infer(self): + with self.assertRaises(ValueError): + TypeVarTuple('Ts', covariant=True, infer_variance=True) + with self.assertRaises(ValueError): + TypeVarTuple('Ts', contravariant=True, infer_variance=True) + + def test_repr_with_variance(self): + Ts = TypeVarTuple('Ts') + self.assertEqual(repr(Ts), '~Ts') + Ts_co = TypeVarTuple('Ts_co', covariant=True) + self.assertEqual(repr(Ts_co), '+Ts_co') + Ts_contra = TypeVarTuple('Ts_contra', contravariant=True) + self.assertEqual(repr(Ts_contra), '-Ts_contra') + Ts_infer = TypeVarTuple('Ts_infer', infer_variance=True) + self.assertEqual(repr(Ts_infer), 'Ts_infer') + def test_unpacked_typevartuple_is_equal_to_itself(self): Ts = TypeVarTuple('Ts') self.assertEqual((*Ts,)[0], (*Ts,)[0]) @@ -1427,16 +1478,16 @@ def test_repr_is_correct(self): class G1(Generic[*Ts]): pass class G2(Generic[Unpack[Ts]]): pass - self.assertEqual(repr(Ts), 'Ts') + self.assertEqual(repr(Ts), '~Ts') - self.assertEqual(repr((*Ts,)[0]), 'typing.Unpack[Ts]') - self.assertEqual(repr(Unpack[Ts]), 'typing.Unpack[Ts]') + self.assertEqual(repr((*Ts,)[0]), 'typing.Unpack[~Ts]') + self.assertEqual(repr(Unpack[Ts]), 'typing.Unpack[~Ts]') - self.assertEqual(repr(tuple[*Ts]), 'tuple[typing.Unpack[Ts]]') - self.assertEqual(repr(Tuple[Unpack[Ts]]), 'typing.Tuple[typing.Unpack[Ts]]') + self.assertEqual(repr(tuple[*Ts]), 'tuple[typing.Unpack[~Ts]]') + self.assertEqual(repr(Tuple[Unpack[Ts]]), 'typing.Tuple[typing.Unpack[~Ts]]') - self.assertEqual(repr(*tuple[*Ts]), '*tuple[typing.Unpack[Ts]]') - self.assertEqual(repr(Unpack[Tuple[Unpack[Ts]]]), 'typing.Unpack[typing.Tuple[typing.Unpack[Ts]]]') + self.assertEqual(repr(*tuple[*Ts]), '*tuple[typing.Unpack[~Ts]]') + self.assertEqual(repr(Unpack[Tuple[Unpack[Ts]]]), 'typing.Unpack[typing.Tuple[typing.Unpack[~Ts]]]') def test_variadic_class_repr_is_correct(self): Ts = TypeVarTuple('Ts') @@ -1475,61 +1526,61 @@ def test_variadic_class_alias_repr_is_correct(self): class A(Generic[Unpack[Ts]]): pass B = A[*Ts] - self.assertEndsWith(repr(B), 'A[typing.Unpack[Ts]]') + self.assertEndsWith(repr(B), 'A[typing.Unpack[~Ts]]') self.assertEndsWith(repr(B[()]), 'A[()]') self.assertEndsWith(repr(B[float]), 'A[float]') self.assertEndsWith(repr(B[float, str]), 'A[float, str]') C = A[Unpack[Ts]] - self.assertEndsWith(repr(C), 'A[typing.Unpack[Ts]]') + self.assertEndsWith(repr(C), 'A[typing.Unpack[~Ts]]') self.assertEndsWith(repr(C[()]), 'A[()]') self.assertEndsWith(repr(C[float]), 'A[float]') self.assertEndsWith(repr(C[float, str]), 'A[float, str]') D = A[*Ts, int] - self.assertEndsWith(repr(D), 'A[typing.Unpack[Ts], int]') + self.assertEndsWith(repr(D), 'A[typing.Unpack[~Ts], int]') self.assertEndsWith(repr(D[()]), 'A[int]') self.assertEndsWith(repr(D[float]), 'A[float, int]') self.assertEndsWith(repr(D[float, str]), 'A[float, str, int]') E = A[Unpack[Ts], int] - self.assertEndsWith(repr(E), 'A[typing.Unpack[Ts], int]') + self.assertEndsWith(repr(E), 'A[typing.Unpack[~Ts], int]') self.assertEndsWith(repr(E[()]), 'A[int]') self.assertEndsWith(repr(E[float]), 'A[float, int]') self.assertEndsWith(repr(E[float, str]), 'A[float, str, int]') F = A[int, *Ts] - self.assertEndsWith(repr(F), 'A[int, typing.Unpack[Ts]]') + self.assertEndsWith(repr(F), 'A[int, typing.Unpack[~Ts]]') self.assertEndsWith(repr(F[()]), 'A[int]') self.assertEndsWith(repr(F[float]), 'A[int, float]') self.assertEndsWith(repr(F[float, str]), 'A[int, float, str]') G = A[int, Unpack[Ts]] - self.assertEndsWith(repr(G), 'A[int, typing.Unpack[Ts]]') + self.assertEndsWith(repr(G), 'A[int, typing.Unpack[~Ts]]') self.assertEndsWith(repr(G[()]), 'A[int]') self.assertEndsWith(repr(G[float]), 'A[int, float]') self.assertEndsWith(repr(G[float, str]), 'A[int, float, str]') H = A[int, *Ts, str] - self.assertEndsWith(repr(H), 'A[int, typing.Unpack[Ts], str]') + self.assertEndsWith(repr(H), 'A[int, typing.Unpack[~Ts], str]') self.assertEndsWith(repr(H[()]), 'A[int, str]') self.assertEndsWith(repr(H[float]), 'A[int, float, str]') self.assertEndsWith(repr(H[float, str]), 'A[int, float, str, str]') I = A[int, Unpack[Ts], str] - self.assertEndsWith(repr(I), 'A[int, typing.Unpack[Ts], str]') + self.assertEndsWith(repr(I), 'A[int, typing.Unpack[~Ts], str]') self.assertEndsWith(repr(I[()]), 'A[int, str]') self.assertEndsWith(repr(I[float]), 'A[int, float, str]') self.assertEndsWith(repr(I[float, str]), 'A[int, float, str, str]') J = A[*Ts, *tuple[str, ...]] - self.assertEndsWith(repr(J), 'A[typing.Unpack[Ts], *tuple[str, ...]]') + self.assertEndsWith(repr(J), 'A[typing.Unpack[~Ts], *tuple[str, ...]]') self.assertEndsWith(repr(J[()]), 'A[*tuple[str, ...]]') self.assertEndsWith(repr(J[float]), 'A[float, *tuple[str, ...]]') self.assertEndsWith(repr(J[float, str]), 'A[float, str, *tuple[str, ...]]') K = A[Unpack[Ts], Unpack[Tuple[str, ...]]] - self.assertEndsWith(repr(K), 'A[typing.Unpack[Ts], typing.Unpack[typing.Tuple[str, ...]]]') + self.assertEndsWith(repr(K), 'A[typing.Unpack[~Ts], typing.Unpack[typing.Tuple[str, ...]]]') self.assertEndsWith(repr(K[()]), 'A[typing.Unpack[typing.Tuple[str, ...]]]') self.assertEndsWith(repr(K[float]), 'A[float, typing.Unpack[typing.Tuple[str, ...]]]') self.assertEndsWith(repr(K[float, str]), 'A[float, str, typing.Unpack[typing.Tuple[str, ...]]]') @@ -1550,9 +1601,9 @@ class G(type(Unpack[Ts])): pass with self.assertRaisesRegex(TypeError, r'Cannot subclass typing\.Unpack'): class H(Unpack): pass - with self.assertRaisesRegex(TypeError, r'Cannot subclass typing.Unpack\[Ts\]'): + with self.assertRaisesRegex(TypeError, r'Cannot subclass typing.Unpack\[~Ts\]'): class I(*Ts): pass - with self.assertRaisesRegex(TypeError, r'Cannot subclass typing.Unpack\[Ts\]'): + with self.assertRaisesRegex(TypeError, r'Cannot subclass typing.Unpack\[~Ts\]'): class J(Unpack[Ts]): pass def test_variadic_class_args_are_correct(self): @@ -5596,13 +5647,13 @@ class TsP(Generic[*Ts, P]): MyCallable[[int], bool]: "MyCallable[[int], bool]", MyCallable[[int, str], bool]: "MyCallable[[int, str], bool]", MyCallable[[int, list[int]], bool]: "MyCallable[[int, list[int]], bool]", - MyCallable[Concatenate[*Ts, P], T]: "MyCallable[typing.Concatenate[typing.Unpack[Ts], ~P], ~T]", + MyCallable[Concatenate[*Ts, P], T]: "MyCallable[typing.Concatenate[typing.Unpack[~Ts], ~P], ~T]", DoubleSpec[P2, P, T]: "DoubleSpec[~P2, ~P, ~T]", DoubleSpec[[int], [str], bool]: "DoubleSpec[[int], [str], bool]", DoubleSpec[[int, int], [str, str], bool]: "DoubleSpec[[int, int], [str, str], bool]", - TsP[*Ts, P]: "TsP[typing.Unpack[Ts], ~P]", + TsP[*Ts, P]: "TsP[typing.Unpack[~Ts], ~P]", TsP[int, str, list[int], []]: "TsP[int, str, list[int], []]", TsP[int, [str, list[int]]]: "TsP[int, [str, list[int]]]", @@ -6837,6 +6888,24 @@ def test_get_type_hints_wrapped_decoratored_func(self): self.assertEqual(gth(ForRefExample.func), expects) self.assertEqual(gth(ForRefExample.nested), expects) + def test_get_type_hints_wrapped_cycle_self(self): + # gh-146553: __wrapped__ self-reference must raise ValueError, + # not loop forever. + def f(x: int) -> str: ... + f.__wrapped__ = f + with self.assertRaisesRegex(ValueError, 'wrapper loop'): + get_type_hints(f) + + def test_get_type_hints_wrapped_cycle_mutual(self): + # gh-146553: mutual __wrapped__ cycle (a -> b -> a) must raise + # ValueError, not loop forever. + def a(): ... + def b(): ... + a.__wrapped__ = b + b.__wrapped__ = a + with self.assertRaisesRegex(ValueError, 'wrapper loop'): + get_type_hints(a) + def test_get_type_hints_annotated(self): def foobar(x: List['X']): ... X = Annotated[int, (1, 10)] @@ -10920,6 +10989,18 @@ def bar(self): self.assertNotIn('__magic__', dir_items) +class DisjointBaseTests(BaseTestCase): + def test_disjoint_base_unmodified(self): + class C: ... + self.assertIs(C, disjoint_base(C)) + + def test_dunder_disjoint_base(self): + @disjoint_base + class C: ... + + self.assertIs(C.__disjoint_base__, True) + + class RevealTypeTests(BaseTestCase): def test_reveal_type(self): obj = object() diff --git a/Lib/test/test_xml_etree.py b/Lib/test/test_xml_etree.py index b380d0276b0169..43864820688bd4 100644 --- a/Lib/test/test_xml_etree.py +++ b/Lib/test/test_xml_etree.py @@ -3190,6 +3190,19 @@ def __deepcopy__(self, memo): self.assertEqual([c.tag for c in children[3:]], [a.tag, b.tag, a.tag, b.tag]) + @support.skip_if_unlimited_stack_size + @support.skip_emscripten_stack_overflow() + @support.skip_wasi_stack_overflow() + def test_deeply_nested_deepcopy(self): + # This should raise a RecursionError and not crash. + # See https://github.com/python/cpython/issues/148801. + root = cur = ET.Element('s') + for _ in range(150_000): + cur = ET.SubElement(cur, 'u') + with support.infinite_recursion(): + with self.assertRaises(RecursionError): + copy.deepcopy(root) + class MutationDeleteElementPath(str): def __new__(cls, elem, *args): @@ -3258,6 +3271,16 @@ def test_findtext_with_mutating(self): e.extend([ET.Element('bar')]) e.findtext(cls(e, 'x')) + def test_findtext_with_mutating_non_none_text(self): + for cls in [MutationDeleteElementPath, MutationClearElementPath]: + with self.subTest(cls): + e = ET.Element('foo') + child = ET.Element('bar') + child.text = str(object()) + e.append(child) + del child + repr(e.findtext(cls(e, 'x'))) + def test_findtext_with_error(self): e = ET.Element('foo') e.extend([ET.Element('bar')]) diff --git a/Lib/typing.py b/Lib/typing.py index e78fb8b71a996c..e7563a53878da5 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -126,6 +126,7 @@ 'cast', 'clear_overloads', 'dataclass_transform', + 'disjoint_base', 'evaluate_forward_ref', 'final', 'get_args', @@ -2485,8 +2486,12 @@ def get_type_hints(obj, globalns=None, localns=None, include_extras=False, else: nsobj = obj # Find globalns for the unwrapped object. + seen = {id(nsobj)} while hasattr(nsobj, '__wrapped__'): nsobj = nsobj.__wrapped__ + if id(nsobj) in seen: + raise ValueError(f'wrapper loop when unwrapping {obj!r}') + seen.add(id(nsobj)) globalns = getattr(nsobj, '__globals__', {}) if localns is None: localns = globalns @@ -2794,6 +2799,29 @@ class Other(Leaf): # Error reported by type checker return f +def disjoint_base(cls): + """This decorator marks a class as a disjoint base. + + Child classes of a disjoint base cannot inherit from other disjoint bases that are + not parent or child classes of the disjoint base. + + For example: + + @disjoint_base + class Disjoint1: pass + + @disjoint_base + class Disjoint2: pass + + class Disjoint3(Disjoint1, Disjoint2): pass # Type checker error + + Type checkers can use knowledge of disjoint bases to detect unreachable code + and determine when two types can overlap. + """ + cls.__disjoint_base__ = True + return cls + + # Some unconstrained type variables. These were initially used by the container types. # They were never meant for export and are now unused, but we keep them around to # avoid breaking compatibility with users who import them. @@ -3122,31 +3150,7 @@ def _namedtuple_mro_entries(bases): NamedTuple.__mro_entries__ = _namedtuple_mro_entries -class _SingletonMeta(type): - def __setattr__(cls, attr, value): - # TypeError is consistent with the behavior of NoneType - raise TypeError( - f"cannot set {attr!r} attribute of immutable type {cls.__name__!r}" - ) - - -class _NoExtraItemsType(metaclass=_SingletonMeta): - """The type of the NoExtraItems singleton.""" - - __slots__ = () - - def __new__(cls): - return globals().get("NoExtraItems") or object.__new__(cls) - - def __repr__(self): - return 'typing.NoExtraItems' - - def __reduce__(self): - return 'NoExtraItems' - -NoExtraItems = _NoExtraItemsType() -del _NoExtraItemsType -del _SingletonMeta +NoExtraItems = sentinel("NoExtraItems") def _get_typeddict_qualifiers(annotation_type): @@ -3588,7 +3592,7 @@ def isatty(self) -> bool: pass @abstractmethod - def read(self, n: int = -1) -> AnyStr: + def read(self, n: int = -1, /) -> AnyStr: pass @abstractmethod @@ -3596,15 +3600,15 @@ def readable(self) -> bool: pass @abstractmethod - def readline(self, limit: int = -1) -> AnyStr: + def readline(self, limit: int = -1, /) -> AnyStr: pass @abstractmethod - def readlines(self, hint: int = -1) -> list[AnyStr]: + def readlines(self, hint: int = -1, /) -> list[AnyStr]: pass @abstractmethod - def seek(self, offset: int, whence: int = 0) -> int: + def seek(self, offset: int, whence: int = 0, /) -> int: pass @abstractmethod @@ -3616,7 +3620,7 @@ def tell(self) -> int: pass @abstractmethod - def truncate(self, size: int | None = None) -> int: + def truncate(self, size: int | None = None, /) -> int: pass @abstractmethod @@ -3624,11 +3628,11 @@ def writable(self) -> bool: pass @abstractmethod - def write(self, s: AnyStr) -> int: + def write(self, s: AnyStr, /) -> int: pass @abstractmethod - def writelines(self, lines: list[AnyStr]) -> None: + def writelines(self, lines: list[AnyStr], /) -> None: pass @abstractmethod @@ -3636,7 +3640,7 @@ def __enter__(self) -> IO[AnyStr]: pass @abstractmethod - def __exit__(self, type, value, traceback) -> None: + def __exit__(self, type, value, traceback, /) -> None: pass @@ -3646,7 +3650,7 @@ class BinaryIO(IO[bytes]): __slots__ = () @abstractmethod - def write(self, s: bytes | bytearray) -> int: + def write(self, s: bytes | bytearray, /) -> int: pass @abstractmethod diff --git a/Makefile.pre.in b/Makefile.pre.in index f869c1f7c93776..2ce53c6a816212 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -290,6 +290,7 @@ LDLIBRARYDIR= @LDLIBRARYDIR@ INSTSONAME= @INSTSONAME@ LIBRARY_DEPS= @LIBRARY_DEPS@ LINK_PYTHON_DEPS=@LINK_PYTHON_DEPS@ +JIT_OBJS= @JIT_SHIM_O@ PY_ENABLE_SHARED= @PY_ENABLE_SHARED@ STATIC_LIBPYTHON= @STATIC_LIBPYTHON@ @@ -469,6 +470,7 @@ PYTHON_OBJS= \ Python/instruction_sequence.o \ Python/intrinsics.o \ Python/jit.o \ + $(JIT_OBJS) \ Python/legacy_tracing.o \ Python/lock.o \ Python/marshal.o \ @@ -558,6 +560,7 @@ OBJECT_OBJS= \ Objects/obmalloc.o \ Objects/picklebufobject.o \ Objects/rangeobject.o \ + Objects/sentinelobject.o \ Objects/setobject.o \ Objects/sliceobject.o \ Objects/structseq.o \ @@ -1214,6 +1217,7 @@ PYTHON_HEADERS= \ $(srcdir)/Include/osdefs.h \ $(srcdir)/Include/osmodule.h \ $(srcdir)/Include/patchlevel.h \ + $(srcdir)/Include/pyabi.h \ $(srcdir)/Include/pyatomic.h \ $(srcdir)/Include/pybuffer.h \ $(srcdir)/Include/pycapsule.h \ @@ -1237,6 +1241,7 @@ PYTHON_HEADERS= \ $(srcdir)/Include/pytypedefs.h \ $(srcdir)/Include/rangeobject.h \ $(srcdir)/Include/refcount.h \ + $(srcdir)/Include/sentinelobject.h \ $(srcdir)/Include/setobject.h \ $(srcdir)/Include/sliceobject.h \ $(srcdir)/Include/structmember.h \ @@ -3203,21 +3208,37 @@ Python/emscripten_trampoline_inner.wasm: $(srcdir)/Python/emscripten_trampoline_ Python/emscripten_trampoline_wasm.c: Python/emscripten_trampoline_inner.wasm $(PYTHON_FOR_REGEN) $(srcdir)/Platforms/emscripten/prepare_external_wasm.py $< $@ getWasmTrampolineModule +JIT_SHIM_BUILD_OBJS= @JIT_SHIM_BUILD_O@ +JIT_BUILD_TARGETS= jit_stencils.h @JIT_STENCILS_H@ $(JIT_SHIM_BUILD_OBJS) +JIT_TARGETS= $(JIT_BUILD_TARGETS) $(filter-out $(JIT_SHIM_BUILD_OBJS),$(JIT_OBJS)) +JIT_GENERATED_STAMP= .jit-stamp + JIT_DEPS = \ $(srcdir)/Tools/jit/*.c \ + $(srcdir)/Tools/jit/*.h \ $(srcdir)/Tools/jit/*.py \ $(srcdir)/Python/executor_cases.c.h \ pyconfig.h -jit_stencils.h @JIT_STENCILS_H@: $(JIT_DEPS) +$(JIT_GENERATED_STAMP): $(JIT_DEPS) @REGEN_JIT_COMMAND@ + @touch $@ + +$(JIT_BUILD_TARGETS): $(JIT_GENERATED_STAMP) + @if test ! -f "$@"; then \ + rm -f $(JIT_GENERATED_STAMP); \ + $(MAKE) $(JIT_GENERATED_STAMP); \ + test -f "$@"; \ + fi + +jit_shim-universal2-apple-darwin.o: jit_shim-aarch64-apple-darwin.o jit_shim-x86_64-apple-darwin.o + lipo -create -output $@ jit_shim-aarch64-apple-darwin.o jit_shim-x86_64-apple-darwin.o Python/jit.o: $(srcdir)/Python/jit.c @JIT_STENCILS_H@ $(CC) -c $(PY_CORE_CFLAGS) -o $@ $< .PHONY: regen-jit -regen-jit: - @REGEN_JIT_COMMAND@ +regen-jit: $(JIT_TARGETS) # Some make's put the object file in the current directory .c.o: @@ -3341,7 +3362,7 @@ clean-profile: clean-retain-profile clean-bolt # gh-141808: The JIT stencils are deliberately kept in clean-profile .PHONY: clean-jit-stencils clean-jit-stencils: - -rm -f jit_stencils*.h + -rm -f $(JIT_TARGETS) $(JIT_GENERATED_STAMP) jit_stencils*.h jit_shim*.o .PHONY: clean clean: clean-profile clean-jit-stencils diff --git a/Misc/NEWS.d/next/Build/2025-09-03-14-55-59.gh-issue-138451.-Qzh2S.rst b/Misc/NEWS.d/next/Build/2025-09-03-14-55-59.gh-issue-138451.-Qzh2S.rst new file mode 100644 index 00000000000000..d83aee08025502 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2025-09-03-14-55-59.gh-issue-138451.-Qzh2S.rst @@ -0,0 +1 @@ +Allow for custom LLVM path using ``LLVM_TOOLS_INSTALL_DIR`` during JIT build. diff --git a/Misc/NEWS.d/next/Build/2026-04-17-21-45-32.gh-issue-148644.vwkknh.rst b/Misc/NEWS.d/next/Build/2026-04-17-21-45-32.gh-issue-148644.vwkknh.rst new file mode 100644 index 00000000000000..a0cc9c9358cb26 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2026-04-17-21-45-32.gh-issue-148644.vwkknh.rst @@ -0,0 +1 @@ +Errors during the PGO training job on Windows are no longer ignored, and a non-zero return code will cause the build to fail. diff --git a/Misc/NEWS.d/next/C_API/2026-04-09-14-45-44.gh-issue-148267.p84kG_.rst b/Misc/NEWS.d/next/C_API/2026-04-09-14-45-44.gh-issue-148267.p84kG_.rst new file mode 100644 index 00000000000000..1ec1afd2cbfeb9 --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2026-04-09-14-45-44.gh-issue-148267.p84kG_.rst @@ -0,0 +1,2 @@ +Using :c:macro:`Py_LIMITED_API` on a non-Windows free-threaded build no +longer needs an extra :c:macro:`Py_GIL_DISABLED`. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-03-23-11-34-37.gh-issue-142186.v8Yp3W.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-03-23-11-34-37.gh-issue-142186.v8Yp3W.rst new file mode 100644 index 00000000000000..4a04658551c444 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-03-23-11-34-37.gh-issue-142186.v8Yp3W.rst @@ -0,0 +1,3 @@ +Global :mod:`sys.monitoring` events can now be turned on and disabled on a +per code object basis. Returning ``DISABLE`` from a callback disables the +event for the entire code object (for the current tool). diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-04-07-20-37-23.gh-issue-148222.uF4D4E.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-04-07-20-37-23.gh-issue-148222.uF4D4E.rst new file mode 100644 index 00000000000000..2c273fc4daba3d --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-04-07-20-37-23.gh-issue-148222.uF4D4E.rst @@ -0,0 +1 @@ +Fix vectorcall support in :class:`types.GenericAlias` when the underlying type does not support the vectorcall protocol. Fix possible leaks in :class:`types.GenericAlias` and :class:`types.UnionType` in case of memory error. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-04-13-23-21-45.gh-issue-145239.pL8qRt.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-04-13-23-21-45.gh-issue-145239.pL8qRt.rst new file mode 100644 index 00000000000000..282b99176642bc --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-04-13-23-21-45.gh-issue-145239.pL8qRt.rst @@ -0,0 +1,3 @@ +Unary plus is now accepted in :keyword:`match` literal patterns, mirroring +the existing support for unary minus. +Patch by Bartosz Sławecki. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-04-17-20-37-02.gh-issue-148653.nbbHMh.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-04-17-20-37-02.gh-issue-148653.nbbHMh.rst new file mode 100644 index 00000000000000..d3242235c6059b --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-04-17-20-37-02.gh-issue-148653.nbbHMh.rst @@ -0,0 +1,2 @@ +Forbid :mod:`marshalling ` recursive code objects, :class:`slice` +and :class:`frozendict` objects which cannot be correctly unmarshalled. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-04-21-06-43-32.gh-issue-148829.GtIrYO.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-04-21-06-43-32.gh-issue-148829.GtIrYO.rst new file mode 100644 index 00000000000000..3d9b4faa6ca443 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-04-21-06-43-32.gh-issue-148829.GtIrYO.rst @@ -0,0 +1,2 @@ +Add :class:`sentinel`, implementing :pep:`661`. PEP by Tal Einat; patch by +Jelle Zijlstra. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-04-21-14-36-44.gh-issue-148820.XhOGhA.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-04-21-14-36-44.gh-issue-148820.XhOGhA.rst new file mode 100644 index 00000000000000..392becaffb73cf --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-04-21-14-36-44.gh-issue-148820.XhOGhA.rst @@ -0,0 +1,5 @@ +Fix a race in :c:type:`!_PyRawMutex` on the free-threaded build where a +``Py_PARK_INTR`` return from ``_PySemaphore_Wait`` could let the waiter +destroy its semaphore before the unlocking thread's +``_PySemaphore_Wakeup`` completed, causing a fatal ``ReleaseSemaphore`` +error. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-04-22-14-55-18.gh-issue-113956.0VEXd6.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-04-22-14-55-18.gh-issue-113956.0VEXd6.rst new file mode 100644 index 00000000000000..54c04bbc28d416 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-04-22-14-55-18.gh-issue-113956.0VEXd6.rst @@ -0,0 +1,4 @@ +Fix a data race in :func:`sys.intern` in the free-threaded build when +interning a string owned by another thread. An interned copy owned by the +current thread is used instead when it is not safe to immortalize the +original. diff --git a/Misc/NEWS.d/next/Documentation/2026-04-17-02-28-55.gh-issue-148663.MHIbRB.rst b/Misc/NEWS.d/next/Documentation/2026-04-17-02-28-55.gh-issue-148663.MHIbRB.rst new file mode 100644 index 00000000000000..0fbe5a699ef0ad --- /dev/null +++ b/Misc/NEWS.d/next/Documentation/2026-04-17-02-28-55.gh-issue-148663.MHIbRB.rst @@ -0,0 +1,2 @@ +Document that :class:`calendar.IllegalMonthError` is a subclass of both +:exc:`ValueError` and :exc:`IndexError` since Python 3.12. diff --git a/Misc/NEWS.d/next/Library/2024-09-09-12-48-37.gh-issue-123853.e-zFxb.rst b/Misc/NEWS.d/next/Library/2024-09-09-12-48-37.gh-issue-123853.e-zFxb.rst new file mode 100644 index 00000000000000..d7204c289369a7 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-09-09-12-48-37.gh-issue-123853.e-zFxb.rst @@ -0,0 +1,3 @@ +Update the table of Windows language code identifiers (LCIDs) used by +:func:`locale.getdefaultlocale` on Windows to protocol version 16.0 +(2024-04-23). diff --git a/Misc/NEWS.d/next/Library/2024-11-24-07-18-40.gh-issue-108951.jyKygP.rst b/Misc/NEWS.d/next/Library/2024-11-24-07-18-40.gh-issue-108951.jyKygP.rst new file mode 100644 index 00000000000000..1696a2dd1728ed --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-11-24-07-18-40.gh-issue-108951.jyKygP.rst @@ -0,0 +1 @@ +Add :meth:`~asyncio.TaskGroup.cancel` which cancels unfinished tasks and exits the group without error. diff --git a/Misc/NEWS.d/next/Library/2025-04-17-15-26-35.gh-issue-132631.IDFZfb.rst b/Misc/NEWS.d/next/Library/2025-04-17-15-26-35.gh-issue-132631.IDFZfb.rst new file mode 100644 index 00000000000000..9cc1d5a389c085 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-04-17-15-26-35.gh-issue-132631.IDFZfb.rst @@ -0,0 +1,2 @@ +Fix "I/O operation on closed file" when parsing JSON Lines file with +:mod:`JSON CLI `. diff --git a/Misc/NEWS.d/next/Library/2025-12-17-02-55-03.gh-issue-108411.up7MAc.rst b/Misc/NEWS.d/next/Library/2025-12-17-02-55-03.gh-issue-108411.up7MAc.rst new file mode 100644 index 00000000000000..95aa41e922684f --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-12-17-02-55-03.gh-issue-108411.up7MAc.rst @@ -0,0 +1,2 @@ +``typing.IO`` and ``typing.BinaryIO`` method arguments are now +positional-only. diff --git a/Misc/NEWS.d/next/Library/2026-02-22-00-00-00.gh-issue-145105.csv-reader-reentrant.rst b/Misc/NEWS.d/next/Library/2026-02-22-00-00-00.gh-issue-145105.csv-reader-reentrant.rst index bc61cc43a5aa33..1c2e06c86f6588 100644 --- a/Misc/NEWS.d/next/Library/2026-02-22-00-00-00.gh-issue-145105.csv-reader-reentrant.rst +++ b/Misc/NEWS.d/next/Library/2026-02-22-00-00-00.gh-issue-145105.csv-reader-reentrant.rst @@ -1,2 +1,2 @@ -Fix crash in :mod:`csv` reader when iterating with a re-entrant iterator -that calls :func:`next` on the same reader from within ``__next__``. +Fix crash in :mod:`csv` reader when iterating with a re-entrant iterator +that calls :func:`next` on the same reader from within ``__next__``. diff --git a/Misc/NEWS.d/next/Library/2026-02-22-19-36-00.gh-issue-145056.TH8nX4.rst b/Misc/NEWS.d/next/Library/2026-02-22-19-36-00.gh-issue-145056.TH8nX4.rst new file mode 100644 index 00000000000000..45be0109677cd1 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-02-22-19-36-00.gh-issue-145056.TH8nX4.rst @@ -0,0 +1 @@ +Add support for :class:`frozendict` in :meth:`dataclasses.asdict` and :meth:`dataclasses.astuple`. diff --git a/Misc/NEWS.d/next/Library/2026-04-07-12-37-53.gh-issue-148207.YhGem4.rst b/Misc/NEWS.d/next/Library/2026-04-07-12-37-53.gh-issue-148207.YhGem4.rst new file mode 100644 index 00000000000000..dd88be0ad25d11 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-04-07-12-37-53.gh-issue-148207.YhGem4.rst @@ -0,0 +1,3 @@ +:class:`typing.TypeVarTuple` now accepts ``bound``, ``covariant``, +``contravariant``, and ``infer_variance`` parameters, matching the interface +of :class:`typing.TypeVar` and :class:`typing.ParamSpec`. diff --git a/Misc/NEWS.d/next/Library/2026-04-13-06-22-27.gh-issue-148464.Bj_NZy.rst b/Misc/NEWS.d/next/Library/2026-04-13-06-22-27.gh-issue-148464.Bj_NZy.rst new file mode 100644 index 00000000000000..85b99531d033b1 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-04-13-06-22-27.gh-issue-148464.Bj_NZy.rst @@ -0,0 +1,3 @@ +Add missing ``__ctype_le/be__`` attributes for +:class:`~ctypes.c_float_complex` and :class:`~ctypes.c_double_complex`. Patch +by Sergey B Kirpichev. diff --git a/Misc/NEWS.d/next/Library/2026-04-15-11-00-39.gh-issue-146553.VGOsoP.rst b/Misc/NEWS.d/next/Library/2026-04-15-11-00-39.gh-issue-146553.VGOsoP.rst new file mode 100644 index 00000000000000..44216318d474a9 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-04-15-11-00-39.gh-issue-146553.VGOsoP.rst @@ -0,0 +1,2 @@ +Fix infinite loop in :func:`typing.get_type_hints` when ``__wrapped__`` +forms a cycle. Patch by Shamil Abdulaev. diff --git a/Misc/NEWS.d/next/Library/2026-04-15-20-32-55.gh-issue-148639.-dwsjB.rst b/Misc/NEWS.d/next/Library/2026-04-15-20-32-55.gh-issue-148639.-dwsjB.rst new file mode 100644 index 00000000000000..d7acdb0983837a --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-04-15-20-32-55.gh-issue-148639.-dwsjB.rst @@ -0,0 +1,2 @@ +Implement :pep:`800`, adding the :deco:`typing.disjoint_base` decorator. +Patch by Jelle Zijlstra. diff --git a/Misc/NEWS.d/next/Library/2026-04-16-13-30-00.gh-issue-148651.ZsTdLk.rst b/Misc/NEWS.d/next/Library/2026-04-16-13-30-00.gh-issue-148651.ZsTdLk.rst new file mode 100644 index 00000000000000..b69f94a17663dc --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-04-16-13-30-00.gh-issue-148651.ZsTdLk.rst @@ -0,0 +1,2 @@ +Fix reference leak in :class:`compression.zstd.ZstdDecompressor` when an +invalid option key is passed. diff --git a/Misc/NEWS.d/next/Library/2026-04-17-16-31-58.gh-issue-148688.vVugFn.rst b/Misc/NEWS.d/next/Library/2026-04-17-16-31-58.gh-issue-148688.vVugFn.rst new file mode 100644 index 00000000000000..1e367716e5a0a7 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-04-17-16-31-58.gh-issue-148688.vVugFn.rst @@ -0,0 +1,2 @@ +:mod:`bz2`, :mod:`compression.zstd`, :mod:`lzma`, :mod:`zlib`: Fix a double +free on memory allocation failure. Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/Library/2026-04-18-21-39-15.gh-issue-148735.siw6DG.rst b/Misc/NEWS.d/next/Library/2026-04-18-21-39-15.gh-issue-148735.siw6DG.rst new file mode 100644 index 00000000000000..db5e94c0ccac50 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-04-18-21-39-15.gh-issue-148735.siw6DG.rst @@ -0,0 +1,3 @@ +:mod:`xml.etree.ElementTree`: Fix a use-after-free in +:meth:`Element.findtext ` when the +element tree is mutated concurrently during the search. diff --git a/Misc/NEWS.d/next/Library/2026-04-20-18-29-21.gh-issue-148801.ROeNqs.rst b/Misc/NEWS.d/next/Library/2026-04-20-18-29-21.gh-issue-148801.ROeNqs.rst new file mode 100644 index 00000000000000..6fcd30e8f057b9 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-04-20-18-29-21.gh-issue-148801.ROeNqs.rst @@ -0,0 +1,2 @@ +:mod:`xml.etree.ElementTree`: Fix a crash in :meth:`Element.__deepcopy__ +` on deeply nested trees. diff --git a/Misc/NEWS.d/next/Library/2026-04-23-07-38-04.gh-issue-148680.___ePl.rst b/Misc/NEWS.d/next/Library/2026-04-23-07-38-04.gh-issue-148680.___ePl.rst new file mode 100644 index 00000000000000..d3790079545a07 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-04-23-07-38-04.gh-issue-148680.___ePl.rst @@ -0,0 +1 @@ +``ForwardRef`` objects that contain internal names to represent known objects now show the ``type_repr`` of the known object rather than the internal ``__annotationlib_name_x__`` name when evaluated as strings. diff --git a/Misc/NEWS.d/next/Security/2026-04-20-15-31-37.gh-issue-148808._Z8JL0.rst b/Misc/NEWS.d/next/Security/2026-04-20-15-31-37.gh-issue-148808._Z8JL0.rst new file mode 100644 index 00000000000000..0b5cf85fedfba1 --- /dev/null +++ b/Misc/NEWS.d/next/Security/2026-04-20-15-31-37.gh-issue-148808._Z8JL0.rst @@ -0,0 +1,3 @@ +Added buffer boundary check when using ``nbytes`` parameter with +:meth:`!asyncio.AbstractEventLoop.sock_recvfrom_into`. Only +relevant for Windows and the :class:`asyncio.ProactorEventLoop`. diff --git a/Misc/NEWS.d/next/Security/2026-04-21-13-46-30.gh-issue-90309.srvj9q.rst b/Misc/NEWS.d/next/Security/2026-04-21-13-46-30.gh-issue-90309.srvj9q.rst new file mode 100644 index 00000000000000..d7d376737e4ad1 --- /dev/null +++ b/Misc/NEWS.d/next/Security/2026-04-21-13-46-30.gh-issue-90309.srvj9q.rst @@ -0,0 +1,3 @@ +Base64-encode values when embedding cookies to JavaScript using the +:meth:`http.cookies.BaseCookie.js_output` method to avoid injection +and escaping. diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index 55eade1c8307ea..0bdc30a0cb3836 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -2222,6 +2222,31 @@ c_void_p_from_param_impl(PyObject *type, PyTypeObject *cls, PyObject *value) return NULL; } +static int +set_stginfo_ffi_type_pointer(StgInfo *stginfo, struct fielddesc *fmt) +{ + if (!fmt->pffi_type->elements) { + stginfo->ffi_type_pointer = *fmt->pffi_type; + } + else { + /* From primitive types - only complex types have the elements + struct field as non-NULL (two element array). */ + assert(fmt->pffi_type->type == FFI_TYPE_COMPLEX); + const size_t els_size = 2 * sizeof(ffi_type *); + stginfo->ffi_type_pointer.size = fmt->pffi_type->size; + stginfo->ffi_type_pointer.alignment = fmt->pffi_type->alignment; + stginfo->ffi_type_pointer.type = fmt->pffi_type->type; + stginfo->ffi_type_pointer.elements = PyMem_Malloc(els_size); + if (!stginfo->ffi_type_pointer.elements) { + PyErr_NoMemory(); + return -1; + } + memcpy(stginfo->ffi_type_pointer.elements, + fmt->pffi_type->elements, els_size); + } + return 0; +} + static PyMethodDef c_void_p_methods[] = {C_VOID_P_FROM_PARAM_METHODDEF {0}}; static PyMethodDef c_char_p_methods[] = {C_CHAR_P_FROM_PARAM_METHODDEF {0}}; static PyMethodDef c_wchar_p_methods[] = {C_WCHAR_P_FROM_PARAM_METHODDEF {0}}; @@ -2266,8 +2291,10 @@ static PyObject *CreateSwappedType(ctypes_state *st, PyTypeObject *type, Py_DECREF(result); return NULL; } - - stginfo->ffi_type_pointer = *fmt->pffi_type; + if (set_stginfo_ffi_type_pointer(stginfo, fmt)) { + Py_DECREF(result); + return NULL; + } stginfo->align = fmt->pffi_type->alignment; stginfo->length = 0; stginfo->size = fmt->pffi_type->size; @@ -2362,18 +2389,8 @@ PyCSimpleType_init(PyObject *self, PyObject *args, PyObject *kwds) if (!stginfo) { goto error; } - - if (!fmt->pffi_type->elements) { - stginfo->ffi_type_pointer = *fmt->pffi_type; - } - else { - const size_t els_size = sizeof(fmt->pffi_type->elements); - stginfo->ffi_type_pointer.size = fmt->pffi_type->size; - stginfo->ffi_type_pointer.alignment = fmt->pffi_type->alignment; - stginfo->ffi_type_pointer.type = fmt->pffi_type->type; - stginfo->ffi_type_pointer.elements = PyMem_Malloc(els_size); - memcpy(stginfo->ffi_type_pointer.elements, - fmt->pffi_type->elements, els_size); + if (set_stginfo_ffi_type_pointer(stginfo, fmt)) { + goto error; } stginfo->align = fmt->pffi_type->alignment; stginfo->length = 0; diff --git a/Modules/_ctypes/cfield.c b/Modules/_ctypes/cfield.c index 4ebca0e0b3db0a..b0dc11fdddcea1 100644 --- a/Modules/_ctypes/cfield.c +++ b/Modules/_ctypes/cfield.c @@ -792,6 +792,44 @@ D_get(void *ptr, Py_ssize_t size) return PyComplex_FromDoubles(x[0], x[1]); } +static PyObject * +D_set_sw(void *ptr, PyObject *value, Py_ssize_t size) +{ + assert(NUM_BITS(size) || (size == 2*sizeof(double))); + Py_complex c = PyComplex_AsCComplex(value); + + if (c.real == -1 && PyErr_Occurred()) { + return NULL; + } +#ifdef WORDS_BIGENDIAN + if (PyFloat_Pack8(c.real, ptr, 1) + || PyFloat_Pack8(c.imag, ptr + sizeof(double), 1)) + { + return NULL; + } +#else + if (PyFloat_Pack8(c.real, ptr, 0) + || PyFloat_Pack8(c.imag, ptr + sizeof(double), 0)) + { + return NULL; + } +#endif + _RET(value); +} + +static PyObject * +D_get_sw(void *ptr, Py_ssize_t size) +{ + assert(NUM_BITS(size) || (size == 2*sizeof(double))); +#ifdef WORDS_BIGENDIAN + return PyComplex_FromDoubles(PyFloat_Unpack8(ptr, 1), + PyFloat_Unpack8(ptr + sizeof(double), 1)); +#else + return PyComplex_FromDoubles(PyFloat_Unpack8(ptr, 0), + PyFloat_Unpack8(ptr + sizeof(double), 0)); +#endif +} + /* F: float complex */ static PyObject * F_set(void *ptr, PyObject *value, Py_ssize_t size) @@ -817,6 +855,44 @@ F_get(void *ptr, Py_ssize_t size) return PyComplex_FromDoubles(x[0], x[1]); } +static PyObject * +F_set_sw(void *ptr, PyObject *value, Py_ssize_t size) +{ + assert(NUM_BITS(size) || (size == 2*sizeof(float))); + Py_complex c = PyComplex_AsCComplex(value); + + if (c.real == -1 && PyErr_Occurred()) { + return NULL; + } +#ifdef WORDS_BIGENDIAN + if (PyFloat_Pack4(c.real, ptr, 1) + || PyFloat_Pack4(c.imag, ptr + sizeof(float), 1)) + { + return NULL; + } +#else + if (PyFloat_Pack4(c.real, ptr, 0) + || PyFloat_Pack4(c.imag, ptr + sizeof(float), 0)) + { + return NULL; + } +#endif + _RET(value); +} + +static PyObject * +F_get_sw(void *ptr, Py_ssize_t size) +{ + assert(NUM_BITS(size) || (size == 2*sizeof(float))); +#ifdef WORDS_BIGENDIAN + return PyComplex_FromDoubles(PyFloat_Unpack4(ptr, 1), + PyFloat_Unpack4(ptr + sizeof(float), 1)); +#else + return PyComplex_FromDoubles(PyFloat_Unpack4(ptr, 0), + PyFloat_Unpack4(ptr + sizeof(float), 0)); +#endif +} + /* G: long double complex */ static PyObject * G_set(void *ptr, PyObject *value, Py_ssize_t size) @@ -1602,7 +1678,9 @@ for base_code, base_c_type in [ #if defined(_Py_FFI_SUPPORT_C_COMPLEX) if (Py_FFI_COMPLEX_AVAILABLE) { TABLE_ENTRY(D, &ffi_type_complex_double); + TABLE_ENTRY_SW(D, &ffi_type_complex_double); TABLE_ENTRY(F, &ffi_type_complex_float); + TABLE_ENTRY_SW(F, &ffi_type_complex_float); TABLE_ENTRY(G, &ffi_type_complex_longdouble); } #endif diff --git a/Modules/_elementtree.c b/Modules/_elementtree.c index e2185c4bd03aad..cbd1e026df2722 100644 --- a/Modules/_elementtree.c +++ b/Modules/_elementtree.c @@ -16,6 +16,7 @@ #endif #include "Python.h" +#include "pycore_ceval.h" // _Py_EnterRecursiveCall() #include "pycore_dict.h" // _PyDict_CopyAsDict() #include "pycore_pyhash.h" // _Py_HashSecret #include "pycore_tuple.h" // _PyTuple_FromPair @@ -572,7 +573,7 @@ element_get_attrib(ElementObject* self) LOCAL(PyObject*) element_get_text(ElementObject* self) { - /* return borrowed reference to text attribute */ + /* return new reference to text attribute */ PyObject *res = self->text; @@ -587,13 +588,13 @@ element_get_text(ElementObject* self) } } - return res; + return Py_NewRef(res); } LOCAL(PyObject*) element_get_tail(ElementObject* self) { - /* return borrowed reference to text attribute */ + /* return new reference to tail attribute */ PyObject *res = self->tail; @@ -608,7 +609,7 @@ element_get_tail(ElementObject* self) } } - return res; + return Py_NewRef(res); } static PyObject* @@ -811,26 +812,31 @@ _elementtree_Element___deepcopy___impl(ElementObject *self, PyObject *memo) /*[clinic end generated code: output=eefc3df50465b642 input=a2d40348c0aade10]*/ { Py_ssize_t i; - ElementObject* element; + ElementObject* element = NULL; PyObject* tag; PyObject* attrib; PyObject* text; PyObject* tail; PyObject* id; + if (_Py_EnterRecursiveCall(" in Element.__deepcopy__")) { + return NULL; + } + PyTypeObject *tp = Py_TYPE(self); elementtreestate *st = get_elementtree_state_by_type(tp); // The deepcopy() helper takes care of incrementing the refcount // of the object to copy so to avoid use-after-frees. tag = deepcopy(st, self->tag, memo); - if (!tag) - return NULL; + if (!tag) { + goto error; + } if (self->extra && self->extra->attrib) { attrib = deepcopy(st, self->extra->attrib, memo); if (!attrib) { Py_DECREF(tag); - return NULL; + goto error; } } else { attrib = NULL; @@ -841,8 +847,9 @@ _elementtree_Element___deepcopy___impl(ElementObject *self, PyObject *memo) Py_DECREF(tag); Py_XDECREF(attrib); - if (!element) - return NULL; + if (!element) { + goto error; + } text = deepcopy(st, JOIN_OBJ(self->text), memo); if (!text) @@ -904,10 +911,12 @@ _elementtree_Element___deepcopy___impl(ElementObject *self, PyObject *memo) if (i < 0) goto error; + _Py_LeaveRecursiveCall(); return (PyObject*) element; error: - Py_DECREF(element); + _Py_LeaveRecursiveCall(); + Py_XDECREF(element); return NULL; } @@ -1350,9 +1359,9 @@ _elementtree_Element_findtext_impl(ElementObject *self, PyTypeObject *cls, PyObject *text = element_get_text((ElementObject *)item); Py_DECREF(item); if (text == Py_None) { + Py_DECREF(text); return Py_GetConstant(Py_CONSTANT_EMPTY_STR); } - Py_XINCREF(text); return text; } Py_DECREF(item); @@ -2055,16 +2064,14 @@ static PyObject* element_text_getter(PyObject *op, void *closure) { ElementObject *self = _Element_CAST(op); - PyObject *res = element_get_text(self); - return Py_XNewRef(res); + return element_get_text(self); } static PyObject* element_tail_getter(PyObject *op, void *closure) { ElementObject *self = _Element_CAST(op); - PyObject *res = element_get_tail(self); - return Py_XNewRef(res); + return element_get_tail(self); } static PyObject* @@ -2307,16 +2314,14 @@ elementiter_next(PyObject *op) continue; gettext: + Py_DECREF(elem); if (!text) { - Py_DECREF(elem); return NULL; } if (text == Py_None) { - Py_DECREF(elem); + Py_DECREF(text); } else { - Py_INCREF(text); - Py_DECREF(elem); rc = PyObject_IsTrue(text); if (rc > 0) return text; diff --git a/Modules/_testcapi/object.c b/Modules/_testcapi/object.c index 9160005e00654f..6e5c8dcbb725fa 100644 --- a/Modules/_testcapi/object.c +++ b/Modules/_testcapi/object.c @@ -555,6 +555,23 @@ pyobject_dump(PyObject *self, PyObject *args) Py_RETURN_NONE; } +static PyObject * +pysentinel_new(PyObject *self, PyObject *args) +{ + const char *name; + const char *module_name = NULL; + if (!PyArg_ParseTuple(args, "s|s", &name, &module_name)) { + return NULL; + } + return PySentinel_New(name, module_name); +} + +static PyObject * +pysentinel_check(PyObject *self, PyObject *obj) +{ + return PyBool_FromLong(PySentinel_Check(obj)); +} + static PyMethodDef test_methods[] = { {"call_pyobject_print", call_pyobject_print, METH_VARARGS}, @@ -585,6 +602,8 @@ static PyMethodDef test_methods[] = { {"clear_managed_dict", clear_managed_dict, METH_O, NULL}, {"is_uniquely_referenced", is_uniquely_referenced, METH_O}, {"pyobject_dump", pyobject_dump, METH_VARARGS}, + {"pysentinel_new", pysentinel_new, METH_VARARGS}, + {"pysentinel_check", pysentinel_check, METH_O}, {NULL}, }; diff --git a/Modules/_testinternalcapi/test_cases.c.h b/Modules/_testinternalcapi/test_cases.c.h index cd579491e4cd2e..8897854078bd45 100644 --- a/Modules/_testinternalcapi/test_cases.c.h +++ b/Modules/_testinternalcapi/test_cases.c.h @@ -5913,7 +5913,7 @@ int og_oparg = (oparg & ~255) | executor->vm_data.oparg; next_instr = this_instr; if (_PyJit_EnterExecutorShouldStopTracing(og_opcode)) { - if (_PyOpcode_Caches[_PyOpcode_Deopt[opcode]]) { + if (_PyOpcode_Caches[_PyOpcode_Deopt[og_opcode]]) { PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter); } opcode = og_opcode; @@ -12500,7 +12500,10 @@ tracer->prev_state.instr_frame = frame; tracer->prev_state.instr_oparg = oparg; tracer->prev_state.instr_stacklevel = PyStackRef_IsNone(frame->f_executable) ? 2 : STACK_LEVEL(); - if (_PyOpcode_Caches[_PyOpcode_Deopt[opcode]]) { + if (_PyOpcode_Caches[_PyOpcode_Deopt[opcode]] + // Branch opcodes use the cache for branch history, not + // specialization counters. Don't reset it. + && !IS_CONDITIONAL_JUMP_OPCODE(opcode)) { (&next_instr[1])->counter = trigger_backoff_counter(); } const _PyOpcodeRecordEntry *record_entry = &_PyOpcode_RecordEntries[opcode]; diff --git a/Modules/_zstd/compressor.c b/Modules/_zstd/compressor.c index f90bc9c5ab58b1..8a3cd182ab1516 100644 --- a/Modules/_zstd/compressor.c +++ b/Modules/_zstd/compressor.c @@ -74,7 +74,7 @@ zstd_contentsize_converter(PyObject *size, unsigned long long *p) if (PyErr_ExceptionMatches(PyExc_OverflowError)) { PyErr_Format(PyExc_ValueError, "size argument should be a positive int less " - "than %ull", ZSTD_CONTENTSIZE_ERROR); + "than %llu", ZSTD_CONTENTSIZE_ERROR); return 0; } return 0; @@ -83,7 +83,7 @@ zstd_contentsize_converter(PyObject *size, unsigned long long *p) *p = ZSTD_CONTENTSIZE_ERROR; PyErr_Format(PyExc_ValueError, "size argument should be a positive int less " - "than %ull", ZSTD_CONTENTSIZE_ERROR); + "than %llu", ZSTD_CONTENTSIZE_ERROR); return 0; } *p = pledged_size; diff --git a/Modules/_zstd/decompressor.c b/Modules/_zstd/decompressor.c index 0186ee92f5b147..46682b483ad06a 100644 --- a/Modules/_zstd/decompressor.c +++ b/Modules/_zstd/decompressor.c @@ -111,6 +111,7 @@ _zstd_set_d_parameters(ZstdDecompressor *self, PyObject *options) int key_v = PyLong_AsInt(key); Py_DECREF(key); if (key_v == -1 && PyErr_Occurred()) { + Py_DECREF(value); return -1; } diff --git a/Modules/arraymodule.c b/Modules/arraymodule.c index a86a7561271b87..b01e92eb8873ba 100644 --- a/Modules/arraymodule.c +++ b/Modules/arraymodule.c @@ -3053,8 +3053,10 @@ array_new(PyTypeObject *type, PyObject *args, PyObject *kwds) len = 0; a = newarrayobject(type, len, descr); - if (a == NULL) + if (a == NULL) { + Py_XDECREF(it); return NULL; + } if (len > 0 && !array_Check(initial, state)) { Py_ssize_t i; @@ -3063,11 +3065,13 @@ array_new(PyTypeObject *type, PyObject *args, PyObject *kwds) PySequence_GetItem(initial, i); if (v == NULL) { Py_DECREF(a); + Py_XDECREF(it); return NULL; } if (setarrayitem(a, i, v) != 0) { Py_DECREF(v); Py_DECREF(a); + Py_XDECREF(it); return NULL; } Py_DECREF(v); @@ -3079,6 +3083,7 @@ array_new(PyTypeObject *type, PyObject *args, PyObject *kwds) v = array_array_frombytes((PyObject *)a, initial); if (v == NULL) { Py_DECREF(a); + Py_XDECREF(it); return NULL; } Py_DECREF(v); @@ -3089,6 +3094,7 @@ array_new(PyTypeObject *type, PyObject *args, PyObject *kwds) wchar_t *ustr = PyUnicode_AsWideCharString(initial, &n); if (ustr == NULL) { Py_DECREF(a); + Py_XDECREF(it); return NULL; } @@ -3109,6 +3115,7 @@ array_new(PyTypeObject *type, PyObject *args, PyObject *kwds) Py_UCS4 *ustr = PyUnicode_AsUCS4Copy(initial); if (ustr == NULL) { Py_DECREF(a); + Py_XDECREF(it); return NULL; } @@ -3136,6 +3143,7 @@ array_new(PyTypeObject *type, PyObject *args, PyObject *kwds) return a; } } + Py_XDECREF(it); PyErr_SetString(PyExc_ValueError, "bad typecode (must be b, B, u, w, h, H, i, I, l, L, q, Q, f or d)"); return NULL; diff --git a/Modules/overlapped.c b/Modules/overlapped.c index 822e1ce4bdc28d..51aee5afd35b6d 100644 --- a/Modules/overlapped.c +++ b/Modules/overlapped.c @@ -1910,6 +1910,11 @@ _overlapped_Overlapped_WSARecvFromInto_impl(OverlappedObject *self, } #endif + if (bufobj->len < (Py_ssize_t)size) { + PyErr_SetString(PyExc_ValueError, "nbytes is greater than the length of the buffer"); + return NULL; + } + wsabuf.buf = bufobj->buf; wsabuf.len = size; diff --git a/Objects/clinic/sentinelobject.c.h b/Objects/clinic/sentinelobject.c.h new file mode 100644 index 00000000000000..51fd35a5979e31 --- /dev/null +++ b/Objects/clinic/sentinelobject.c.h @@ -0,0 +1,34 @@ +/*[clinic input] +preserve +[clinic start generated code]*/ + +#include "pycore_modsupport.h" // _PyArg_CheckPositional() + +static PyObject * +sentinel_new_impl(PyTypeObject *type, PyObject *name); + +static PyObject * +sentinel_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) +{ + PyObject *return_value = NULL; + PyTypeObject *base_tp = &PySentinel_Type; + PyObject *name; + + if ((type == base_tp || type->tp_init == base_tp->tp_init) && + !_PyArg_NoKeywords("sentinel", kwargs)) { + goto exit; + } + if (!_PyArg_CheckPositional("sentinel", PyTuple_GET_SIZE(args), 1, 1)) { + goto exit; + } + if (!PyUnicode_Check(PyTuple_GET_ITEM(args, 0))) { + _PyArg_BadArgument("sentinel", "argument 1", "str", PyTuple_GET_ITEM(args, 0)); + goto exit; + } + name = PyTuple_GET_ITEM(args, 0); + return_value = sentinel_new_impl(type, name); + +exit: + return return_value; +} +/*[clinic end generated code: output=7f28fc0bf0259cba input=a9049054013a1b77]*/ diff --git a/Objects/clinic/typevarobject.c.h b/Objects/clinic/typevarobject.c.h index bd4c7a0e64fd49..d2f350a3487f08 100644 --- a/Objects/clinic/typevarobject.c.h +++ b/Objects/clinic/typevarobject.c.h @@ -517,13 +517,15 @@ paramspec_has_default(PyObject *self, PyObject *Py_UNUSED(ignored)) } PyDoc_STRVAR(typevartuple__doc__, -"typevartuple(name, *, default=typing.NoDefault)\n" +"typevartuple(name, *, bound=None, covariant=False, contravariant=False,\n" +" infer_variance=False, default=typing.NoDefault)\n" "--\n" "\n" "Create a new TypeVarTuple with the given name."); static PyObject * -typevartuple_impl(PyTypeObject *type, PyObject *name, +typevartuple_impl(PyTypeObject *type, PyObject *name, PyObject *bound, + int covariant, int contravariant, int infer_variance, PyObject *default_value); static PyObject * @@ -532,7 +534,7 @@ typevartuple(PyTypeObject *type, PyObject *args, PyObject *kwargs) PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 2 + #define NUM_KEYWORDS 6 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD @@ -541,7 +543,7 @@ typevartuple(PyTypeObject *type, PyObject *args, PyObject *kwargs) } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) .ob_hash = -1, - .ob_item = { &_Py_ID(name), &_Py_ID(default), }, + .ob_item = { &_Py_ID(name), &_Py_ID(bound), &_Py_ID(covariant), &_Py_ID(contravariant), &_Py_ID(infer_variance), &_Py_ID(default), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -550,18 +552,22 @@ typevartuple(PyTypeObject *type, PyObject *args, PyObject *kwargs) # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"name", "default", NULL}; + static const char * const _keywords[] = {"name", "bound", "covariant", "contravariant", "infer_variance", "default", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "typevartuple", .kwtuple = KWTUPLE, }; #undef KWTUPLE - PyObject *argsbuf[2]; + PyObject *argsbuf[6]; PyObject * const *fastargs; Py_ssize_t nargs = PyTuple_GET_SIZE(args); Py_ssize_t noptargs = nargs + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - 1; PyObject *name; + PyObject *bound = Py_None; + int covariant = 0; + int contravariant = 0; + int infer_variance = 0; PyObject *default_value = &_Py_NoDefaultStruct; fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, @@ -577,9 +583,42 @@ typevartuple(PyTypeObject *type, PyObject *args, PyObject *kwargs) if (!noptargs) { goto skip_optional_kwonly; } - default_value = fastargs[1]; + if (fastargs[1]) { + bound = fastargs[1]; + if (!--noptargs) { + goto skip_optional_kwonly; + } + } + if (fastargs[2]) { + covariant = PyObject_IsTrue(fastargs[2]); + if (covariant < 0) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_kwonly; + } + } + if (fastargs[3]) { + contravariant = PyObject_IsTrue(fastargs[3]); + if (contravariant < 0) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_kwonly; + } + } + if (fastargs[4]) { + infer_variance = PyObject_IsTrue(fastargs[4]); + if (infer_variance < 0) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_kwonly; + } + } + default_value = fastargs[5]; skip_optional_kwonly: - return_value = typevartuple_impl(type, name, default_value); + return_value = typevartuple_impl(type, name, bound, covariant, contravariant, infer_variance, default_value); exit: return return_value; @@ -764,4 +803,4 @@ typealias_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=67ab9a5d1869f2c9 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=2e7dd170924d92e5 input=a9049054013a1b77]*/ diff --git a/Objects/genericaliasobject.c b/Objects/genericaliasobject.c index 7aef56cf4e93b8..e3bc8eb2739e3f 100644 --- a/Objects/genericaliasobject.c +++ b/Objects/genericaliasobject.c @@ -242,7 +242,6 @@ _Py_make_parameters(PyObject *args) len += needed; if (_PyTuple_Resize(¶meters, len) < 0) { Py_DECREF(subparams); - Py_DECREF(parameters); Py_XDECREF(tuple_args); return NULL; } @@ -650,7 +649,7 @@ ga_vectorcall(PyObject *self, PyObject *const *args, size_t nargsf, PyObject *kwnames) { gaobject *alias = (gaobject *) self; - PyObject *obj = PyVectorcall_Function(alias->origin)(alias->origin, args, nargsf, kwnames); + PyObject *obj = PyObject_Vectorcall(alias->origin, args, nargsf, kwnames); return set_orig_class(obj, self); } diff --git a/Objects/genobject.c b/Objects/genobject.c index 2895833b4ff933..d628889afc6dc8 100644 --- a/Objects/genobject.c +++ b/Objects/genobject.c @@ -496,7 +496,7 @@ gen_close(PyObject *self, PyObject *args) } if (is_resume(frame->instr_ptr)) { - bool no_unwind_tools = _PyEval_NoToolsForUnwind(_PyThreadState_GET()); + bool no_unwind_tools = _PyEval_NoToolsForUnwind(_PyThreadState_GET(), frame); /* We can safely ignore the outermost try block * as it is automatically generated to handle * StopIteration. */ @@ -1110,9 +1110,6 @@ make_gen(PyTypeObject *type, PyFunctionObject *func) return (PyObject *)gen; } -static PyObject * -compute_cr_origin(int origin_depth, _PyInterpreterFrame *current_frame); - PyObject * _Py_MakeCoro(PyFunctionObject *func) { @@ -1150,7 +1147,7 @@ _Py_MakeCoro(PyFunctionObject *func) assert(frame); assert(_PyFrame_IsIncomplete(frame)); frame = _PyFrame_GetFirstComplete(frame->previous); - PyObject *cr_origin = compute_cr_origin(origin_depth, frame); + PyObject *cr_origin = _PyCoro_ComputeOrigin(origin_depth, frame); ((PyCoroObject *)coro)->cr_origin_or_finalizer = cr_origin; if (!cr_origin) { Py_DECREF(coro); @@ -1535,8 +1532,8 @@ PyTypeObject _PyCoroWrapper_Type = { 0, /* tp_free */ }; -static PyObject * -compute_cr_origin(int origin_depth, _PyInterpreterFrame *current_frame) +PyObject * +_PyCoro_ComputeOrigin(int origin_depth, _PyInterpreterFrame *current_frame) { _PyInterpreterFrame *frame = current_frame; /* First count how many frames we have */ @@ -1581,7 +1578,7 @@ PyCoro_New(PyFrameObject *f, PyObject *name, PyObject *qualname) if (origin_depth == 0) { ((PyCoroObject *)coro)->cr_origin_or_finalizer = NULL; } else { - PyObject *cr_origin = compute_cr_origin(origin_depth, _PyEval_GetFrame()); + PyObject *cr_origin = _PyCoro_ComputeOrigin(origin_depth, _PyEval_GetFrame()); ((PyCoroObject *)coro)->cr_origin_or_finalizer = cr_origin; if (!cr_origin) { Py_DECREF(coro); diff --git a/Objects/object.c b/Objects/object.c index 3166254f6f640b..e6a764435bc292 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -2597,6 +2597,7 @@ static PyTypeObject* static_types[] = { &PyRange_Type, &PyReversed_Type, &PySTEntry_Type, + &PySentinel_Type, &PySeqIter_Type, &PySetIter_Type, &PySet_Type, @@ -2768,9 +2769,9 @@ _Py_SetImmortalUntracked(PyObject *op) return; } #ifdef Py_GIL_DISABLED - op->ob_tid = _Py_UNOWNED_TID; - op->ob_ref_local = _Py_IMMORTAL_REFCNT_LOCAL; - op->ob_ref_shared = 0; + _Py_atomic_store_uintptr_relaxed(&op->ob_tid, _Py_UNOWNED_TID); + _Py_atomic_store_uint32_relaxed(&op->ob_ref_local, _Py_IMMORTAL_REFCNT_LOCAL); + _Py_atomic_store_ssize_relaxed(&op->ob_ref_shared, 0); _Py_atomic_or_uint8(&op->ob_gc_bits, _PyGC_BITS_DEFERRED); #elif SIZEOF_VOID_P > 4 op->ob_flags = _Py_IMMORTAL_FLAGS; diff --git a/Objects/sentinelobject.c b/Objects/sentinelobject.c new file mode 100644 index 00000000000000..e7e9f60e3edfbe --- /dev/null +++ b/Objects/sentinelobject.c @@ -0,0 +1,196 @@ +/* Sentinel object implementation */ + +#include "Python.h" +#include "descrobject.h" // PyMemberDef +#include "pycore_ceval.h" // _PyThreadState_GET() +#include "pycore_interpframe.h" // _PyFrame_IsIncomplete() +#include "pycore_object.h" // _PyObject_GC_TRACK/UNTRACK() +#include "pycore_stackref.h" // PyStackRef_AsPyObjectBorrow() +#include "pycore_tuple.h" // _PyTuple_FromPair +#include "pycore_typeobject.h" // _Py_BaseObject_RichCompare() +#include "pycore_unionobject.h" // _Py_union_type_or() + +typedef struct { + PyObject_HEAD + PyObject *name; + PyObject *module; +} sentinelobject; + +#define sentinelobject_CAST(op) \ + (assert(PySentinel_Check(op)), _Py_CAST(sentinelobject *, (op))) + +/*[clinic input] +class sentinel "sentinelobject *" "&PySentinel_Type" +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=8b88f8268d3b5775]*/ + +#include "clinic/sentinelobject.c.h" + + +static PyObject * +caller(void) +{ + _PyInterpreterFrame *f = _PyThreadState_GET()->current_frame; + if (f == NULL || PyStackRef_IsNull(f->f_funcobj)) { + assert(!PyErr_Occurred()); + Py_RETURN_NONE; + } + PyFunctionObject *func = _PyFrame_GetFunction(f); + assert(PyFunction_Check(func)); + PyObject *r = PyFunction_GetModule((PyObject *)func); + if (!r) { + assert(!PyErr_Occurred()); + Py_RETURN_NONE; + } + return Py_NewRef(r); +} + +static PyObject * +sentinel_new_with_module(PyTypeObject *type, PyObject *name, PyObject *module) +{ + assert(PyUnicode_Check(name)); + + sentinelobject *self = PyObject_GC_New(sentinelobject, type); + if (self == NULL) { + return NULL; + } + self->name = Py_NewRef(name); + self->module = Py_NewRef(module); + _PyObject_GC_TRACK(self); + return (PyObject *)self; +} + +/*[clinic input] +@classmethod +sentinel.__new__ as sentinel_new + + name: object(subclass_of='&PyUnicode_Type') + / +[clinic start generated code]*/ + +static PyObject * +sentinel_new_impl(PyTypeObject *type, PyObject *name) +/*[clinic end generated code: output=4af55c6048bed30d input=3ab75704f39c119c]*/ +{ + PyObject *module = caller(); + PyObject *self = sentinel_new_with_module(type, name, module); + Py_DECREF(module); + return self; +} + +PyObject * +PySentinel_New(const char *name, const char *module_name) +{ + PyObject *name_obj = PyUnicode_FromString(name); + if (name_obj == NULL) { + return NULL; + } + PyObject *module_obj = module_name == NULL + ? Py_None + : PyUnicode_FromString(module_name); + if (module_obj == NULL) { + Py_DECREF(name_obj); + return NULL; + } + + PyObject *sentinel = sentinel_new_with_module( + &PySentinel_Type, name_obj, module_obj); + Py_DECREF(module_obj); + Py_DECREF(name_obj); + return sentinel; +} + +static int +sentinel_clear(PyObject *op) +{ + sentinelobject *self = sentinelobject_CAST(op); + Py_CLEAR(self->name); + Py_CLEAR(self->module); + return 0; +} + +static void +sentinel_dealloc(PyObject *op) +{ + _PyObject_GC_UNTRACK(op); + (void)sentinel_clear(op); + Py_TYPE(op)->tp_free(op); +} + +static int +sentinel_traverse(PyObject *op, visitproc visit, void *arg) +{ + sentinelobject *self = sentinelobject_CAST(op); + Py_VISIT(self->name); + Py_VISIT(self->module); + return 0; +} + +static PyObject * +sentinel_repr(PyObject *op) +{ + sentinelobject *self = sentinelobject_CAST(op); + return Py_NewRef(self->name); +} + +static PyObject * +sentinel_copy(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + return Py_NewRef(self); +} + +static PyObject * +sentinel_deepcopy(PyObject *self, PyObject *Py_UNUSED(memo)) +{ + return Py_NewRef(self); +} + +static PyObject * +sentinel_reduce(PyObject *op, PyObject *Py_UNUSED(ignored)) +{ + sentinelobject *self = sentinelobject_CAST(op); + return Py_NewRef(self->name); +} + +static PyMethodDef sentinel_methods[] = { + {"__copy__", sentinel_copy, METH_NOARGS, NULL}, + {"__deepcopy__", sentinel_deepcopy, METH_O, NULL}, + {"__reduce__", sentinel_reduce, METH_NOARGS, NULL}, + {NULL, NULL} +}; + +static PyMemberDef sentinel_members[] = { + {"__name__", Py_T_OBJECT_EX, offsetof(sentinelobject, name), Py_READONLY}, + {"__module__", Py_T_OBJECT_EX, offsetof(sentinelobject, module), Py_READONLY}, + {NULL} +}; + +static PyNumberMethods sentinel_as_number = { + .nb_or = _Py_union_type_or, +}; + +PyDoc_STRVAR(sentinel_doc, +"sentinel(name, /)\n" +"--\n\n" +"Create a unique sentinel object with the given name."); + +PyTypeObject PySentinel_Type = { + PyVarObject_HEAD_INIT(&PyType_Type, 0) + .tp_name = "sentinel", + .tp_basicsize = sizeof(sentinelobject), + .tp_dealloc = sentinel_dealloc, + .tp_repr = sentinel_repr, + .tp_as_number = &sentinel_as_number, + .tp_hash = PyObject_GenericHash, + .tp_getattro = PyObject_GenericGetAttr, + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_IMMUTABLETYPE + | Py_TPFLAGS_HAVE_GC, + .tp_doc = sentinel_doc, + .tp_traverse = sentinel_traverse, + .tp_clear = sentinel_clear, + .tp_richcompare = _Py_BaseObject_RichCompare, + .tp_methods = sentinel_methods, + .tp_members = sentinel_members, + .tp_new = sentinel_new, + .tp_free = PyObject_GC_Del, +}; diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 08b95cfbc6ce59..fb3c7101410683 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -5878,7 +5878,13 @@ PyType_GetModuleByToken_DuringGC(PyTypeObject *type, const void *token) PyObject * PyType_GetModuleByToken(PyTypeObject *type, const void *token) { - PyObject *mod = PyType_GetModuleByToken_DuringGC(type, token); + return Py_XNewRef(PyType_GetModuleByDef(type, (PyModuleDef *)token)); +} + +PyObject * +PyType_GetModuleByDef(PyTypeObject *type, PyModuleDef *def) +{ + PyObject *mod = PyType_GetModuleByToken_DuringGC(type, def); if (!mod) { PyErr_Format( PyExc_TypeError, @@ -5886,14 +5892,6 @@ PyType_GetModuleByToken(PyTypeObject *type, const void *token) type->tp_name); return NULL; } - return Py_NewRef(mod); -} - -PyObject * -PyType_GetModuleByDef(PyTypeObject *type, PyModuleDef *def) -{ - PyObject *mod = PyType_GetModuleByToken(type, def); - Py_XDECREF(mod); // return borrowed ref return mod; } diff --git a/Objects/typevarobject.c b/Objects/typevarobject.c index c2b8ee43119cb1..cdc0ea42eac263 100644 --- a/Objects/typevarobject.c +++ b/Objects/typevarobject.c @@ -36,8 +36,12 @@ typedef struct { typedef struct { PyObject_HEAD PyObject *name; + PyObject *bound; PyObject *default_value; PyObject *evaluate_default; + bool covariant; + bool contravariant; + bool infer_variance; } typevartupleobject; typedef struct { @@ -1524,6 +1528,7 @@ typevartuple_dealloc(PyObject *self) typevartupleobject *tvt = typevartupleobject_CAST(self); Py_XDECREF(tvt->name); + Py_XDECREF(tvt->bound); Py_XDECREF(tvt->default_value); Py_XDECREF(tvt->evaluate_default); PyObject_ClearManagedDict(self); @@ -1555,16 +1560,28 @@ static PyObject * typevartuple_repr(PyObject *self) { typevartupleobject *tvt = typevartupleobject_CAST(self); - return Py_NewRef(tvt->name); + + if (tvt->infer_variance) { + return Py_NewRef(tvt->name); + } + + char variance = tvt->covariant ? '+' : tvt->contravariant ? '-' : '~'; + return PyUnicode_FromFormat("%c%U", variance, tvt->name); } static PyMemberDef typevartuple_members[] = { {"__name__", _Py_T_OBJECT, offsetof(typevartupleobject, name), Py_READONLY}, + {"__bound__", _Py_T_OBJECT, offsetof(typevartupleobject, bound), Py_READONLY}, + {"__covariant__", Py_T_BOOL, offsetof(typevartupleobject, covariant), Py_READONLY}, + {"__contravariant__", Py_T_BOOL, offsetof(typevartupleobject, contravariant), Py_READONLY}, + {"__infer_variance__", Py_T_BOOL, offsetof(typevartupleobject, infer_variance), Py_READONLY}, {0} }; static typevartupleobject * -typevartuple_alloc(PyObject *name, PyObject *module, PyObject *default_value) +typevartuple_alloc(PyObject *name, PyObject *bound, PyObject *default_value, + bool covariant, bool contravariant, bool infer_variance, + PyObject *module) { PyTypeObject *tp = _PyInterpreterState_GET()->cached_objects.typevartuple_type; typevartupleobject *tvt = PyObject_GC_New(typevartupleobject, tp); @@ -1572,6 +1589,10 @@ typevartuple_alloc(PyObject *name, PyObject *module, PyObject *default_value) return NULL; } tvt->name = Py_NewRef(name); + tvt->bound = Py_XNewRef(bound); + tvt->covariant = covariant; + tvt->contravariant = contravariant; + tvt->infer_variance = infer_variance; tvt->default_value = Py_XNewRef(default_value); tvt->evaluate_default = NULL; _PyObject_GC_TRACK(tvt); @@ -1590,21 +1611,46 @@ typevartuple.__new__ name: object(subclass_of="&PyUnicode_Type") * + bound: object = None + covariant: bool = False + contravariant: bool = False + infer_variance: bool = False default as default_value: object(c_default="&_Py_NoDefaultStruct") = typing.NoDefault Create a new TypeVarTuple with the given name. [clinic start generated code]*/ static PyObject * -typevartuple_impl(PyTypeObject *type, PyObject *name, +typevartuple_impl(PyTypeObject *type, PyObject *name, PyObject *bound, + int covariant, int contravariant, int infer_variance, PyObject *default_value) -/*[clinic end generated code: output=9d6b76dfe95aae51 input=e149739929a866d0]*/ +/*[clinic end generated code: output=40bc9ca10f64e392 input=56e28c725a8da40b]*/ { + if (covariant && contravariant) { + PyErr_SetString(PyExc_ValueError, "Bivariant types are not supported."); + return NULL; + } + if (infer_variance && (covariant || contravariant)) { + PyErr_SetString(PyExc_ValueError, "Variance cannot be specified with infer_variance."); + return NULL; + } + if (Py_IsNone(bound)) { + bound = NULL; + } + if (bound != NULL) { + bound = type_check(bound, "Bound must be a type."); + if (bound == NULL) { + return NULL; + } + } PyObject *module = caller(); if (module == NULL) { + Py_XDECREF(bound); return NULL; } - PyObject *result = (PyObject *)typevartuple_alloc(name, module, default_value); + PyObject *result = (PyObject *)typevartuple_alloc( + name, bound, default_value, covariant, contravariant, infer_variance, module); + Py_XDECREF(bound); Py_DECREF(module); return result; } @@ -1688,6 +1734,7 @@ typevartuple_traverse(PyObject *self, visitproc visit, void *arg) Py_VISIT(Py_TYPE(self)); typevartupleobject *tvt = typevartupleobject_CAST(self); Py_VISIT(tvt->name); + Py_VISIT(tvt->bound); Py_VISIT(tvt->default_value); Py_VISIT(tvt->evaluate_default); return PyObject_VisitManagedDict(self, visit, arg); @@ -1698,6 +1745,7 @@ typevartuple_clear(PyObject *self) { typevartupleobject *tvt = typevartupleobject_CAST(self); Py_CLEAR(tvt->name); + Py_CLEAR(tvt->bound); Py_CLEAR(tvt->default_value); Py_CLEAR(tvt->evaluate_default); PyObject_ClearManagedDict(self); @@ -1829,7 +1877,7 @@ PyObject * _Py_make_typevartuple(PyThreadState *Py_UNUSED(ignored), PyObject *v) { assert(PyUnicode_Check(v)); - return (PyObject *)typevartuple_alloc(v, NULL, NULL); + return (PyObject *)typevartuple_alloc(v, NULL, NULL, false, false, true, NULL); } static PyObject * diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index d2569132998acc..9aee7120c811de 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -589,6 +589,14 @@ _PyUnicode_CheckConsistency(PyObject *op, int check_content) { #define CHECK(expr) \ do { if (!(expr)) { _PyObject_ASSERT_FAILED_MSG(op, Py_STRINGIFY(expr)); } } while (0) +#ifdef Py_GIL_DISABLED +# define CHECK_IF_GIL(expr) (void)(expr) +# define CHECK_IF_FT(expr) CHECK(expr) +#else +# define CHECK_IF_GIL(expr) CHECK(expr) +# define CHECK_IF_FT(expr) (void)(expr) +#endif + assert(op != NULL); CHECK(PyUnicode_Check(op)); @@ -669,11 +677,9 @@ _PyUnicode_CheckConsistency(PyObject *op, int check_content) /* Check interning state */ #ifdef Py_DEBUG - // Note that we do not check `_Py_IsImmortal(op)`, since stable ABI - // extensions can make immortal strings mortal (but with a high enough - // refcount). - // The other way is extremely unlikely (worth a potential failed assertion - // in a debug build), so we do check `!_Py_IsImmortal(op)`. + // Note that we do not check `_Py_IsImmortal(op)` in the GIL-enabled build + // since stable ABI extensions can make immortal strings mortal (but with a + // high enough refcount). switch (PyUnicode_CHECK_INTERNED(op)) { case SSTATE_NOT_INTERNED: if (ascii->state.statically_allocated) { @@ -683,18 +689,20 @@ _PyUnicode_CheckConsistency(PyObject *op, int check_content) // are static but use SSTATE_NOT_INTERNED } else { - CHECK(!_Py_IsImmortal(op)); + CHECK_IF_GIL(!_Py_IsImmortal(op)); } break; case SSTATE_INTERNED_MORTAL: CHECK(!ascii->state.statically_allocated); - CHECK(!_Py_IsImmortal(op)); + CHECK_IF_GIL(!_Py_IsImmortal(op)); break; case SSTATE_INTERNED_IMMORTAL: CHECK(!ascii->state.statically_allocated); + CHECK_IF_FT(_Py_IsImmortal(op)); break; case SSTATE_INTERNED_IMMORTAL_STATIC: CHECK(ascii->state.statically_allocated); + CHECK_IF_FT(_Py_IsImmortal(op)); break; default: Py_UNREACHABLE(); @@ -14208,6 +14216,18 @@ immortalize_interned(PyObject *s) FT_ATOMIC_STORE_UINT8(_PyUnicode_STATE(s).interned, SSTATE_INTERNED_IMMORTAL); } +#ifdef Py_GIL_DISABLED +static bool +can_immortalize_safely(PyObject *s) +{ + if (_Py_IsOwnedByCurrentThread(s) || _Py_IsImmortal(s)) { + return true; + } + Py_ssize_t shared = _Py_atomic_load_ssize(&s->ob_ref_shared); + return _Py_REF_IS_MERGED(shared); +} +#endif + static /* non-null */ PyObject* intern_common(PyInterpreterState *interp, PyObject *s /* stolen */, bool immortalize) @@ -14236,11 +14256,16 @@ intern_common(PyInterpreterState *interp, PyObject *s /* stolen */, // no, go on break; case SSTATE_INTERNED_MORTAL: +#ifndef Py_GIL_DISABLED // yes but we might need to make it immortal if (immortalize) { immortalize_interned(s); } return s; +#else + // not fully interned yet; fall through to the locking path + break; +#endif default: // all done return s; @@ -14305,6 +14330,23 @@ intern_common(PyInterpreterState *interp, PyObject *s /* stolen */, Py_DECREF(r); } #endif + +#ifdef Py_GIL_DISABLED + // Immortalization writes to the refcount fields non-atomically. That + // races with Py_INCREF / Py_DECREF on the thread that owns `s`. If we + // don't own it (and its refcount hasn't been merged), intern a copy + // we own instead. + if (!can_immortalize_safely(s)) { + PyObject *copy = _PyUnicode_Copy(s); + if (copy == NULL) { + PyErr_Clear(); + return s; + } + Py_DECREF(s); + s = copy; + } +#endif + FT_MUTEX_LOCK(INTERN_MUTEX); PyObject *t; { diff --git a/Objects/unionobject.c b/Objects/unionobject.c index d33d581f049c5b..0f6b1e44bc2402 100644 --- a/Objects/unionobject.c +++ b/Objects/unionobject.c @@ -245,6 +245,7 @@ is_unionable(PyObject *obj) { if (obj == Py_None || PyType_Check(obj) || + PySentinel_Check(obj) || _PyGenericAlias_Check(obj) || _PyUnion_Check(obj) || Py_IS_TYPE(obj, &_PyTypeAlias_Type)) { diff --git a/PCbuild/_freeze_module.vcxproj b/PCbuild/_freeze_module.vcxproj index 38236922a523db..953973a2ad32df 100644 --- a/PCbuild/_freeze_module.vcxproj +++ b/PCbuild/_freeze_module.vcxproj @@ -158,6 +158,7 @@ + diff --git a/PCbuild/_freeze_module.vcxproj.filters b/PCbuild/_freeze_module.vcxproj.filters index 73861dbb0c9e7e..13db4d93f54518 100644 --- a/PCbuild/_freeze_module.vcxproj.filters +++ b/PCbuild/_freeze_module.vcxproj.filters @@ -400,6 +400,9 @@ Source Files + + Source Files + Source Files diff --git a/PCbuild/build.bat b/PCbuild/build.bat index 8fb2f096c93c0e..9d2f032f5a9355 100644 --- a/PCbuild/build.bat +++ b/PCbuild/build.bat @@ -170,16 +170,20 @@ if "%do_pgo%"=="true" ( del /s "%dir%\*.pgc" del /s "%dir%\..\Lib\*.pyc" set conf=PGUpdate - if "%clean%"=="false" ( - echo on - call "%dir%\..\python.bat" %pgo_job% - @echo off - call :Kill - set target=Build - ) + if "%clean%"=="false" goto :RunPgoJob ) goto :Build +:RunPgoJob +echo on +call "%dir%\..\python.bat" %pgo_job% +@echo off +set pgo_errorlevel=%ERRORLEVEL% +call :Kill +if %pgo_errorlevel% NEQ 0 exit /B %pgo_errorlevel% +set target=Build +goto :Build + :Kill echo on %MSBUILD% "%dir%\pythoncore.vcxproj" /t:KillPython %verbose%^ diff --git a/PCbuild/pyproject.props b/PCbuild/pyproject.props index 94ae718d58c4ba..f79608e1d58dbc 100644 --- a/PCbuild/pyproject.props +++ b/PCbuild/pyproject.props @@ -12,8 +12,9 @@ $(IntDir.Replace(`\\`, `\`)) $(Py_IntDir)\$(MajorVersionNumber)$(MinorVersionNumber)_frozen\ $(Py_IntDir)\$(MajorVersionNumber)$(MinorVersionNumber)$(ArchName)_$(Configuration)\zlib-ng\ - $(Py_IntDir)\$(MajorVersionNumber)$(MinorVersionNumber)_$(Configuration) - $(Py_IntDir)\$(MajorVersionNumber)$(MinorVersionNumber)_PGInstrument + $(Py_IntDir)\$(MajorVersionNumber)$(MinorVersionNumber)_$(Configuration)\ + $(Py_IntDir)\$(MajorVersionNumber)$(MinorVersionNumber)_PGInstrument\ + $(GeneratedJitStencilsDir.Replace(`\\`, `\`)) $(ProjectName) $(TargetName)$(PyDebugExt) false diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index 61bee29c0af3d6..fb9217fee8bd73 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -115,6 +115,9 @@ version.lib;ws2_32.lib;pathcch.lib;bcrypt.lib;%(AdditionalDependencies) zlib-ng$(PyDebugExt).lib;%(AdditionalDependencies) + $(GeneratedJitStencilsDir)jit_shim-aarch64-pc-windows-msvc.o;%(AdditionalDependencies) + $(GeneratedJitStencilsDir)jit_shim-i686-pc-windows-msvc.o;%(AdditionalDependencies) + $(GeneratedJitStencilsDir)jit_shim-x86_64-pc-windows-msvc.o;%(AdditionalDependencies) @@ -359,6 +362,7 @@ + @@ -380,6 +384,7 @@ + @@ -557,6 +562,7 @@ + diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index 664788e69af19a..1e1d085cd75511 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -156,6 +156,9 @@ Include + + Include + Include @@ -219,6 +222,9 @@ Include + + Include + Include @@ -1271,6 +1277,9 @@ Objects + + Objects + Objects diff --git a/PCbuild/regen.targets b/PCbuild/regen.targets index 41af9cacfb912b..9552e73ef6a2ec 100644 --- a/PCbuild/regen.targets +++ b/PCbuild/regen.targets @@ -35,6 +35,9 @@ <_JITOutputs Include="$(GeneratedJitStencilsDir)jit_stencils-aarch64-pc-windows-msvc.h" Condition="$(Platform) == 'ARM64'"/> <_JITOutputs Include="$(GeneratedJitStencilsDir)jit_stencils-i686-pc-windows-msvc.h" Condition="$(Platform) == 'Win32'"/> <_JITOutputs Include="$(GeneratedJitStencilsDir)jit_stencils-x86_64-pc-windows-msvc.h" Condition="$(Platform) == 'x64'"/> + <_JITOutputs Include="$(GeneratedJitStencilsDir)jit_shim-aarch64-pc-windows-msvc.o" Condition="$(Platform) == 'ARM64'"/> + <_JITOutputs Include="$(GeneratedJitStencilsDir)jit_shim-i686-pc-windows-msvc.o" Condition="$(Platform) == 'Win32'"/> + <_JITOutputs Include="$(GeneratedJitStencilsDir)jit_shim-x86_64-pc-windows-msvc.o" Condition="$(Platform) == 'x64'"/> <_CasesSources Include="$(PySourcePath)Python\bytecodes.c;$(PySourcePath)Python\optimizer_bytecodes.c;"/> <_CasesOutputs Include="$(PySourcePath)Python\generated_cases.c.h;$(PySourcePath)Include\opcode_ids.h;$(PySourcePath)Include\internal\pycore_uop_ids.h;$(PySourcePath)Python\opcode_targets.h;$(PySourcePath)Include\internal\pycore_opcode_metadata.h;$(PySourcePath)Include\internal\pycore_uop_metadata.h;$(PySourcePath)Python\optimizer_cases.c.h;$(PySourcePath)Lib\_opcode_metadata.py"/> <_SbomSources Include="$(PySourcePath)PCbuild\get_externals.bat" /> @@ -129,7 +132,7 @@ x86_64-pc-windows-msvc $(JITArgs) --debug - + diff --git a/Parser/parser.c b/Parser/parser.c index f853d309de9180..c55c081dfc3d8e 100644 --- a/Parser/parser.c +++ b/Parser/parser.c @@ -9066,7 +9066,7 @@ complex_number_rule(Parser *p) return _res; } -// signed_number: NUMBER | '-' NUMBER +// signed_number: NUMBER | '+' NUMBER | '-' NUMBER static expr_ty signed_number_rule(Parser *p) { @@ -9107,6 +9107,33 @@ signed_number_rule(Parser *p) D(fprintf(stderr, "%*c%s signed_number[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "NUMBER")); } + { // '+' NUMBER + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> signed_number[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'+' NUMBER")); + Token * _literal; + expr_ty number; + if ( + (_literal = _PyPegen_expect_token(p, 14)) // token='+' + && + (number = _PyPegen_number_token(p)) // NUMBER + ) + { + D(fprintf(stderr, "%*c+ signed_number[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'+' NUMBER")); + _res = number; + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { + p->error_indicator = 1; + p->level--; + return NULL; + } + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s signed_number[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'+' NUMBER")); + } { // '-' NUMBER if (p->error_indicator) { p->level--; @@ -9149,7 +9176,7 @@ signed_number_rule(Parser *p) return _res; } -// signed_real_number: real_number | '-' real_number +// signed_real_number: real_number | '+' real_number | '-' real_number static expr_ty signed_real_number_rule(Parser *p) { @@ -9190,6 +9217,33 @@ signed_real_number_rule(Parser *p) D(fprintf(stderr, "%*c%s signed_real_number[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "real_number")); } + { // '+' real_number + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> signed_real_number[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'+' real_number")); + Token * _literal; + expr_ty real; + if ( + (_literal = _PyPegen_expect_token(p, 14)) // token='+' + && + (real = real_number_rule(p)) // real_number + ) + { + D(fprintf(stderr, "%*c+ signed_real_number[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'+' real_number")); + _res = real; + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { + p->error_indicator = 1; + p->level--; + return NULL; + } + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s signed_real_number[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'+' real_number")); + } { // '-' real_number if (p->error_indicator) { p->level--; @@ -9275,7 +9329,7 @@ real_number_rule(Parser *p) return _res; } -// imaginary_number: NUMBER +// imaginary_number: NUMBER | '+' NUMBER static expr_ty imaginary_number_rule(Parser *p) { @@ -9312,6 +9366,33 @@ imaginary_number_rule(Parser *p) D(fprintf(stderr, "%*c%s imaginary_number[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "NUMBER")); } + { // '+' NUMBER + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> imaginary_number[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'+' NUMBER")); + Token * _literal; + expr_ty imag; + if ( + (_literal = _PyPegen_expect_token(p, 14)) // token='+' + && + (imag = _PyPegen_number_token(p)) // NUMBER + ) + { + D(fprintf(stderr, "%*c+ imaginary_number[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'+' NUMBER")); + _res = _PyPegen_ensure_imaginary ( p , imag ); + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { + p->error_indicator = 1; + p->level--; + return NULL; + } + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s imaginary_number[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'+' NUMBER")); + } _res = NULL; done: p->level--; diff --git a/Platforms/emscripten/README.md b/Platforms/emscripten/README.md index 017bb3c8977d26..ce230c4b74a273 100644 --- a/Platforms/emscripten/README.md +++ b/Platforms/emscripten/README.md @@ -186,8 +186,8 @@ await createEmscriptenModule({ are not shipped. All other modules are bundled as pre-compiled ``pyc`` files. - In-memory file system (MEMFS) is not persistent and limited. -- Test modules are disabled by default. Use ``--enable-test-modules`` build - test modules like ``_testcapi``. +- Test modules are built by default. Use ``--disable-test-modules`` to disable + building test modules like ``_testcapi``. ## Detecting Emscripten builds diff --git a/Platforms/emscripten/web_example/index.html b/Platforms/emscripten/web_example/index.html index 9c89c9c0ed3bf5..3a207b92015451 100644 --- a/Platforms/emscripten/web_example/index.html +++ b/Platforms/emscripten/web_example/index.html @@ -663,9 +663,9 @@

Simple REPL for Python WASM

The simple REPL provides a limited Python experience in the browser. - Tools/wasm/README.md + Platforms/emscripten/README.md contains a list of known limitations and issues. Networking, subprocesses, and threading are not available. @@ -679,9 +679,9 @@

Simple REPL for Python WASM

your browser instead of using server.py as described in - Tools/wasm/README.md + Platforms/emscripten/README.md .

diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index fec64e1ff9d25f..35b30a243318cc 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -1840,15 +1840,17 @@ hash as builtin_hash obj: object / -Return the hash value for the given object. +Return the integer hash value for the given object. -Two objects that compare equal must also have the same hash value, but the -reverse is not necessarily true. +Two objects that compare equal must also have the same hash value, but +the reverse is not necessarily true. Hash values may differ between +Python processes. Not all objects are hashable; calling hash() on an +unhashable object raises TypeError. [clinic start generated code]*/ static PyObject * builtin_hash(PyObject *module, PyObject *obj) -/*[clinic end generated code: output=237668e9d7688db7 input=58c48be822bf9c54]*/ +/*[clinic end generated code: output=237668e9d7688db7 input=70a242ff65f6717c]*/ { Py_hash_t x; @@ -3553,6 +3555,7 @@ _PyBuiltin_Init(PyInterpreterState *interp) SETBUILTIN("object", &PyBaseObject_Type); SETBUILTIN("range", &PyRange_Type); SETBUILTIN("reversed", &PyReversed_Type); + SETBUILTIN("sentinel", &PySentinel_Type); SETBUILTIN("set", &PySet_Type); SETBUILTIN("slice", &PySlice_Type); SETBUILTIN("staticmethod", &PyStaticMethod_Type); diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 7de889b93b71a7..59db0eb399b121 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -3529,7 +3529,7 @@ dummy_func( int og_oparg = (oparg & ~255) | executor->vm_data.oparg; next_instr = this_instr; if (_PyJit_EnterExecutorShouldStopTracing(og_opcode)) { - if (_PyOpcode_Caches[_PyOpcode_Deopt[opcode]]) { + if (_PyOpcode_Caches[_PyOpcode_Deopt[og_opcode]]) { PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter); } opcode = og_opcode; @@ -6541,7 +6541,10 @@ dummy_func( tracer->prev_state.instr_frame = frame; tracer->prev_state.instr_oparg = oparg; tracer->prev_state.instr_stacklevel = PyStackRef_IsNone(frame->f_executable) ? 2 : STACK_LEVEL(); - if (_PyOpcode_Caches[_PyOpcode_Deopt[opcode]]) { + if (_PyOpcode_Caches[_PyOpcode_Deopt[opcode]] + // Branch opcodes use the cache for branch history, not + // specialization counters. Don't reset it. + && !IS_CONDITIONAL_JUMP_OPCODE(opcode)) { (&next_instr[1])->counter = trigger_backoff_counter(); } diff --git a/Python/ceval.c b/Python/ceval.c index 03bc5229565966..506ea591c385c0 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -1305,7 +1305,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int } #ifdef _Py_TIER2 #ifdef _Py_JIT -_PyJitEntryFuncPtr _Py_jit_entry = _Py_LazyJitShim; +_PyJitEntryFuncPtr _Py_jit_entry = _PyJIT; #else _PyJitEntryFuncPtr _Py_jit_entry = _PyTier2Interpreter; #endif @@ -2406,15 +2406,16 @@ void _PyEval_MonitorRaise(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *instr) { - if (no_tools_for_global_event(tstate, PY_MONITORING_EVENT_RAISE)) { + if (no_tools_for_local_event(tstate, frame, PY_MONITORING_EVENT_RAISE)) { return; } do_monitor_exc(tstate, frame, instr, PY_MONITORING_EVENT_RAISE); } bool -_PyEval_NoToolsForUnwind(PyThreadState *tstate) { - return no_tools_for_global_event(tstate, PY_MONITORING_EVENT_PY_UNWIND); +_PyEval_NoToolsForUnwind(PyThreadState *tstate, _PyInterpreterFrame *frame) +{ + return no_tools_for_local_event(tstate, frame, PY_MONITORING_EVENT_PY_UNWIND); } diff --git a/Python/ceval.h b/Python/ceval.h index bb5f7ddb857246..0437ab85c5a668 100644 --- a/Python/ceval.h +++ b/Python/ceval.h @@ -367,7 +367,7 @@ no_tools_for_global_event(PyThreadState *tstate, int event) static inline bool no_tools_for_local_event(PyThreadState *tstate, _PyInterpreterFrame *frame, int event) { - assert(event < _PY_MONITORING_LOCAL_EVENTS); + assert(event < _PY_MONITORING_UNGROUPED_EVENTS); _PyCoMonitoringData *data = _PyFrame_GetCode(frame)->_co_monitoring; if (data) { return data->active_monitors.tools[event] == 0; @@ -382,7 +382,7 @@ monitor_handled(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, PyObject *exc) { - if (no_tools_for_global_event(tstate, PY_MONITORING_EVENT_EXCEPTION_HANDLED)) { + if (no_tools_for_local_event(tstate, frame, PY_MONITORING_EVENT_EXCEPTION_HANDLED)) { return 0; } return _Py_call_instrumentation_arg(tstate, PY_MONITORING_EVENT_EXCEPTION_HANDLED, frame, instr, exc); @@ -393,7 +393,7 @@ monitor_throw(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *instr) { - if (no_tools_for_global_event(tstate, PY_MONITORING_EVENT_PY_THROW)) { + if (no_tools_for_local_event(tstate, frame, PY_MONITORING_EVENT_PY_THROW)) { return; } do_monitor_exc(tstate, frame, instr, PY_MONITORING_EVENT_PY_THROW); @@ -403,7 +403,7 @@ static void monitor_reraise(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *instr) { - if (no_tools_for_global_event(tstate, PY_MONITORING_EVENT_RERAISE)) { + if (no_tools_for_local_event(tstate, frame, PY_MONITORING_EVENT_RERAISE)) { return; } do_monitor_exc(tstate, frame, instr, PY_MONITORING_EVENT_RERAISE); @@ -431,7 +431,7 @@ monitor_unwind(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *instr) { - if (no_tools_for_global_event(tstate, PY_MONITORING_EVENT_PY_UNWIND)) { + if (no_tools_for_local_event(tstate, frame, PY_MONITORING_EVENT_PY_UNWIND)) { return; } do_monitor_exc(tstate, frame, instr, PY_MONITORING_EVENT_PY_UNWIND); diff --git a/Python/clinic/bltinmodule.c.h b/Python/clinic/bltinmodule.c.h index c8c141f863d26a..e6b845cd375d73 100644 --- a/Python/clinic/bltinmodule.c.h +++ b/Python/clinic/bltinmodule.c.h @@ -826,10 +826,12 @@ PyDoc_STRVAR(builtin_hash__doc__, "hash($module, obj, /)\n" "--\n" "\n" -"Return the hash value for the given object.\n" +"Return the integer hash value for the given object.\n" "\n" -"Two objects that compare equal must also have the same hash value, but the\n" -"reverse is not necessarily true."); +"Two objects that compare equal must also have the same hash value, but\n" +"the reverse is not necessarily true. Hash values may differ between\n" +"Python processes. Not all objects are hashable; calling hash() on an\n" +"unhashable object raises TypeError."); #define BUILTIN_HASH_METHODDEF \ {"hash", (PyCFunction)builtin_hash, METH_O, builtin_hash__doc__}, @@ -1380,4 +1382,4 @@ builtin_issubclass(PyObject *module, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=1c3327da8885bb8e input=a9049054013a1b77]*/ +/*[clinic end generated code: output=f1fc836a63d89826 input=a9049054013a1b77]*/ diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index e84886ed04020c..dccee0e4a3b110 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -5913,7 +5913,7 @@ int og_oparg = (oparg & ~255) | executor->vm_data.oparg; next_instr = this_instr; if (_PyJit_EnterExecutorShouldStopTracing(og_opcode)) { - if (_PyOpcode_Caches[_PyOpcode_Deopt[opcode]]) { + if (_PyOpcode_Caches[_PyOpcode_Deopt[og_opcode]]) { PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter); } opcode = og_opcode; @@ -12497,7 +12497,10 @@ tracer->prev_state.instr_frame = frame; tracer->prev_state.instr_oparg = oparg; tracer->prev_state.instr_stacklevel = PyStackRef_IsNone(frame->f_executable) ? 2 : STACK_LEVEL(); - if (_PyOpcode_Caches[_PyOpcode_Deopt[opcode]]) { + if (_PyOpcode_Caches[_PyOpcode_Deopt[opcode]] + // Branch opcodes use the cache for branch history, not + // specialization counters. Don't reset it. + && !IS_CONDITIONAL_JUMP_OPCODE(opcode)) { (&next_instr[1])->counter = trigger_backoff_counter(); } const _PyOpcodeRecordEntry *record_entry = &_PyOpcode_RecordEntries[opcode]; diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 256e2a3d3a2df0..51bcbfdb3b6c55 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -185,6 +185,12 @@ opcode_has_event(int opcode) ); } +uint8_t +_PyCode_Deinstrument(uint8_t opcode) +{ + return DE_INSTRUMENT[opcode]; +} + static inline bool is_instrumented(int opcode) { @@ -197,7 +203,7 @@ is_instrumented(int opcode) static inline bool monitors_equals(_Py_LocalMonitors a, _Py_LocalMonitors b) { - for (int i = 0; i < _PY_MONITORING_LOCAL_EVENTS; i++) { + for (int i = 0; i < _PY_MONITORING_UNGROUPED_EVENTS; i++) { if (a.tools[i] != b.tools[i]) { return false; } @@ -210,7 +216,7 @@ static inline _Py_LocalMonitors monitors_sub(_Py_LocalMonitors a, _Py_LocalMonitors b) { _Py_LocalMonitors res; - for (int i = 0; i < _PY_MONITORING_LOCAL_EVENTS; i++) { + for (int i = 0; i < _PY_MONITORING_UNGROUPED_EVENTS; i++) { res.tools[i] = a.tools[i] & ~b.tools[i]; } return res; @@ -221,7 +227,7 @@ static inline _Py_LocalMonitors monitors_and(_Py_LocalMonitors a, _Py_LocalMonitors b) { _Py_LocalMonitors res; - for (int i = 0; i < _PY_MONITORING_LOCAL_EVENTS; i++) { + for (int i = 0; i < _PY_MONITORING_UNGROUPED_EVENTS; i++) { res.tools[i] = a.tools[i] & b.tools[i]; } return res; @@ -237,7 +243,7 @@ static inline _Py_LocalMonitors local_union(_Py_GlobalMonitors a, _Py_LocalMonitors b) { _Py_LocalMonitors res; - for (int i = 0; i < _PY_MONITORING_LOCAL_EVENTS; i++) { + for (int i = 0; i < _PY_MONITORING_UNGROUPED_EVENTS; i++) { res.tools[i] = a.tools[i] | b.tools[i]; } return res; @@ -246,7 +252,7 @@ local_union(_Py_GlobalMonitors a, _Py_LocalMonitors b) static inline bool monitors_are_empty(_Py_LocalMonitors m) { - for (int i = 0; i < _PY_MONITORING_LOCAL_EVENTS; i++) { + for (int i = 0; i < _PY_MONITORING_UNGROUPED_EVENTS; i++) { if (m.tools[i]) { return false; } @@ -257,7 +263,7 @@ monitors_are_empty(_Py_LocalMonitors m) static inline bool multiple_tools(_Py_LocalMonitors *m) { - for (int i = 0; i < _PY_MONITORING_LOCAL_EVENTS; i++) { + for (int i = 0; i < _PY_MONITORING_UNGROUPED_EVENTS; i++) { if (_Py_popcount32(m->tools[i]) > 1) { return true; } @@ -269,7 +275,7 @@ static inline _PyMonitoringEventSet get_local_events(_Py_LocalMonitors *m, int tool_id) { _PyMonitoringEventSet result = 0; - for (int e = 0; e < _PY_MONITORING_LOCAL_EVENTS; e++) { + for (int e = 0; e < _PY_MONITORING_UNGROUPED_EVENTS; e++) { if ((m->tools[e] >> tool_id) & 1) { result |= (1 << e); } @@ -330,12 +336,6 @@ _PyInstruction_GetLength(PyCodeObject *code, int offset) return 1 + _PyOpcode_Caches[inst.op.code]; } -static inline uint8_t -get_original_opcode(_PyCoLineInstrumentationData *line_data, int index) -{ - return line_data->data[index*line_data->bytes_per_entry]; -} - static inline uint8_t * get_original_opcode_ptr(_PyCoLineInstrumentationData *line_data, int index) { @@ -401,7 +401,7 @@ dump_instrumentation_data_lines(PyCodeObject *code, _PyCoLineInstrumentationData fprintf(out, ", lines = NULL"); } else { - int opcode = get_original_opcode(lines, i); + int opcode = _PyCode_GetOriginalOpcode(lines, i); int line_delta = get_line_delta(lines, i); if (opcode == 0) { fprintf(out, ", lines = {original_opcode = No LINE (0), line_delta = %d)", line_delta); @@ -453,7 +453,7 @@ static void dump_local_monitors(const char *prefix, _Py_LocalMonitors monitors, FILE*out) { fprintf(out, "%s monitors:\n", prefix); - for (int event = 0; event < _PY_MONITORING_LOCAL_EVENTS; event++) { + for (int event = 0; event < _PY_MONITORING_UNGROUPED_EVENTS; event++) { fprintf(out, " Event %d: Tools %x\n", event, monitors.tools[event]); } } @@ -571,7 +571,7 @@ sanity_check_instrumentation(PyCodeObject *code) } if (opcode == INSTRUMENTED_LINE) { CHECK(data->lines); - opcode = get_original_opcode(data->lines, i); + opcode = _PyCode_GetOriginalOpcode(data->lines, i); CHECK(valid_opcode(opcode)); CHECK(opcode != END_FOR); CHECK(opcode != RESUME); @@ -588,7 +588,7 @@ sanity_check_instrumentation(PyCodeObject *code) * *and* we are executing a INSTRUMENTED_LINE instruction * that has de-instrumented itself, then we will execute * an invalid INSTRUMENTED_INSTRUCTION */ - CHECK(get_original_opcode(data->lines, i) != INSTRUMENTED_INSTRUCTION); + CHECK(_PyCode_GetOriginalOpcode(data->lines, i) != INSTRUMENTED_INSTRUCTION); } if (opcode == INSTRUMENTED_INSTRUCTION) { CHECK(data->per_instruction_opcodes[i] != 0); @@ -603,7 +603,7 @@ sanity_check_instrumentation(PyCodeObject *code) } CHECK(active_monitors.tools[event] != 0); } - if (data->lines && get_original_opcode(data->lines, i)) { + if (data->lines && _PyCode_GetOriginalOpcode(data->lines, i)) { int line1 = compute_line(code, get_line_delta(data->lines, i)); int line2 = _PyCode_CheckLineNumber(i*sizeof(_Py_CODEUNIT), &range); CHECK(line1 == line2); @@ -655,7 +655,7 @@ _Py_GetBaseCodeUnit(PyCodeObject *code, int i) return inst; } if (opcode == INSTRUMENTED_LINE) { - opcode = get_original_opcode(code->_co_monitoring->lines, i); + opcode = _PyCode_GetOriginalOpcode(code->_co_monitoring->lines, i); } if (opcode == INSTRUMENTED_INSTRUCTION) { opcode = code->_co_monitoring->per_instruction_opcodes[i]; @@ -714,7 +714,7 @@ de_instrument_line(PyCodeObject *code, _Py_CODEUNIT *bytecode, _PyCoMonitoringDa return; } _PyCoLineInstrumentationData *lines = monitoring->lines; - int original_opcode = get_original_opcode(lines, i); + int original_opcode = _PyCode_GetOriginalOpcode(lines, i); if (original_opcode == INSTRUMENTED_INSTRUCTION) { set_original_opcode(lines, i, monitoring->per_instruction_opcodes[i]); } @@ -1102,8 +1102,10 @@ get_tools_for_instruction(PyCodeObject *code, PyInterpreterState *interp, int i, event == PY_MONITORING_EVENT_C_RETURN); event = PY_MONITORING_EVENT_CALL; } + assert(_PY_MONITORING_IS_UNGROUPED_EVENT(event)); + CHECK(debug_check_sanity(interp, code)); if (PY_MONITORING_IS_INSTRUMENTED_EVENT(event)) { - CHECK(debug_check_sanity(interp, code)); + /* Instrumented events use per-instruction tool bitmaps. */ if (code->_co_monitoring->tools) { tools = code->_co_monitoring->tools[i]; } @@ -1112,7 +1114,9 @@ get_tools_for_instruction(PyCodeObject *code, PyInterpreterState *interp, int i, } } else { - tools = interp->monitors.tools[event]; + /* Other (non-instrumented) events are not tied to specific instructions; + * use the code-object-level active_monitors bitmap instead. */ + tools = code->_co_monitoring->active_monitors.tools[event]; } return tools; } @@ -1139,6 +1143,25 @@ static const char *const event_names [] = { [PY_MONITORING_EVENT_STOP_ITERATION] = "STOP_ITERATION", }; +/* Disable an "other" (non-instrumented) event (e.g. PY_UNWIND) for a single + * tool on this code object. Must be called with the world stopped or the + * code lock held. */ +static void +remove_local_tool(PyCodeObject *code, PyInterpreterState *interp, + int event, int tool) +{ + ASSERT_WORLD_STOPPED_OR_LOCKED(code); + assert(_PY_MONITORING_IS_UNGROUPED_EVENT(event)); + assert(!PY_MONITORING_IS_INSTRUMENTED_EVENT(event)); + assert(code->_co_monitoring); + code->_co_monitoring->local_monitors.tools[event] &= ~(1 << tool); + /* Recompute active_monitors for this event as the union of global and + * (now updated) local monitors. */ + code->_co_monitoring->active_monitors.tools[event] = + interp->monitors.tools[event] | + code->_co_monitoring->local_monitors.tools[event]; +} + static int call_instrumentation_vector( _Py_CODEUNIT *instr, PyThreadState *tstate, int event, @@ -1183,7 +1206,18 @@ call_instrumentation_vector( } else { /* DISABLE */ - if (!PY_MONITORING_IS_INSTRUMENTED_EVENT(event)) { + if (PY_MONITORING_IS_INSTRUMENTED_EVENT(event)) { + _PyEval_StopTheWorld(interp); + remove_tools(code, offset, event, 1 << tool); + _PyEval_StartTheWorld(interp); + } + else if (_PY_MONITORING_IS_UNGROUPED_EVENT(event)) { + /* Other (non-instrumented) event: disable for this code object. */ + _PyEval_StopTheWorld(interp); + remove_local_tool(code, interp, event, tool); + _PyEval_StartTheWorld(interp); + } + else { PyErr_Format(PyExc_ValueError, "Cannot disable %s events. Callback removed.", event_names[event]); @@ -1192,12 +1226,6 @@ call_instrumentation_vector( err = -1; break; } - else { - PyInterpreterState *interp = tstate->interp; - _PyEval_StopTheWorld(interp); - remove_tools(code, offset, event, 1 << tool); - _PyEval_StartTheWorld(interp); - } } } Py_DECREF(arg2_obj); @@ -1391,7 +1419,7 @@ _Py_call_instrumentation_line(PyThreadState *tstate, _PyInterpreterFrame* frame, Py_DECREF(line_obj); uint8_t original_opcode; done: - original_opcode = get_original_opcode(line_data, i); + original_opcode = _PyCode_GetOriginalOpcode(line_data, i); assert(original_opcode != 0); assert(original_opcode != INSTRUMENTED_LINE); assert(_PyOpcode_Deopt[original_opcode] == original_opcode); @@ -1464,7 +1492,7 @@ initialize_tools(PyCodeObject *code) int opcode = instr->op.code; assert(opcode != ENTER_EXECUTOR); if (opcode == INSTRUMENTED_LINE) { - opcode = get_original_opcode(code->_co_monitoring->lines, i); + opcode = _PyCode_GetOriginalOpcode(code->_co_monitoring->lines, i); } if (opcode == INSTRUMENTED_INSTRUCTION) { opcode = code->_co_monitoring->per_instruction_opcodes[i]; @@ -1681,7 +1709,7 @@ update_instrumentation_data(PyCodeObject *code, PyInterpreterState *interp) _Py_LocalMonitors *local_monitors = &code->_co_monitoring->local_monitors; for (int i = 0; i < PY_MONITORING_TOOL_IDS; i++) { if (code->_co_monitoring->tool_versions[i] != interp->monitoring_tool_versions[i]) { - for (int j = 0; j < _PY_MONITORING_LOCAL_EVENTS; j++) { + for (int j = 0; j < _PY_MONITORING_UNGROUPED_EVENTS; j++) { local_monitors->tools[j] &= ~(1 << i); } } @@ -1849,7 +1877,7 @@ force_instrument_lock_held(PyCodeObject *code, PyInterpreterState *interp) if (removed_line_tools) { _PyCoLineInstrumentationData *line_data = code->_co_monitoring->lines; for (int i = code->_co_firsttraceable; i < code_len;) { - if (get_original_opcode(line_data, i)) { + if (_PyCode_GetOriginalOpcode(line_data, i)) { remove_line_tools(code, i, removed_line_tools); } i += _PyInstruction_GetLength(code, i); @@ -1876,7 +1904,7 @@ force_instrument_lock_held(PyCodeObject *code, PyInterpreterState *interp) if (new_line_tools) { _PyCoLineInstrumentationData *line_data = code->_co_monitoring->lines; for (int i = code->_co_firsttraceable; i < code_len;) { - if (get_original_opcode(line_data, i)) { + if (_PyCode_GetOriginalOpcode(line_data, i)) { add_line_tools(code, i, new_line_tools); } i += _PyInstruction_GetLength(code, i); @@ -1977,7 +2005,7 @@ static void set_local_events(_Py_LocalMonitors *m, int tool_id, _PyMonitoringEventSet events) { assert(0 <= tool_id && tool_id < PY_MONITORING_TOOL_IDS); - for (int e = 0; e < _PY_MONITORING_LOCAL_EVENTS; e++) { + for (int e = 0; e < _PY_MONITORING_UNGROUPED_EVENTS; e++) { uint8_t *tools = &m->tools[e]; int val = (events >> e) & 1; *tools &= ~(1 << tool_id); @@ -2037,7 +2065,7 @@ _PyMonitoring_SetLocalEvents(PyCodeObject *code, int tool_id, _PyMonitoringEvent assert(0 <= tool_id && tool_id < PY_MONITORING_TOOL_IDS); PyInterpreterState *interp = _PyInterpreterState_GET(); - assert(events < (1 << _PY_MONITORING_LOCAL_EVENTS)); + assert(events < (1 << _PY_MONITORING_UNGROUPED_EVENTS)); if (code->_co_firsttraceable >= Py_SIZE(code)) { PyErr_Format(PyExc_SystemError, "cannot instrument shim code object '%U'", code->co_name); return -1; @@ -2373,7 +2401,7 @@ monitoring_get_local_events_impl(PyObject *module, int tool_id, _PyMonitoringEventSet event_set = 0; _PyCoMonitoringData *data = ((PyCodeObject *)code)->_co_monitoring; if (data != NULL) { - for (int e = 0; e < _PY_MONITORING_LOCAL_EVENTS; e++) { + for (int e = 0; e < _PY_MONITORING_UNGROUPED_EVENTS; e++) { if ((data->local_monitors.tools[e] >> tool_id) & 1) { event_set |= (1 << e); } @@ -2416,7 +2444,7 @@ monitoring_set_local_events_impl(PyObject *module, int tool_id, event_set &= ~(1 << PY_MONITORING_EVENT_BRANCH); event_set |= (1 << PY_MONITORING_EVENT_BRANCH_RIGHT) | (1 << PY_MONITORING_EVENT_BRANCH_LEFT); } - if (event_set < 0 || event_set >= (1 << _PY_MONITORING_LOCAL_EVENTS)) { + if (event_set < 0 || event_set >= (1 << _PY_MONITORING_UNGROUPED_EVENTS)) { PyErr_Format(PyExc_ValueError, "invalid local event set 0x%x", event_set); return NULL; } diff --git a/Python/jit.c b/Python/jit.c index af75acf1ff2bb3..26e01b25d48c04 100644 --- a/Python/jit.c +++ b/Python/jit.c @@ -60,8 +60,6 @@ jit_error(const char *message) PyErr_Format(PyExc_RuntimeWarning, "JIT %s (%d)", message, hint); } -static size_t _Py_jit_shim_size = 0; - static int address_in_executor_array(_PyExecutorObject **ptrs, size_t count, uintptr_t addr) { @@ -104,13 +102,6 @@ _PyJIT_AddressInJitCode(PyInterpreterState *interp, uintptr_t addr) if (interp == NULL) { return 0; } - if (_Py_jit_entry != _Py_LazyJitShim && _Py_jit_shim_size != 0) { - uintptr_t start = (uintptr_t)_Py_jit_entry; - uintptr_t end = start + _Py_jit_shim_size; - if (addr >= start && addr < end) { - return 1; - } - } if (address_in_executor_array(interp->executor_ptrs, interp->executor_count, addr)) { return 1; } @@ -727,75 +718,6 @@ _PyJIT_Compile(_PyExecutorObject *executor, const _PyUOpInstruction trace[], siz return 0; } -/* One-off compilation of the jit entry shim - * We compile this once only as it effectively a normal - * function, but we need to use the JIT because it needs - * to understand the jit-specific calling convention. - * Don't forget to call _PyJIT_Fini later! - */ -static _PyJitEntryFuncPtr -compile_shim(void) -{ - _PyExecutorObject dummy; - const StencilGroup *group; - size_t code_size = 0; - size_t data_size = 0; - jit_state state = {0}; - group = &shim; - code_size += group->code_size; - data_size += group->data_size; - combine_symbol_mask(group->trampoline_mask, state.trampolines.mask); - combine_symbol_mask(group->got_mask, state.got_symbols.mask); - // Round up to the nearest page: - size_t page_size = get_page_size(); - assert((page_size & (page_size - 1)) == 0); - size_t code_padding = DATA_ALIGN - ((code_size + state.trampolines.size) & (DATA_ALIGN - 1)); - size_t padding = page_size - ((code_size + state.trampolines.size + code_padding + data_size + state.got_symbols.size) & (page_size - 1)); - size_t total_size = code_size + state.trampolines.size + code_padding + data_size + state.got_symbols.size + padding; - unsigned char *memory = jit_alloc(total_size); - if (memory == NULL) { - return NULL; - } - unsigned char *code = memory; - state.trampolines.mem = memory + code_size; - unsigned char *data = memory + code_size + state.trampolines.size + code_padding; - state.got_symbols.mem = data + data_size; - // Compile the shim, which handles converting between the native - // calling convention and the calling convention used by jitted code - // (which may be different for efficiency reasons). - group = &shim; - group->emit(code, data, &dummy, NULL, &state); - code += group->code_size; - data += group->data_size; - assert(code == memory + code_size); - assert(data == memory + code_size + state.trampolines.size + code_padding + data_size); - if (mark_executable(memory, total_size)) { - jit_free(memory, total_size); - return NULL; - } - _Py_jit_shim_size = total_size; - return (_PyJitEntryFuncPtr)memory; -} - -static PyMutex lazy_jit_mutex = { 0 }; - -_Py_CODEUNIT * -_Py_LazyJitShim( - _PyExecutorObject *executor, _PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate -) { - PyMutex_Lock(&lazy_jit_mutex); - if (_Py_jit_entry == _Py_LazyJitShim) { - _PyJitEntryFuncPtr shim = compile_shim(); - if (shim == NULL) { - PyMutex_Unlock(&lazy_jit_mutex); - Py_FatalError("Cannot allocate core JIT code"); - } - _Py_jit_entry = shim; - } - PyMutex_Unlock(&lazy_jit_mutex); - return _Py_jit_entry(executor, frame, stack_pointer, tstate); -} - // Free executor's memory allocated with _PyJIT_Compile void _PyJIT_Free(_PyExecutorObject *executor) @@ -812,22 +734,4 @@ _PyJIT_Free(_PyExecutorObject *executor) } } -// Free shim memory allocated with compile_shim -void -_PyJIT_Fini(void) -{ - PyMutex_Lock(&lazy_jit_mutex); - unsigned char *memory = (unsigned char *)_Py_jit_entry; - size_t size = _Py_jit_shim_size; - if (size) { - _Py_jit_entry = _Py_LazyJitShim; - _Py_jit_shim_size = 0; - if (jit_free(memory, size)) { - PyErr_FormatUnraisable("Exception ignored while " - "freeing JIT entry code"); - } - } - PyMutex_Unlock(&lazy_jit_mutex); -} - #endif // _Py_JIT diff --git a/Python/lock.c b/Python/lock.c index 752a5899e088a5..af136fefd299d3 100644 --- a/Python/lock.c +++ b/Python/lock.c @@ -248,7 +248,16 @@ _PyRawMutex_LockSlow(_PyRawMutex *m) // Wait for us to be woken up. Note that we still have to lock the // mutex ourselves: it is NOT handed off to us. - _PySemaphore_Wait(&waiter.sema, -1); + // + // Loop until we observe an actual wakeup. A return of Py_PARK_INTR + // could otherwise let us exit _PySemaphore_Wait and destroy + // `waiter.sema` while _PyRawMutex_UnlockSlow's matching + // _PySemaphore_Wakeup is still pending, since the unlocker has + // already CAS-removed us from the waiter list without any handshake. + int res; + do { + res = _PySemaphore_Wait(&waiter.sema, -1); + } while (res != Py_PARK_OK); } _PySemaphore_Destroy(&waiter.sema); diff --git a/Python/marshal.c b/Python/marshal.c index b60a36e128cd9f..dace22da0d4a94 100644 --- a/Python/marshal.c +++ b/Python/marshal.c @@ -382,7 +382,6 @@ static int w_ref(PyObject *v, char *flag, WFILE *p) { _Py_hashtable_entry_t *entry; - int w; if (p->version < 3 || p->hashtable == NULL) return 0; /* not writing object references */ @@ -399,20 +398,28 @@ w_ref(PyObject *v, char *flag, WFILE *p) entry = _Py_hashtable_get_entry(p->hashtable, v); if (entry != NULL) { /* write the reference index to the stream */ - w = (int)(uintptr_t)entry->value; + uintptr_t w = (uintptr_t)entry->value; + if (w & 0x80000000LU) { + PyErr_Format(PyExc_ValueError, "cannot marshal recursion %T objects", v); + goto err; + } /* we don't store "long" indices in the dict */ - assert(0 <= w && w <= 0x7fffffff); + assert(w <= 0x7fffffff); w_byte(TYPE_REF, p); - w_long(w, p); + w_long((int)w, p); return 1; } else { - size_t s = p->hashtable->nentries; + size_t w = p->hashtable->nentries; /* we don't support long indices */ - if (s >= 0x7fffffff) { + if (w >= 0x7fffffff) { PyErr_SetString(PyExc_ValueError, "too many objects"); goto err; } - w = (int)s; + // Corresponding code should call w_complete() after + // writing the object. + if (PyCode_Check(v) || PySlice_Check(v) || PyFrozenDict_CheckExact(v)) { + w |= 0x80000000LU; + } if (_Py_hashtable_set(p->hashtable, Py_NewRef(v), (void *)(uintptr_t)w) < 0) { Py_DECREF(v); @@ -426,6 +433,27 @@ w_ref(PyObject *v, char *flag, WFILE *p) return 1; } +static void +w_complete(PyObject *v, WFILE *p) +{ + if (p->version < 3 || p->hashtable == NULL) { + return; + } + if (_PyObject_IsUniquelyReferenced(v)) { + return; + } + + _Py_hashtable_entry_t *entry = _Py_hashtable_get_entry(p->hashtable, v); + if (entry == NULL) { + return; + } + assert(entry != NULL); + uintptr_t w = (uintptr_t)entry->value; + assert(w & 0x80000000LU); + w &= ~0x80000000LU; + entry->value = (void *)(uintptr_t)w; +} + static void w_complex_object(PyObject *v, char flag, WFILE *p); @@ -599,6 +627,9 @@ w_complex_object(PyObject *v, char flag, WFILE *p) w_object(value, p); } w_object((PyObject *)NULL, p); + if (PyFrozenDict_CheckExact(v)) { + w_complete(v, p); + } } else if (PyAnySet_CheckExact(v)) { PyObject *value; @@ -684,6 +715,7 @@ w_complex_object(PyObject *v, char flag, WFILE *p) w_object(co->co_linetable, p); w_object(co->co_exceptiontable, p); Py_DECREF(co_code); + w_complete(v, p); } else if (PyObject_CheckBuffer(v)) { /* Write unknown bytes-like objects as a bytes object */ @@ -709,6 +741,7 @@ w_complex_object(PyObject *v, char flag, WFILE *p) w_object(slice->start, p); w_object(slice->stop, p); w_object(slice->step, p); + w_complete(v, p); } else { W_TYPE(TYPE_UNKNOWN, p); @@ -1433,9 +1466,19 @@ r_object(RFILE *p) case TYPE_DICT: case TYPE_FROZENDICT: v = PyDict_New(); - R_REF(v); - if (v == NULL) + if (v == NULL) { break; + } + if (type == TYPE_DICT) { + R_REF(v); + } + else { + idx = r_ref_reserve(flag, p); + if (idx < 0) { + Py_CLEAR(v); + break; + } + } for (;;) { PyObject *key, *val; key = r_object(p); @@ -1458,13 +1501,7 @@ r_object(RFILE *p) Py_CLEAR(v); } if (type == TYPE_FROZENDICT && v != NULL) { - PyObject *frozendict = PyFrozenDict_New(v); - if (frozendict != NULL) { - Py_SETREF(v, frozendict); - } - else { - Py_CLEAR(v); - } + Py_SETREF(v, PyFrozenDict_New(v)); } retval = v; break; diff --git a/Python/optimizer.c b/Python/optimizer.c index 60f3e541be25cf..a389c0f4072817 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -551,8 +551,6 @@ dynamic_exit_uop[MAX_UOP_ID + 1] = { }; -#define CONFIDENCE_RANGE 1000 -#define CONFIDENCE_CUTOFF 333 #ifdef Py_DEBUG #define DPRINTF(level, ...) \ @@ -600,6 +598,54 @@ add_to_trace( ((uint32_t)((INSTR) - ((_Py_CODEUNIT *)(CODE)->co_code_adaptive))) +/* Branch penalty: 0 for a fully biased branch and FITNESS_BRANCH_BALANCED for + * a balanced or fully off-trace branch. This keeps any single branch from + * consuming more than one balanced-branch cost. + */ +static inline int +compute_branch_penalty(uint16_t history) +{ + bool branch_taken = history & 1; + int taken_count = _Py_popcount32((uint32_t)history); + int on_trace_count = branch_taken ? taken_count : 16 - taken_count; + int off_trace = 16 - on_trace_count; + int penalty = off_trace * FITNESS_BRANCH_BALANCED / 8; + if (penalty > FITNESS_BRANCH_BALANCED) { + penalty = FITNESS_BRANCH_BALANCED; + } + return penalty; +} + +/* Compute exit quality for the current trace position. + * Higher values mean better places to stop the trace. */ +static inline int32_t +compute_exit_quality(_Py_CODEUNIT *target_instr, int opcode, + const _PyJitTracerState *tracer) +{ + if (target_instr == tracer->initial_state.close_loop_instr) { + return EXIT_QUALITY_CLOSE_LOOP; + } + else if (target_instr->op.code == ENTER_EXECUTOR) { + return EXIT_QUALITY_ENTER_EXECUTOR; + } + else if (opcode == JUMP_BACKWARD_JIT || + opcode == JUMP_BACKWARD || + opcode == JUMP_BACKWARD_NO_INTERRUPT) { + return EXIT_QUALITY_BACKWARD_EDGE; + } + else if (_PyOpcode_Caches[_PyOpcode_Deopt[opcode]] > 0) { + return EXIT_QUALITY_SPECIALIZABLE; + } + return EXIT_QUALITY_DEFAULT; +} + +/* Frame penalty: (MAX_ABSTRACT_FRAME_DEPTH-1) pushes exhaust fitness. */ +static inline int32_t +compute_frame_penalty(uint16_t fitness_initial) +{ + return (int32_t)fitness_initial / (MAX_ABSTRACT_FRAME_DEPTH - 1) + 1; +} + static int is_terminator(const _PyUOpInstruction *uop) { @@ -736,13 +782,11 @@ _PyJit_translate_single_bytecode_to_trace( DPRINTF(2, "Unsupported: oparg too large\n"); unsupported: { - // Rewind to previous instruction and replace with _EXIT_TRACE. _PyUOpInstruction *curr = uop_buffer_last(trace); while (curr->opcode != _SET_IP && uop_buffer_length(trace) > 2) { trace->next--; curr = uop_buffer_last(trace); } - assert(curr->opcode == _SET_IP || uop_buffer_length(trace) == 2); if (curr->opcode == _SET_IP) { int32_t old_target = (int32_t)uop_get_target(curr); curr->opcode = _DEOPT; @@ -765,11 +809,28 @@ _PyJit_translate_single_bytecode_to_trace( return 1; } + // Stop the trace if fitness has dropped below the exit quality threshold. + _PyJitTracerTranslatorState *ts = &tracer->translator_state; + int32_t eq = compute_exit_quality(target_instr, opcode, tracer); + DPRINTF(3, "Fitness check: %s(%d) fitness=%d, exit_quality=%d, depth=%d\n", + _PyOpcode_OpName[opcode], oparg, ts->fitness, eq, ts->frame_depth); + + if (ts->fitness < eq) { + // Heuristic exit: leave operand1=0 so the side exit increments chain_depth. + ADD_TO_TRACE(_EXIT_TRACE, 0, 0, target); + OPT_STAT_INC(fitness_terminated_traces); + DPRINTF(2, "Fitness terminated: %s(%d) fitness=%d < exit_quality=%d\n", + _PyOpcode_OpName[opcode], oparg, ts->fitness, eq); + goto done; + } + + // Snapshot remaining space so the later fitness charge reflects all buffer + // space this bytecode consumed, including reserved tail slots. + int32_t remaining_before = uop_buffer_remaining_space(trace); + // One for possible _DEOPT, one because _CHECK_VALIDITY itself might _DEOPT trace->end -= 2; - const struct opcode_macro_expansion *expansion = &_PyOpcode_macro_expansion[opcode]; - assert(opcode != ENTER_EXECUTOR && opcode != EXTENDED_ARG); assert(!_PyErr_Occurred(tstate)); @@ -790,13 +851,11 @@ _PyJit_translate_single_bytecode_to_trace( // _GUARD_IP leads to an exit. trace->end -= needs_guard_ip; +#if Py_DEBUG + const struct opcode_macro_expansion *expansion = &_PyOpcode_macro_expansion[opcode]; int space_needed = expansion->nuops + needs_guard_ip + 2 + (!OPCODE_HAS_NO_SAVE_IP(opcode)); - if (uop_buffer_remaining_space(trace) < space_needed) { - DPRINTF(2, "No room for expansions and guards (need %d, got %d)\n", - space_needed, uop_buffer_remaining_space(trace)); - OPT_STAT_INC(trace_too_long); - goto done; - } + assert(uop_buffer_remaining_space(trace) > space_needed); +#endif ADD_TO_TRACE(_CHECK_VALIDITY, 0, 0, target); @@ -818,6 +877,12 @@ _PyJit_translate_single_bytecode_to_trace( assert(jump_happened ? (next_instr == computed_jump_instr) : (next_instr == computed_next_instr)); uint32_t uopcode = BRANCH_TO_GUARD[opcode - POP_JUMP_IF_FALSE][jump_happened]; ADD_TO_TRACE(uopcode, 0, 0, INSTR_IP(jump_happened ? computed_next_instr : computed_jump_instr, old_code)); + int bp = compute_branch_penalty(target_instr[1].cache); + tracer->translator_state.fitness -= bp; + DPRINTF(3, " branch penalty: -%d (history=0x%04x, taken=%d) -> fitness=%d\n", + bp, target_instr[1].cache, jump_happened, + tracer->translator_state.fitness); + break; } case JUMP_BACKWARD_JIT: @@ -825,29 +890,9 @@ _PyJit_translate_single_bytecode_to_trace( case JUMP_BACKWARD_NO_JIT: case JUMP_BACKWARD: ADD_TO_TRACE(_CHECK_PERIODIC, 0, 0, target); - _Py_FALLTHROUGH; + break; case JUMP_BACKWARD_NO_INTERRUPT: - { - if ((next_instr != tracer->initial_state.close_loop_instr) && - (next_instr != tracer->initial_state.start_instr) && - uop_buffer_length(&tracer->code_buffer) > CODE_SIZE_NO_PROGRESS && - // For side exits, we don't want to terminate them early. - tracer->initial_state.exit == NULL && - // These are coroutines, and we want to unroll those usually. - opcode != JUMP_BACKWARD_NO_INTERRUPT) { - // We encountered a JUMP_BACKWARD but not to the top of our own loop. - // We don't want to continue tracing as we might get stuck in the - // inner loop. Instead, end the trace where the executor of the - // inner loop might start and let the traces rejoin. - OPT_STAT_INC(inner_loop); - ADD_TO_TRACE(_EXIT_TRACE, 0, 0, target); - uop_buffer_last(trace)->operand1 = true; // is_control_flow - DPRINTF(2, "JUMP_BACKWARD not to top ends trace %p %p %p\n", next_instr, - tracer->initial_state.close_loop_instr, tracer->initial_state.start_instr); - goto done; - } break; - } case RESUME: case RESUME_CHECK: @@ -948,6 +993,39 @@ _PyJit_translate_single_bytecode_to_trace( assert(next->op.code == STORE_FAST); operand = next->op.arg; } + else if (uop == _PUSH_FRAME) { + _PyJitTracerTranslatorState *ts_depth = &tracer->translator_state; + ts_depth->frame_depth++; + assert(ts_depth->frame_depth < MAX_ABSTRACT_FRAME_DEPTH); + int32_t frame_penalty = compute_frame_penalty(tstate->interp->opt_config.fitness_initial); + ts_depth->fitness -= frame_penalty; + DPRINTF(3, " _PUSH_FRAME: depth=%d, penalty=-%d -> fitness=%d\n", + ts_depth->frame_depth, frame_penalty, + ts_depth->fitness); + } + else if (uop == _RETURN_VALUE || uop == _RETURN_GENERATOR || uop == _YIELD_VALUE) { + _PyJitTracerTranslatorState *ts_depth = &tracer->translator_state; + int32_t frame_penalty = compute_frame_penalty(tstate->interp->opt_config.fitness_initial); + if (ts_depth->frame_depth <= 0) { + // Returning past the traced root is normal for guarded + // caller continuation. Charge a small penalty so these + // paths still terminate. + int32_t underflow_penalty = frame_penalty / 4; + ts_depth->fitness -= underflow_penalty; + DPRINTF(3, " %s: underflow penalty=-%d -> fitness=%d\n", + _PyOpcode_uop_name[uop], underflow_penalty, + ts_depth->fitness); + } + else { + // Symmetric with push: net-zero frame impact. + ts_depth->fitness += frame_penalty; + ts_depth->frame_depth--; + DPRINTF(3, " %s: return reward=+%d, depth=%d -> fitness=%d\n", + _PyOpcode_uop_name[uop], frame_penalty, + ts_depth->frame_depth, + ts_depth->fitness); + } + } else if (_PyUop_Flags[uop] & HAS_RECORDS_VALUE_FLAG) { PyObject *recorded_value = tracer->prev_state.recorded_values[record_idx]; tracer->prev_state.recorded_values[record_idx] = NULL; @@ -990,13 +1068,20 @@ _PyJit_translate_single_bytecode_to_trace( ADD_TO_TRACE(_JUMP_TO_TOP, 0, 0, 0); goto done; } - DPRINTF(2, "Trace continuing\n"); + // Charge fitness by trace-buffer capacity consumed for this bytecode, + // including both emitted uops and tail reservations. + { + int32_t slots_used = remaining_before - uop_buffer_remaining_space(trace); + tracer->translator_state.fitness -= slots_used; + DPRINTF(3, " per-insn cost: -%d -> fitness=%d\n", slots_used, + tracer->translator_state.fitness); + } + DPRINTF(2, "Trace continuing (fitness=%d)\n", tracer->translator_state.fitness); return 1; done: DPRINTF(2, "Trace done\n"); if (!is_terminator(uop_buffer_last(trace))) { ADD_TO_TRACE(_EXIT_TRACE, 0, 0, target); - uop_buffer_last(trace)->operand1 = true; // is_control_flow } return 0; } @@ -1077,6 +1162,13 @@ _PyJit_TryInitializeTracing( assert(curr_instr->op.code == JUMP_BACKWARD_JIT || curr_instr->op.code == RESUME_CHECK_JIT || (exit != NULL)); tracer->initial_state.jump_backward_instr = curr_instr; + const _PyOptimizationConfig *cfg = &tstate->interp->opt_config; + _PyJitTracerTranslatorState *ts = &tracer->translator_state; + ts->fitness = cfg->fitness_initial; + ts->frame_depth = 0; + DPRINTF(3, "Fitness init: chain_depth=%d, fitness=%d\n", + chain_depth, ts->fitness); + tracer->is_tracing = true; return 1; } diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index 6e4882143fbe50..daebef4a04320b 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -300,7 +300,7 @@ dummy_func(void) { // narrowing unlocks a meaningful downstream win: // - NB_TRUE_DIVIDE: enables the specialized float path below. // - NB_REMAINDER: lets the float result type propagate. - // NB_POWER is excluded — speculative guards there regressed + // NB_POWER is excluded: speculative guards there regressed // test_power_type_depends_on_input_values (GH-127844). if (is_truediv || is_remainder) { if (!sym_has_type(rhs) diff --git a/Python/parking_lot.c b/Python/parking_lot.c index 99c1ad848be795..8823d77719cb9a 100644 --- a/Python/parking_lot.c +++ b/Python/parking_lot.c @@ -61,7 +61,9 @@ _PySemaphore_Init(_PySemaphore *sema) NULL // unnamed ); if (!sema->platform_sem) { - Py_FatalError("parking_lot: CreateSemaphore failed"); + _Py_FatalErrorFormat(__func__, + "parking_lot: CreateSemaphore failed (error: %u)", + GetLastError()); } #elif defined(_Py_USE_SEMAPHORES) if (sem_init(&sema->platform_sem, /*pshared=*/0, /*value=*/0) < 0) { @@ -141,8 +143,8 @@ _PySemaphore_Wait(_PySemaphore *sema, PyTime_t timeout) } else { _Py_FatalErrorFormat(__func__, - "unexpected error from semaphore: %u (error: %u)", - wait, GetLastError()); + "unexpected error from semaphore: %u (error: %u, handle: %p)", + wait, GetLastError(), sema->platform_sem); } #elif defined(_Py_USE_SEMAPHORES) int err; @@ -230,7 +232,9 @@ _PySemaphore_Wakeup(_PySemaphore *sema) { #if defined(MS_WINDOWS) if (!ReleaseSemaphore(sema->platform_sem, 1, NULL)) { - Py_FatalError("parking_lot: ReleaseSemaphore failed"); + _Py_FatalErrorFormat(__func__, + "parking_lot: ReleaseSemaphore failed (error: %u, handle: %p)", + GetLastError(), sema->platform_sem); } #elif defined(_Py_USE_SEMAPHORES) int err = sem_post(&sema->platform_sem); diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index d9fc28475a489d..0a88e32bb6b65e 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -37,9 +37,6 @@ #include "pycore_uniqueid.h" // _PyObject_FinalizeUniqueIdPool() #include "pycore_warnings.h" // _PyWarnings_InitState() #include "pycore_weakref.h" // _PyWeakref_GET_REF() -#ifdef _Py_JIT -#include "pycore_jit.h" // _PyJIT_Fini() -#endif #if defined(PYMALLOC_USE_HUGEPAGES) && defined(MS_WINDOWS) #include @@ -1641,18 +1638,12 @@ Py_InitializeFromConfig(const PyConfig *config) void Py_InitializeEx(int install_sigs) { - PyStatus status; - - status = _PyRuntime_Initialize(); - if (_PyStatus_EXCEPTION(status)) { - Py_ExitStatusException(status); - } - if (Py_IsInitialized()) { /* bpo-33932: Calling Py_Initialize() twice does nothing. */ return; } + PyStatus status; PyConfig config; _PyConfig_InitCompatConfig(&config); @@ -2537,11 +2528,6 @@ _Py_Finalize(_PyRuntimeState *runtime) finalize_interp_clear(tstate); -#ifdef _Py_JIT - /* Free JIT shim memory */ - _PyJIT_Fini(); -#endif - #ifdef Py_TRACE_REFS /* Display addresses (& refcnts) of all objects still alive. * An address can be used to find the repr of the object, printed diff --git a/Python/pystate.c b/Python/pystate.c index d6a26f3339b863..2df24597e65785 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -489,11 +489,6 @@ free_interpreter(PyInterpreterState *interp) static inline int check_interpreter_whence(long); #endif -extern _Py_CODEUNIT * -_Py_LazyJitShim( - struct _PyExecutorObject *exec, _PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate -); - /* Get the interpreter state to a minimal consistent state. Further init happens in pylifecycle.c before it can be used. All fields not initialized here are expected to be zeroed out, @@ -635,6 +630,11 @@ init_interpreter(PyInterpreterState *interp, "PYTHON_JIT_SIDE_EXIT_INITIAL_BACKOFF", SIDE_EXIT_INITIAL_BACKOFF, 0, MAX_BACKOFF); + // Trace fitness configuration + init_policy(&interp->opt_config.fitness_initial, + "PYTHON_JIT_FITNESS_INITIAL", + FITNESS_INITIAL, EXIT_QUALITY_CLOSE_LOOP, UOP_MAX_TRACE_LENGTH - 1); + interp->opt_config.specialization_enabled = !is_env_enabled("PYTHON_SPECIALIZATION_OFF"); interp->opt_config.uops_optimize_enabled = !is_env_disabled("PYTHON_UOPS_OPTIMIZE"); if (interp != &runtime->_main_interpreter) { diff --git a/Python/pystats.c b/Python/pystats.c index a057ad884566d8..2fac2db1b738c7 100644 --- a/Python/pystats.c +++ b/Python/pystats.c @@ -274,6 +274,7 @@ print_optimization_stats(FILE *out, OptimizationStats *stats) fprintf(out, "Optimization low confidence: %" PRIu64 "\n", stats->low_confidence); fprintf(out, "Optimization unknown callee: %" PRIu64 "\n", stats->unknown_callee); fprintf(out, "Executors invalidated: %" PRIu64 "\n", stats->executors_invalidated); + fprintf(out, "Optimization fitness terminated: %" PRIu64 "\n", stats->fitness_terminated_traces); print_histogram(out, "Trace length", stats->trace_length_hist); print_histogram(out, "Trace run length", stats->trace_run_length_hist); diff --git a/Tools/build/generate_levenshtein_examples.py b/Tools/build/generate_levenshtein_examples.py index 30dcc7cf1a1479..2396c8040ca539 100644 --- a/Tools/build/generate_levenshtein_examples.py +++ b/Tools/build/generate_levenshtein_examples.py @@ -13,7 +13,7 @@ _CASE_COST = 1 -def _substitution_cost(ch_a, ch_b): +def _substitution_cost(ch_a: str, ch_b: str) -> int: if ch_a == ch_b: return 0 if ch_a.lower() == ch_b.lower(): @@ -22,7 +22,7 @@ def _substitution_cost(ch_a, ch_b): @lru_cache(None) -def levenshtein(a, b): +def levenshtein(a: str, b: str) -> int: if not a or not b: return (len(a) + len(b)) * _MOVE_COST option1 = levenshtein(a[:-1], b[:-1]) + _substitution_cost(a[-1], b[-1]) @@ -31,7 +31,7 @@ def levenshtein(a, b): return min(option1, option2, option3) -def main(): +def main() -> None: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('output_path', metavar='FILE', type=str) parser.add_argument('--overwrite', dest='overwrite', action='store_const', @@ -48,7 +48,7 @@ def main(): ) return - examples = set() + examples: set[tuple[str, str, int]] = set() # Create a lot of non-empty examples, which should end up with a Gauss-like # distribution for even costs (moves) and odd costs (case substitutions). while len(examples) < 9990: diff --git a/Tools/build/mypy.ini b/Tools/build/mypy.ini index 7d341afd1cd48b..5465e2d4b6171f 100644 --- a/Tools/build/mypy.ini +++ b/Tools/build/mypy.ini @@ -9,6 +9,7 @@ files = Tools/build/consts_getter.py, Tools/build/deepfreeze.py, Tools/build/generate-build-details.py, + Tools/build/generate_levenshtein_examples.py, Tools/build/generate_sbom.py, Tools/build/generate_stdlib_module_names.py, Tools/build/verify_ensurepip_wheels.py, diff --git a/Tools/c-analyzer/cpython/globals-to-fix.tsv b/Tools/c-analyzer/cpython/globals-to-fix.tsv index 74ca562824012b..db575d870be5c5 100644 --- a/Tools/c-analyzer/cpython/globals-to-fix.tsv +++ b/Tools/c-analyzer/cpython/globals-to-fix.tsv @@ -83,6 +83,7 @@ Objects/picklebufobject.c - PyPickleBuffer_Type - Objects/rangeobject.c - PyLongRangeIter_Type - Objects/rangeobject.c - PyRangeIter_Type - Objects/rangeobject.c - PyRange_Type - +Objects/sentinelobject.c - PySentinel_Type - Objects/setobject.c - PyFrozenSet_Type - Objects/setobject.c - PySetIter_Type - Objects/setobject.c - PySet_Type - diff --git a/Tools/jit/README.md b/Tools/jit/README.md index 8eadb3349ba6da..9361f39dcc64f2 100644 --- a/Tools/jit/README.md +++ b/Tools/jit/README.md @@ -9,7 +9,12 @@ Python 3.11 or newer is required to build the JIT. The JIT compiler does not require end users to install any third-party dependencies, but part of it must be *built* using LLVM[^why-llvm]. You are *not* required to build the rest of CPython using LLVM, or even the same version of LLVM (in fact, this is uncommon). -LLVM version 21 is the officially supported version. You can modify if needed using the `LLVM_VERSION` env var during configure. Both `clang` and `llvm-readobj` need to be installed and discoverable (version suffixes, like `clang-19`, are okay). It's highly recommended that you also have `llvm-objdump` available, since this allows the build script to dump human-readable assembly for the generated code. +LLVM version 21 is the officially supported version. Both `clang` and `llvm-readobj` need to be installed and discoverable (version suffixes, like `clang-21`, are okay). It's highly recommended that you also have `llvm-objdump` available, since this allows the build script to dump human-readable assembly for the generated code. + +You can customize the LLVM configuration using environment variables before running configure: + +- LLVM_VERSION: Specify a different LLVM version (default: 21) +- LLVM_TOOLS_INSTALL_DIR: Point to a specific LLVM installation prefix when multiple installations exist (the tools are expected in `

/bin`) It's easy to install all of the required tools: @@ -62,7 +67,7 @@ choco install llvm --version=21.1.0 ### Dev Containers -If you are working on CPython in a [Codespaces instance](https://devguide.python.org/getting-started/setup-building/#using-codespaces), there's no +If you are working on CPython in a [Codespaces instance](https://devguide.python.org/getting-started/setup-building/#using-codespaces), there's no need to install LLVM as the Fedora 43 base image includes LLVM 21 out of the box. ## Building @@ -81,3 +86,8 @@ If you're looking for information on how to update the JIT build dependencies, s [^pep-744]: [PEP 744](https://peps.python.org/pep-0744/) [^why-llvm]: Clang is specifically needed because it's the only C compiler with support for guaranteed tail calls (`musttail`), which are required by CPython's continuation-passing-style approach to JIT compilation. Since LLVM also includes other functionalities we need (namely, object file parsing and disassembly), it's convenient to only support one toolchain at this time. + +### Understanding JIT behavior + +The [example_trace_dump.py](./example_trace_dump.py) script will (when configured as described in the script) dump out the +executors for a range of tiny programs to show the behavior of the JIT front-end. \ No newline at end of file diff --git a/Tools/jit/_llvm.py b/Tools/jit/_llvm.py index a4aaacdf41249d..601752bf1f6396 100644 --- a/Tools/jit/_llvm.py +++ b/Tools/jit/_llvm.py @@ -80,7 +80,18 @@ async def _get_brew_llvm_prefix(llvm_version: str, *, echo: bool = False) -> str @_async_cache -async def _find_tool(tool: str, llvm_version: str, *, echo: bool = False) -> str | None: +async def _find_tool( + tool: str, + llvm_version: str, + llvm_tools_install_dir: str | None, + *, + echo: bool = False, +) -> str | None: + # Explicitly defined LLVM installation location + if llvm_tools_install_dir: + path = os.path.join(llvm_tools_install_dir, "bin", tool) + if await _check_tool_version(path, llvm_version, echo=echo): + return path # Unversioned executables: path = tool if await _check_tool_version(path, llvm_version, echo=echo): @@ -114,10 +125,11 @@ async def maybe_run( args: typing.Iterable[str], echo: bool = False, llvm_version: str = _LLVM_VERSION, + llvm_tools_install_dir: str | None = None, ) -> str | None: """Run an LLVM tool if it can be found. Otherwise, return None.""" - path = await _find_tool(tool, llvm_version, echo=echo) + path = await _find_tool(tool, llvm_version, llvm_tools_install_dir, echo=echo) return path and await _run(path, args, echo=echo) @@ -126,10 +138,17 @@ async def run( args: typing.Iterable[str], echo: bool = False, llvm_version: str = _LLVM_VERSION, + llvm_tools_install_dir: str | None = None, ) -> str: """Run an LLVM tool if it can be found. Otherwise, raise RuntimeError.""" - output = await maybe_run(tool, args, echo=echo, llvm_version=llvm_version) + output = await maybe_run( + tool, + args, + echo=echo, + llvm_version=llvm_version, + llvm_tools_install_dir=llvm_tools_install_dir, + ) if output is None: raise RuntimeError(f"Can't find {tool}-{llvm_version}!") return output diff --git a/Tools/jit/_targets.py b/Tools/jit/_targets.py index fd5c143b8a812f..15cac3de3fe11f 100644 --- a/Tools/jit/_targets.py +++ b/Tools/jit/_targets.py @@ -53,9 +53,16 @@ class _Target(typing.Generic[_S, _R]): cflags: str = "" frame_pointers: bool = False llvm_version: str = _llvm._LLVM_VERSION + llvm_tools_install_dir: str | None = None known_symbols: dict[str, int] = dataclasses.field(default_factory=dict) pyconfig_dir: pathlib.Path = pathlib.Path.cwd().resolve() + def _compile_args(self) -> list[str]: + return list(self.args) + + def _shim_compile_args(self) -> list[str]: + return [] + def _get_nop(self) -> bytes: if re.fullmatch(r"aarch64-.*", self.triple): nop = b"\x1f\x20\x03\xd5" @@ -85,7 +92,11 @@ async def _parse(self, path: pathlib.Path) -> _stencils.StencilGroup: group = _stencils.StencilGroup() args = ["--disassemble", "--reloc", f"{path}"] output = await _llvm.maybe_run( - "llvm-objdump", args, echo=self.verbose, llvm_version=self.llvm_version + "llvm-objdump", + args, + echo=self.verbose, + llvm_version=self.llvm_version, + llvm_tools_install_dir=self.llvm_tools_install_dir, ) if output is not None: # Make sure that full paths don't leak out (for reproducibility): @@ -105,7 +116,11 @@ async def _parse(self, path: pathlib.Path) -> _stencils.StencilGroup: f"{path}", ] output = await _llvm.run( - "llvm-readobj", args, echo=self.verbose, llvm_version=self.llvm_version + "llvm-readobj", + args, + echo=self.verbose, + llvm_version=self.llvm_version, + llvm_tools_install_dir=self.llvm_tools_install_dir, ) # --elf-output-style=JSON is only *slightly* broken on Mach-O... output = output.replace("PrivateExtern\n", "\n") @@ -130,12 +145,8 @@ def _handle_relocation( ) -> _stencils.Hole: raise NotImplementedError(type(self)) - async def _compile( - self, opname: str, c: pathlib.Path, tempdir: pathlib.Path - ) -> _stencils.StencilGroup: - s = tempdir / f"{opname}.s" - o = tempdir / f"{opname}.o" - args_s = [ + def _base_clang_args(self, opname: str, tempdir: pathlib.Path) -> list[str]: + return [ f"--target={self.triple}", "-DPy_BUILD_CORE_MODULE", "-D_DEBUG" if self.debug else "-DNDEBUG", @@ -158,48 +169,88 @@ async def _compile( # generates better code than -O2 (and -O2 usually generates better # code than -O3). As a nice benefit, it uses less memory too: "-Os", - "-S", # Shorten full absolute file paths in the generated code (like the # __FILE__ macro and assert failure messages) for reproducibility: f"-ffile-prefix-map={CPYTHON}=.", f"-ffile-prefix-map={tempdir}=.", - # This debug info isn't necessary, and bloats out the JIT'ed code. - # We *may* be able to re-enable this, process it, and JIT it for a - # nicer debugging experience... but that needs a lot more research: - "-fno-asynchronous-unwind-tables", # Don't call built-in functions that we can't find or patch: "-fno-builtin", # Don't call stack-smashing canaries that we can't find or patch: "-fno-stack-protector", "-std=c11", + ] + + async def _build_stencil_group( + self, opname: str, c: pathlib.Path, tempdir: pathlib.Path + ) -> _stencils.StencilGroup: + s = tempdir / f"{opname}.s" + o = tempdir / f"{opname}.o" + args_s = self._base_clang_args(opname, tempdir) + args_s += [ + "-S", + # Stencils do not need unwind info, and the optimizer does not + # preserve .cfi_* directives correctly. On Darwin, + # -fno-asynchronous-unwind-tables alone still leaves synchronous + # unwind directives in the assembly, so disable both forms here. + "-fno-unwind-tables", + "-fno-asynchronous-unwind-tables", "-o", f"{s}", f"{c}", ] - is_shim = opname == "shim" if self.frame_pointers: - frame_pointer = "all" if is_shim else "reserved" - args_s += ["-Xclang", f"-mframe-pointer={frame_pointer}"] - args_s += self.args + args_s += ["-Xclang", "-mframe-pointer=reserved"] + args_s += self._compile_args() # Allow user-provided CFLAGS to override any defaults args_s += shlex.split(self.cflags) await _llvm.run( - "clang", args_s, echo=self.verbose, llvm_version=self.llvm_version + "clang", + args_s, + echo=self.verbose, + llvm_version=self.llvm_version, + llvm_tools_install_dir=self.llvm_tools_install_dir, ) - if not is_shim: - self.optimizer( - s, - label_prefix=self.label_prefix, - symbol_prefix=self.symbol_prefix, - re_global=self.re_global, - frame_pointers=self.frame_pointers, - ).run() + self.optimizer( + s, + label_prefix=self.label_prefix, + symbol_prefix=self.symbol_prefix, + re_global=self.re_global, + frame_pointers=self.frame_pointers, + ).run() args_o = [f"--target={self.triple}", "-c", "-o", f"{o}", f"{s}"] await _llvm.run( - "clang", args_o, echo=self.verbose, llvm_version=self.llvm_version + "clang", + args_o, + echo=self.verbose, + llvm_version=self.llvm_version, + llvm_tools_install_dir=self.llvm_tools_install_dir, ) return await self._parse(o) + async def _build_shim_object(self, output: pathlib.Path) -> None: + with tempfile.TemporaryDirectory() as tempdir: + work = pathlib.Path(tempdir).resolve() + args_o = self._base_clang_args("shim", work) + args_o += self._shim_compile_args() + args_o += [ + "-c", + # The linked shim is a real function in the final binary, so + # keep unwind info for debuggers and stack walkers. + "-fasynchronous-unwind-tables", + ] + if self.frame_pointers: + args_o += ["-Xclang", "-mframe-pointer=all"] + args_o += self._compile_args() + args_o += shlex.split(self.cflags) + args_o += ["-o", f"{output}", f"{TOOLS_JIT / 'shim.c'}"] + await _llvm.run( + "clang", + args_o, + echo=self.verbose, + llvm_version=self.llvm_version, + llvm_tools_install_dir=self.llvm_tools_install_dir, + ) + async def _build_stencils(self) -> dict[str, _stencils.StencilGroup]: generated_cases = PYTHON_EXECUTOR_CASES_C_H.read_text() cases_and_opnames = sorted( @@ -214,8 +265,6 @@ async def _build_stencils(self) -> dict[str, _stencils.StencilGroup]: with tempfile.TemporaryDirectory() as tempdir: work = pathlib.Path(tempdir).resolve() async with asyncio.TaskGroup() as group: - coro = self._compile("shim", TOOLS_JIT / "shim.c", work) - tasks.append(group.create_task(coro, name="shim")) template = TOOLS_JIT_TEMPLATE_C.read_text() for case, opname in cases_and_opnames: # Write out a copy of the template with *only* this case @@ -225,7 +274,7 @@ async def _build_stencils(self) -> dict[str, _stencils.StencilGroup]: # all of the other cases): c = work / f"{opname}.c" c.write_text(template.replace("CASE", case)) - coro = self._compile(opname, c, work) + coro = self._build_stencil_group(opname, c, work) tasks.append(group.create_task(coro, name=opname)) stencil_groups = {task.get_name(): task.result() for task in tasks} for stencil_group in stencil_groups.values(): @@ -239,8 +288,9 @@ def build( comment: str = "", force: bool = False, jit_stencils: pathlib.Path, + jit_shim_object: pathlib.Path, ) -> None: - """Build jit_stencils.h in the given directory.""" + """Build jit_stencils.h and the shim object in the given directory.""" jit_stencils.parent.mkdir(parents=True, exist_ok=True) if not self.stable: warning = f"JIT support for {self.triple} is still experimental!" @@ -254,8 +304,10 @@ def build( not force and jit_stencils.exists() and jit_stencils.read_text().startswith(digest) + and jit_shim_object.exists() ): return + ASYNCIO_RUNNER.run(self._build_shim_object(jit_shim_object)) stencil_groups = ASYNCIO_RUNNER.run(self._build_stencils()) jit_stencils_new = jit_stencils.parent / "jit_stencils.h.new" try: @@ -279,6 +331,13 @@ def build( class _COFF( _Target[_schema.COFFSection, _schema.COFFRelocation] ): # pylint: disable = too-few-public-methods + def _shim_compile_args(self) -> list[str]: + # The linked shim is part of pythoncore, not a shared extension. + # On Windows, Py_BUILD_CORE_MODULE makes public APIs import from + # pythonXY.lib, which creates a self-dependency when linking + # pythoncore.dll. Build the shim with builtin/core semantics. + return ["-UPy_BUILD_CORE_MODULE", "-DPy_BUILD_CORE_BUILTIN"] + def _handle_section( self, section: _schema.COFFSection, group: _stencils.StencilGroup ) -> None: @@ -379,6 +438,10 @@ class _COFF64(_COFF): symbol_prefix = "" re_global = re.compile(r'\s*\.def\s+(?P