Merge branch '3.14' into list-repr-null-3.14

Author: serhiy-storchaka

Doc/tools/extensions/c_annotations.py

@@ -308,27 +308,27 @@ def _unstable_api_annotation() -> nodes.admonition:
 def _threadsafety_annotation(level: str) -> nodes.emphasis:
     match level:
         case "incompatible":
-            display = sphinx_gettext("Not safe to call from multiple threads.")
+            display = sphinx_gettext("Not safe to call from multiple threads")
             reftarget = "threadsafety-level-incompatible"
         case "compatible":
             display = sphinx_gettext(
                 "Safe to call from multiple threads"
-                " with external synchronization only."
+                " with external synchronization only"
             )
             reftarget = "threadsafety-level-compatible"
         case "distinct":
             display = sphinx_gettext(
                 "Safe to call without external synchronization"
-                " on distinct objects."
+                " on distinct objects"
             )
             reftarget = "threadsafety-level-distinct"
         case "shared":
             display = sphinx_gettext(
-                "Safe for concurrent use on the same object."
+                "Safe for concurrent use on the same object"
             )
             reftarget = "threadsafety-level-shared"
         case "atomic":
-            display = sphinx_gettext("Atomic.")
+            display = sphinx_gettext("Atomic")
             reftarget = "threadsafety-level-atomic"
         case _:
             raise AssertionError(f"Unknown thread safety level {level!r}")
@@ -340,9 +340,11 @@ def _threadsafety_annotation(level: str) -> nodes.emphasis:
         reftype="ref",
         refexplicit="True",
     )
-    prefix = sphinx_gettext("Thread safety:") + " "
+    prefix = " " + sphinx_gettext("Thread safety:") + " "
     classes = ["threadsafety", f"threadsafety-{level}"]
-    return nodes.emphasis("", prefix, ref_node, classes=classes)
+    return nodes.emphasis(
+        "", prefix, ref_node, nodes.Text("."), classes=classes
+    )
 
 
 def _return_value_annotation(result_refs: int | None) -> nodes.emphasis:

Include/internal/pycore_call.h

@@ -98,6 +98,14 @@ _PyObject_CallMethodIdOneArg(PyObject *self, _Py_Identifier *name, PyObject *arg
 }
 
 
+extern PyObject *_PyObject_VectorcallPrepend(
+    PyThreadState *tstate,
+    PyObject *callable,
+    PyObject *arg,
+    PyObject *const *args,
+    size_t nargsf,
+    PyObject *kwnames);
+
 /* === Vectorcall protocol (PEP 590) ============================= */
 
 // Call callable using tp_call. Arguments are like PyObject_Vectorcall(),

Include/internal/pycore_ceval.h

@@ -383,6 +383,11 @@ extern int _PyRunRemoteDebugger(PyThreadState *tstate);
 #define SPECIAL___AEXIT__   3
 #define SPECIAL_MAX   3
 
+PyAPI_FUNC(_PyStackRef)
+_Py_LoadAttr_StackRefSteal(
+    PyThreadState *tstate, _PyStackRef owner,
+    PyObject *name, _PyStackRef *self_or_null);
+
 #ifdef __cplusplus
 }
 #endif

Include/internal/pycore_function.h

@@ -47,6 +47,17 @@ static inline PyObject* _PyFunction_GET_BUILTINS(PyObject *func) {
 #define _PyFunction_GET_BUILTINS(func) _PyFunction_GET_BUILTINS(_PyObject_CAST(func))
 
 
+/* Get the callable wrapped by a classmethod.
+   Returns a borrowed reference.
+   The caller must ensure 'cm' is a classmethod object. */
+extern PyObject *_PyClassMethod_GetFunc(PyObject *cm);
+
+/* Get the callable wrapped by a staticmethod.
+   Returns a borrowed reference.
+   The caller must ensure 'sm' is a staticmethod object. */
+extern PyObject *_PyStaticMethod_GetFunc(PyObject *sm);
+
+
 #ifdef __cplusplus
 }
 #endif

Include/internal/pycore_object.h

@@ -897,6 +897,9 @@ extern PyObject *_PyType_LookupRefAndVersion(PyTypeObject *, PyObject *,
 extern unsigned int
 _PyType_LookupStackRefAndVersion(PyTypeObject *type, PyObject *name, _PyStackRef *out);
 
+extern int _PyObject_GetMethodStackRef(PyThreadState *ts, _PyStackRef *self,
+                                       PyObject *name, _PyStackRef *method);
+
 // Cache the provided init method in the specialization cache of type if the
 // provided type version matches the current version of the type.
 //

Include/internal/pycore_stackref.h

@@ -127,6 +127,13 @@ _PyStackRef_FromPyObjectSteal(PyObject *obj, const char *filename, int linenumbe
 }
 #define PyStackRef_FromPyObjectSteal(obj) _PyStackRef_FromPyObjectSteal(_PyObject_CAST(obj), __FILE__, __LINE__)
 
+static inline _PyStackRef
+_PyStackRef_FromPyObjectBorrow(PyObject *obj, const char *filename, int linenumber)
+{
+    return _Py_stackref_create(obj, filename, linenumber);
+}
+#define PyStackRef_FromPyObjectBorrow(obj) _PyStackRef_FromPyObjectBorrow(_PyObject_CAST(obj), __FILE__, __LINE__)
+
 static inline _PyStackRef
 _PyStackRef_FromPyObjectImmortal(PyObject *obj, const char *filename, int linenumber)
 {
@@ -320,6 +327,14 @@ _PyStackRef_FromPyObjectSteal(PyObject *obj)
 }
 #   define PyStackRef_FromPyObjectSteal(obj) _PyStackRef_FromPyObjectSteal(_PyObject_CAST(obj))
 
+static inline _PyStackRef
+PyStackRef_FromPyObjectBorrow(PyObject *obj)
+{
+    assert(obj != NULL);
+    assert(((uintptr_t)obj & Py_TAG_BITS) == 0);
+    return (_PyStackRef){ .bits = (uintptr_t)obj | Py_TAG_DEFERRED };
+}
+
 static inline bool
 PyStackRef_IsHeapSafe(_PyStackRef stackref)
 {
@@ -538,6 +553,13 @@ PyStackRef_FromPyObjectSteal(PyObject *obj)
     return ref;
 }
 
+static inline _PyStackRef
+PyStackRef_FromPyObjectBorrow(PyObject *obj)
+{
+    assert(obj != NULL);
+    return (_PyStackRef){ .bits = (uintptr_t)obj | Py_TAG_REFCNT };
+}
+
 static inline _PyStackRef
 PyStackRef_FromPyObjectStealMortal(PyObject *obj)
 {
@@ -753,6 +775,17 @@ _PyThreadState_PopCStackRef(PyThreadState *tstate, _PyCStackRef *ref)
     PyStackRef_XCLOSE(ref->ref);
 }
 
+static inline _PyStackRef
+_PyThreadState_PopCStackRefSteal(PyThreadState *tstate, _PyCStackRef *ref)
+{
+#ifdef Py_GIL_DISABLED
+    _PyThreadStateImpl *tstate_impl = (_PyThreadStateImpl *)tstate;
+    assert(tstate_impl->c_stack_refs == ref);
+    tstate_impl->c_stack_refs = ref->next;
+#endif
+    return ref->ref;
+}
+
 #ifdef Py_GIL_DISABLED
 
 static inline int

Lib/test/test_capi/test_unicode.py

@@ -1751,13 +1751,15 @@ def test_basic(self):
         writer.write_utf8(b'var', -1)
 
         # test PyUnicodeWriter_WriteChar()
-        writer.write_char('=')
+        writer.write_char(ord('='))
 
         # test PyUnicodeWriter_WriteSubstring()
         writer.write_substring("[long]", 1, 5)
+        # CRASHES writer.write_substring(NULL, 0, 0)
 
         # test PyUnicodeWriter_WriteStr()
         writer.write_str(" value ")
+        # CRASHES writer.write_str(NULL)
 
         # test PyUnicodeWriter_WriteRepr()
         writer.write_repr("repr")
@@ -1772,22 +1774,38 @@ def test_repr_null(self):
         writer.write_repr_true(NULL)
         self.assertEqual(writer.finish(),
                          "var=<NULL>")
+    def test_write_char(self):
+        writer = self.create_writer(0)
+        writer.write_char(0)
+        writer.write_char(ord('$'))
+        writer.write_char(0x20ac)
+        writer.write_char(0x10_ffff)
+        self.assertRaises(ValueError, writer.write_char, 0x11_0000)
+        self.assertRaises(ValueError, writer.write_char, 0xFFFF_FFFF)
+        self.assertEqual(writer.finish(),
+                         "\0$\u20AC\U0010FFFF")
 
     def test_utf8(self):
         writer = self.create_writer(0)
         writer.write_utf8(b"ascii", -1)
-        writer.write_char('-')
+        writer.write_char(ord('-'))
         writer.write_utf8(b"latin1=\xC3\xA9", -1)
-        writer.write_char('-')
+        writer.write_char(ord('-'))
         writer.write_utf8(b"euro=\xE2\x82\xAC", -1)
-        writer.write_char('.')
+        writer.write_char(ord('.'))
+        writer.write_utf8(NULL, 0)
+        # CRASHES writer.write_utf8(NULL, 1)
+        # CRASHES writer.write_utf8(NULL, -1)
         self.assertEqual(writer.finish(),
                          "ascii-latin1=\xE9-euro=\u20AC.")
 
     def test_ascii(self):
         writer = self.create_writer(0)
         writer.write_ascii(b"Hello ", -1)
         writer.write_ascii(b"", 0)
+        writer.write_ascii(NULL, 0)
+        # CRASHES writer.write_ascii(NULL, 1)
+        # CRASHES writer.write_ascii(NULL, -1)
         writer.write_ascii(b"Python! <truncated>", 6)
         self.assertEqual(writer.finish(), "Hello Python")
 
@@ -1804,6 +1822,9 @@ def test_recover_utf8_error(self):
         # write fails with an invalid string
         with self.assertRaises(UnicodeDecodeError):
             writer.write_utf8(b"invalid\xFF", -1)
+        with self.assertRaises(UnicodeDecodeError):
+            s = "truncated\u20AC".encode()
+            writer.write_utf8(s, len(s) - 1)
 
         # retry write with a valid string
         writer.write_utf8(b"valid", -1)
@@ -1815,13 +1836,19 @@ def test_decode_utf8(self):
         # test PyUnicodeWriter_DecodeUTF8Stateful()
         writer = self.create_writer(0)
         writer.decodeutf8stateful(b"ign\xFFore", -1, b"ignore")
-        writer.write_char('-')
+        writer.write_char(ord('-'))
         writer.decodeutf8stateful(b"replace\xFF", -1, b"replace")
-        writer.write_char('-')
+        writer.write_char(ord('-'))
 
         # incomplete trailing UTF-8 sequence
         writer.decodeutf8stateful(b"incomplete\xC3", -1, b"replace")
 
+        writer.decodeutf8stateful(NULL, 0, b"replace")
+        # CRASHES writer.decodeutf8stateful(NULL, 1, b"replace")
+        # CRASHES writer.decodeutf8stateful(NULL, -1, b"replace")
+        with self.assertRaises(UnicodeDecodeError):
+            writer.decodeutf8stateful(b"default\xFF", -1, NULL)
+
         self.assertEqual(writer.finish(),
                          "ignore-replace\uFFFD-incomplete\uFFFD")
 
@@ -1832,12 +1859,12 @@ def test_decode_utf8_consumed(self):
         # valid string
         consumed = writer.decodeutf8stateful(b"text", -1, b"strict", True)
         self.assertEqual(consumed, 4)
-        writer.write_char('-')
+        writer.write_char(ord('-'))
 
         # non-ASCII
         consumed = writer.decodeutf8stateful(b"\xC3\xA9-\xE2\x82\xAC", 6, b"strict", True)
         self.assertEqual(consumed, 6)
-        writer.write_char('-')
+        writer.write_char(ord('-'))
 
         # invalid UTF-8 (consumed is 0 on error)
         with self.assertRaises(UnicodeDecodeError):
@@ -1846,54 +1873,92 @@ def test_decode_utf8_consumed(self):
         # ignore error handler
         consumed = writer.decodeutf8stateful(b"more\xFF", -1, b"ignore", True)
         self.assertEqual(consumed, 5)
-        writer.write_char('-')
+        writer.write_char(ord('-'))
 
         # incomplete trailing UTF-8 sequence
         consumed = writer.decodeutf8stateful(b"incomplete\xC3", -1, b"ignore", True)
         self.assertEqual(consumed, 10)
+        writer.write_char(ord('-'))
+
+        consumed = writer.decodeutf8stateful(NULL, 0, b"replace", True)
+        self.assertEqual(consumed, 0)
+        # CRASHES writer.decodeutf8stateful(NULL, 1, b"replace", True)
+        # CRASHES writer.decodeutf8stateful(NULL, -1, b"replace", True)
+        consumed = writer.decodeutf8stateful(b"default\xC3", -1, NULL, True)
+        self.assertEqual(consumed, 7)
 
-        self.assertEqual(writer.finish(), "text-\xE9-\u20AC-more-incomplete")
+        self.assertEqual(writer.finish(), "text-\xE9-\u20AC-more-incomplete-default")
 
     def test_widechar(self):
+        from _testcapi import SIZEOF_WCHAR_T
+
+        if SIZEOF_WCHAR_T == 2:
+            encoding = 'utf-16le' if sys.byteorder == 'little' else 'utf-16be'
+        elif SIZEOF_WCHAR_T == 4:
+            encoding = 'utf-32le' if sys.byteorder == 'little' else 'utf-32be'
+
         writer = self.create_writer(0)
-        writer.write_widechar("latin1=\xE9")
-        writer.write_widechar("-")
-        writer.write_widechar("euro=\u20AC")
-        writer.write_char("-")
-        writer.write_widechar("max=\U0010ffff")
-        writer.write_char('.')
+        writer.write_widechar("latin1=\xE9".encode(encoding))
+        writer.write_char(ord("-"))
+        writer.write_widechar("euro=\u20AC".encode(encoding))
+        writer.write_char(ord("-"))
+        writer.write_widechar("max=\U0010ffff".encode(encoding))
+        writer.write_char(ord("-"))
+        writer.write_widechar("zeroes=".encode(encoding).ljust(SIZEOF_WCHAR_T * 10, b'\0'),
+                              10)
+        writer.write_char(ord('.'))
+
+        if SIZEOF_WCHAR_T == 4:
+            invalid = (b'\x00\x00\x11\x00' if sys.byteorder == 'little' else
+                       b'\x00\x11\x00\x00')
+            with self.assertRaises(ValueError):
+                writer.write_widechar("invalid=".encode(encoding) + invalid)
+        writer.write_widechar(b'', -5)
+        writer.write_widechar(NULL, 0)
+        # CRASHES writer.write_widechar(NULL, 1)
+        # CRASHES writer.write_widechar(NULL, -1)
+
         self.assertEqual(writer.finish(),
-                         "latin1=\xE9-euro=\u20AC-max=\U0010ffff.")
+                         "latin1=\xE9-euro=\u20AC-max=\U0010ffff-zeroes=\0\0\0.")
 
     def test_ucs4(self):
+        encoding = 'utf-32le' if sys.byteorder == 'little' else 'utf-32be'
+
         writer = self.create_writer(0)
-        writer.write_ucs4("ascii IGNORED", 5)
-        writer.write_char("-")
-        writer.write_ucs4("latin1=\xe9", 8)
-        writer.write_char("-")
-        writer.write_ucs4("euro=\u20ac", 6)
-        writer.write_char("-")
-        writer.write_ucs4("max=\U0010ffff", 5)
-        writer.write_char(".")
+        writer.write_ucs4("ascii IGNORED".encode(encoding), 5)
+        writer.write_char(ord("-"))
+        writer.write_ucs4("latin1=\xe9".encode(encoding))
+        writer.write_char(ord("-"))
+        writer.write_ucs4("euro=\u20ac".encode(encoding))
+        writer.write_char(ord("-"))
+        writer.write_ucs4("max=\U0010ffff".encode(encoding))
+        writer.write_char(ord("."))
         self.assertEqual(writer.finish(),
                          "ascii-latin1=\xE9-euro=\u20AC-max=\U0010ffff.")
 
         # Test some special characters
         writer = self.create_writer(0)
         # Lone surrogate character
-        writer.write_ucs4("lone\uDC80", 5)
-        writer.write_char("-")
+        writer.write_ucs4("lone\uDC80".encode(encoding, 'surrogatepass'))
+        writer.write_char(ord("-"))
         # Surrogate pair
-        writer.write_ucs4("pair\uDBFF\uDFFF", 5)
-        writer.write_char("-")
-        writer.write_ucs4("null[\0]", 7)
+        writer.write_ucs4("pair\uD83D\uDC0D".encode(encoding, 'surrogatepass'))
+        writer.write_char(ord("-"))
+        writer.write_ucs4("null[\0]".encode(encoding), 7)
+        invalid = (b'\x00\x00\x11\x00' if sys.byteorder == 'little' else
+                   b'\x00\x11\x00\x00')
+        # CRASHES writer.write_ucs4("invalid".encode(encoding) + invalid)
+        writer.write_ucs4(NULL, 0)
+        # CRASHES writer.write_ucs4(NULL, 1)
         self.assertEqual(writer.finish(),
-                         "lone\udc80-pair\udbff-null[\0]")
+                         "lone\udc80-pair\ud83d\udc0d-null[\x00]")
 
         # invalid size
         writer = self.create_writer(0)
         with self.assertRaises(ValueError):
-            writer.write_ucs4("text", -1)
+            writer.write_ucs4("text".encode(encoding), -1)
+        self.assertRaises(ValueError, writer.write_ucs4, b'', -1)
+        self.assertRaises(ValueError, writer.write_ucs4, NULL, -1)
 
     def test_substring_empty(self):
         writer = self.create_writer(0)
@@ -1919,7 +1984,7 @@ def test_format(self):
         from ctypes import c_int
         writer = self.create_writer(0)
         self.writer_format(writer, b'%s %i', b'abc', c_int(123))
-        writer.write_char('.')
+        writer.write_char(ord('.'))
         self.assertEqual(writer.finish(), 'abc 123.')
 
     def test_recover_error(self):