[GR-76039] Fix Windows native long buffer formats

PullRequest: graalpython/4578

Author: timfel

graalpython/com.oracle.graal.python.test/src/tests/cpyext/test_shutdown.py

@@ -47,6 +47,7 @@
 import sys
 from unittest import skipIf
 from tests import compile_module_from_string
+from tests.util import run_subprocess_with_graalpy_startup_retry
 
 GRAALPY = sys.implementation.name == 'graalpy'
 
@@ -65,7 +66,7 @@
 
 # Test that running Py_DECREF in native global destructor doesn't crash
 def test_normal_exit():
-    subprocess.run(COMMAND, check=True, env=ENV)
+    run_subprocess_with_graalpy_startup_retry(COMMAND, check=True, env=ENV)
 
 
 def test_sigterm():

graalpython/com.oracle.graal.python.test/src/tests/test_memoryview.py

@@ -1,9 +1,10 @@
-# Copyright (c) 2018, 2024, Oracle and/or its affiliates.
+# Copyright (c) 2018, 2026, Oracle and/or its affiliates.
 # Copyright (C) 1996-2017 Python Software Foundation
 #
 # Licensed under the PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2
 
 import sys
+import struct
 import unittest
 
 from tests.util import assert_raises
@@ -93,15 +94,16 @@ def test_slice():
                                         assert e1 == e2
 
 def test_unpack():
+    long_bytes = b'\xaa' * struct.calcsize('l')
     assert memoryview(b'\xaa')[0] == 170
     assert memoryview(b'\xaa').cast('B')[0] == 170
     assert memoryview(b'\xaa').cast('b')[0] == -86
     assert memoryview(b'\xaa\xaa').cast('H')[0] == 43690
     assert memoryview(b'\xaa\xaa').cast('h')[0] == -21846
     assert memoryview(b'\xaa\xaa\xaa\xaa').cast('I')[0] == 2863311530
     assert memoryview(b'\xaa\xaa\xaa\xaa').cast('i')[0] == -1431655766
-    assert memoryview(b'\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa').cast('L')[0] == 12297829382473034410
-    assert memoryview(b'\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa').cast('l')[0] == -6148914691236517206
+    assert memoryview(long_bytes).cast('L')[0] == struct.unpack('L', long_bytes)[0]
+    assert memoryview(long_bytes).cast('l')[0] == struct.unpack('l', long_bytes)[0]
     assert memoryview(b'\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa').cast('Q')[0] == 12297829382473034410
     assert memoryview(b'\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa').cast('q')[0] == -6148914691236517206
     assert memoryview(b'\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa').cast('N')[0] == 12297829382473034410
@@ -114,6 +116,7 @@ def test_unpack():
     assert memoryview(b'\xaa').cast('c')[0] == b'\xaa'
 
 def test_pack():
+    long_bytes = b'\xaa' * struct.calcsize('l')
     b = bytearray(1)
     memoryview(b).cast('B')[0] = 170
     assert b == b'\xaa'
@@ -132,12 +135,12 @@ def test_pack():
     b = bytearray(4)
     memoryview(b).cast('i')[0] = -1431655766
     assert b == b'\xaa\xaa\xaa\xaa'
-    b = bytearray(8)
-    memoryview(b).cast('L')[0] = 12297829382473034410
-    assert b == b'\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa'
-    b = bytearray(8)
-    memoryview(b).cast('l')[0] = -6148914691236517206
-    assert b == b'\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa'
+    b = bytearray(struct.calcsize('L'))
+    memoryview(b).cast('L')[0] = struct.unpack('L', long_bytes)[0]
+    assert b == long_bytes
+    b = bytearray(struct.calcsize('l'))
+    memoryview(b).cast('l')[0] = struct.unpack('l', long_bytes)[0]
+    assert b == long_bytes
     b = bytearray(8)
     memoryview(b).cast('Q')[0] = 12297829382473034410
     assert b == b'\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa'

graalpython/com.oracle.graal.python.test/src/tests/test_subprocess.py

@@ -11,6 +11,8 @@
 from subprocess import CalledProcessError
 from tempfile import mkdtemp
 
+from tests.util import run_subprocess_with_graalpy_startup_retry
+
 POSIX_BACKEND_IS_JAVA = sys.implementation.name == "graalpy" and __graalpython__.posix_module_backend() == 'java'
 
 def test_os_pipe():
@@ -228,54 +230,52 @@ def test_subprocess_inherits_environ(self):
     @unittest.skipIf(sys.platform == 'win32', "TODO the cmd replacement breaks the test")
     def test_graal_python_args(self):
         if sys.implementation.name == "graalpy":
-            import subprocess
-
             def env_with_graal_python_args(args):
                 env = os.environ.copy()
                 env["GRAAL_PYTHON_ARGS"] = args
                 return env
 
             env = env_with_graal_python_args("-c 12")
-            result = subprocess.run([sys.executable], env=env)
-            self.assertEqual(0, result.returncode)
+            result = run_subprocess_with_graalpy_startup_retry([sys.executable], env=env, text=True)
+            self.assertEqual(0, result.returncode, result.stderr)
 
             env = env_with_graal_python_args("-c 'print(12)'")
-            result = subprocess.check_output([sys.executable], env=env, text=True)
+            result = run_subprocess_with_graalpy_startup_retry([sys.executable], env=env, text=True, check=True).stdout
             self.assertEqual('12\n', result)
 
             env = env_with_graal_python_args("""-c 'print("Hello world")'""")
-            result = subprocess.check_output([sys.executable], env=env, text=True)
+            result = run_subprocess_with_graalpy_startup_retry([sys.executable], env=env, text=True, check=True).stdout
             self.assertEqual('Hello world\n', result)
 
             env = env_with_graal_python_args("""-c ""'print("Hello world")'""""")
-            result = subprocess.check_output([sys.executable], env=env, text=True)
+            result = run_subprocess_with_graalpy_startup_retry([sys.executable], env=env, text=True, check=True).stdout
             self.assertEqual('Hello world\n', result)
 
             env = env_with_graal_python_args(r"""-c 'print(\'"Hello world"\')'""")
-            result = subprocess.check_output([sys.executable], env=env, text=True)
+            result = run_subprocess_with_graalpy_startup_retry([sys.executable], env=env, text=True, check=True).stdout
             self.assertEqual('"Hello world"\n', result)
 
             env = env_with_graal_python_args("""\v-c\vprint('"Hello world"')""")
-            result = subprocess.check_output([sys.executable], env=env, text=True)
+            result = run_subprocess_with_graalpy_startup_retry([sys.executable], env=env, text=True, check=True).stdout
             self.assertEqual('"Hello world"\n', result)
 
             env = env_with_graal_python_args("""\v-c\vprint('Hello', "world")""")
-            result = subprocess.check_output([sys.executable], env=env, text=True)
+            result = run_subprocess_with_graalpy_startup_retry([sys.executable], env=env, text=True, check=True).stdout
             self.assertEqual('Hello world\n', result)
 
             # check that the subprocess receives the args and thus it should fail because it recurses
             args = """\v-c\vimport os\nprint(os.environ.get("GRAAL_PYTHON_ARGS"))"""
             env = env_with_graal_python_args(args)
-            result = subprocess.check_output([sys.executable], env=env, text=True)
+            result = run_subprocess_with_graalpy_startup_retry([sys.executable], env=env, text=True, check=True).stdout
             self.assertEqual(f"{args}\n", result)
 
             # check that the subprocess does not receive the args when we end with \v
             env = env_with_graal_python_args("""\v-c\vimport os\nprint(os.environ.get("GRAAL_PYTHON_ARGS"))\v""")
-            result = subprocess.check_output([sys.executable], env=env, text=True)
+            result = run_subprocess_with_graalpy_startup_retry([sys.executable], env=env, text=True, check=True).stdout
             self.assertEqual('None\n', result)
 
             # check that the subprocess receives an empty arg
             args = """\v-c\vimport sys\nprint(repr(sys.argv))\va1\v\va3"""
             env = env_with_graal_python_args(args)
-            result = subprocess.check_output([sys.executable], env=env, text=True)
+            result = run_subprocess_with_graalpy_startup_retry([sys.executable], env=env, text=True, check=True).stdout
             self.assertEqual("['-c', 'a1', '', 'a3']\n", result)

graalpython/com.oracle.graal.python.test/src/tests/util.py

@@ -36,10 +36,13 @@
 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 # SOFTWARE.
+import subprocess
 import sys
+import time
 import unittest
 
 IS_BYTECODE_DSL = sys.implementation.name == 'graalpy' and __graalpython__.is_bytecode_dsl_interpreter
+TRANSIENT_GRAALPY_STARTUP_BLOCKING_IO = "ERROR: BlockingIOError: [Errno 11] Resource temporarily unavailable"
 
 
 def _is_sandboxed():
@@ -108,3 +111,31 @@ def assert_raises(err, fn, *args, err_check=None, **kwargs):
             else:
                 assert err_check(e)
     assert raised
+
+
+def _contains_transient_graalpy_startup_blocking_io(output):
+    if output is None:
+        return False
+    if isinstance(output, bytes):
+        return TRANSIENT_GRAALPY_STARTUP_BLOCKING_IO.encode() in output
+    return TRANSIENT_GRAALPY_STARTUP_BLOCKING_IO in output
+
+
+def run_subprocess_with_graalpy_startup_retry(args, *, attempts=5, retry_delay=0.2, **kwargs):
+    unsupported_kwargs = {"stdout", "stderr", "capture_output"} & kwargs.keys()
+    if unsupported_kwargs:
+        raise TypeError(f"unsupported keyword arguments: {', '.join(sorted(unsupported_kwargs))}")
+    check = kwargs.pop("check", False)
+    for attempt in range(attempts):
+        result = subprocess.run(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, **kwargs)
+        if result.returncode == 0 or not (
+            _contains_transient_graalpy_startup_blocking_io(result.stdout) or
+            _contains_transient_graalpy_startup_blocking_io(result.stderr)
+        ):
+            break
+        if attempt + 1 < attempts:
+            time.sleep(retry_delay)
+            retry_delay *= 2
+    if check and result.returncode != 0:
+        raise subprocess.CalledProcessError(result.returncode, result.args, output=result.stdout, stderr=result.stderr)
+    return result

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/util/BufferFormat.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2020, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2020, 2026, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * The Universal Permissive License (UPL), Version 1.0
@@ -44,14 +44,16 @@
 import static com.oracle.graal.python.util.PythonUtils.toTruffleStringUncached;
 import static com.oracle.graal.python.util.PythonUtils.tsLiteral;
 
+import com.oracle.graal.python.PythonLanguage;
+import com.oracle.graal.python.annotations.PythonOS;
 import com.oracle.truffle.api.strings.TruffleString;
 
 /**
  * This enum represents formats used by {@code array} and {@code memoryview}. The correspondence
  * between the type specifier string and {@link BufferFormat} is not 1 to 1. Multiple specifiers may
- * represent the same format (e.g. both {@code L} and {@code Q} on 64 bit platforms both map to
- * {@link BufferFormat#UINT_64}) format. Therefore it is necessary to keep the original specifier
- * string around for error messages.
+ * represent the same format (e.g. both {@code L} and {@code Q} on 64 bit non-Windows platforms both
+ * map to {@link BufferFormat#UINT_64}) format. Therefore it is necessary to keep the original
+ * specifier string around for error messages.
  */
 public enum BufferFormat {
     UINT_8(1, 0, "B"),
@@ -140,9 +142,11 @@ private static BufferFormat fromCharCommon(char fmtchar) {
             case 'i':
                 return INT_32;
             case 'L':
+                return PythonLanguage.getPythonOS() == PythonOS.PLATFORM_WIN32 ? UINT_32 : UINT_64;
             case 'Q':
                 return UINT_64;
             case 'l':
+                return PythonLanguage.getPythonOS() == PythonOS.PLATFORM_WIN32 ? INT_32 : INT_64;
             case 'q':
                 return INT_64;
             case 'e':