Support pyodide xbuildenv install-emscripten on Windows (#322)

* Support `pyodide xbuildenv install-emscripten` on Windows

* test_install_emscripten - update tests

* Add Windows basic integration test CI

* windows-integration-test - test emcc compilation

* windows-integration-test - add integration test trigger condition

* windows-integration-test - bump `test` job node to 24 for consistency

* build_env - document emsdk_env script usage for both platforms

* windows-integration-test - use `pwsh` instead of `powershell`

`powershell` is using powershell 5.1, instead of powershell 7.4 and 5.1
is using utf16 by default, while 7.4 is utf8 and utf8 is more unix
compatible and doesn't conflict with emcc.

* xbuildenv - use`GIT_DIR=.` when applying emscripten patches

In `git apply` there's no `--no-index` option or any other to prevent
`git` from crawling up from `emsdk/upstream/emscripten`to `emsdk` folder
to find `.git` there and then fail to apply patch. Annoyingly, it
doesn't produce errors and even doesn't produce any logs without
`--verbose`.

Providing `GIT_DIR=.` explicitly prevent `.git` from searching git repo
and applies patch cleanly.

* windows-integration-test - use Python 3.13

To test against the most recent pyodide - missed some patches not
being applied, because they weren't present in pyodide 0.27.7
that's used for Python 3.12.

* xbuildenv - sort patches before applying

Since `Path.glob` doesn't guarantee consistent order.

Author: Andrej730

.github/workflows/main.yml

@@ -64,6 +64,53 @@ jobs:
           if-no-files-found: error
           include-hidden-files: true
 
+  windows-integration-test:
+    runs-on: windows-latest
+    needs: [check-integration-test-trigger]
+    if: needs.check-integration-test-trigger.outputs.run-integration-test
+    defaults:
+      run:
+        shell: pwsh
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          fetch-depth: 0
+
+      - name: Setup Python
+        uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
+        with:
+          python-version: "3.13"
+
+      - name: Set up Node.js
+        uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0
+        with:
+          node-version: "24"
+
+      - name: Install package
+        run: |
+          python -m pip install --upgrade pip
+          pip install -e .
+
+      - name: Install xbuildenv and emscripten
+        run: |
+          pyodide xbuildenv install
+          pyodide xbuildenv install-emscripten
+
+      - name: Test emcc
+        run: |
+          $EMSDK_DIR = pyodide config get emsdk_dir
+          . "$EMSDK_DIR\emsdk_env.ps1"
+          $code = @"
+          #include <stdio.h>
+          int main() {
+            printf("emcc works\n");
+            return 0;
+          }
+          "@
+          $code | Out-File test.c
+          emcc test.c -o test.js
+          node test.js
+
   check-integration-test-trigger:
     name: test-integration-test-trigger
     runs-on: ubuntu-latest

pyodide_build/build_env.py

@@ -4,6 +4,7 @@
 import functools
 import os
 import re
+import shutil
 import subprocess
 import sys
 from collections.abc import Iterator, Sequence
@@ -14,7 +15,12 @@
 from packaging.tags import Tag, compatible_tags, cpython_tags
 
 from pyodide_build import __version__
-from pyodide_build.common import default_xbuildenv_path, search_pyproject_toml, to_bool
+from pyodide_build.common import (
+    IS_WIN,
+    default_xbuildenv_path,
+    search_pyproject_toml,
+    to_bool,
+)
 
 RUST_BUILD_PRELUDE = """
 rustup default ${RUST_TOOLCHAIN}
@@ -300,8 +306,11 @@ def emscripten_version() -> str:
 
 def get_emscripten_version_info() -> str:
     """Extracted for testing purposes."""
+    emcc = shutil.which("emcc")
+    if not emcc:
+        raise FileNotFoundError
     return subprocess.run(
-        ["emcc", "-v"], capture_output=True, encoding="utf8", check=True
+        [emcc, "-v"], capture_output=True, encoding="utf8", check=True
     ).stderr
 
 
@@ -317,35 +326,51 @@ def get_emscripten_version_info() -> str:
 
 def activate_emscripten_env(emsdk_dir: Path) -> dict[str, str]:
     """
-    Source emsdk_env.sh and return the resulting environment variables.
+    Source the emsdk_env script (emsdk_env.sh on Unix, emsdk_env.bat on Windows)
+    and return the resulting environment variables.
 
     Parameters
     ----------
     emsdk_dir
-        Path to the emsdk directory containing emsdk_env.sh
+        Path to the emsdk directory containing the emsdk_env script
 
     Returns
     -------
     dict[str, str]
-        Dictionary of environment variables set by emsdk_env.sh
+        Dictionary of environment variables set by the emsdk_env script
     """
-    emsdk_env_script = emsdk_dir / "emsdk_env.sh"
+    emsdk_env_script_filename = "emsdk_env.bat" if IS_WIN else "emsdk_env.sh"
+    emsdk_env_script = emsdk_dir / emsdk_env_script_filename
     if not emsdk_env_script.exists():
-        raise FileNotFoundError(f"emsdk_env.sh not found at {emsdk_env_script}")
-
-    # Source emsdk_env.sh and capture the resulting environment
-    result = subprocess.run(
-        ["bash", "-c", f"source {emsdk_env_script} > /dev/null 2>&1 && env"],
-        capture_output=True,
-        encoding="utf8",
-        check=True,
-    )
+        raise FileNotFoundError(
+            f"{emsdk_env_script_filename} not found at {emsdk_env_script}"
+        )
+
+    # Source the emsdk_env script and capture the resulting environment
+    if IS_WIN:
+        # Passing args as a string, as otherwise shell is misinterpreting the command with quotes,
+        # resulting in no output.
+        result = subprocess.run(
+            f'cmd /c call "{emsdk_env_script}" > nul 2>&1 && set',
+            capture_output=True,
+            encoding="utf8",
+            check=True,
+        )
+    else:
+        result = subprocess.run(
+            ["bash", "-c", f'source "{emsdk_env_script}" > /dev/null 2>&1 && env'],
+            capture_output=True,
+            encoding="utf8",
+            check=True,
+        )
 
     # Parse the environment variables from output
     env_vars: dict[str, str] = {}
     for line in result.stdout.splitlines():
         if "=" in line:
             key, _, value = line.partition("=")
+            # On Windows it's 'Path'.
+            key = key.upper()
             if key in EMSDK_ENV_VARS:
                 env_vars[key] = value
 

pyodide_build/cli/xbuildenv.py

@@ -3,7 +3,7 @@
 import click
 
 from pyodide_build.build_env import get_build_flag, local_versions
-from pyodide_build.common import default_xbuildenv_path
+from pyodide_build.common import IS_WIN, default_xbuildenv_path
 from pyodide_build.views import MetadataView
 from pyodide_build.xbuildenv import CrossBuildEnvManager
 from pyodide_build.xbuildenv_releases import (
@@ -295,4 +295,8 @@ def _install_emscripten(
     emsdk_dir = manager.install_emscripten(version, force=force)
 
     print("Installing emsdk complete.")
-    print(f"Use `source {emsdk_dir}/emsdk_env.sh` to set up the environment.")
+    if IS_WIN:
+        cmd = f"{emsdk_dir}/emsdk_env.bat"
+    else:
+        cmd = f"source {emsdk_dir}/emsdk_env.sh"
+    print(f"Use `{cmd}` to set up the environment.")