Arm backend: Add public API validation and BC tests to CI

Introduce BC tests and add those as well as the validation of the public
API manifest to CI in the pull job.

Change-Id: I1815223a13109e225f2b9e0f9914b504799db2ae
Signed-off-by: Sebastian Larsson <sebastian.larsson@arm.com>

Author: Sebastian-Larsson

.github/workflows/pull.yml

@@ -743,6 +743,33 @@ jobs:
         # Test test_arm_baremetal.sh with test
         backends/arm/test/test_arm_baremetal.sh "${ARM_TEST}"
 
+  test-arm-backend-public-api-backward-compatibility:
+    name: test-arm-backend-public-api-backward-compatibility
+    uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main
+    permissions:
+      id-token: write
+      contents: read
+    with:
+      runner: linux.2xlarge.memory
+      docker-image: ci-image:executorch-ubuntu-22.04-arm-sdk
+      submodules: 'recursive'
+      ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
+      timeout: 120
+      script: |
+        # The generic Linux job chooses to use base env, not the one setup by the image
+        CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]")
+        conda activate "${CONDA_ENV}"
+
+        source .ci/scripts/utils.sh
+        install_executorch "--use-pt-pinned-commit"
+
+        .ci/scripts/setup-arm-baremetal-tools.sh --enable-mlsdk-deps --install-mlsdk-deps-with-pip
+        source examples/arm/arm-scratch/setup_path.sh
+
+        backends/arm/scripts/public_api_manifest/validate_all_public_api_manifests.sh
+
+        python backends/arm/test/public_api_bc/run_public_api_bc_scenarios.py
+
   test-llama-runner-qnn-linux:
     name: test-llama-runner-qnn-linux
     uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main

backends/arm/scripts/pre-push

@@ -71,33 +71,9 @@ run_docgen_check() {
 }
 
 run_public_api_validator() {
-    echo -e "${INFO} Validating Arm public API manifests"
-
-    manifest_failures=0
-    for manifest_path in backends/arm/public_api_manifests/api_manifest_*.toml; do
-        if [[ ! -f "${manifest_path}" ]]; then
-            continue
-        fi
-        manifest_name="${manifest_path##*/}"
-        echo
-        echo "=== ${manifest_name} ==="
-        validator_output=$(
-            python backends/arm/scripts/public_api_manifest/validate_public_api_manifest.py \
-                --manifest "${manifest_path}" 2>&1
-        )
-        validator_status=$?
-        printf '%s\n' "${validator_output}"
-        if [[ ${validator_status} -ne 0 ]]; then
-            manifest_failures=$((manifest_failures + 1))
-            FAILED=1
-        fi
-    done
-
-    echo
-    if [[ ${manifest_failures} -eq 0 ]]; then
-        echo -e "${SUCCESS} Arm public API manifests OK"
-    else
-        echo -e "${ERROR} ${manifest_failures} manifest(s) failed validation"
+    if ! backends/arm/scripts/public_api_manifest/validate_all_public_api_manifests.sh; then
+        echo -e "${ERROR} Arm public API manifest validation failed"
+        FAILED=1
     fi
 }
 

backends/arm/scripts/public_api_manifest/README.md

@@ -44,7 +44,9 @@ To validate the running manifest directly, run
 To validate all manifests, use `backends/arm/scripts/pre-push`. This is the
 check that must pass before the change is ready to merge.
 
-Manifest validation only checks the API surface and signatures.
+Manifest validation only checks the API surface and signatures. Workflow-level
+backward compatibility is covered separately by the scenario runner described
+below.
 
 Running-manifest validation uses exact signature matching. Any intentional API
 change must update `api_manifest_running.toml`.
@@ -55,6 +57,35 @@ adding a trailing optional parameter is accepted for static manifests, while
 removing a parameter, reordering parameters, or adding a new required
 parameter still fails validation.
 
+## Backward-compatibility scenarios
+
+Workflow-level backward compatibility is checked by
+`python backends/arm/test/public_api_bc/run_public_api_bc_scenarios.py`.
+
+The runner hardcodes the current canonical public API workflow scripts:
+
+- `backends/arm/test/public_api_bc/test_ethosu_flow.py`
+- `backends/arm/test/public_api_bc/test_vgf_fp_flow.py`
+- `backends/arm/test/public_api_bc/test_vgf_int_flow.py`
+
+These scripts should be updated continuously to reflect the current public API.
+The runner materializes those same paths into a temporary harness and executes
+them there with pytest so they import the latest installed
+`executorch.backends.arm` package instead of the repository source tree.
+
+The rolling support window is controlled by the `OLDEST_SUPPORTED_REF` constant
+in `backends/arm/test/public_api_bc/run_public_api_bc_scenarios.py`:
+
+- If `OLDEST_SUPPORTED_REF` is empty, the runner uses the current workspace.
+  This is the bootstrap mode until a release contains the scenario scripts.
+- Once a release contains the scripts, the release epic should update
+  `OLDEST_SUPPORTED_REF` to the oldest still-supported release ref.
+- At that point the runner uses `git show <ref>:<path>` to fetch the old
+  release's scripts and run them against the latest code.
+
+When an old release falls out of the support window, update
+`OLDEST_SUPPORTED_REF` to the next newer supported release. That is how the
+backward-compatibility window rolls forward.
 Reasons for passing validation may include:
 - Adding a new API symbol and adding it to the running manifest.
 - Removing an API that was marked as deprecated and no longer exists in any

backends/arm/scripts/public_api_manifest/validate_all_public_api_manifests.sh

@@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+# Copyright 2026 Arm Limited and/or its affiliates.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+set -u
+
+echo "Validating Arm public API manifests"
+
+manifest_failures=0
+for manifest_path in backends/arm/public_api_manifests/api_manifest_*.toml; do
+    if [[ ! -f "${manifest_path}" ]]; then
+        continue
+    fi
+    manifest_name="${manifest_path##*/}"
+    echo
+    echo "=== ${manifest_name} ==="
+    validator_output=$(
+        python backends/arm/scripts/public_api_manifest/validate_public_api_manifest.py \
+            --manifest "${manifest_path}" 2>&1
+    )
+    validator_status=$?
+    printf '%s\n' "${validator_output}"
+    if [[ ${validator_status} -ne 0 ]]; then
+        manifest_failures=$((manifest_failures + 1))
+    fi
+done
+
+echo
+if [[ ${manifest_failures} -eq 0 ]]; then
+    echo "Arm public API manifests OK"
+else
+    echo "${manifest_failures} manifest(s) failed validation"
+    exit 1
+fi

backends/arm/test/public_api_bc/run_public_api_bc_scenarios.py

@@ -0,0 +1,104 @@
+# Copyright 2026 Arm Limited and/or its affiliates.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+"""Run Arm public API backward-compatibility scenarios."""
+
+from __future__ import annotations
+
+import os
+import shutil
+import sys
+import tempfile
+from pathlib import Path
+from subprocess import run  # nosec B404
+
+
+REPO_ROOT = Path(__file__).resolve().parents[4]
+PYTEST_CONFIG = Path("backends/arm/test/pytest.ini")
+SCENARIO_FILES = (
+    Path("backends/arm/test/public_api_bc/test_ethosu_flow.py"),
+    Path("backends/arm/test/public_api_bc/test_vgf_fp_flow.py"),
+    Path("backends/arm/test/public_api_bc/test_vgf_int_flow.py"),
+)
+# Leave empty until a release contains these scenario files. The release epic
+# should update this to the oldest still-supported release ref.
+OLDEST_SUPPORTED_REF = ""
+
+
+def _resolve_git() -> str:
+    git = shutil.which("git")
+    if git is None:
+        raise RuntimeError("Could not find git in PATH")
+    return git
+
+
+GIT = _resolve_git()
+
+
+def _materialize_file(repo_relative_path: Path, output_root: Path) -> Path:
+    destination_path = output_root / repo_relative_path
+    destination_path.parent.mkdir(parents=True, exist_ok=True)
+
+    if OLDEST_SUPPORTED_REF:
+        result = run(  # nosec B603
+            [
+                GIT,
+                "show",
+                f"{OLDEST_SUPPORTED_REF}:{repo_relative_path.as_posix()}",
+            ],
+            cwd=REPO_ROOT,
+            check=True,
+            capture_output=True,
+            text=True,
+        )
+        destination_path.write_text(result.stdout, encoding="utf-8")
+        return destination_path
+
+    source_path = REPO_ROOT / repo_relative_path
+    if not source_path.is_file():
+        raise FileNotFoundError(f"Missing scenario file: {source_path}")
+
+    shutil.copy2(source_path, destination_path)
+    return destination_path
+
+
+def _run_pytest(entrypoints: list[Path], output_root: Path) -> None:
+    env = os.environ.copy()
+    env.setdefault("PYTEST_DISABLE_PLUGIN_AUTOLOAD", "1")
+    run(  # nosec B603
+        [
+            sys.executable,
+            "-m",
+            "pytest",
+            "--config-file",
+            str(REPO_ROOT / PYTEST_CONFIG),
+            *[str(path) for path in entrypoints],
+        ],
+        cwd=output_root,
+        env=env,
+        check=True,
+    )
+
+
+def main() -> None:
+    with tempfile.TemporaryDirectory(
+        prefix="arm-public-api-bc-",
+        ignore_cleanup_errors=True,
+    ) as temporary_dir:
+        materialized_root = Path(temporary_dir)
+        entrypoints = [
+            _materialize_file(repo_relative_path, materialized_root)
+            for repo_relative_path in SCENARIO_FILES
+        ]
+
+        source_name = OLDEST_SUPPORTED_REF or "workspace snapshot"
+        print("Materialized Arm public API BC scenarios:")
+        print(f"  source: {source_name}")
+        print(f"  root:   {materialized_root}")
+
+        _run_pytest(entrypoints, materialized_root)
+
+
+if __name__ == "__main__":
+    main()

backends/arm/test/public_api_bc/test_ethosu_flow.py

@@ -0,0 +1,129 @@
+# Copyright 2026 Arm Limited and/or its affiliates.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+from __future__ import annotations
+
+from pathlib import Path
+
+import torch
+from executorch.backends.arm import (
+    EthosUBackend,
+    EthosUCompileSpec,
+    EthosUPartitioner,
+    EthosUQuantizer,
+    get_symmetric_a16w8_quantization_config,
+    get_symmetric_quantization_config,
+)
+from executorch.exir import ExecutorchBackendConfig, to_edge_transform_and_lower
+from executorch.extension.export_util.utils import save_pte_program
+from torchao.quantization.pt2e.quantize_pt2e import convert_pt2e, prepare_pt2e
+
+
+class TinyConvRelu(torch.nn.Module):
+    def __init__(self) -> None:
+        super().__init__()
+        self.conv = torch.nn.Conv2d(3, 4, kernel_size=3)
+        self.relu = torch.nn.ReLU()
+
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        return self.relu(self.conv(x))
+
+
+class TinyAdd(torch.nn.Module):
+    def forward(self, x: torch.Tensor, y: torch.Tensor) -> torch.Tensor:
+        return x + y
+
+
+def _configured_compile_spec(tmp_path: Path) -> EthosUCompileSpec:
+    compile_spec = EthosUCompileSpec(
+        "ethos-u55-128",
+        system_config="Ethos_U55_High_End_Embedded",
+        memory_mode="Shared_Sram",
+    )
+
+    assert compile_spec == EthosUCompileSpec(
+        "ethos-u55-128",
+        system_config="Ethos_U55_High_End_Embedded",
+        memory_mode="Shared_Sram",
+    )
+    assert "EthosUCompileSpec" in repr(compile_spec)
+
+    compile_spec.dump_intermediate_artifacts_to(str(tmp_path / "ethosu_intermediates"))
+    returned = compile_spec.dump_debug_info(EthosUCompileSpec.DebugMode.TOSA)
+    assert returned is compile_spec
+    return compile_spec
+
+
+def _exercise_quantizer_api(compile_spec: EthosUCompileSpec) -> None:
+    quantizer = EthosUQuantizer(compile_spec)
+    symmetric_config = get_symmetric_quantization_config(is_per_channel=False)
+    a16w8_config = get_symmetric_a16w8_quantization_config(is_per_channel=False)
+
+    quantizer.set_global(symmetric_config)
+    quantizer.set_io(a16w8_config)
+    quantizer.set_module_name("conv", symmetric_config)
+    quantizer.set_module_type(torch.nn.ReLU, symmetric_config)
+
+    example_inputs = (torch.randn(1, 3, 8, 8),)
+    graph_module = torch.export.export(TinyConvRelu().eval(), example_inputs).module(
+        check_guards=False
+    )
+    transformed = quantizer.transform_for_annotation(graph_module)
+    annotated = quantizer.annotate(transformed)
+    quantizer.validate(annotated)
+
+
+def _build_quantized_program(compile_spec: EthosUCompileSpec):
+    model = TinyAdd().eval()
+    example_inputs = (
+        torch.ones(1, 1, 1, 1),
+        torch.ones(1, 1, 1, 1),
+    )
+    exported_program = torch.export.export(model, example_inputs)
+    graph_module = exported_program.module(check_guards=False)
+
+    quantizer = EthosUQuantizer(compile_spec)
+    quantizer.set_global(get_symmetric_quantization_config())
+
+    prepared = prepare_pt2e(graph_module, quantizer)
+    prepared(*example_inputs)
+    converted = convert_pt2e(prepared)
+
+    return torch.export.export(converted, example_inputs)
+
+
+def test_u55_u85_public_api_scenario(tmp_path: Path) -> None:
+    backend = EthosUBackend()
+    assert isinstance(backend, EthosUBackend)
+
+    compile_spec = _configured_compile_spec(tmp_path)
+    _exercise_quantizer_api(compile_spec)
+
+    partitioner = EthosUPartitioner(compile_spec)
+    quantized_program_for_partition = _build_quantized_program(compile_spec)
+    ops_to_preserve, filter_fn = partitioner.ops_to_not_decompose(
+        quantized_program_for_partition
+    )
+    partition_result = partitioner.partition(quantized_program_for_partition)
+
+    assert isinstance(ops_to_preserve, list)
+    assert filter_fn is None or callable(filter_fn)
+    assert partition_result.tagged_exported_program is quantized_program_for_partition
+
+    quantized_program = _build_quantized_program(compile_spec)
+    edge_manager = to_edge_transform_and_lower(
+        programs=quantized_program,
+        partitioner=[partitioner],
+    )
+    executorch_program_manager = edge_manager.to_executorch(
+        config=ExecutorchBackendConfig(extract_delegate_segments=False)
+    )
+
+    pte_path = tmp_path / "ethosu_public_api_bc.pte"
+    save_pte_program(executorch_program_manager, str(pte_path))
+
+    assert pte_path.is_file()
+    assert pte_path.stat().st_size > 0
+    assert any((tmp_path / "ethosu_intermediates").rglob("*"))