Arm backend: Add manifest validation and add Backwards compabillety testing (#18813)

Author: Sebastian-Larsson

.github/workflows/pull.yml

@@ -816,6 +816,33 @@ jobs:
         # Test test_arm_backend.sh with test
         backends/arm/test/test_arm_backend.sh "${ARM_TEST}"
 
+  test-arm-backend-public-api-backward-compatibility:
+    name: test-arm-backend-public-api-backward-compatibility
+    uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main
+    permissions:
+      id-token: write
+      contents: read
+    with:
+      runner: linux.2xlarge.memory
+      docker-image: ci-image:executorch-ubuntu-22.04-arm-sdk
+      submodules: 'recursive'
+      ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
+      timeout: 120
+      script: |
+        # The generic Linux job chooses to use base env, not the one setup by the image
+        CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]")
+        conda activate "${CONDA_ENV}"
+
+        source .ci/scripts/utils.sh
+        install_executorch "--use-pt-pinned-commit"
+
+        .ci/scripts/setup-arm-baremetal-tools.sh --enable-mlsdk-deps --install-mlsdk-deps-with-pip
+        source examples/arm/arm-scratch/setup_path.sh
+
+        backends/arm/scripts/public_api_manifest/validate_all_public_api_manifests.sh
+
+        python backends/arm/test/public_api_bc/run_public_api_bc_scenarios.py
+
   test-llama-runner-qnn-linux:
     name: test-llama-runner-qnn-linux
     uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main

backends/arm/TARGETS

@@ -119,6 +119,15 @@ runtime.python_library(
         "//executorch/exir:lib",
     ],
 )
+runtime.python_library(
+    name = "public_api",
+    srcs = ["__init__.py"],
+    deps = [
+        ":ethosu",
+        ":vgf",
+        "//executorch/backends/arm/quantizer:lib",
+    ],
+)
 
 runtime.python_library(
     name = "process_node",

backends/arm/public_api_manifests/api_manifest_running.toml

@@ -4,7 +4,7 @@
 # LICENSE file in the root directory of this source tree.
 #
 # This file is generated by
-# backends/arm/scripts/generate_public_api_manifest.py
+# backends/arm/scripts/public_api_manifest/generate_public_api_manifest.py
 
 [python]
 

backends/arm/scripts/pre-push

@@ -70,6 +70,13 @@ run_docgen_check() {
     fi
 }
 
+run_public_api_validator() {
+    if ! backends/arm/scripts/public_api_manifest/validate_all_public_api_manifests.sh; then
+        echo -e "${ERROR} Arm public API manifest validation failed"
+        FAILED=1
+    fi
+}
+
 # This list of imperative verbs was compiled from the entire list of Executorch
 # commits. It should be fairly exhaustive, but add more verbs if you find one
 # that's missing.
@@ -149,7 +156,6 @@ for COMMIT in ${COMMITS}; do
             fi
         done
     fi
-
     # Check license headers
     # We do a simple check of if all committed headers contain
     # "$current_year Arm". This does not guarantee OK in ci but should be ok
@@ -312,6 +318,8 @@ else
     echo -e "${INFO} Skipping Arm docgen (no public API inputs changed)"
 fi
 
+run_public_api_validator
+
 if [[ $FAILED ]]; then
     echo -e "${INFO} Fix your commit message errors with"\
         "'git commit --amend' or 'git commit --fixup=<SHA>'"

backends/arm/scripts/public_api_manifest/BUCK

@@ -0,0 +1,22 @@
+# Copyright 2026 Arm Limited and/or its affiliates.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+load("@fbcode_macros//build_defs:build_file_migration.bzl", "fbcode_target", "non_fbcode_target")
+load("@fbsource//xplat/executorch/build:runtime_wrapper.bzl", "runtime")
+
+oncall("executorch")
+
+fbcode_target(_kind = runtime.python_library,
+    name = "public_api_manifest",
+    srcs = [
+        "__init__.py",
+        "generate_public_api_manifest.py",
+        "validate_public_api_manifest.py",
+    ],
+    deps = [
+        "//executorch/backends/arm:public_api",
+        "fbsource//third-party/pypi/tomli:tomli",
+    ],
+)

backends/arm/scripts/public_api_manifest/README.md

@@ -0,0 +1,105 @@
+# Manifests
+
+Manifests are used to track the current public API of the Arm backend. They are
+generated with
+`python backends/arm/scripts/public_api_manifest/generate_public_api_manifest.py`.
+
+## Running manifest
+
+There is always one running manifest which has the main purpose of tracking the
+API surface inbetween releases.
+
+## Static manifests
+
+At any given time there may be up to two static manifests. These are generated
+in conjunction with a release and are used to track the API surface of that
+release. The main purpose of these is to make sure backwards compatibility.
+
+A static manifest may never be changed. It belongs to a release and must be kept
+as is.
+
+A static manifest should not live longer than 2 releases. It may then be
+removed.
+
+# On release
+
+With each release, check that the running manifest is up to date and reflects
+the API surface of the release. Then, copy the running manifest to a new static
+manifest for the release. This can be done by running
+`cp <running_manifest> <static_manifest>`. The new static manifest should be
+named according to the release, e.g. `api_manifest_1_3.toml` for release 1.3 and
+so on. If there are now more than two static manifests, remove the oldest one in
+the same commit.
+
+# API changes
+
+When introducing an API change, the running manifest must be updated to reflect
+the change. This is done by running the manifest generation script,
+`python backends/arm/scripts/public_api_manifest/generate_public_api_manifest.py`.
+This updates the running manifest.
+
+To validate the running manifest directly, run
+`python backends/arm/scripts/public_api_manifest/validate_public_api_manifest.py`.
+
+To validate all manifests, use `backends/arm/scripts/pre-push`. This is the
+check that must pass before the change is ready to merge.
+
+Manifest validation only checks the API surface and signatures. Workflow-level
+backward compatibility is covered separately by the scenario runner described
+below.
+
+Running-manifest validation uses exact signature matching. Any intentional API
+change must update `api_manifest_running.toml`.
+
+Static-manifest validation uses backward-compatibility matching. The old
+release signature must still be callable against the current API. For example,
+adding a trailing optional parameter is accepted for static manifests, while
+removing a parameter, reordering parameters, or adding a new required
+parameter still fails validation.
+
+## Backward-compatibility scenarios
+
+Workflow-level backward compatibility is checked by
+`python backends/arm/test/public_api_bc/run_public_api_bc_scenarios.py`.
+
+The runner hardcodes the current canonical public API workflow scripts:
+
+- `backends/arm/test/public_api_bc/test_ethosu_flow.py`
+- `backends/arm/test/public_api_bc/test_vgf_fp_flow.py`
+- `backends/arm/test/public_api_bc/test_vgf_int_flow.py`
+
+These scripts should be updated continuously to reflect the current public API.
+The runner materializes those same paths into a temporary harness and executes
+them there with pytest so they import the latest installed
+`executorch.backends.arm` package instead of the repository source tree.
+
+The rolling support window is controlled by the `OLDEST_SUPPORTED_REF` constant
+in `backends/arm/test/public_api_bc/run_public_api_bc_scenarios.py`:
+
+- If `OLDEST_SUPPORTED_REF` is empty, the runner uses the current workspace.
+  This is the bootstrap mode until a release contains the scenario scripts.
+- Once a release contains the scripts, the release epic should update
+  `OLDEST_SUPPORTED_REF` to the oldest still-supported release ref.
+- At that point the runner uses `git show <ref>:<path>` to fetch the old
+  release's scripts and run them against the latest code.
+
+When an old release falls out of the support window, update
+`OLDEST_SUPPORTED_REF` to the next newer supported release. That is how the
+backward-compatibility window rolls forward.
+Reasons for passing validation may include:
+- Adding a new API symbol and adding it to the running manifest.
+- Removing an API that was marked as deprecated and no longer exists in any
+  manifest.
+- Deprecated symbols do not break backward compatibility with static
+  manifests.
+- Deprecating a symbol removes it from the running manifest, but it can only be
+  removed fully once it no longer appears in any static manifest.
+- Extending a static-manifest signature in a backward-compatible way, such as
+  adding a trailing optional parameter.
+
+Reasons for failing validation may include:
+- Removing an API symbol without deprecation.
+- Changing a running-manifest signature without regenerating the running
+  manifest.
+- Changing a static-manifest signature in a non-backward-compatible way.
+- New API symbol added but not added to the running manifest.

backends/arm/scripts/public_api_manifest/__init__.py

@@ -0,0 +1,4 @@
+# Copyright 2026 Arm Limited and/or its affiliates.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.

…/scripts/generate_public_api_manifest.py …manifest/generate_public_api_manifest.pybackends/arm/scripts/generate_public_api_manifest.py renamed to backends/arm/scripts/public_api_manifest/generate_public_api_manifest.py backends/arm/scripts/generate_public_api_manifest.py renamed to backends/arm/scripts/public_api_manifest/generate_public_api_manifest.py

@@ -23,10 +23,10 @@
 # LICENSE file in the root directory of this source tree.
 #
 # This file is generated by
-# backends/arm/scripts/generate_public_api_manifest.py
+# backends/arm/scripts/public_api_manifest/generate_public_api_manifest.py
 """
 MANIFEST_PATH = (
-    Path(__file__).resolve().parents[1]
+    Path(__file__).resolve().parents[2]
     / "public_api_manifests"
     / "api_manifest_running.toml"
 )
@@ -84,8 +84,10 @@ def _collect_entry(
     path: str,
     obj: object,
     entries: dict[str, dict[str, str]],
+    *,
+    include_deprecated: bool = False,
 ) -> None:
-    if _is_unstable_api(obj):
+    if _is_unstable_api(obj) and not include_deprecated:
         return
     entries[path] = {"kind": _api_kind(obj), "signature": _api_signature(path, obj)}
     if not inspect.isclass(obj):
@@ -99,13 +101,25 @@ def _collect_entry(
             continue
         member = getattr(obj, name)
         if inspect.isclass(member) or callable(member):
-            _collect_entry(f"{path}.{name}", member, entries)
+            _collect_entry(
+                f"{path}.{name}",
+                member,
+                entries,
+                include_deprecated=include_deprecated,
+            )
 
 
-def _collect_public_api() -> dict[str, dict[str, str]]:
+def _collect_public_api(
+    *, include_deprecated: bool = False
+) -> dict[str, dict[str, str]]:
     entries: dict[str, dict[str, str]] = {}
     for name in sorted(LAZY_IMPORTS):
-        _collect_entry(name, getattr(arm, name), entries)
+        _collect_entry(
+            name,
+            getattr(arm, name),
+            entries,
+            include_deprecated=include_deprecated,
+        )
     return entries
 
 
@@ -127,9 +141,10 @@ def _render_manifest(entries: dict[str, dict[str, str]]) -> str:
 def generate_manifest_from_init(
     *,
     repo_path: Path | None = None,
+    include_deprecated: bool = False,
 ) -> str:
     del repo_path
-    return _render_manifest(_collect_public_api())
+    return _render_manifest(_collect_public_api(include_deprecated=include_deprecated))
 
 
 def main() -> None:

backends/arm/scripts/public_api_manifest/validate_all_public_api_manifests.sh

@@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+# Copyright 2026 Arm Limited and/or its affiliates.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+set -u
+
+echo "Validating Arm public API manifests"
+
+manifest_failures=0
+for manifest_path in backends/arm/public_api_manifests/api_manifest_*.toml; do
+    if [[ ! -f "${manifest_path}" ]]; then
+        continue
+    fi
+    manifest_name="${manifest_path##*/}"
+    echo
+    echo "=== ${manifest_name} ==="
+    validator_output=$(
+        python backends/arm/scripts/public_api_manifest/validate_public_api_manifest.py \
+            --manifest "${manifest_path}" 2>&1
+    )
+    validator_status=$?
+    printf '%s\n' "${validator_output}"
+    if [[ ${validator_status} -ne 0 ]]; then
+        manifest_failures=$((manifest_failures + 1))
+    fi
+done
+
+echo
+if [[ ${manifest_failures} -eq 0 ]]; then
+    echo "Arm public API manifests OK"
+else
+    echo "${manifest_failures} manifest(s) failed validation"
+    exit 1
+fi