azure-pipelines.yml

# Deterministic Stage 3 (executor) safe-output E2E pipeline.
#
# Unlike the agentic smoke suite in tests/safe-outputs/ (which drives Stage 3 by
# having an LLM emit each safe output), this pipeline is fully deterministic and
# has no agent in the loop. A TypeScript harness
# (scripts/ado-script/src/executor-e2e/) sets up preconditions via the ADO REST
# API, crafts the executor's NDJSON input directly, runs the real `ado-aw
# execute` binary built from this checkout, asserts the effect via REST, and
# cleans up. On any failure it files a GitHub issue on the configured issue
# repository and fails the build.
#
# This is a hand-authored ADO pipeline (NOT compiled from an agent markdown).
# Register it in msazuresphere/AgentPlayground — see tests/executor-e2e/README.md.

trigger: none
pr:
  branches:
    include:
      - main
  paths:
    include:
      - src/safe_outputs/**
      - scripts/ado-script/src/executor-e2e/**
      - tests/executor-e2e/**

schedules:
  - cron: "0 5 * * *"
    displayName: Daily deterministic executor E2E
    branches:
      include:
        - main
    always: true

pool:
  name: AZS-1ES-L-Playground-ubuntu-22.04

variables:
  # Keep YAML defaults under private names so same-named pipeline/definition UI
  # variables (EXECUTOR_E2E_ADO_REPO, E2E_QUEUE_PIPELINE_ID, E2E_WIKI_NAME,
  # CRATES_IO_FEED) are not shadowed by this block.
  EFFECTIVE_EXECUTOR_E2E_ADO_REPO: $[ coalesce(variables['EXECUTOR_E2E_ADO_REPO'], 'agent-definitions') ]
  EFFECTIVE_E2E_QUEUE_PIPELINE_ID: $[ coalesce(variables['E2E_QUEUE_PIPELINE_ID'], '') ]
  EFFECTIVE_E2E_WIKI_NAME: $[ coalesce(variables['E2E_WIKI_NAME'], '') ]
  # Internal sparse crates.io mirror (Azure Artifacts). Declared in the cargo
  # config below so CargoAuthenticate@0 can attach credentials, and reused as
  # RustInstaller's crates.io override so the two never drift.
  EFFECTIVE_CRATES_IO_FEED: $[ coalesce(variables['CRATES_IO_FEED'], 'sparse+https://pkgs.dev.azure.com/msazuresphere/AgentPlayground/_packaging/cargo/Cargo/index/') ]
  # GitHub repo that failure issues are filed against. NOT defined here on
  # purpose: a YAML `variables:` entry SHADOWS a same-named pipeline (definition)
  # variable, so hardcoding it would block per-pipeline overrides. Set
  # EXECUTOR_E2E_ISSUE_REPO as a pipeline/definition variable to redirect issue
  # filing (e.g. to a fork you can write to); the harness defaults to
  # githubnext/ado-aw when it is unset.
  # Optional-precondition scenarios: empty by default so queue-build / wiki
  # skip gracefully. Override at queue time (or via a variable group) to enable.
  # EXECUTOR_E2E_GITHUB_TOKEN must be provided as a SECRET pipeline variable
  # scoped to Issues:read/write on the configured issue repository.

steps:
  - checkout: self
    fetchDepth: 1
    displayName: Checkout ado-aw

  - script: |
      set -euo pipefail
      mkdir -p .cargo
      # CargoAuthenticate@0 only attaches credentials to feeds that are
      # DECLARED in the config file it is given. The checked-in
      # .cargo/config.toml carries only the Windows MSVC stack setting (inert
      # on Linux), so append the internal crates.io source replacement here so
      # the auth task has something to authenticate and `cargo build` fetches
      # crates from the internal mirror.
      {
        printf '\n[registries]\ncargo = { index = "%s" }\n\n' "$(EFFECTIVE_CRATES_IO_FEED)"
        printf '[source.crates-io]\nreplace-with = "cargo"\n'
      } >> .cargo/config.toml
      echo "----- .cargo/config.toml -----"
      cat .cargo/config.toml
    displayName: Write cargo config for internal crates.io feed

  - task: CargoAuthenticate@0
    inputs:
      configFile: ".cargo/config.toml"
    displayName: Authenticate with cargo (internal feeds)

  - task: RustInstaller@1
    inputs:
      rustVersion: ms-stable
      toolchainFeed: "https://pkgs.dev.azure.com/msazuresphere/AgentPlayground/_packaging/AgentPlaygroundRustTools%40Local/nuget/v3/index.json"
      cratesIoFeedOverride: "$(EFFECTIVE_CRATES_IO_FEED)"
    displayName: Install Rust toolchain

  - script: |
      set -euo pipefail
      cargo build --release --bin ado-aw
      echo "##vso[task.setvariable variable=ADO_AW_BIN]$(Build.SourcesDirectory)/target/release/ado-aw"
    displayName: Build ado-aw (release, from checkout)

  - task: UseNode@1
    inputs:
      version: "20.x"
    displayName: Use Node.js 20

  - script: |
      set -euo pipefail
      npm ci
      npm run build:executor-e2e
    workingDirectory: scripts/ado-script
    displayName: Build executor-e2e harness (not shipped in ado-script.zip)

  - task: AzureCLI@2
    displayName: Acquire ADO token (SC_WRITE_TOKEN)
    inputs:
      azureSubscription: agent-playground-write
      scriptType: bash
      scriptLocation: inlineScript
      addSpnToEnvironment: true
      inlineScript: |
        ADO_TOKEN=$(az account get-access-token \
          --resource 499b84ac-1321-427f-aa17-267ca6975798 \
          --query accessToken -o tsv)
        echo "##vso[task.setvariable variable=SC_WRITE_TOKEN;issecret=true]$ADO_TOKEN"

  - script: |
      set -euo pipefail
      node scripts/ado-script/test-bin/executor-e2e.js
    displayName: Run deterministic executor E2E scenarios
    env:
      # ADO auth + context. SYSTEM_COLLECTIONURI / SYSTEM_TEAMPROJECT /
      # BUILD_BUILDID are auto-injected; SYSTEM_ACCESSTOKEN must be mapped
      # explicitly so the harness (and the ado-aw binary it spawns) can write.
      SYSTEM_ACCESSTOKEN: $(SC_WRITE_TOKEN)
      EXECUTOR_E2E_ADO_AW_BIN: $(ADO_AW_BIN)
      EXECUTOR_E2E_ADO_REPO: $(EFFECTIVE_EXECUTOR_E2E_ADO_REPO)
      EXECUTOR_E2E_ISSUE_REPO: $(EXECUTOR_E2E_ISSUE_REPO)
      # Secret PAT for filing failure issues on the configured repository.
      EXECUTOR_E2E_GITHUB_TOKEN: $(EXECUTOR_E2E_GITHUB_TOKEN)
      # Optional-precondition scenarios (skipped when unset):
      E2E_QUEUE_PIPELINE_ID: $(EFFECTIVE_E2E_QUEUE_PIPELINE_ID)
      E2E_WIKI_NAME: $(EFFECTIVE_E2E_WIKI_NAME)