Azure
diff --git a/‎eng/common/pipelines/live-eval.yml‎
Lines changed: 43 additions & 0 deletions b/‎eng/common/pipelines/live-eval.yml‎
Lines changed: 43 additions & 0 deletions
diff --git a/‎eng/common/pipelines/skill-eval.yml‎
Lines changed: 38 additions & 0 deletions b/‎eng/common/pipelines/skill-eval.yml‎
Lines changed: 38 additions & 0 deletions
diff --git a/‎eng/common/pipelines/templates/jobs/build-mcp.yml‎
Lines changed: 43 additions & 0 deletions b/‎eng/common/pipelines/templates/jobs/build-mcp.yml‎
Lines changed: 43 additions & 0 deletions
diff --git a/‎eng/common/pipelines/templates/jobs/eval-shard.yml‎
Lines changed: 141 additions & 0 deletions b/‎eng/common/pipelines/templates/jobs/eval-shard.yml‎
Lines changed: 141 additions & 0 deletions
diff --git a/‎eng/common/pipelines/templates/jobs/eval-summarize.yml‎
Lines changed: 45 additions & 0 deletions b/‎eng/common/pipelines/templates/jobs/eval-summarize.yml‎
Lines changed: 45 additions & 0 deletions
diff --git a/‎eng/common/pipelines/templates/jobs/generate-eval-matrix.yml‎
Lines changed: 48 additions & 0 deletions b/‎eng/common/pipelines/templates/jobs/generate-eval-matrix.yml‎
Lines changed: 48 additions & 0 deletions
@@ -0,0 +1,43 @@
+# Live-tier eval CI: nightly end-to-end run of the live workflow scenarios against the real
+# azsdk-cli MCP and real Azure DevOps (writes confined to a test area).
+
+# Nightly only — no CI/PR trigger.
+trigger: none
+pr: none
+
+schedules:
+  - cron: '0 9 * * *'        # 09:00 UTC daily
+    displayName: 'Nightly live eval'
+    branches:
+      include:
+        - main
+    always: true             # run even when main has not changed
+
+variables:
+  # Managed-pool image selection (LINUXPOOL/LINUXVMIMAGE). Repo-local.
+  - template: /eng/pipelines/templates/variables/image.yml
+  # Provides the secret azuresdk-copilot-github-pat, mapped into GITHUB_TOKEN in the invoke step.
+  - group: AzSDK_Eval_Variable_group
+
+extends:
+  template: /eng/common/pipelines/templates/stages/archetype-eval.yml
+  parameters:
+    # Shared mock/live builder; select the live tier (real Cli MCP).
+    mcpSetupTemplate: /eng/common/pipelines/templates/steps/eval-mcp-setup.yml
+    TestType: live
+    vallyRoot: tools/azsdk-cli/Azure.Sdk.Tools.Vally
+    evalGlobs:
+      - 'evals/workflow-scenarios/live/*.eval.yaml'
+    # Run each shard under AzureCLI@2 so the real MCP's DevOps calls are authenticated.
+    UseAzSdkAuthentication: true
+    failOnFailedTests: true
+    # Live scenarios are end-to-end, so give each shard more headroom than the report-only tiers.
+    shardTimeoutInMinutes: 45
+    # Pass-rate gate for `vally eval`. This tier gates (failOnFailedTests: true), so the threshold
+    # is the real bar the nightly run must clear; raise it here as live coverage stabilizes.
+    threshold: 0.8
+    # This repo needs no repo-specific setup (the live MCP is built by the common BuildMcp job).
+    # A spec/language repo that must start its own bot / server / MCP copies the example hook and
+    # points these at it — see eng/common/pipelines/templates/steps/eval-hook-example.yml.
+    # preEvalTemplate: /eng/pipelines/eval/start-my-bot.yml
+    # postEvalTemplate: /eng/pipelines/eval/stop-my-bot.yml
@@ -0,0 +1,38 @@
+# Skill-compliance eval CI: runs the per-skill Vally evals under .github/skills (one job per skill).
+
+trigger:
+  branches:
+    include:
+      - main
+  paths:
+    include:
+      - .github/skills/**
+      # Retrigger when the mock MCP's tool catalog changes (it can move results).
+      - tools/azsdk-cli/Azure.Sdk.Tools.Mock/**
+      - eng/common/pipelines/skill-eval.yml
+      - eng/common/pipelines/templates/jobs/**
+      - eng/common/pipelines/templates/steps/eval-invoke.yml
+      - eng/common/pipelines/templates/steps/eval-mcp-setup.yml
+      - eng/common/pipelines/templates/stages/archetype-eval.yml
+      - eng/common/scripts/eval/**
+
+pr: none
+
+variables:
+  # Managed-pool image selection (LINUXPOOL/LINUXVMIMAGE). Repo-local.
+  - template: /eng/pipelines/templates/variables/image.yml
+  # Provides the secret azuresdk-copilot-github-pat, mapped into GITHUB_TOKEN in the invoke step.
+  - group: AzSDK_Eval_Variable_group
+
+extends:
+  template: /eng/common/pipelines/templates/stages/archetype-eval.yml
+  parameters:
+    # Shared mock/live builder; select the mock tier.
+    mcpSetupTemplate: /eng/common/pipelines/templates/steps/eval-mcp-setup.yml
+    TestType: mock
+    vallyRoot: .github/skills
+    # Single-level glob excludes azure-typespec-author/evaluate/ (its own benchmark pipeline).
+    evalGlobs:
+      - '*/evals/*.eval.yaml'
+    # Per-shard job timeout (report-only tier).
+    shardTimeoutInMinutes: 20
@@ -0,0 +1,43 @@
+# BuildMcp job: builds the MCP server(s) via the swap-point mcpSetupTemplate and ships the
+# artifacts/mcp tree as the `mcp-servers` artifact every shard re-stages.
+
+parameters:
+  - name: mcpSetupTemplate
+    type: string
+  - name: TestType
+    # Which MCP tier to build (mock/live), forwarded to templates that build both tiers
+    # (e.g. eval-mcp-setup.yml). Use 'none' for param-free repo templates.
+    type: string
+    default: none
+  - name: toolsRepo
+    # Optional azure-sdk-tools repository-resource alias, forwarded to the MCP template when this
+    # pipeline runs OUTSIDE azure-sdk-tools (cross-repo build). Blank in the tools repo.
+    type: string
+    default: ''
+
+jobs:
+  - job: BuildMcp
+    displayName: 'Build + publish MCP server(s)'
+    pool:
+      name: $(LINUXPOOL)
+      image: $(LINUXVMIMAGE)
+      os: linux
+    # 1ES publishes via templateContext.outputs, not the `publish` shortcut.
+    templateContext:
+      outputs:
+        - output: pipelineArtifact
+          displayName: 'Publish MCP server artifact'
+          artifactName: mcp-servers
+          targetPath: $(Build.SourcesDirectory)/artifacts/mcp
+    steps:
+      - checkout: self
+        fetchDepth: 1
+
+      # Pass TestType only when not 'none', so param-free repo templates still resolve.
+      - ${{ if ne(parameters.TestType, 'none') }}:
+        - template: ${{ parameters.mcpSetupTemplate }}
+          parameters:
+            TestType: ${{ parameters.TestType }}
+            toolsRepo: ${{ parameters.toolsRepo }}
+      - ${{ else }}:
+        - template: ${{ parameters.mcpSetupTemplate }}
@@ -0,0 +1,141 @@
+# Fan-out shard job: one leg per shard runs its evals against the prebuilt MCP and publishes
+# per-leg JUnit (always) + transcripts (on failure). Pass/fail is the eval VERDICT, not the
+# `vally` exit code (vally can exit 1 on a teardown flake after a pass).
+
+parameters:
+  - name: vallyRoot
+    # Required (no default) so each repo passes its own eval root explicitly.
+    type: string
+  - name: evalInvokeTemplate
+    # Swap point: runs `vally eval` for the shard (the common template is auth-conditional).
+    type: string
+    default: /eng/common/pipelines/templates/steps/eval-invoke.yml
+  - name: UseAzSdkAuthentication
+    # Forwarded to the invoke template: true runs the shard under AzureCLI@2.
+    type: boolean
+    default: false
+  - name: threshold
+    # Pass-rate gate forwarded to the invoke template (`vally eval --threshold` + shard verdict).
+    type: number
+    default: 0.8
+  - name: preEvalTemplate
+    # Optional repo-specific setup run before the eval (e.g. start a bot/agent). Placeholder by default.
+    type: string
+    default: ''
+  - name: postEvalTemplate
+    # Optional repo-specific teardown run after the eval. Placeholder by default.
+    type: string
+    default: ''
+  - name: shardTimeoutInMinutes
+    # Per-shard job timeout. Must be a parameter (not a runtime variable) so 1ES resolves it at compile time.
+    type: number
+    default: 20
+
+jobs:
+  - job: RunShard
+    displayName: 'Eval'
+    pool:
+      name: $(LINUXPOOL)
+      image: $(LINUXVMIMAGE)
+      os: linux
+    # A red shard shouldn't abort its siblings; the rollup gates in Summary.
+    continueOnError: true
+    timeoutInMinutes: ${{ parameters.shardTimeoutInMinutes }}
+    strategy:
+      matrix: $[ stageDependencies.Prepare.generate_eval_matrix.outputs['detect.matrix'] ]
+      maxParallel: 10
+    templateContext:
+      outputs:
+        # Artifact names carry $(System.JobAttempt) so "Rerun failed jobs" publishes a fresh
+        # name instead of colliding with the prior attempt's artifact (which fails the rerun).
+        # The Summary job keeps only the highest attempt per shard so retries supersede cleanly.
+        - output: pipelineArtifact
+          displayName: 'Publish $(shardName) JUnit'
+          artifactName: 'eval-result-$(shardName)-$(System.JobAttempt)'
+          targetPath: $(Build.ArtifactStagingDirectory)/junit
+          condition: always()
+        - output: pipelineArtifact
+          displayName: 'Publish $(shardName) transcripts (on failure)'
+          artifactName: 'eval-debug-$(shardName)-$(System.JobAttempt)'
+          targetPath: $(Build.ArtifactStagingDirectory)/debug
+          condition: failed()
+    steps:
+      - checkout: self
+        fetchDepth: 1
+
+      - task: UseNode@1
+        displayName: 'Use Node.js 22.x'
+        inputs:
+          version: 22.x
+
+      # 1ES Official image blocks registry.npmjs.org; restore via the azure-sdk npm mirror.
+      - template: /eng/common/pipelines/templates/steps/create-authenticated-npmrc.yml
+        parameters:
+          npmrcPath: $(Build.SourcesDirectory)/.npmrc
+          registryUrl: https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-tools/npm/registry/
+
+      # Installs vally-cli + its pinned copilot-sdk executor into node_modules.
+      - script: npm ci --userconfig $(Build.SourcesDirectory)/.npmrc
+        displayName: 'Install Vally CLI + copilot-sdk (pinned)'
+        workingDirectory: eng/common/scripts/eval
+
+      - download: current
+        artifact: mcp-servers
+        displayName: 'Download MCP server(s)'
+
+      - script: |
+          mkdir -p "$(Build.SourcesDirectory)/artifacts/mcp"
+          cp -r "$(Pipeline.Workspace)/mcp-servers/." "$(Build.SourcesDirectory)/artifacts/mcp/"
+        displayName: 'Stage MCP server(s) at expected path'
+
+      # Optional repo-specific setup (e.g. start a bot/agent) before the eval runs.
+      - ${{ if parameters.preEvalTemplate }}:
+        - template: ${{ parameters.preEvalTemplate }}
+
+      # Clone only the git fixtures this shard's evals reference (Vally won't clone them).
+      - script: |
+          patterns=()
+          set -f
+          for tok in $EVAL_ARGS; do
+            [ "$tok" = "-e" ] && continue
+            patterns+=(--pattern "$tok")
+          done
+          set +f
+          node "$(Build.SourcesDirectory)/eng/common/scripts/eval/init-eval-git-fixtures.js" \
+            --eval-root "$(Build.SourcesDirectory)/${{ parameters.vallyRoot }}" \
+            "${patterns[@]}"
+        displayName: 'Prime eval git fixtures'
+        env:
+          EVAL_ARGS: $(evalArgs)
+
+      # Runs the eval via the invoke template (auth-conditional in the common default).
+      - template: ${{ parameters.evalInvokeTemplate }}
+        parameters:
+          vallyRoot: ${{ parameters.vallyRoot }}
+          UseAzSdkAuthentication: ${{ parameters.UseAzSdkAuthentication }}
+          threshold: ${{ parameters.threshold }}
+
+      # Optional repo-specific teardown after the eval.
+      - ${{ if parameters.postEvalTemplate }}:
+        - template: ${{ parameters.postEvalTemplate }}
+
+      # JUnit is tiny — stage it always; heavy transcripts stage separately on failure.
+      # Also create the debug dir here so the failed()-gated debug publish always has a target
+      # (otherwise the publish errors "Path does not exist" when the dir was never created).
+      - script: |
+          mkdir -p "$(Build.ArtifactStagingDirectory)/junit" "$(Build.ArtifactStagingDirectory)/debug"
+          find "$(Build.SourcesDirectory)/artifacts/vally-results/$(shardName)" -name '*.junit.xml' -exec cp {} "$(Build.ArtifactStagingDirectory)/junit/" \;
+        displayName: 'Collect $(shardName) JUnit'
+        condition: always()
+
+      # On a real failure keep transcripts; strip JUnit so Summary can't double-count it.
+      - script: |
+          src="$(Build.SourcesDirectory)/artifacts/vally-results/$(shardName)"
+          dest="$(Build.ArtifactStagingDirectory)/debug"
+          mkdir -p "$dest"
+          if [ -d "$src" ]; then
+            cp -r "$src/." "$dest/"
+            find "$dest" -name '*.junit.xml' -delete
+          fi
+        displayName: 'Collect $(shardName) transcripts (on failure)'
+        condition: failed()
@@ -0,0 +1,45 @@
+# Summary job: merges every shard's JUnit into one rollup (Markdown summary + Tests tab) and
+# gates on it when failOnFailedTests is true.
+
+parameters:
+  - name: failOnFailedTests
+    type: boolean
+    default: false
+
+jobs:
+  - job: Summarize
+    displayName: 'Publish + gate rollup'
+    pool:
+      name: $(LINUXPOOL)
+      image: $(LINUXVMIMAGE)
+      os: linux
+    steps:
+      - task: UseNode@1
+        displayName: 'Use Node.js 22.x'
+        inputs:
+          version: 22.x
+
+      # Match *.junit.xml only so stray XML (e.g. a DLL's doc XML) can't fake a shard.
+      - task: DownloadPipelineArtifact@2
+        displayName: 'Download all shard results'
+        inputs:
+          buildType: 'current'
+          itemPattern: '**/*.junit.xml'
+          targetPath: $(Pipeline.Workspace)/eval-results
+
+      # Renders the rollup to the run's Summary tab (##vso[task.uploadsummary]).
+      - script: |
+          node "$(Build.SourcesDirectory)/eng/common/scripts/eval/build-eval-summary.js" \
+            --results-root "$(Pipeline.Workspace)/eval-results" \
+            --output-path "$(Build.ArtifactStagingDirectory)/eval-summary.md"
+        displayName: 'Render Markdown rollup'
+        condition: always()
+
+      - task: PublishTestResults@2
+        displayName: 'Publish eval rollup'
+        inputs:
+          testResultsFormat: 'JUnit'
+          testResultsFiles: '$(Pipeline.Workspace)/eval-results/**/*.junit.xml'
+          mergeTestResults: true
+          failTaskOnFailedTests: ${{ parameters.failOnFailedTests }}
+          testRunTitle: 'Vally evals'
@@ -0,0 +1,48 @@
+# Globs the eval files into a fan-out matrix (output var detect.matrix). Runs in Prepare
+# alongside BuildMcp; the Eval stage's RunShard job reads it cross-stage.
+
+parameters:
+  - name: vallyRoot
+    type: string
+  - name: evalGlobs
+    type: object
+    default: []
+
+jobs:
+  # Job/step/output names are fixed: the Eval stage reads the matrix cross-stage at
+  # stageDependencies.Prepare.generate_eval_matrix.outputs['detect.matrix'].
+  - job: generate_eval_matrix
+    displayName: 'Glob eval files -> matrix'
+    pool:
+      name: $(LINUXPOOL)
+      image: $(LINUXVMIMAGE)
+      os: linux
+    steps:
+      - checkout: self
+        fetchDepth: 1
+
+      - task: UseNode@1
+        displayName: 'Use Node.js 22.x'
+        inputs:
+          version: 22.x
+
+      # set -f stops the shell expanding the glob `*` before patterns reach the script.
+      - script: |
+          patterns=()
+          if [ -n "$EVAL_PATTERNS" ]; then
+            set -f
+            for p in $EVAL_PATTERNS; do
+              patterns+=(--pattern "$p")
+            done
+            set +f
+          fi
+          base="$(Build.SourcesDirectory)/${{ parameters.vallyRoot }}"
+          node "$(Build.SourcesDirectory)/eng/common/scripts/eval/collect-stimuli.js" \
+            --eval-root "$base" \
+            --path-base "$base" \
+            "${patterns[@]}" \
+            --output-variable matrix
+        name: detect
+        displayName: 'Build fan-out matrix'
+        env:
+          EVAL_PATTERNS: ${{ join(' ', parameters.evalGlobs) }}