Sync eng/common directory with azure-sdk-tools for PR 16402 (#49878)

* Rename azsdk-evals directory to evals

* Flatten eval suite directory layout

---------

Co-authored-by: helen229 <gaoh@microsoft.com>

Author: azure-sdk-automation[bot]

eng/common/pipelines/live-eval.yml

@@ -25,9 +25,9 @@ extends:
     # Shared mock/live builder; select the live tier (real Cli MCP).
     mcpSetupTemplate: /eng/common/pipelines/templates/steps/eval-mcp-setup.yml
     TestType: live
-    vallyRoot: azsdk-evals
+    vallyRoot: evals
     evalGlobs:
-      - 'evals/workflow-scenarios/live/*.eval.yaml'
+      - 'workflows/live/*.eval.yaml'
     # Run each shard under AzureCLI@2 so the real MCP's DevOps calls are authenticated.
     UseAzSdkAuthentication: true
     failOnFailedTests: true

eng/common/pipelines/templates/stages/archetype-eval.yml

@@ -15,8 +15,8 @@ parameters:
     # collect-stimuli.js mirrors this list as its no-args fallback — keep them in sync.
     type: object
     default:
-      - 'evals/tools/*.eval.yaml'
-      - 'evals/workflow-scenarios/mock/*.eval.yaml'
+      - 'tools/*.eval.yaml'
+      - 'workflows/mock/*.eval.yaml'
   - name: mcpSetupTemplate
     # SWAP POINT: builds/installs the MCP server(s) under artifacts/mcp/<name>. Required (no default):
     # each repo passes its own build template. This repo uses the shared mock/live builder

eng/common/pipelines/workflow-eval.yml

@@ -1,13 +1,13 @@
 # Hermetic workflow-scenario eval CI: runs the unit-tool + mock workflow-scenario evals in
-# azsdk-evals against the mock MCP (live tier runs in live-eval.yml).
+# evals against the mock MCP (live tier runs in live-eval.yml).
 
 trigger:
   branches:
     include:
       - main
   paths:
     include:
-      - azsdk-evals/**
+      - evals/**
       - tools/azsdk-cli/Azure.Sdk.Tools.Mock/**
       # Cli tool catalog + skills feed the eval results, so changes here must retrigger.
       - tools/azsdk-cli/Azure.Sdk.Tools.Cli/**
@@ -25,7 +25,7 @@ pr:
       - main
   paths:
     include:
-      - azsdk-evals/**
+      - evals/**
       - tools/azsdk-cli/Azure.Sdk.Tools.Mock/**
       - tools/azsdk-cli/Azure.Sdk.Tools.Cli/**
       - .github/skills/**
@@ -48,7 +48,7 @@ extends:
     # Shared mock/live builder; select the mock tier.
     mcpSetupTemplate: /eng/common/pipelines/templates/steps/eval-mcp-setup.yml
     TestType: mock
-    vallyRoot: azsdk-evals
+    vallyRoot: evals
     # Per-shard job timeout (report-only tier).
     shardTimeoutInMinutes: 20
     # Pass-rate gate for `vally eval`. This tier is report-only (failOnFailedTests defaults false),

eng/common/scripts/allow-relative-links.txt

@@ -9,5 +9,5 @@
 AGENTS.md
 # Allow relative links for all files under the eng folder (e.g. engineering system scripts and templates).
 eng/**
-# Allow relative links for the top-level eval suite (azsdk-evals is a cross-cutting asset at repo root).
-azsdk-evals/**
+# Allow relative links for the top-level eval suite (evals is a cross-cutting asset at repo root).
+evals/**

eng/common/scripts/eval/collect-stimuli.ts

@@ -10,8 +10,8 @@ import { globFiles } from "./lib/glob.ts";
 
 // Fallback patterns when run with no --pattern. Mirrors archetype-eval.yml's evalGlobs default.
 const DEFAULT_PATTERNS = [
-  "evals/tools/*.eval.yaml",
-  "evals/workflow-scenarios/mock/*.eval.yaml",
+  "tools/*.eval.yaml",
+  "workflows/mock/*.eval.yaml",
 ];
 
 const SANITIZE = /[^A-Za-z0-9]/g;

eng/common/scripts/eval/init-eval-git-fixtures.ts

@@ -9,8 +9,8 @@ import { globFiles } from "./lib/glob.ts";
 import { syncRepo } from "./sync-eval-git-repo.ts";
 
 export const DEFAULT_PATTERNS = [
-  "evals/tools/*.eval.yaml",
-  "evals/workflow-scenarios/mock/*.eval.yaml",
+  "tools/*.eval.yaml",
+  "workflows/mock/*.eval.yaml",
 ];
 
 // Repos we know how to clone efficiently (cone-sparse to the spec folders the fixtures touch).

eng/common/scripts/eval/lib/glob.ts

@@ -62,7 +62,7 @@ function walk(dir, segments, out) {
  * glob `pattern`. Results are sorted for deterministic ordering.
  *
  * @param {string} root Absolute or relative base directory to glob from.
- * @param {string} pattern Forward-slashed glob relative to `root` (e.g. "evals/tools/*.eval.yaml").
+ * @param {string} pattern Forward-slashed glob relative to `root` (e.g. "tools/*.eval.yaml").
  * @returns {string[]} Sorted absolute file paths.
  */
 export function globFiles(root, pattern) {

eng/common/scripts/eval/test/collect-stimuli.test.ts

@@ -31,19 +31,19 @@ describe("collect-stimuli (default discovery)", () => {
   before(() => {
     root = fs.mkdtempSync(path.join(os.tmpdir(), "vally-matrix-"));
     writeFile(
-      path.join(root, "evals/tools/prompt-to-tool-github.eval.yaml"),
+      path.join(root, "tools/prompt-to-tool-github.eval.yaml"),
       "tags:\n  area: github"
     );
     writeFile(
-      path.join(root, "evals/tools/add-arm-resource.eval.yaml"),
+      path.join(root, "tools/add-arm-resource.eval.yaml"),
       "tags:\n  area: typespec"
     );
     writeFile(
-      path.join(root, "evals/workflow-scenarios/mock/rename-client-property.eval.yaml"),
+      path.join(root, "workflows/mock/rename-client-property.eval.yaml"),
       "tags:\n  area: typespec"
     );
     writeFile(
-      path.join(root, "evals/workflow-scenarios/live/release-planner.eval.yaml"),
+      path.join(root, "workflows/live/release-planner.eval.yaml"),
       "tags:\n  area: release-plan"
     );
   });
@@ -70,7 +70,7 @@ describe("collect-stimuli (default discovery)", () => {
     const matrix = buildMatrix({ roots: [root] });
     for (const entry of Object.values(matrix)) {
       assert.doesNotMatch(entry.evalArgs, /\\/);
-      assert.match(entry.evalArgs, /^-e evals\//);
+      assert.match(entry.evalArgs, /^-e (tools|workflows)\//);
     }
   });
 
@@ -94,15 +94,15 @@ describe("collect-stimuli (area grouping)", () => {
   before(() => {
     root = fs.mkdtempSync(path.join(os.tmpdir(), "vally-matrix-area-"));
     writeFile(
-      path.join(root, "evals/tools/prompt-to-tool-github.eval.yaml"),
+      path.join(root, "tools/prompt-to-tool-github.eval.yaml"),
       "tags:\n  area: github"
     );
     writeFile(
-      path.join(root, "evals/tools/add-arm-resource.eval.yaml"),
+      path.join(root, "tools/add-arm-resource.eval.yaml"),
       "tags:\n  area: typespec"
     );
     writeFile(
-      path.join(root, "evals/workflow-scenarios/mock/rename-client-property.eval.yaml"),
+      path.join(root, "workflows/mock/rename-client-property.eval.yaml"),
       "tags:\n  area: typespec"
     );
   });
@@ -127,11 +127,11 @@ describe("collect-stimuli (area grouping)", () => {
     const collideRoot = fs.mkdtempSync(path.join(os.tmpdir(), "vally-matrix-area-collide-"));
     try {
       writeFile(
-        path.join(collideRoot, "evals/tools/a.eval.yaml"),
+        path.join(collideRoot, "tools/a.eval.yaml"),
         "tags:\n  area: release-plan"
       );
       writeFile(
-        path.join(collideRoot, "evals/tools/b.eval.yaml"),
+        path.join(collideRoot, "tools/b.eval.yaml"),
         "tags:\n  area: release_plan"
       );
       assert.throws(
@@ -149,8 +149,8 @@ describe("collect-stimuli (area with an untagged eval)", () => {
 
   before(() => {
     root = fs.mkdtempSync(path.join(os.tmpdir(), "vally-matrix-ut-"));
-    writeFile(path.join(root, "evals/tools/tagged.eval.yaml"), "tags:\n  area: github");
-    writeFile(path.join(root, "evals/tools/untagged.eval.yaml"), "no tags here");
+    writeFile(path.join(root, "tools/tagged.eval.yaml"), "tags:\n  area: github");
+    writeFile(path.join(root, "tools/untagged.eval.yaml"), "no tags here");
   });
 
   after(() => fs.rmSync(root, { recursive: true, force: true }));
@@ -159,7 +159,7 @@ describe("collect-stimuli (area with an untagged eval)", () => {
     const { result: matrix } = withWarnings((warn) =>
       buildMatrix({
         roots: [root],
-        patterns: ["evals/tools/*.eval.yaml"],
+        patterns: ["tools/*.eval.yaml"],
         warn,
       })
     );
@@ -172,7 +172,7 @@ describe("collect-stimuli (area with an untagged eval)", () => {
     const { result: matrix } = withWarnings((warn) =>
       buildMatrix({
         roots: [root],
-        patterns: ["evals/tools/*.eval.yaml"],
+        patterns: ["tools/*.eval.yaml"],
         warn,
       })
     );
@@ -183,7 +183,7 @@ describe("collect-stimuli (area with an untagged eval)", () => {
     const { warnings } = withWarnings((warn) =>
       buildMatrix({
         roots: [root],
-        patterns: ["evals/tools/*.eval.yaml"],
+        patterns: ["tools/*.eval.yaml"],
         warn,
       })
     );
@@ -197,11 +197,11 @@ describe("collect-stimuli (overlapping patterns)", () => {
   before(() => {
     root = fs.mkdtempSync(path.join(os.tmpdir(), "vally-matrix-overlap-"));
     writeFile(
-      path.join(root, "evals/tools/add-arm-resource.eval.yaml"),
+      path.join(root, "tools/add-arm-resource.eval.yaml"),
       "tags:\n  area: typespec"
     );
     writeFile(
-      path.join(root, "evals/tools/prompt-to-tool-github.eval.yaml"),
+      path.join(root, "tools/prompt-to-tool-github.eval.yaml"),
       "tags:\n  area: github"
     );
   });
@@ -211,7 +211,7 @@ describe("collect-stimuli (overlapping patterns)", () => {
   it("does not emit a duplicate -e flag for a file matched by multiple patterns", () => {
     const matrix = buildMatrix({
       roots: [root],
-      patterns: ["evals/tools/*.eval.yaml", "evals/tools/add-arm-resource.eval.yaml"],
+      patterns: ["tools/*.eval.yaml", "tools/add-arm-resource.eval.yaml"],
     });
     const count = (matrix.area_typespec.evalArgs.match(/add-arm-resource/g) || []).length;
     assert.equal(count, 1);
@@ -226,11 +226,11 @@ describe("collect-stimuli (multiple eval roots: repo + common)", () => {
     repoRoot = fs.mkdtempSync(path.join(os.tmpdir(), "vally-matrix-repo-"));
     commonRoot = fs.mkdtempSync(path.join(os.tmpdir(), "vally-matrix-common-"));
     writeFile(
-      path.join(repoRoot, "evals/tools/repo-specific.eval.yaml"),
+      path.join(repoRoot, "tools/repo-specific.eval.yaml"),
       "tags:\n  area: repo"
     );
     writeFile(
-      path.join(commonRoot, "evals/tools/shared-scenario.eval.yaml"),
+      path.join(commonRoot, "tools/shared-scenario.eval.yaml"),
       "tags:\n  area: shared"
     );
   });
@@ -243,7 +243,7 @@ describe("collect-stimuli (multiple eval roots: repo + common)", () => {
   it("collects evals from both roots into one matrix", () => {
     const matrix = buildMatrix({
       roots: [repoRoot, commonRoot],
-      patterns: ["evals/tools/*.eval.yaml"],
+      patterns: ["tools/*.eval.yaml"],
     });
     const keys = Object.keys(matrix);
     assert.equal(keys.length, 2);
@@ -254,11 +254,11 @@ describe("collect-stimuli (multiple eval roots: repo + common)", () => {
   it("computes each file's relative path against its own root", () => {
     const matrix = buildMatrix({
       roots: [repoRoot, commonRoot],
-      patterns: ["evals/tools/*.eval.yaml"],
+      patterns: ["tools/*.eval.yaml"],
     });
     assert.equal(
       matrix.area_shared.evalArgs,
-      "-e evals/tools/shared-scenario.eval.yaml"
+      "-e tools/shared-scenario.eval.yaml"
     );
   });
 });
@@ -275,11 +275,11 @@ describe("collect-stimuli (pathBase anchors scattered roots to one run root)", (
     runRoot = path.join(parent, "project");
     scatteredRoot = path.join(parent, "extra");
     writeFile(
-      path.join(runRoot, "evals/tools/in-project.eval.yaml"),
+      path.join(runRoot, "tools/in-project.eval.yaml"),
       "tags:\n  area: inproject"
     );
     writeFile(
-      path.join(scatteredRoot, "evals/out-of-tree.eval.yaml"),
+      path.join(scatteredRoot, "workflows/out-of-tree.eval.yaml"),
       "tags:\n  area: scattered"
     );
   });
@@ -290,28 +290,28 @@ describe("collect-stimuli (pathBase anchors scattered roots to one run root)", (
     const matrix = buildMatrix({
       roots: [runRoot, scatteredRoot],
       pathBase: runRoot,
-      patterns: ["evals/**/*.eval.yaml", "evals/*.eval.yaml"],
+      patterns: ["tools/**/*.eval.yaml", "workflows/*.eval.yaml"],
     });
     // The in-project file stays a simple relative path; the scattered one walks up.
     assert.equal(
       matrix.area_inproject.evalArgs,
-      "-e evals/tools/in-project.eval.yaml"
+      "-e tools/in-project.eval.yaml"
     );
     assert.equal(
       matrix.area_scattered.evalArgs,
-      "-e ../extra/evals/out-of-tree.eval.yaml"
+      "-e ../extra/workflows/out-of-tree.eval.yaml"
     );
   });
 
   it("falls back to per-root relative paths when no pathBase is given", () => {
     const matrix = buildMatrix({
       roots: [scatteredRoot],
-      patterns: ["evals/*.eval.yaml"],
+      patterns: ["workflows/*.eval.yaml"],
     });
     // Without a base, the path is relative to the root it was found under (no `../`).
     assert.equal(
       matrix.area_scattered.evalArgs,
-      "-e evals/out-of-tree.eval.yaml"
+      "-e workflows/out-of-tree.eval.yaml"
     );
   });
 });

eng/common/scripts/eval/test/init-eval-git-fixtures.test.ts

@@ -17,12 +17,12 @@ describe("getEvalGitFixtures discovery", () => {
   before(() => {
     // Throwaway eval tree so the tests do not depend on real eval content.
     root = fs.mkdtempSync(path.join(os.tmpdir(), "vally-fixtures-test-"));
-    fs.mkdirSync(path.join(root, "evals/tools"), { recursive: true });
-    fs.mkdirSync(path.join(root, "evals/workflow-scenarios/mock"), { recursive: true });
+    fs.mkdirSync(path.join(root, "tools"), { recursive: true });
+    fs.mkdirSync(path.join(root, "workflows/mock"), { recursive: true });
 
     // A unit eval with NO git fixture.
     fs.writeFileSync(
-      path.join(root, "evals/tools/prompt-to-tool-github.eval.yaml"),
+      path.join(root, "tools/prompt-to-tool-github.eval.yaml"),
       "tags:\n  area: github\nstimuli:\n  - name: x\n"
     );
 
@@ -45,7 +45,7 @@ describe("getEvalGitFixtures discovery", () => {
       "",
     ].join("\n");
     fs.writeFileSync(
-      path.join(root, "evals/workflow-scenarios/mock/release-planner-workflows.eval.yaml"),
+      path.join(root, "workflows/mock/release-planner-workflows.eval.yaml"),
       mock
     );
   });
@@ -76,7 +76,7 @@ describe("getEvalGitFixtures discovery", () => {
   });
 
   it("is a no-op when the scanned suite declares no git fixtures", () => {
-    const fixtures = getEvalGitFixtures({ root, patterns: ["evals/tools/*.eval.yaml"] });
+    const fixtures = getEvalGitFixtures({ root, patterns: ["tools/*.eval.yaml"] });
     assert.equal(fixtures.length, 0);
   });
 
@@ -90,7 +90,7 @@ describe("getEvalGitFixtures discovery", () => {
       "        source: ../../../../../../artifacts/specs-cache/some-other-repo",
       "",
     ].join("\n");
-    const file = path.join(root, "evals/workflow-scenarios/mock/no-ref.eval.yaml");
+    const file = path.join(root, "workflows/mock/no-ref.eval.yaml");
     fs.writeFileSync(file, noRef);
     try {
       const fixtures = dedupeFixtures(getEvalGitFixtures({ root }));
@@ -111,9 +111,9 @@ describe("getEvalGitFixtures discovery", () => {
 // contain the Vally suite they skip.
 describe("Folder-level invariant for real git fixtures", () => {
   const repoRoot = path.resolve(here, "../../../../..");
-  const vallyRoot = path.join(repoRoot, "azsdk-evals");
-  const vallyEvals = path.join(vallyRoot, "evals");
-  const present = fs.existsSync(vallyEvals);
+  const vallyRoot = path.join(repoRoot, "evals");
+  const evalRoots = [path.join(vallyRoot, "tools"), path.join(vallyRoot, "workflows")];
+  const present = evalRoots.some((root) => fs.existsSync(root));
 
   const expectedCacheRoot = path
     .join(repoRoot, "artifacts", "specs-cache")
@@ -122,7 +122,7 @@ describe("Folder-level invariant for real git fixtures", () => {
   function collectRealFixtures() {
     const srcRegex = /^\s*source:\s*(\.\.\S+)/gm;
     const results = [];
-    const stack = [vallyEvals];
+    const stack = evalRoots.filter((root) => fs.existsSync(root));
     while (stack.length > 0) {
       const dir = stack.pop();
       for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {