Merge branch 'main' into storagefiles

Author: l0lawrence

.github/skills/emitter-package-update/SKILL.md

@@ -70,7 +70,12 @@ npx npm-check-updates --packageFile eng/emitter-package.json -u
 > (cross-checked against `packages/typespec-python/package.json` on `main`) — and edit
 > `eng/emitter-package.json` by hand to that version.
 
-Align `@azure-tools/openai-typespec` and `@typespec/openapi3` with the versions pinned in [azure-rest-api-specs/package.json](https://github.com/Azure/azure-rest-api-specs/blob/main/package.json) to ensure consistency between the emitter and the spec repo. Read the spec repo's current values for those two packages and set them to match in `eng/emitter-package.json` (do not assume specific version numbers — use whatever the spec repo currently pins).
+Align the packages listed in `alignment-sources.json` with the versions pinned in each
+configured source repo package file to ensure consistency between the emitter and its
+upstream sources. For each source, read `packageJsonUrl`, set every package listed in
+that source's `packages` array to the version currently pinned there, and do not assume
+specific version numbers. To add more libraries for alignment later, add their package
+names to the relevant `packages` array in `alignment-sources.json`.
 
 If a specific version was requested, pin `@azure-tools/typespec-python` to that exact
 version in `eng/emitter-package.json` (overriding what npm-check-updates picked).

.github/skills/emitter-package-update/alignment-sources.json

@@ -0,0 +1,11 @@
+{
+  "azure-rest-api-specs": {
+    "packageJsonUrl": "https://github.com/Azure/azure-rest-api-specs/blob/main/package.json",
+    "packages": [
+      "@azure-tools/openai-typespec",
+      "@typespec/openapi3",
+      "@azure-tools/typespec-liftr-base",
+      "@azure-tools/typespec-azure-portal-core"
+    ]
+  }
+}

.github/workflows/api-consistency.yml

@@ -53,6 +53,7 @@ jobs:
           API_MD_CHANGED_FILE: .artifacts/changed_package_dirs.txt
           API_MD_PACKAGES_FILE: .artifacts/affected_package_dirs.txt
           API_MD_MISMATCHES_FILE: .artifacts/mismatched_api_files.txt
+          API_MD_MISMATCH_DETAILS_FILE: .artifacts/mismatched_api_details.txt
           API_MD_MISSING_FILE: .artifacts/missing_api_files.txt
         with:
           script: |

.github/workflows/src/api-md-consistency/api-md-consistency.js

@@ -34,6 +34,15 @@ function readLines(fileRelativePath, workspace) {
     .filter((line) => Boolean(line));
 }
 
+function readText(fileRelativePath, workspace) {
+  const fullPath = path.join(workspace, fileRelativePath);
+  if (!fs.existsSync(fullPath)) {
+    return "";
+  }
+
+  return fs.readFileSync(fullPath, "utf-8").trim();
+}
+
 function formatIssueSection(title, apiFiles) {
   if (!apiFiles.length) {
     return "";
@@ -55,6 +64,16 @@ function formatIssueSection(title, apiFiles) {
   return lines.join("\n");
 }
 
+function logMismatchDetails(core, mismatchDetails) {
+  if (!mismatchDetails) {
+    return;
+  }
+
+  core.startGroup("API.md mismatch details");
+  core.info(mismatchDetails);
+  core.endGroup();
+}
+
 export default async function apiMdConsistency({ core }) {
   const workspace = process.env.GITHUB_WORKSPACE || process.cwd();
 
@@ -81,6 +100,7 @@ export default async function apiMdConsistency({ core }) {
 
   const mismatches = readLines(process.env.API_MD_MISMATCHES_FILE, workspace);
   const missing = readLines(process.env.API_MD_MISSING_FILE, workspace);
+  const mismatchDetails = readText(process.env.API_MD_MISMATCH_DETAILS_FILE, workspace);
 
   const mismatchCount = mismatches.length;
   const missingCount = missing.length;
@@ -91,12 +111,15 @@ export default async function apiMdConsistency({ core }) {
   core.setOutput("issue_count", String(issueCount));
 
   if (issueCount > 0) {
+    logMismatchDetails(core, mismatchDetails);
+
     const messageParts = [
       "Generated api.md or api.metadata.yml does not match the committed files, or required API files are missing, for one or more affected packages.",
       "api.metadata.yml must be committed alongside api.md, and selected metadata fields are part of pass/fail gating.",
       "",
       formatIssueSection("Mismatched packages:", mismatches),
       formatIssueSection("Missing required API files:", missing),
+      mismatchDetails ? ["Mismatch details:", mismatchDetails].join("\n") : "",
       "To regenerate api.md and api.metadata.yml locally, run the command shown for each package from the repository root.",
     ].filter((part) => part !== "");
 

.github/workflows/src/api-md-consistency/find_mismatches.js

@@ -5,6 +5,8 @@ import fs from "fs";
 import { appendGithubOutput, envPath, getDefaultLogger, readLines, runAsync, writeLines } from "./common.js";
 import { loadAdapter, loadWorkflowConfig } from "./adapter_config.js";
 
+const MAX_DIFF_LINES = 200;
+
 /**
  * Parse a simple key: value YAML file into an object.
  * Only handles flat scalar mappings (no nesting, no multi-line values).
@@ -20,6 +22,36 @@ function parseSimpleYaml(text) {
   return result;
 }
 
+function truncateLines(text, maxLines) {
+  const lines = text.split(/\r?\n/);
+  if (lines.length <= maxLines) {
+    return text.trimEnd();
+  }
+
+  return `${lines.slice(0, maxLines).join("\n")}\n... diff truncated after ${maxLines} lines ...`;
+}
+
+function formatMetadataDiff(metadataFile, keys, committed, current) {
+  const lines = [`${metadataFile} gated metadata differences:`];
+  for (const key of keys) {
+    if (current[key] !== committed[key]) {
+      lines.push(`  ${key}:`);
+      lines.push(`    committed: ${committed[key] ?? "<missing>"}`);
+      lines.push(`    generated:  ${current[key] ?? "<missing>"}`);
+    }
+  }
+  return lines.join("\n");
+}
+
+async function formatApiDiff(apiFile) {
+  const diffResult = await runAsync("git", ["diff", "--no-ext-diff", "--unified=3", "--", apiFile], {
+    check: false,
+    maxBuffer: 1024 * 1024 * 10,
+  });
+  const diff = diffResult.stdout || diffResult.stderr;
+  return diff ? truncateLines(diff, MAX_DIFF_LINES) : `${apiFile} changed, but no diff output was captured.`;
+}
+
 async function main() {
   const config = loadWorkflowConfig();
   const adapter = await loadAdapter(config.adapter);
@@ -31,10 +63,12 @@ async function main() {
   const packagesFile = envPath("API_MD_PACKAGES_FILE", ".artifacts/affected_package_dirs.txt");
   const mismatchesFile = envPath("API_MD_MISMATCHES_FILE", ".artifacts/mismatched_api_files.txt");
   const missingFile = envPath("API_MD_MISSING_FILE", ".artifacts/missing_api_files.txt");
+  const mismatchDetailsFile = envPath("API_MD_MISMATCH_DETAILS_FILE", ".artifacts/mismatched_api_details.txt");
   const packages = readLines(packagesFile);
 
   const mismatches = [];
   const missing = [];
+  const mismatchDetails = [];
   for (const pkgDir of packages) {
     const apiFile = `${pkgDir}/api.md`;
     const metadataFile = `${pkgDir}/api.metadata.yml`;
@@ -72,6 +106,7 @@ async function main() {
         const mismatch = keys.some((key) => current[key] !== committed[key]);
         if (mismatch) {
           mismatches.push(metadataFile);
+          mismatchDetails.push(formatMetadataDiff(metadataFile, keys, committed, current));
         }
       }
     }
@@ -82,11 +117,13 @@ async function main() {
     });
     if (quietDiffResult.status !== 0) {
       mismatches.push(apiFile);
+      mismatchDetails.push(await formatApiDiff(apiFile));
     }
   }
 
   writeLines(mismatchesFile, mismatches);
   writeLines(missingFile, missing);
+  writeLines(mismatchDetailsFile, mismatchDetails);
   appendGithubOutput("mismatch_count", mismatches.length);
   appendGithubOutput("missing_count", missing.length);
   appendGithubOutput("issue_count", mismatches.length + missing.length);

.vscode/cspell.json

@@ -301,6 +301,7 @@
     "ivars",
     "jwks",
     "kashifkhan",
+    "liftr",
     "keybak",
     "kube",
     "kubeconfig",
@@ -1663,18 +1664,6 @@
         "Igog",
       ]
     },
-    {
-      "filename": "sdk/deviceupdate/azure-iot-deviceupdate/**",
-      "words": [
-        "wday",
-        "mday",
-        "unflattened",
-        "deseralize",
-        "astimezone",
-        "ctxt",
-        "Nify"
-      ]
-    },
     {
       "filename": "sdk/cosmos/azure-cosmos/**",
       "words": [

doc/analyze_check_versions.md

@@ -5,10 +5,9 @@ This table is to clarify the currently pinned version of tools we run in CI and
 
 | Tool | Current Version | Next Version | Next Version Merge Date |
 |------|-----------------|--------------|-------------------------|
-Pylint | 4.0.4 | 4.0.4 | 2026-07-13 |
+Pylint | 4.0.4 | 4.0.6 | 2026-10-05 |
 Pylint Guidelines Checker | 0.5.7 | 0.5.7 | 2026-07-13 |
-MyPy | 1.19.1 | 1.19.1 | 2026-07-13 |
-Pyright | 1.1.407 | 1.1.407 | 2026-07-13 |
+MyPy | 1.19.1 | 2.1.0 | 2026-10-05 |
+Pyright | 1.1.407 | 1.1.411 | 2026-10-05 |
 Sphinx | 8.2.0 | N/A | N/A |
 Black | 24.4.0 | N/A | N/A |
-Ruff | 0.15.11 | N/A | N/A |

eng/common/TestResources/SubConfig-Helpers.ps1

@@ -82,6 +82,11 @@ function ShouldMarkValueAsSecret([string]$serviceName, [string]$key, [string]$va
         "SERVICE_MANAGEMENT_URL",
         "ENDPOINT_SUFFIX",
         "SERVICE_DIRECTORY",
+        "RUST_TEST_THREADS",
+        "RUST_BACKTRACE",
+        "COSMOS_RUSTFLAGS",
+        "DATABASE_NAME",
+        "ACCOUNT_HOST",
         # This is used in many places and is harder to extract from the base subscription config, so hardcode it for now.
         "STORAGE_ENDPOINT_SUFFIX",
         # Parameters

eng/common/pipelines/live-eval.yml

@@ -0,0 +1,43 @@
+# Live-tier eval CI: nightly end-to-end run of the live workflow scenarios against the real
+# azsdk-cli MCP and real Azure DevOps (writes confined to a test area).
+
+# Nightly only — no CI/PR trigger.
+trigger: none
+pr: none
+
+schedules:
+  - cron: '0 9 * * *'        # 09:00 UTC daily
+    displayName: 'Nightly live eval'
+    branches:
+      include:
+        - main
+    always: true             # run even when main has not changed
+
+variables:
+  # Managed-pool image selection (LINUXPOOL/LINUXVMIMAGE). Repo-local.
+  - template: /eng/pipelines/templates/variables/image.yml
+  # Provides the secret azuresdk-copilot-github-pat, mapped into GITHUB_TOKEN in the invoke step.
+  - group: AzSDK_Eval_Variable_group
+
+extends:
+  template: /eng/common/pipelines/templates/stages/archetype-eval.yml
+  parameters:
+    # Shared mock/live builder; select the live tier (real Cli MCP).
+    mcpSetupTemplate: /eng/common/pipelines/templates/steps/eval-mcp-setup.yml
+    TestType: live
+    vallyRoot: tools/azsdk-cli/Azure.Sdk.Tools.Vally
+    evalGlobs:
+      - 'evals/workflow-scenarios/live/*.eval.yaml'
+    # Run each shard under AzureCLI@2 so the real MCP's DevOps calls are authenticated.
+    UseAzSdkAuthentication: true
+    failOnFailedTests: true
+    # Live scenarios are end-to-end, so give each shard more headroom than the report-only tiers.
+    shardTimeoutInMinutes: 45
+    # Pass-rate gate for `vally eval`. This tier gates (failOnFailedTests: true), so the threshold
+    # is the real bar the nightly run must clear; raise it here as live coverage stabilizes.
+    threshold: 0.8
+    # This repo needs no repo-specific setup (the live MCP is built by the common BuildMcp job).
+    # A spec/language repo that must start its own bot / server / MCP copies the example hook and
+    # points these at it — see eng/common/pipelines/templates/steps/eval-hook-example.yml.
+    # preEvalTemplate: /eng/pipelines/eval/start-my-bot.yml
+    # postEvalTemplate: /eng/pipelines/eval/stop-my-bot.yml

eng/common/pipelines/skill-eval.yml

@@ -0,0 +1,38 @@
+# Skill-compliance eval CI: runs the per-skill Vally evals under .github/skills (one job per skill).
+
+trigger:
+  branches:
+    include:
+      - main
+  paths:
+    include:
+      - .github/skills/**
+      # Retrigger when the mock MCP's tool catalog changes (it can move results).
+      - tools/azsdk-cli/Azure.Sdk.Tools.Mock/**
+      - eng/common/pipelines/skill-eval.yml
+      - eng/common/pipelines/templates/jobs/**
+      - eng/common/pipelines/templates/steps/eval-invoke.yml
+      - eng/common/pipelines/templates/steps/eval-mcp-setup.yml
+      - eng/common/pipelines/templates/stages/archetype-eval.yml
+      - eng/common/scripts/eval/**
+
+pr: none
+
+variables:
+  # Managed-pool image selection (LINUXPOOL/LINUXVMIMAGE). Repo-local.
+  - template: /eng/pipelines/templates/variables/image.yml
+  # Provides the secret azuresdk-copilot-github-pat, mapped into GITHUB_TOKEN in the invoke step.
+  - group: AzSDK_Eval_Variable_group
+
+extends:
+  template: /eng/common/pipelines/templates/stages/archetype-eval.yml
+  parameters:
+    # Shared mock/live builder; select the mock tier.
+    mcpSetupTemplate: /eng/common/pipelines/templates/steps/eval-mcp-setup.yml
+    TestType: mock
+    vallyRoot: .github/skills
+    # Single-level glob excludes azure-typespec-author/evaluate/ (its own benchmark pipeline).
+    evalGlobs:
+      - '*/evals/*.eval.yaml'
+    # Per-shard job timeout (report-only tier).
+    shardTimeoutInMinutes: 20