diff --git a/evals/azure-advisor/eval.yaml b/evals/azure-advisor/eval.yaml new file mode 100644 index 000000000..91710ab7a --- /dev/null +++ b/evals/azure-advisor/eval.yaml @@ -0,0 +1,305 @@ +# Vally eval config for the azure-advisor skill. +# +# Replaces the legacy Jest trigger tests (tests/azure-advisor/triggers.test.ts), +# which are deprecated per tests/README.md. This suite expresses the skill's +# scenarios as Vally stimuli graded against a real LLM agent run. +# +# Coverage (per tests/AGENTS.md): +# - Routing stimuli measure skill-invocation rate across the documented WHEN +# trigger phrases (5 runs each, 80% threshold), with early termination once the +# skill is invoked to keep runs cheap. +# - Response-quality stimuli (runs: 1) check the assistant output mentions Advisor +# recommendations and does not surface fatal errors. + +name: azure-advisor-integration-eval +description: | + Integration evaluation for the azure-advisor skill. Verifies routing for Advisor + review, recommendation-check, health-check, and audit prompts via skill-invocation + rate (5 runs, 80% threshold), plus response-quality checks for Advisor recommendation + output. + +tags: + type: integration + skill: azure-advisor + +config: + runs: 5 + timeout: "10m" + executor: integration-test-agent-runner + model: claude-sonnet-4.6 + +scoring: + threshold: 0.8 + +stimuli: + # ═══════════════════════════════════════════ + # Skill routing prompts + # ═══════════════════════════════════════════ + + # ── run-advisor-review (smoke) ── + - name: "Run an Advisor review" + prompt: "Run an Azure Advisor review of my subscription" + tags: + type: integration + skill: azure-advisor + tier: smoke + cost: llm + area: routing + earlyTerminate: '[{"type":"skill-call","skill":"azure-advisor"},{"type":"tool-call-count","count":3}]' + graders: + - type: skill-invocation + config: + required: + - azure-advisor + - type: output-not-matches + config: + pattern: "(?i)fatal error|unhandled exception|stack trace" + + # ── check-recommendations ── + - name: "Check Advisor recommendations" + prompt: "Check my Azure Advisor recommendations" + tags: + type: integration + skill: azure-advisor + tier: full + cost: llm + area: routing + earlyTerminate: '[{"type":"skill-call","skill":"azure-advisor"},{"type":"tool-call-count","count":3}]' + graders: + - type: skill-invocation + config: + required: + - azure-advisor + - type: output-not-matches + config: + pattern: "(?i)fatal error|unhandled exception|stack trace" + + # ── what-does-advisor-say ── + - name: "What does Advisor say about my subscription" + prompt: "What does Advisor say about my Azure subscription?" + tags: + type: integration + skill: azure-advisor + tier: full + cost: llm + area: routing + earlyTerminate: '[{"type":"skill-call","skill":"azure-advisor"},{"type":"tool-call-count","count":3}]' + graders: + - type: skill-invocation + config: + required: + - azure-advisor + - type: output-not-matches + config: + pattern: "(?i)fatal error|unhandled exception|stack trace" + + # ── advisor-health-check ── + - name: "Azure Advisor health check" + prompt: "Give me an Azure Advisor health check" + tags: + type: integration + skill: azure-advisor + tier: full + cost: llm + area: routing + earlyTerminate: '[{"type":"skill-call","skill":"azure-advisor"},{"type":"tool-call-count","count":3}]' + graders: + - type: skill-invocation + config: + required: + - azure-advisor + - type: output-not-matches + config: + pattern: "(?i)fatal error|unhandled exception|stack trace" + + # ── audit-with-advisor ── + - name: "Audit resources with Advisor" + prompt: "Audit my Azure resources with Advisor" + tags: + type: integration + skill: azure-advisor + tier: full + cost: llm + area: routing + earlyTerminate: '[{"type":"skill-call","skill":"azure-advisor"},{"type":"tool-call-count","count":3}]' + graders: + - type: skill-invocation + config: + required: + - azure-advisor + - type: output-not-matches + config: + pattern: "(?i)fatal error|unhandled exception|stack trace" + + # ═══════════════════════════════════════════ + # Boundary (negative) routing — adjacent services from + # the skill's DO NOT USE FOR section must NOT route to azure-advisor. + # `disallowed` asserts the skill stays out of these requests. + # ═══════════════════════════════════════════ + + # ── boundary-cost ── + - name: "Cost query does not route to advisor" + prompt: "Analyze my Azure subscription costs and spending trends" + tags: + type: integration + skill: azure-advisor + tier: smoke + cost: llm + area: routing + graders: + - type: skill-invocation + config: + disallowed: + - azure-advisor + - type: output-not-matches + config: + pattern: "(?i)fatal error|unhandled exception|stack trace" + + # ── boundary-diagnostics ── + - name: "Diagnostics query does not route to advisor" + prompt: "My App Service keeps returning 500 errors, help me troubleshoot it" + tags: + type: integration + skill: azure-advisor + tier: full + cost: llm + area: routing + graders: + - type: skill-invocation + config: + disallowed: + - azure-advisor + - type: output-not-matches + config: + pattern: "(?i)fatal error|unhandled exception|stack trace" + + # ── boundary-rbac ── + - name: "RBAC query does not route to advisor" + prompt: "Grant my team Reader access to the production resource group" + tags: + type: integration + skill: azure-advisor + tier: full + cost: llm + area: routing + graders: + - type: skill-invocation + config: + disallowed: + - azure-advisor + - type: output-not-matches + config: + pattern: "(?i)fatal error|unhandled exception|stack trace" + + # ── boundary-keyvault ── + - name: "Key Vault query does not route to advisor" + prompt: "Store a new secret in my Azure Key Vault" + tags: + type: integration + skill: azure-advisor + tier: full + cost: llm + area: routing + graders: + - type: skill-invocation + config: + disallowed: + - azure-advisor + - type: output-not-matches + config: + pattern: "(?i)fatal error|unhandled exception|stack trace" + + # ── boundary-appservice ── + - name: "App Service deploy does not route to advisor" + prompt: "Deploy my web app to Azure App Service" + tags: + type: integration + skill: azure-advisor + tier: full + cost: llm + area: routing + graders: + - type: skill-invocation + config: + disallowed: + - azure-advisor + - type: output-not-matches + config: + pattern: "(?i)fatal error|unhandled exception|stack trace" + + # ═══════════════════════════════════════════ + # Tool-call trajectory — validate behavior up to the point the agent + # invokes an Advisor MCP tool. Early-terminates as soon as an `advisor_` + # tool is called (or after a small tool-call cap as a safety net), then the + # `tool-calls` grader asserts an `advisor_` tool was actually executed. + # ═══════════════════════════════════════════ + + # ── review-calls-advisor-tool ── + - name: "Advisor review reaches an advisor tool call" + prompt: "Run an Azure Advisor review of my subscription" + tags: + type: integration + skill: azure-advisor + tier: smoke + cost: llm + area: behavior + earlyTerminate: '[{"type":"tool-call-match","toolPattern":"advisor_","argsPattern":".*"},{"type":"tool-call-count","count":8}]' + graders: + - type: skill-invocation + config: + required: + - azure-advisor + - type: tool-calls + config: + required: + - name: "advisor_" + - type: output-not-matches + config: + pattern: "(?i)fatal error|unhandled exception|stack trace" + + # ── check-recommendations-calls-advisor-tool ── + - name: "Recommendation check reaches an advisor tool call" + prompt: "Check my Azure Advisor recommendations" + tags: + type: integration + skill: azure-advisor + tier: full + cost: llm + area: behavior + earlyTerminate: '[{"type":"tool-call-match","toolPattern":"advisor_","argsPattern":".*"},{"type":"tool-call-count","count":8}]' + graders: + - type: skill-invocation + config: + required: + - azure-advisor + - type: tool-calls + config: + required: + - name: "advisor_" + - type: output-not-matches + config: + pattern: "(?i)fatal error|unhandled exception|stack trace" + + # ═══════════════════════════════════════════ + # Response quality tests + # ═══════════════════════════════════════════ + + # ── review-mentions-recommendations ── + - name: "Advisor review mentions recommendations" + prompt: "Run an Azure Advisor review of my subscription" + config: + runs: 1 + tags: + type: integration + skill: azure-advisor + tier: full + cost: llm + area: response-quality + graders: + - type: output-matches + config: + pattern: "(?i)recommendation|advisor" + - type: completed + - type: output-not-matches + config: + pattern: "(?i)fatal error|unhandled exception|stack trace" diff --git a/plugin/skills/azure-advisor/SKILL.md b/plugin/skills/azure-advisor/SKILL.md new file mode 100644 index 000000000..cefee34ca --- /dev/null +++ b/plugin/skills/azure-advisor/SKILL.md @@ -0,0 +1,44 @@ +--- +name: azure-advisor +description: "Azure Advisor reviews resources and provides recommendations using Azure MCP advisor_* tools. WHEN: \"run an advisor review\", \"check my Azure advisor recommendations\", \"summarize advisor findings\", \"what does Advisor say about my subscription\", \"give me an advisor health check\", \"audit my Azure resources with Advisor\". USE FOR: read-only catalog, recommendations, and IaC fixes. DO NOT USE FOR: changing resources, billing (use azure-cost), or non-Advisor issues (use azure-diagnostics)." +license: MIT +metadata: + author: Microsoft + version: "0.0.0-placeholder" +--- + +# Azure Advisor Skill + +Azure Advisor is a **product area** with multiple capabilities. This skill routes a +user's intent to the right capability and runs it using whichever `advisor_*` MCP tools +the connected Azure MCP server exposes. Routing is by *capability* (catalog, +recommendations, summary, IaC fix), not by hard-coded tool names, so the skill stays +useful as new advisor tools land. + +## Pre-Execution Requirements + +Inspect the available `advisor_*` MCP tools and their parameters before running a +capability. Match a tool when its name **contains** `advisor_` (i.e. `*advisor_*`), not +only when it *starts with* it — MCP clients prepend a server-name prefix (e.g. +`azure-mcp-advisor_recommendation_list`). Match by capability description, not by a fixed +name list — see the shared [Capability Routing](references/capability-routing.md) reference. + +## Shared References + +These product-area references are reused by **every** capability below. Read the +relevant one before acting: + +| Reference | Purpose | +|-----------|---------| +| [Capability Routing](references/capability-routing.md) | Resolve which `advisor_*` MCP tool to call for each capability (catalog, recommendations, summary, IaC fix). | +| [Subscription Discovery](references/subscription-discovery.md) | Resolve a single target subscription, or enumerate and classify all subscriptions by environment, without hardcoding. | +| [Resource Scope Discovery](references/resource-discovery.md) | Narrow a review to the resources defined in this repo (resource group / type / id) without hardcoding. | + +## Capabilities + +Route the user's request to the matching capability. **Use these instead of the main +skill when they match the task:** + +| Capability | When to Use | Reference | +|-----------|-------------|-----------| +| **review** | Run a holistic, read-only Advisor sweep across one subscription — or **all** subscriptions classified by environment (dev/staging/prod) — probing the catalog, pulling active recommendations, aggregating by category/impact, spotlighting high-impact items, and proposing IaC fix snippets. | [review](review/review.md) | diff --git a/plugin/skills/azure-advisor/references/capability-routing.md b/plugin/skills/azure-advisor/references/capability-routing.md new file mode 100644 index 000000000..096da3fbc --- /dev/null +++ b/plugin/skills/azure-advisor/references/capability-routing.md @@ -0,0 +1,37 @@ +# Shared Reference — Advisor Capability Routing + +> **Shared across all `azure-advisor` capabilities.** Any capability that needs to +> pick an `advisor_*` MCP tool should link here instead of re-defining the table. +> Match tools by *capability description*, never by hard-coded name. + +The connected Azure MCP server may expose a changing set of `advisor_*` tools. Resolve +which tool to invoke at each step from this capability table. + +## Tool name matching + +Match a tool when its name **contains** `advisor_` (i.e. `*advisor_*`) — **not** only +when it *starts with* `advisor_`. MCP clients prepend the **server name** to every tool, +so the same Advisor tool surfaces under a different prefix depending on the host: + +| Client | Example tool name | +|---|---| +| Copilot CLI | `azure-mcp-advisor_recommendation_list` | +| Other hosts | `advisor_recommendation_list` | + +A strict *starts-with* check would reject `azure-mcp-advisor_recommendation_list` and the +skill would wrongly report "no Advisor tools found". Always substring-match on `advisor_`. + +| Capability | Look for an `advisor_*` tool whose description says... | Required input | +|---|---|---| +| **Metadata / catalog** | "list recommendation types / categories / impact levels / supported values" | tenant context (no subscription needed) | +| **Active recommendations** | "list Advisor recommendations in a subscription" | subscription (resource group optional, filters optional) | +| **Aggregation / summary** | "summarize / group / aggregate Advisor recommendations" | subscription + group-by field | +| **IaC remediation** | "apply Advisor recommendations to IaC / ARM / Bicep / Terraform" | a resource type identifier | + +## Resolution rules + +- If a step's capability has **no matching tool**, skip it and note that in the chat + output (e.g. "no aggregation tool available, presenting raw list"). +- If **multiple tools** match, prefer the one with the more specific description. +- **Never** substitute a tool whose name does **not** contain `advisor_` for a missing + capability — report the gap and skip instead. diff --git a/plugin/skills/azure-advisor/references/resource-discovery.md b/plugin/skills/azure-advisor/references/resource-discovery.md new file mode 100644 index 000000000..66fe8fef9 --- /dev/null +++ b/plugin/skills/azure-advisor/references/resource-discovery.md @@ -0,0 +1,66 @@ +# Shared Reference — Repo Resource Scope Discovery + +> **Shared across all `azure-advisor` capabilities.** Any capability that wants to +> narrow Advisor results to the resources this repository actually deploys should link +> here instead of re-defining discovery logic. + +Advisor tools query a **whole subscription** by default. This skill's repo-scoped review +defaults to narrowing the query to the resources the repo defines, rather than dumping the +entire subscription. + +A repo almost always defines **several** resources of **multiple** types — collect them +**all**, never stop after the first. Only when the repo defines no Azure resources at all +should you fall back to a full-subscription review and say so. **Never** hardcode a +resource group, resource type, or resource id. + +## What to extract + +Scan the workspace's Infrastructure-as-Code and config files for any of these scoping +signals, in this order. Collect every distinct hit (a repo may define several): + +1. **Resource group(s)** — the strongest scope signal: + - `azure.yaml` → `resourceGroup:` / `resourceGroupName:` key + - `.azure/*/config.json` → `resourceGroup` field + - `*.bicepparam` / `*.parameters.json` → a `resourceGroup` / `resourceGroupName` parameter + - `.env*` files → read **only** the `AZURE_RESOURCE_GROUP` / `AZURE_RESOURCEGROUP` line; never load, echo, or summarize other `.env*` contents (they routinely hold secrets) + - `main.tf` / `*.tf` → `azurerm_resource_group` `name =` value (or a `resource_group_name` local/variable default) +2. **Resource type(s)** — derive from the IaC resource declarations: + - Bicep `resource x 'Microsoft./@...'` → `Microsoft./` + - ARM `"type": "Microsoft./"` + - Terraform `resource "azurerm_"` → map to its `Microsoft.*` provider type + - Bicep and ARM give the provider type **literally** (extract it as-is). The Terraform + `azurerm_` → `Microsoft.*` mapping must be **derived** and is not always obvious + (e.g. `azurerm_linux_function_app` → `Microsoft.Web/sites`). If a Terraform type mapping + is uncertain, **do not guess** — fall back to the resource group filter for that resource + rather than risk a wrong type filter that hides real recommendations. +3. **Specific resource id(s)** — only when a fully-qualified ARM id is present in + params/env (`/subscriptions/.../resourceGroups/.../providers/...`). + +## How to apply the scope + +Pass the discovered signal(s) to the **active recommendations** and **summary** +capabilities as filters (see [Capability Routing](capability-routing.md)): + +| Discovered signal | Filter to pass | +|---|---| +| Resource group | the tool's resource-group input | +| Resource type(s) | the tool's resource-type filter — review **once per type** when the tool takes a single type, so **every** discovered type is covered | +| Specific resource id | the tool's resource (resource-id) filter | + +Apply **all** discovered signals — the review covers the **union** of every resource group +and resource type the repo defines. Where signals overlap, prefer the narrowest available +for a given resource (a specific resource id beats its type, which beats its group), but +never drop a resource type just because a broader signal also exists. + +If the recommendations tool has **no** matching scope filter, pull the subscription list +unfiltered and **post-filter** in-context to the discovered resource groups / types / ids +before aggregating or spotlighting. + +## Resolution rules + +- If **no** scope signal is found, run the **full-subscription** review and note + "no repo resource scope found — reviewed the whole subscription" in the summary. +- Always mention the *source* of the resolved scope in the final chat summary + (e.g. "Scoped to resource group `rg-shop-prod` from `infra/main.parameters.json`"). +- Scope is a **filter**, not a guarantee — Advisor only returns findings for resources + that are actually deployed, so a repo resource with no Advisor data simply won't appear. diff --git a/plugin/skills/azure-advisor/references/subscription-discovery.md b/plugin/skills/azure-advisor/references/subscription-discovery.md new file mode 100644 index 000000000..8555cca58 --- /dev/null +++ b/plugin/skills/azure-advisor/references/subscription-discovery.md @@ -0,0 +1,62 @@ +# Shared Reference — Subscription Discovery + +> **Shared across all `azure-advisor` capabilities.** Any capability that operates on a +> subscription should link here instead of re-defining discovery logic. + +## Default — discover every subscription the repo references + +Auto-discovery is the point of this skill: **do not make the user name the scope**. Scan +the workspace and collect **every distinct** subscription id you find — a repo commonly +pins a different subscription per environment, so **don't stop at the first hit**. +Subscriptions **must never be hardcoded**. Scan all of: + +- `azure.yaml` → `subscriptionId:` key, plus any per-environment azd configs +- `.azure/*/config.json` → `subscriptionId` field (one per azd environment) +- `*.bicepparam` / `*.parameters.json`, incl. `infra/**/main.parameters.json` → + `subscriptionId` / `subscription` parameter +- `.env*` files → read **only** the `AZURE_SUBSCRIPTION_ID` line; never load, echo, or + summarize other `.env*` contents (they routinely hold secrets) +- environment variable `AZURE_SUBSCRIPTION_ID` + +Then **classify each** discovered subscription by environment (see table below) and carry +the `(id, name, environment, source)` tuple forward so every per-subscription result can +be attributed and grouped in the summary. If only one subscription is found, the review +simply runs once. Always mention each *source* in the final chat summary +(e.g. "dev sub from `infra/main.parameters.json`, prod sub from `.azure/prod/config.json`"). + +## Widen to the whole tenant — on request or as fallback + +Enumerate **all** subscriptions the signed-in identity can access when either: + +- the user explicitly asks for a tenant-wide / org-wide sweep ("all my subscriptions", + "every subscription in the tenant"), **or** +- repo discovery above found nothing. + +Invoke the Azure MCP **subscription-list** tool (a tool whose name contains `subscription` +and whose description says "list Azure subscriptions"). Do **not** hardcode ids. Classify +each result with the same table below. If no subscription-list tool is available and repo +discovery also found nothing, fall back to **Ask the user**. + +## Ask the user — last resort + +Only if repo discovery yields nothing **and** tenant enumeration is unavailable. Do not +guess. Say which files were scanned and what was missing. + +## Classify each subscription by environment + +Use the first signal that matches: + +- **Tags** — an `Environment` / `env` tag value (e.g. `Production`, `Staging`, `Dev`). +- **Name / config keywords** (case-insensitive substring) when no tag exists — the + subscription name, or the azd environment / param-file name it came from: + + | Bucket | Match any of | + |---|---| + | **prod** | `prod`, `production`, `prd`, `live` | + | **staging** | `stag`, `staging`, `stg`, `uat`, `preprod`, `pre-prod` | + | **test** | `test`, `qa`, `sit` | + | **dev** | `dev`, `development`, `sandbox`, `sbx` | + | **other** | nothing above matched — list under "Unclassified" | + +Never invent an environment; if ambiguous, place it in **other** and say so. + diff --git a/plugin/skills/azure-advisor/review/review.md b/plugin/skills/azure-advisor/review/review.md new file mode 100644 index 000000000..c09c73745 --- /dev/null +++ b/plugin/skills/azure-advisor/review/review.md @@ -0,0 +1,176 @@ +# Azure Advisor Review + +Drive an end-to-end Azure Advisor sweep using whichever `advisor_*` MCP tools the +connected Azure MCP server exposes. Designed to stay useful as new advisor tools +land — it routes by *capability* (catalog, recommendations, summary, IaC fix), +not by hard-coded tool name lists. + +## When to Use This Sub-Skill + +Use this sub-skill when the user wants to: + +- Get an overall picture of Advisor findings for an Azure subscription +- Sweep **all** subscriptions (dev, staging, prod, …), classify them by environment, and get recommendations grouped per environment +- See "what categories / impact levels does Advisor surface here?" before drilling in +- Combine the recommendation catalog (metadata) with the active recommendation list +- Produce a single grouped/aggregated view ("top N by impact", "by resource type") +- Quickly check whether their tenant even has Advisor data yet (new/empty subs) + +This is a **read-only** review. Even if an `apply`-style advisor tool exists, this +sub-skill only proposes IaC fix snippets — it never modifies cloud state. + +## Shared References + +This capability builds on two product-area references shared by every `azure-advisor` +capability — read them before running the workflow: + +- **[Capability Routing](../references/capability-routing.md)** — how to pick the right + `advisor_*` MCP tool by capability (catalog, recommendations, summary, IaC fix). +- **[Subscription Discovery](../references/subscription-discovery.md)** — how to resolve + the target subscription from repo config / env without hardcoding. +- **[Resource Scope Discovery](../references/resource-discovery.md)** — how to narrow the + review to the resources this repo deploys (resource group / type / id) without + hardcoding. + +## Workflow + +### Step 1 — Discover and classify subscription(s) + +Run [Subscription Discovery](../references/subscription-discovery.md). By **default**, +auto-discover **every** subscription the repo references (a repo often pins a different +subscription per environment) and classify each into an environment bucket (prod / +staging / test / dev / other) — do **not** wait for the user to name the scope. Then run +Steps 2–5 **once per discovered subscription**. + +- If discovery finds only one subscription, the review simply runs once. +- Widen to a tenant-wide enumeration (subscription-list tool) only when the user explicitly + asks for it, or when repo discovery finds nothing. + +Mention each subscription's *source* and the environment breakdown in the final chat +summary. Never hardcode ids. + +### Step 1b — Resolve repo resource scope + +Run [Resource Scope Discovery](../references/resource-discovery.md) to find the resource +group(s), resource **type(s)**, and resource id(s) this repo defines. Collect **every** +distinct signal — a repo usually defines **multiple** resource types, so do **not** stop +after the first. Save them all with their *source(s)*. Only if the repo defines no Azure +resources at all, fall back to a full-subscription review and note that in the summary. + +### Step 2 — Probe the catalog (metadata) + +Invoke the **metadata / catalog** capability (see +[Capability Routing](../references/capability-routing.md)) with no filter to learn what +recommendation categories, impacts, and resource types Advisor knows about for this +tenant. Cache the result — later steps can reference category/impact values from it. + +If this call returns an empty list, the tenant likely lacks Advisor coverage — report +that and stop early. + +### Step 3 — Pull active recommendations + +Invoke the **active recommendations** capability with the resolved subscription. If +Step 1b found a repo scope, pass it as the matching filter (resource group / resource +type / resource id) per [Capability Routing](../references/capability-routing.md); if the +tool has no such filter, pull unfiltered and **post-filter** the list to the discovered +scope in-context. Otherwise pull the whole subscription. Limit output to a manageable +page if the tool supports it. + +### Step 4 — Aggregate (if available) + +If a **summary / aggregation** capability exists, call it twice (applying the same Step 1b +scope filter, if any): + +- Once grouped by **category** (Cost, Security, Reliability, etc.) +- Once grouped by **impact** (High, Medium, Low) + +If no aggregation tool exists, derive the same two breakdowns locally from the Step 3 +list. + +### Step 5 — Spotlight high-impact items + +From the Step 3 results, pick up to **5 distinct High-impact recommendations** across +different resource types. For each, if an **IaC remediation** capability exists and +the recommendation's resource type matches a supported type, fetch a fix snippet. + +### Step 6 — Compose chat summary + +Reply in chat with this structure (no files written). + +**Single subscription:** + +```text +## Azure Advisor Review + +**Subscription:** (from ) +**Scope:** + +### Snapshot +- Total active recommendations: X +- By category: Cost A | Security B | Reliability C | Performance D | Operational E +- By impact: High H | Medium M | Low L + +### High-impact spotlight (up to 5) +1. + +... + +### Notes +- Tools used: +- Tools skipped (no capability match): +``` + +**All subscriptions** — group the report by environment bucket (prod first, then staging, +test, dev, other), with a roll-up at the top: + +```text +## Azure Advisor Review — All Subscriptions + +**Scope:** S subscriptions across E environments +**Totals:** High H | Medium M | Low L · Cost A | Security B | Reliability C | Performance D | Operational E + +### prod (n subscriptions) +- (): High h | Medium m | Low l — top category + - High-impact spotlight (up to 5): + +... + +### staging (n subscriptions) +... + +### Unclassified (n subscriptions) +... + +### Notes +- Subscriptions enumerated via: +- Tools used: +- Tools skipped (no capability match): +- Subscriptions with no Advisor coverage: +``` + +Do **not** write any file to disk. The summary lives only in the chat response. + +## Constraints + +- ✅ **Always** discover the subscription from repo files / env first; ask only as last resort. +- ✅ **Always** auto-discover **every** subscription the repo references and classify each by environment before reviewing; never make the user name the scope and never hardcode the list. +- ✅ **Always** include every advisor_* tool name actually invoked in the "Tools used" line so the user can see what the skill chose. +- ✅ **Always** mention which steps were skipped because no matching capability was found. +- ❌ **Never** hardcode a subscription id, tenant id, or resource group. +- ❌ **Never** modify Azure state — this sub-skill is read + suggest only. +- ❌ **Never** call a tool whose name does not contain `advisor_` as a substitute; if no capability matches, report it and skip. (Match on the substring `advisor_` — clients prepend a server-name prefix like `azure-mcp-`.) +- ❌ **Never** write the summary to a file unless the user explicitly asks for one in their prompt. +- ❌ **Never** write helper scripts, scratch files, or parsing utilities to disk (no `.tmp/*.js`, no `.tmp/*.json`, no temp files of any kind). Reason over MCP tool responses directly in-context. Prefer aggregating via the Aggregation / summary capability (see [capability-routing.md](../references/capability-routing.md)) over computing counts yourself. +- ❌ **Never** shell out to `node`, `python`, `pwsh`, `powershell`, `jq`, or any other interpreter to read, parse, group, or count MCP tool responses. The tool responses are already in your context — reason over them directly. If you need server-side aggregation, use the Aggregation / summary capability from [capability-routing.md](../references/capability-routing.md). + +## Error Handling + +| Symptom | Probable cause | Action | +|---|---|---| +| No tool name contains `advisor_` | MCP server not configured / not running, **or** a strict starts-with match rejected prefixed names | Substring-match on `advisor_` (names look like `azure-mcp-advisor_*`). If still none, tell user to check `.vscode/mcp.json`, that `azmcp.exe` is reachable, and that tools are exposed individually (`--mode all`). | +| Tenant-wide enumeration requested but no subscription-list tool | Azure MCP subscription tool not exposed | Fall back to repo-discovered subscriptions; if none, ask the user. | +| Recommendations tool has no resource-scope filter | Tool only accepts a subscription | Pull unfiltered and post-filter the list to the Step 1b scope in-context. | +| Catalog call returns empty | Tenant has no Advisor coverage yet | Stop after Step 2; report empty tenant. | +| Recommendation list 401/403 | Auth not scoped to subscription | Tell user to run `az login` and verify subscription access. | +| Aggregation tool errors on `group-by` | Field name not in supported list | Re-call with a value drawn from Step 2's catalog. | +| IaC tool says "unknown resource" | Resource type not in its supported list | Skip the fix for that recommendation; keep the rest. | diff --git a/plugin/skills/azure-advisor/version.json b/plugin/skills/azure-advisor/version.json new file mode 100644 index 000000000..af73f6419 --- /dev/null +++ b/plugin/skills/azure-advisor/version.json @@ -0,0 +1,6 @@ +{ + "version": "1.1", + "pathFilters": [ + "." + ] +} diff --git a/tests/skills.json b/tests/skills.json index d20b46697..3cd0843c5 100644 --- a/tests/skills.json +++ b/tests/skills.json @@ -2,6 +2,7 @@ "skills": [ "airunway-aks-setup", "appinsights-instrumentation", + "azure-advisor", "azure-ai", "azure-aigateway", "azure-cloud-migrate", @@ -32,6 +33,6 @@ "integrationTestSchedule": { "0 5 * * 2-6": "microsoft-foundry", "0 8 * * 2-6": "azure-deploy", - "0 12 * * 2-6": "airunway-aks-setup,appinsights-instrumentation,azure-ai,azure-aigateway,azure-cloud-migrate,azure-compliance,azure-compute,azure-cost,azure-diagnostics,azure-enterprise-infra-planner,azure-hosted-copilot-sdk,azure-kubernetes,azure-kusto,azure-messaging,azure-prepare,azure-quotas,azure-rbac,azure-resource-lookup,azure-resource-visualizer,azure-storage,azure-upgrade,azure-validate,entra-agent-id,entra-app-registration,azure-reliability,python-appservice-deploy" + "0 12 * * 2-6": "airunway-aks-setup,appinsights-instrumentation,azure-advisor,azure-ai,azure-aigateway,azure-cloud-migrate,azure-compliance,azure-compute,azure-cost,azure-diagnostics,azure-enterprise-infra-planner,azure-hosted-copilot-sdk,azure-kubernetes,azure-kusto,azure-messaging,azure-prepare,azure-quotas,azure-rbac,azure-resource-lookup,azure-resource-visualizer,azure-storage,azure-upgrade,azure-validate,entra-agent-id,entra-app-registration,azure-reliability,python-appservice-deploy" } } \ No newline at end of file