Skip to content

Commit 4e6f133

Browse files
JennyLawranceJennyLawrance
authored
Merge pull request #1734 from paulyuk/paulyuk/sandboxes-skill-eval-wins
docs(skills/sandboxes): make routing reliable for common tasks
2 parents 3a801c8 + 82bbd3c commit 4e6f133

6 files changed

Lines changed: 302 additions & 18 deletions

File tree

‎plugin/scripts/README.md‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
# plugin/scripts
2+
3+
Maintenance scripts for the plugin's skills.
4+
5+
## `verify-aca-verbs.mjs`
6+
7+
Walks every `*.md` file under `plugin/skills/`, extracts every `aca <verb1> <verb2>`
8+
invocation from shell code fences, and runs `aca <verb1> <verb2> --help` against
9+
the real `aca` binary on `PATH`. Fails the run if any verb pair does not exist.
10+
11+
This catches **fabricated commands** before they ship — e.g. `aca sandbox-group
12+
connector add` (not a real command; the group is `aca sandboxgroup` with no hyphen).
13+
14+
### Run it
15+
16+
```bash
17+
# Requires Node 18+ and the aca CLI on PATH.
18+
# Install: https://aka.ms/aca-cli-install (bash) or https://aka.ms/aca-cli-install-ps (Windows)
19+
node plugin/scripts/verify-aca-verbs.mjs
20+
21+
# Or scope to a single skill:
22+
node plugin/scripts/verify-aca-verbs.mjs plugin/skills/sandboxes
23+
```
24+
25+
### Exit codes
26+
27+
| Code | Meaning |
28+
| ---- | ------- |
29+
| 0 | Every verb pair resolves against the real `aca` binary. |
30+
| 1 | At least one verb pair does not exist (fabrication). |
31+
| 2 | `aca` is not on `PATH` — skipped, not failed (CI-friendly). |

‎plugin/scripts/verify-aca-verbs.mjs‎

Lines changed: 107 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,107 @@
1+
#!/usr/bin/env node
2+
// Validate every `aca <verb1> <verb2>` invocation in plugin/skills/**/*.md
3+
// against the real `aca` binary on PATH. Catches fabricated commands
4+
// (e.g. the hyphenated `aca sandbox-group …` family) before they ship.
5+
//
6+
// Usage: node plugin/scripts/verify-aca-verbs.mjs [path-to-skill-dir]
7+
// Exits: 0 if all verbs exist, 1 if any are broken, 2 if `aca` is not
8+
// on PATH (skips with a warning so CI without the toolchain
9+
// doesn't fail).
10+
11+
import { promises as fs } from 'node:fs';
12+
import { spawnSync } from 'node:child_process';
13+
import path from 'node:path';
14+
import { fileURLToPath } from 'node:url';
15+
16+
const __dirname = path.dirname(fileURLToPath(import.meta.url));
17+
const SCAN_ROOT = path.resolve(process.argv[2] ?? path.join(__dirname, '..', 'skills'));
18+
const SHELL_FENCES = new Set([
19+
'bash', 'sh', 'shell', 'console', 'azurecli', 'powershell', 'pwsh', 'ps1', 'ps',
20+
]);
21+
22+
async function walk(dir, out = []) {
23+
let ents;
24+
try { ents = await fs.readdir(dir, { withFileTypes: true }); }
25+
catch (e) { if (e.code === 'ENOENT') return out; throw e; }
26+
for (const ent of ents) {
27+
const full = path.join(dir, ent.name);
28+
if (ent.isDirectory()) await walk(full, out);
29+
else if (ent.isFile() && ent.name.endsWith('.md')) out.push(full);
30+
}
31+
return out;
32+
}
33+
34+
// Extract every aca invocation inside a shell code fence.
35+
// Returns [{ file, line, v1, v2 }].
36+
function extractFromMarkdown(file, text) {
37+
const out = [];
38+
const lines = text.split(/\r?\n/);
39+
let fenceLang = null;
40+
for (let i = 0; i < lines.length; i++) {
41+
const fenceOpen = lines[i].match(/^```([a-zA-Z0-9_-]*)/);
42+
if (fenceOpen) {
43+
fenceLang = fenceLang === null ? (fenceOpen[1] || '').toLowerCase() : null;
44+
continue;
45+
}
46+
if (fenceLang === null || !SHELL_FENCES.has(fenceLang)) continue;
47+
// Strip leading prompts and trailing line continuations.
48+
let line = lines[i].replace(/\r/g, '')
49+
.replace(/^(?:PS[^>]*>|\$|#|>)\s+/, '')
50+
.replace(/[\\`]\s*$/, '').trim();
51+
if (!/^aca(\s|$)/.test(line)) continue;
52+
// Skip subshell contents so flags don't get attributed to aca.
53+
line = line.replace(/\$\([^()]*\)/g, '_').replace(/`[^`]*`/g, '_');
54+
const tokens = line.split(/\s+/);
55+
const v1 = tokens[1] && /^[a-z][-a-z]*$/.test(tokens[1]) ? tokens[1] : null;
56+
const v2 = tokens[2] && /^[a-z][-a-z]*$/.test(tokens[2]) ? tokens[2] : null;
57+
if (v1) out.push({ file, line: i + 1, v1, v2 });
58+
}
59+
return out;
60+
}
61+
62+
const helpCache = new Map();
63+
function verbExists(args) {
64+
const key = args.join(' ');
65+
if (helpCache.has(key)) return helpCache.get(key);
66+
const r = spawnSync('aca', [...args, '--help'], { encoding: 'utf8', timeout: 15000, windowsHide: true });
67+
const ok = r.status === 0;
68+
helpCache.set(key, ok);
69+
return ok;
70+
}
71+
72+
// Probe for `aca` once; if missing, skip with exit 2.
73+
const probe = spawnSync('aca', ['--help'], { encoding: 'utf8', timeout: 15000, windowsHide: true });
74+
if (probe.error || probe.status !== 0) {
75+
console.warn(`[verify-aca-verbs] WARNING: 'aca' is not on PATH (skipping). Install from https://aka.ms/aca-cli-install to enable.`);
76+
process.exit(2);
77+
}
78+
79+
const files = await walk(SCAN_ROOT);
80+
const invocations = [];
81+
for (const f of files) invocations.push(...extractFromMarkdown(f, await fs.readFile(f, 'utf8')));
82+
83+
const seen = new Map(); // key -> [{file, line}]
84+
for (const inv of invocations) {
85+
const key = inv.v2 ? `${inv.v1} ${inv.v2}` : inv.v1;
86+
if (!seen.has(key)) seen.set(key, []);
87+
seen.get(key).push(inv);
88+
}
89+
90+
const broken = [];
91+
for (const [key, sites] of [...seen.entries()].sort()) {
92+
const args = key.split(' ');
93+
if (!verbExists(args)) broken.push({ key, sites });
94+
}
95+
96+
const total = seen.size;
97+
console.log(`[verify-aca-verbs] Scanned ${files.length} markdown files, extracted ${total} unique aca verb pairs.`);
98+
if (broken.length === 0) {
99+
console.log(`[verify-aca-verbs] OK: all ${total} verb pairs exist in the real 'aca' binary.`);
100+
process.exit(0);
101+
}
102+
console.error(`[verify-aca-verbs] FAIL: ${broken.length} of ${total} verb pairs do not exist:`);
103+
for (const b of broken) {
104+
console.error(` X aca ${b.key}`);
105+
for (const s of b.sites) console.error(` at ${path.relative(process.cwd(), s.file)}:${s.line}`);
106+
}
107+
process.exit(1);

‎plugin/skills/sandboxes/SKILL.md‎

Lines changed: 76 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,8 +19,9 @@ description: |
1919
2020
Triggers: install aca, install aca cli, setup aca, aca doctor, aca
2121
login, command not found: aca, create sandbox, sandbox group, aca
22-
cli, aca sandbox, exec in sandbox, microVM, code interpreter, agent
23-
swarm, host mcp.
22+
cli, aca sandbox, exec in sandbox, sandbox shell, mount volume,
23+
expose port, fs write, egress allow, suspend sandbox, snapshot
24+
sandbox, microVM, code interpreter, agent swarm, host mcp.
2425
---
2526

2627
# Sandboxes
@@ -46,6 +47,58 @@ folder.
4647
> do not touch sandboxes. If you see `az containerapp sandbox …` in a
4748
> snippet, it's wrong.
4849
50+
> ⚠️ **There is no `aca sandbox-group` (hyphenated) command group.**
51+
> Every group-level verb is `aca sandboxgroup …` (no hyphen):
52+
> `aca sandboxgroup create / get / list / delete / role create /
53+
> identity assign / secret upsert / disk list / volume create / snapshot …`.
54+
> The hyphenated `--sandbox-group <name>` is the *flag* you pass to
55+
> top-level commands to select the default group — it is not a verb.
56+
> If you see `aca sandbox-group …` in a snippet (including for MCP
57+
> connectors, credentials, or triggers), it's wrong / fabricated.
58+
59+
## ⚠ Required cues per response type (non-negotiable)
60+
61+
Before composing your answer, identify the user's intent and ensure the
62+
cues below are present. These are **acceptance criteria, not
63+
suggestions** — a response is incomplete if it omits any cue for its
64+
intent.
65+
66+
| User intent | Cues your response MUST include |
67+
|---|---|
68+
| **Install the `aca` CLI** (any OS) | (1) The curl/iwr one-liner from [references/install.md](references/install.md) — use the `https://aka.ms/aca-cli-install` (Linux/macOS) and `https://aka.ms/aca-cli-install-ps` (Windows) short URLs. (2) `aca --version` + `az login` + `aca doctor` (`aca` delegates auth to `az login` — same Entra identity). (3) The explicit sentence: **"this same install path is also used inside sandboxes and containers for agent-driven self-installs."** |
69+
| **Bootstrap a sandbox group (one-time setup)** | The 4-step flow: `az login` → `aca sandboxgroup create --name <g> --location <region> --set-config` → `aca sandboxgroup role create --role "Container Apps SandboxGroup Data Owner" --principal-id $(az ad signed-in-user show --query id -o tsv)` → `aca doctor`. **`--set-config` is required** so subsequent `aca sandbox …` commands don't need `--group` on every call. Treat green `aca doctor` as the gate before doing anything else. |
70+
| **Create a sandbox (imperative)** | Minimum: `aca sandbox create --disk ubuntu`. Common knobs: `--cpu 2000m`, `--memory 4096Mi`, `--env "K=V"`, `--labels "name=dev,role=worker"`. Capture the printed ID into `SANDBOX_ID=$(aca sandbox create --disk ubuntu -o json \| jq -r .id)` for reuse. For config that should live in source control, use the manifest flow (see the row below) instead. |
71+
| **Apply / deploy a sandbox manifest** | The full 3-command flow: `aca sandbox init` → `aca sandbox validate --file sandbox.yaml` → `aca sandbox apply --file sandbox.yaml`. Always `--file` (no `-f` short flag). State that **the manifest pattern is the recommended path for CI/CD and reproducibility**, in contrast to imperative `aca sandbox create`. If no manifest is present, run `aca sandbox init` — don't ask for a path. |
72+
| **Scaffold / generate a sandbox manifest** | Run (or show) `aca sandbox init`. Mention the commonly edited fields (`disk`, `resources`, `lifecycle.autoSuspendPolicy`, `egressPolicy`, plus `ports`, `env`, `labels` as needed). Mention `aca sandbox schema` as the way to dump the JSON Schema for editor autocomplete. |
73+
| **Run a command or open a shell in a sandbox** | Two distinct verbs: `aca sandbox exec --id "$SANDBOX_ID" -c "<command>"` for one-shot commands (returns stdout/stderr); `aca sandbox shell --id "$SANDBOX_ID"` for an interactive PTY. **Anti-cue:** `ssh` does not work — there is no SSH daemon inside the sandbox. `aca sandbox exec` / `shell` is the only path. |
74+
| **Delete a sandbox** | `aca sandbox delete --id "$SANDBOX_ID" --yes`. **Always recommend snapshotting first** if there is any state worth preserving (`aca sandbox snapshot --id "$SANDBOX_ID" --name <snap>`) — delete is destructive. To delete by label selector: `aca sandbox list -l "name=<n>" -o json \| jq -r '.[].id' \| xargs -I{} aca sandbox delete --id {} --yes`. |
75+
| **Read / write / copy files in a sandbox** | The `aca sandbox fs` family — `fs write --id "$SANDBOX_ID" --path /remote/p --file ./local` to upload, `fs cat --id "$SANDBOX_ID" --path /remote/p` to read, plus `fs ls / stat / mkdir / rm [--recursive]` for management. **Don't** suggest `scp` / `rsync` / shared filesystems — there is no SSH, and `fs` is the only data-plane file transport. |
76+
| **Expose a port — public preview (anonymous)** | The two-step shape: `URL=$(aca sandbox port add --id "$SANDBOX_ID" --port <p> --anonymous -o json \| jq -r .url)`, then hit `$URL`. **State explicitly** that anonymous = anyone with the URL can reach it (public preview only). Remove with `aca sandbox port remove --id "$SANDBOX_ID" --port <p>`. For per-user gating use the Entra row below. |
77+
| **Expose a port with email / Entra auth** | The `aca sandbox port add --id "$SANDBOX_ID" --port <p> --email <email>` command. **The Entra gotcha:** the email must be the user's Entra `mail` value — for some tenants the alias / UPN differs and won't work. Recommend `az ad signed-in-user show --query mail -o tsv` to fetch it. |
78+
| **Mount a shared volume** | Two-step: (1) at the group: `aca sandboxgroup volume create --name <v> --type AzureBlob` (multi-attach, shared) or `--type DataDisk` (single-attach, high-perf block). (2) at the sandbox: `aca sandbox mount --id "$SANDBOX_ID" --volume <v> --path /mnt/<v>`. State that **the volume lives at the group level**; sandboxes attach it at runtime. |
79+
| **Lock down network egress (deny-default + allow-list)** | The canonical form: `aca sandbox egress set --id "$SANDBOX_ID" --default Deny --rule "*.github.com:Allow" --traffic-inspection Full`. Multiple `--rule "host:Allow"` flags accumulate. Inspect current policy with `aca sandbox egress show --id "$SANDBOX_ID"`. For production agent code, **always recommend `--default Deny`** with an explicit allow-list. |
80+
| **Use a non-default disk image** | List published images first: `aca sandboxgroup disk list-public`, then `aca sandbox create --disk <name>`. To bake your own from an OCI image: `aca sandboxgroup disk create --image docker.io/library/alpine:3.19 --name <my-disk>`, then `aca sandbox create --disk-id <id>`. **Flag distinction:** `--disk` takes the public name; `--disk-id` takes the resource ID of a private/committed disk. |
81+
| **Suspend, resume, or set auto-suspend** | Manual: `aca sandbox stop --id "$SANDBOX_ID"` suspends (preserves memory + disk); `aca sandbox resume --id "$SANDBOX_ID"` does sub-second restore. Idle policy: `aca sandbox lifecycle set --id "$SANDBOX_ID" --auto-suspend <seconds>` (default 300s = 5 min). State that **suspended sandboxes incur storage cost only, no compute** — this is the primary cost lever. |
82+
| **Snapshot / commit a sandbox** | Per-sandbox: `aca sandbox snapshot --id "$SANDBOX_ID" --name <snap>`, then boot replicas with `aca sandbox create --snapshot <snap>`. Group-level CRUD: `aca sandboxgroup snapshot list / get / delete --selector "name=<snap>"`. **Strongly recommend snapshotting BEFORE `aca sandbox delete`** to preserve state. Use `--name`, never `--image`. Disk-only baking is `aca sandbox commit … --name <disk>`. |
83+
| **Anything in the "When NOT to use this skill" table below** | A one-paragraph redirect to the right tool or official docs. **Do NOT** run the out-of-scope tool's commands. **Do NOT** walk through options. **Do NOT** ask follow-up questions about the out-of-scope tool. Bow out cleanly. |
84+
85+
## When **NOT** to use this skill (hard reject + redirect)
86+
87+
If the user's task is **not about ACA Sandboxes**, refuse and redirect
88+
in one short reply — don't run any commands, don't walk through options,
89+
don't ask clarifying questions about the out-of-scope tool. The skill
90+
activated by mistake; bow out cleanly.
91+
92+
| User asks about… | Reply pattern |
93+
|---|---|
94+
| `azd init`, `azd up`, `azd deploy`, project scaffolding | "That's the Azure Developer CLI (`azd`), not ACA Sandboxes. See the [azd docs](https://learn.microsoft.com/azure/developer/azure-developer-cli/). Sandboxes don't have an `init`/`up`/`deploy` command and aren't a project bootstrapper." |
95+
| `az acr build`, `docker build`, registry pushes | "That's Azure Container Registry / Docker, not ACA Sandboxes. See the [`az acr` docs](https://learn.microsoft.com/cli/azure/acr). Sandboxes consume disk images, not container images." |
96+
| Cosmos / SQL / data-plane queries to other Azure services | "That's the relevant data service (Cosmos DB, Azure SQL, etc.), not ACA Sandboxes. Use that service's CLI / SDK / portal." |
97+
| Listing Kubernetes pods, AKS cluster ops, `kubectl` | "That's AKS / Kubernetes, not ACA Sandboxes. Use `kubectl` or the AKS docs. Sandboxes are individual microVMs, not a Kubernetes cluster." |
98+
| Deploying a Function App, App Service site, full Container App (Apps/Jobs) | "That's Azure Functions / App Service / Container Apps (apps and jobs), not Sandboxes. Use those products' deployment docs." |
99+
100+
**Never** start running the out-of-scope tool's commands "just to help." A one-paragraph redirect is the correct, complete answer.
101+
49102
## Get started
50103

51104
| | Where |
@@ -60,6 +113,27 @@ After install, always confirm setup with `aca doctor` — it resolves
60113
subscription / RG / group / region / role and tells you which check
61114
is red.
62115

116+
## Try asking
117+
118+
Once the skill is loaded, paste any of these into your agent. Each one
119+
exercises a different capability — together they show the canonical
120+
shape for the most common sandbox tasks (and they double as a routing
121+
smoke test if you're testing changes to this skill).
122+
123+
| Try saying | What you should get back |
124+
|---|---|
125+
| *"install the aca cli"* | the `aka.ms/aca-cli-install` one-liner + `aca --version` + `az login` + `aca doctor` |
126+
| *"set up a sandbox group from scratch"* | the full 4-step bootstrap (group create + Data Owner role + `aca doctor` gate) |
127+
| *"create an ubuntu sandbox and run uname -a in it"* | `aca sandbox create` with ID capture, then `aca sandbox exec` |
128+
| *"how do I ssh into my sandbox?"* | corrective answer — no SSH daemon; use `aca sandbox shell` or `exec` |
129+
| *"copy data.csv into my sandbox"* | `aca sandbox fs write --path … --file …` (and the anti-`scp` note) |
130+
| *"expose port 8080 publicly"* | `aca sandbox port add --anonymous -o json \| jq -r .url` |
131+
| *"mount a shared volume on two sandboxes"* | `aca sandboxgroup volume create --type AzureBlob` + `aca sandbox mount` |
132+
| *"restrict outbound traffic to github.com only"* | `aca sandbox egress set --default Deny --rule "*.github.com:Allow"` |
133+
| *"snapshot my sandbox before I tear it down"* | `aca sandbox snapshot --name <s>` followed by `aca sandbox delete --yes` |
134+
| *"suspend my sandbox to save money"* | `aca sandbox stop`/`resume` plus `aca sandbox lifecycle set --auto-suspend` |
135+
| *"give me a YAML manifest for a 2 vCPU sandbox"* | `aca sandbox init` → edit → `aca sandbox validate --file` → `aca sandbox apply --file` |
136+
63137
## Capabilities
64138

65139
Everything the platform exposes. Each row is the starting point — open

‎plugin/skills/sandboxes/references/install.md‎

Lines changed: 13 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,41 +1,45 @@
11
# Install the `aca` CLI
22

33
The `aca` CLI is the primary surface for sandboxes. It delegates
4-
authentication to the Azure CLI (`az login`).
4+
authentication to the Azure CLI — sign in once with `az login` and the
5+
same Entra identity is used by `aca`.
56

67
## Linux / macOS
78

89
```bash
9-
curl -fsSL https://raw.githubusercontent.com/microsoft/azure-container-apps/main/docs/early/aca-cli/install.sh | sh
10+
# This same one-liner is also the install path used INSIDE a sandbox or
11+
# container for agent-driven self-installs — no package manager needed.
12+
curl -fsSL https://aka.ms/aca-cli-install | sh
1013
```
1114

1215
Pin a specific version:
1316

1417
```bash
15-
curl -fsSL https://raw.githubusercontent.com/microsoft/azure-container-apps/main/docs/early/aca-cli/install.sh \
18+
curl -fsSL https://aka.ms/aca-cli-install \
1619
| ACA_VERSION=aca-cli-v0.1.0-early-access sh
1720
```
1821

1922
## Windows (PowerShell)
2023

2124
```powershell
22-
irm https://raw.githubusercontent.com/microsoft/azure-container-apps/main/docs/early/aca-cli/install.ps1 | iex
25+
# This same one-liner is also the install path used INSIDE a Windows
26+
# sandbox or container for agent-driven self-installs.
27+
irm https://aka.ms/aca-cli-install-ps | iex
2328
```
2429

2530
Pin a specific version:
2631

2732
```powershell
28-
& ([scriptblock]::Create((irm https://raw.githubusercontent.com/microsoft/azure-container-apps/main/docs/early/aca-cli/install.ps1))) -Version aca-cli-v0.1.0-early-access
33+
& ([scriptblock]::Create((irm https://aka.ms/aca-cli-install-ps))) -Version aca-cli-v0.1.0-early-access
2934
```
3035

3136
## Verify
3237

3338
```bash
3439
aca --version
35-
# aca 1.0.0-beta.1
3640
```
3741

38-
Then log in and run the doctor:
42+
Then log in (`aca` delegates auth to `az login`) and run the doctor:
3943

4044
```bash
4145
az login
@@ -46,12 +50,12 @@ aca doctor
4650

4751
```bash
4852
# Linux / macOS
49-
curl -fsSL https://raw.githubusercontent.com/microsoft/azure-container-apps/main/docs/early/aca-cli/install.sh | sh -s -- --uninstall
53+
curl -fsSL https://aka.ms/aca-cli-install | sh -s -- --uninstall
5054
```
5155

5256
```powershell
5357
# Windows
54-
& ([scriptblock]::Create((irm https://raw.githubusercontent.com/microsoft/azure-container-apps/main/docs/early/aca-cli/install.ps1))) -Uninstall
58+
& ([scriptblock]::Create((irm https://aka.ms/aca-cli-install-ps))) -Uninstall
5559
```
5660

5761
## Supported platforms

0 commit comments

Comments
 (0)