Update agent skills to 4e74b31990bde519c276a6648ff59f50e49ff861

.gen.json

@@ -1,3 +1,3 @@
 {
-  "source_commit": "654461451d03b1cac88e8b2c30f15060d27f6295"
+  "source_commit": "4e74b31990bde519c276a6648ff59f50e49ff861"
 }

.github/workflows/validate-manifest.yml

@@ -35,6 +35,13 @@ jobs:
         with:
           python-version: '3.11'
 
+      # The Databricks CLI's files-channel installer reads the committed
+      # manifest.json from main before fetching each listed file. Check this
+      # before the dry-run regeneration below, otherwise CI can mask a stale
+      # manifest entry that would 404 for `databricks aitools install`.
+      - name: Validate committed CLI manifest references
+        run: python3 scripts/skills.py validate-committed-manifest
+
       # This repo is published as a source-only mirror from an internal source of
       # truth. Generated artifacts (the plugins/ bundle, manifest.json, the
       # marketplace catalogs, the rendered hook/routing files) are regenerated and

CONTRIBUTING.md

@@ -60,6 +60,8 @@ python3 scripts/skills.py validate
 
 - Every skill has `agents/openai.yaml`.
 - Every skill ships `assets/databricks.svg` + `assets/databricks.png` byte-identical to the repo-root source.
+- Any committed `manifest.json` points only at skill directories and files that
+  exist in the repo, so CLI installs from `main` cannot 404 on stale entries.
 - `manifest.json` matches what `scripts/skills.py generate` would produce (stable skills' `repo_dir` stays `skills`; the CLI files-channel fetches the root `skills/`).
 - Every stable skill has a `plugin.meta.json` "skills" entry (and vice versa).
 - Every target's `plugin.json` + each root `marketplace.json` catalog is byte-identical to what the generator produces from `plugin.meta.json` (no drift across the four targets).
@@ -245,4 +247,4 @@ commit automatically with `git commit -s`:
 
 ```
 git commit -s -m "Your commit message"
-```
+```

README.md

@@ -241,7 +241,14 @@ This approach:
 
 ### Manifest Management
 
-`manifest.json` is **generated** by `scripts/skills.py` from the skill directories and frontmatter. Do not edit it by hand. CI rejects manual changes via two checks: content drift (parsed dict doesn't match what `generate` would produce) and canonical form (on-disk bytes don't match `json.dumps(..., indent=2, sort_keys=True)`).
+`manifest.json` is **generated** by `scripts/skills.py` from the skill
+directories and frontmatter. Do not edit it by hand. CI rejects manual changes
+via two checks: content drift (parsed dict doesn't match what `generate` would
+produce) and canonical form (on-disk bytes don't match
+`json.dumps(..., indent=2, sort_keys=True)`). CI also checks the committed
+manifest's file references before dry-run regeneration, because the CLI reads
+that committed file from `main`; a stale entry for a moved or deleted skill
+would otherwise make installs fetch missing files.
 
 Sync assets and regenerate the manifest after adding or updating skills:
 
@@ -255,6 +262,12 @@ Validate that assets and manifest are up to date (used by CI):
 python3 scripts/skills.py validate
 ```
 
+To check only that a committed manifest points at files that exist in the repo:
+
+```bash
+python3 scripts/skills.py validate-committed-manifest
+```
+
 The manifest is consumed by the CLI to discover available skills.
 
 ### Plugin manifest management

metaplugin/plugin.meta.json

@@ -186,7 +186,7 @@
       { "label": "Python SDK / Databricks Connect / REST API", "skill": "databricks-python-sdk" },
       { "label": "Spark Structured Streaming", "skill": "databricks-spark-structured-streaming" },
       { "label": "Synthetic / test data generation (Faker)", "skill": "databricks-synthetic-data-gen" },
-      { "label": "Unity Catalog system tables (lineage, audit, billing)", "skill": "databricks-unity-catalog" },
+      { "label": "Unity Catalog governance, access control & system tables", "skill": "databricks-unity-catalog" },
       { "label": "Synthetic PDF generation for RAG", "skill": "databricks-unstructured-pdf-generation" },
       { "label": "Zerobus streaming ingest", "skill": "databricks-zerobus-ingest" }
     ],

scripts/skills.py

@@ -50,6 +50,7 @@
     _build_stable_entry,
     _build_experimental_entry,
     serialize_manifest,
+    check_manifest_file_references,
     validate_manifest,
 )
 from skillsgen.plugins import (

scripts/skillsgen/cli.py

@@ -8,7 +8,7 @@
 from skillsgen.common import META_FILE, load_meta
 from skillsgen.discovery import check_codex_metadata, ensure_codex_metadata
 from skillsgen.generate import generate_all
-from skillsgen.manifest import validate_manifest
+from skillsgen.manifest import check_manifest_file_references, validate_manifest
 from skillsgen.plugins import (
     check_generated_plugins,
     check_meta_skill_coverage,
@@ -47,11 +47,13 @@ def main() -> None:
         "mode",
         nargs="?",
         default="generate",
-        choices=["sync", "generate", "validate"],
+        choices=["sync", "generate", "validate", "validate-committed-manifest"],
         help=(
             "sync: ensure every skill has assets + agents/openai.yaml. "
             "generate: sync + (re)build manifest.json (default). "
-            "validate: check assets, metadata, and manifest are up to date."
+            "validate: check assets, metadata, and manifest are up to date. "
+            "validate-committed-manifest: before regenerating, check that any "
+            "committed manifest.json points only at files present in the repo."
         ),
     )
 
@@ -83,6 +85,23 @@ def main() -> None:
                 f"Generated {result['bundle']} file(s) in the plugins/databricks/ bundle"
             )
 
+        case "validate-committed-manifest":
+            errors = check_manifest_file_references(repo_root)
+            if errors:
+                print(
+                    "ERROR: committed manifest.json points at missing or invalid "
+                    "skill files:",
+                    file=sys.stderr,
+                )
+                for err in errors:
+                    print(f"  - {err}", file=sys.stderr)
+                sys.exit(1)
+
+            if (repo_root / "manifest.json").exists():
+                print("Committed manifest.json references existing skill files.")
+            else:
+                print("No committed manifest.json found; skipped file-reference check.")
+
         case "validate":
             ok = True
 

scripts/skillsgen/manifest.py

@@ -4,6 +4,7 @@
 import sys
 from pathlib import Path
 
+from skillsgen.common import _norm_rel_path
 from skillsgen.discovery import (
     EXPERIMENTAL_REPO_DIR,
     STABLE_REPO_DIR,
@@ -91,6 +92,93 @@ def serialize_manifest(manifest: dict) -> str:
     return json.dumps(manifest, indent=2, sort_keys=True) + "\n"
 
 
+def _is_safe_manifest_path(path: str) -> bool:
+    path = _norm_rel_path(path)
+    rel = Path(path)
+    return bool(path) and not rel.is_absolute() and ".." not in rel.parts
+
+
+def check_manifest_file_references(repo_root: Path) -> list[str]:
+    """Validate that a committed manifest cannot point the CLI at 404s.
+
+    The publish repo can carry a generated manifest between releases while
+    source skills change on main. The CLI's files-channel installer reads that
+    committed manifest directly, so validate every advertised skill directory
+    and file before CI's dry-run regeneration masks stale entries.
+    """
+    manifest_path = repo_root / "manifest.json"
+    if not manifest_path.exists():
+        return []
+
+    try:
+        manifest = json.loads(manifest_path.read_text())
+    except json.JSONDecodeError as exc:
+        return [f"manifest.json is not valid JSON: {exc}"]
+
+    skills = manifest.get("skills")
+    if not isinstance(skills, dict):
+        return ['manifest.json "skills" must be an object.']
+
+    errors: list[str] = []
+    for name, entry in sorted(skills.items()):
+        if not isinstance(entry, dict):
+            errors.append(f'manifest.json skills["{name}"] must be an object.')
+            continue
+
+        repo_dir = entry.get("repo_dir")
+        if not isinstance(repo_dir, str):
+            errors.append(
+                f'manifest.json skills["{name}"].repo_dir must be a string.'
+            )
+            continue
+
+        repo_dir = _norm_rel_path(repo_dir)
+        if repo_dir not in {STABLE_REPO_DIR, EXPERIMENTAL_REPO_DIR}:
+            errors.append(
+                f'manifest.json skills["{name}"].repo_dir is {repo_dir!r}; '
+                f"expected {STABLE_REPO_DIR!r} or {EXPERIMENTAL_REPO_DIR!r}."
+            )
+            continue
+
+        if not _is_safe_manifest_path(name) or Path(name).name != name:
+            errors.append(
+                f"manifest.json skill name {name!r} is not a safe directory name."
+            )
+            continue
+
+        skill_root = repo_root / repo_dir / name
+        if not skill_root.is_dir():
+            errors.append(
+                f"manifest.json lists skill '{name}' at {repo_dir}/{name}, "
+                "but that directory does not exist. Regenerate manifest.json "
+                "or restore the skill."
+            )
+            continue
+
+        files = entry.get("files")
+        if not isinstance(files, list):
+            errors.append(f'manifest.json skills["{name}"].files must be an array.')
+            continue
+
+        for file_rel in files:
+            if not isinstance(file_rel, str) or not _is_safe_manifest_path(file_rel):
+                errors.append(
+                    f"manifest.json lists unsafe file path {file_rel!r} "
+                    f"for skill '{name}'."
+                )
+                continue
+
+            file_rel = _norm_rel_path(file_rel)
+            if not (skill_root / file_rel).is_file():
+                errors.append(
+                    f"manifest.json lists file '{file_rel}' for skill '{name}', "
+                    f"but {repo_dir}/{name}/{file_rel} does not exist. "
+                    "Regenerate manifest.json or restore the file."
+                )
+
+    return errors
+
+
 def validate_manifest(repo_root: Path) -> bool:
     """Validate that manifest.json is up to date AND in canonical sorted form."""
     manifest_path = repo_root / "manifest.json"
@@ -99,6 +187,16 @@ def validate_manifest(repo_root: Path) -> bool:
         print("ERROR: manifest.json does not exist", file=sys.stderr)
         return False
 
+    reference_errors = check_manifest_file_references(repo_root)
+    if reference_errors:
+        print(
+            "ERROR: manifest.json points at missing or invalid skill files:",
+            file=sys.stderr,
+        )
+        for err in reference_errors:
+            print(f"  - {err}", file=sys.stderr)
+        return False
+
     current_text = manifest_path.read_text()
     current_manifest = json.loads(current_text)
     expected_manifest = generate_manifest(repo_root)