feat: add supply chain content integrity for external skill manifests

AlexU-A · claude · AlexU-A · commit b86d97576c87 · 2026-03-14T10:56:52.000-04:00
Refresh script now fetches skill content, shows unified diffs, and
requires --approve to write updates. Verify script gains --verify-content
flag to fetch and re-hash content against stored content_sha256.

Co-Authored-By: Claude Opus 4.6 (1M context) &lt;noreply@anthropic.com&gt;
diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml
@@ -15,3 +15,5 @@ jobs:
       - run: pip install pyyaml
       - run: python3 scripts/refresh_external_skills.py --check
       - run: python3 scripts/verify_no_copied_skills.py
+        # Note: add --verify-content for full content integrity checks;
+        # omitted by default to avoid CI dependency on GitHub raw content availability.
diff --git a/CLAUDE.md b/CLAUDE.md
@@ -15,9 +15,11 @@ A router agent (`js-critic-router`) dispatches to the correct critic based on fr
 ## Commands
 
 ```bash
-python3 scripts/refresh_external_skills.py
-python3 scripts/refresh_external_skills.py --check
-python3 scripts/verify_no_copied_skills.py
+python3 scripts/refresh_external_skills.py              # dry-run: show diffs
+python3 scripts/refresh_external_skills.py --approve     # apply pin + hash updates
+python3 scripts/refresh_external_skills.py --check       # CI: fail if updates needed
+python3 scripts/verify_no_copied_skills.py               # validate manifest structure
+python3 scripts/verify_no_copied_skills.py --verify-content  # fetch + hash verification
 python3 scripts/run_benchmark.py --all
 python3 scripts/aggregate_stability.py
 ```
@@ -29,6 +31,15 @@ python3 scripts/aggregate_stability.py
 - Evidence required for CRITICAL/MAJOR findings.
 - Shared JS-core rubric + critic-specific rubrics.
 
+## Supply Chain Security
+
+External skills are loaded by reference (pinned commit SHA + content SHA-256 hash).
+
+- `content_sha256` in manifests stores the SHA-256 of each skill's SKILL.md at the pinned commit.
+- `refresh_external_skills.py` shows content diffs and requires `--approve` to write updates.
+- `verify_no_copied_skills.py --verify-content` fetches and re-hashes to detect tampering.
+- CI validates manifest structure; full content verification available via `--verify-content`.
+
 ## Benchmark Infrastructure
 
 - Fixtures: `research/benchmarks/fixtures/{react,next,react-native}/` (8 per critic)
diff --git a/scripts/refresh_external_skills.py b/scripts/refresh_external_skills.py
@@ -1,6 +1,10 @@
 #!/usr/bin/env python3
 import argparse
+import difflib
+import hashlib
 import subprocess
+import urllib.request
+import urllib.error
 from datetime import datetime, timezone
 from pathlib import Path
 
@@ -17,9 +21,40 @@ def get_head_sha(repo_url: str) -> str:
     return out.split()[0]
 
 
+def fetch_skill_content(repo_url: str, commit: str, skill_name: str) -> str | None:
+    """Fetch SKILL.md content from raw.githubusercontent.com. Returns None on failure."""
+    owner_repo = repo_url.removeprefix('https://github.com/')
+    raw_url = f'https://raw.githubusercontent.com/{owner_repo}/{commit}/{skill_name}/SKILL.md'
+    try:
+        with urllib.request.urlopen(raw_url, timeout=15) as resp:
+            return resp.read().decode('utf-8')
+    except urllib.error.HTTPError as e:
+        print(f'  WARNING: HTTP {e.code} fetching {raw_url}')
+        return None
+    except Exception as e:
+        print(f'  WARNING: Failed to fetch {raw_url}: {e}')
+        return None
+
+
+def sha256_of(content: str) -> str:
+    return hashlib.sha256(content.encode('utf-8')).hexdigest()
+
+
+def show_diff(skill_id: str, old_content: str | None, new_content: str | None) -> None:
+    old_lines = (old_content or '(content unavailable)\n').splitlines(keepends=True)
+    new_lines = (new_content or '(content unavailable)\n').splitlines(keepends=True)
+    diff = list(difflib.unified_diff(old_lines, new_lines, fromfile=f'{skill_id} (old)', tofile=f'{skill_id} (new)'))
+    if diff:
+        print(f'\n--- Diff for {skill_id} ---')
+        print(''.join(diff), end='')
+    else:
+        print(f'  (no content diff for {skill_id})')
+
+
 def main() -> int:
     parser = argparse.ArgumentParser(description='Refresh pinned commits in all critic manifests.')
     parser.add_argument('--check', action='store_true', help='Fail if updates are needed without writing files.')
+    parser.add_argument('--approve', action='store_true', help='Apply pin and content hash updates (writes manifests).')
     args = parser.parse_args()
 
     changes = []
@@ -30,15 +65,40 @@ def main() -> int:
         skills = data.get('skills', [])
         checked += len(skills)
         local_changes = []
+
         for s in skills:
             latest = get_head_sha(s['repo_url'])
             current = s.get('pinned_commit', '')
-            if latest != current:
+            if latest == current:
+                continue
+
+            skill_name = s['id'].split('/')[-1]
+            print(f'\nSkill {s["id"]}: {current or "-"} -> {latest}')
+
+            old_content = fetch_skill_content(s['repo_url'], current, skill_name) if current else None
+            new_content = fetch_skill_content(s['repo_url'], latest, skill_name)
+
+            if not args.check:
+                show_diff(s['id'], old_content, new_content)
+
+            if new_content is not None:
+                new_hash = sha256_of(new_content)
+                print(f'  content_sha256 (new): {new_hash}')
+            else:
+                new_hash = None
+                print(f'  WARNING: Could not fetch new content — content_sha256 will be unverified.')
+
+            local_changes.append((s['id'], current, latest, new_hash))
+            changes.append((manifest, s['id'], current, latest, new_hash))
+
+            if args.approve:
                 s['pinned_commit'] = latest
-                local_changes.append((s['id'], current, latest))
-                changes.append((manifest, s['id'], current, latest))
+                if new_hash is not None:
+                    s['content_sha256'] = new_hash
+                else:
+                    s.pop('content_sha256', None)
 
-        if local_changes and not args.check:
+        if local_changes and args.approve:
             data['generated_at'] = datetime.now(timezone.utc).replace(microsecond=0).isoformat().replace('+00:00', 'Z')
             manifest.write_text(yaml.safe_dump(data, sort_keys=False), encoding='utf-8')
 
@@ -52,10 +112,11 @@ def main() -> int:
     ]
 
     if changes:
-        lines.append('| Manifest | Skill | Old | New |')
-        lines.append('|---|---|---|---|')
-        for m, sid, old, new in changes:
-            lines.append(f'| `{m.relative_to(ROOT)}` | `{sid}` | `{old or "-"}` | `{new}` |')
+        lines.append('| Manifest | Skill | Old | New | Hash Verified |')
+        lines.append('|---|---|---|---|---|')
+        for m, sid, old, new, new_hash in changes:
+            verified = 'yes' if new_hash else '# UNVERIFIED'
+            lines.append(f'| `{m.relative_to(ROOT)}` | `{sid}` | `{old or "-"}` | `{new}` | {verified} |')
     else:
         lines.append('No pin changes detected.')
 
@@ -66,6 +127,9 @@ def main() -> int:
         print(f'Manifest updates required for {len(changes)} skill entries.')
         return 1
 
+    if changes and not args.approve and not args.check:
+        print('\nRun with --approve to apply these changes.')
+
     print(f'Checked {checked} skill entries across {len(MANIFESTS)} manifests.')
     print(f'Wrote report: {REPORT}')
     return 0
diff --git a/scripts/verify_no_copied_skills.py b/scripts/verify_no_copied_skills.py
@@ -1,7 +1,11 @@
 #!/usr/bin/env python3
+import argparse
+import hashlib
 import re
 import subprocess
 import sys
+import urllib.request
+import urllib.error
 from pathlib import Path
 
 import yaml
@@ -11,18 +15,48 @@
 
 ID_RE = re.compile(r'^[^/]+/[^/]+/[^/]+$')
 SHA_RE = re.compile(r'^[0-9a-f]{40}$')
+SHA256_RE = re.compile(r'^[0-9a-f]{64}$')
 
 
 def fail(msg: str) -> None:
     print(f'ERROR: {msg}')
     raise SystemExit(1)
 
 
+def fetch_skill_content(repo_url: str, commit: str, skill_name: str) -> str | None:
+    """Fetch SKILL.md content from raw.githubusercontent.com. Returns None on failure."""
+    owner_repo = repo_url.removeprefix('https://github.com/')
+    raw_url = f'https://raw.githubusercontent.com/{owner_repo}/{commit}/{skill_name}/SKILL.md'
+    try:
+        with urllib.request.urlopen(raw_url, timeout=15) as resp:
+            return resp.read().decode('utf-8')
+    except urllib.error.HTTPError as e:
+        print(f'  WARNING: HTTP {e.code} fetching {raw_url}')
+        return None
+    except Exception as e:
+        print(f'  WARNING: Failed to fetch {raw_url}: {e}')
+        return None
+
+
+def sha256_of(content: str) -> str:
+    return hashlib.sha256(content.encode('utf-8')).hexdigest()
+
+
 def main() -> int:
+    parser = argparse.ArgumentParser(description='Verify no copied skills in manifests.')
+    parser.add_argument(
+        '--verify-content',
+        action='store_true',
+        help='Fetch skill content from raw.githubusercontent.com and verify content_sha256 hashes.',
+    )
+    args = parser.parse_args()
+
     if not MANIFESTS:
         fail('No external-skills-manifest.yaml files found.')
 
     seen = {}
+    content_errors = 0
+
     for manifest in MANIFESTS:
         data = yaml.safe_load(manifest.read_text(encoding='utf-8')) or {}
         skills = data.get('skills', [])
@@ -57,6 +91,32 @@ def main() -> int:
             if s.get('status') not in {'active', 'deprecated'}:
                 fail(f'Invalid status for {sid}: {s.get("status")}')
 
+            # Validate content_sha256 format if present
+            stored_hash = s.get('content_sha256')
+            if stored_hash is not None:
+                if not SHA256_RE.match(str(stored_hash)):
+                    fail(f'Invalid content_sha256 format for {sid} (must be 64-char hex): {stored_hash}')
+
+            # Optionally verify content hash against live content
+            if args.verify_content:
+                skill_name = sid.split('/')[-1]
+                commit = s.get('pinned_commit', '')
+                repo_url = s.get('repo_url', '')
+                content = fetch_skill_content(repo_url, commit, skill_name)
+                if content is None:
+                    print(f'  WARNING: Could not fetch content for {sid} — skipping hash verification.')
+                elif stored_hash is None:
+                    print(f'  WARNING: No content_sha256 stored for {sid} — run refresh_external_skills.py --approve to populate.')
+                else:
+                    actual_hash = sha256_of(content)
+                    if actual_hash != stored_hash:
+                        print(f'ERROR: content_sha256 mismatch for {sid}')
+                        print(f'  expected: {stored_hash}')
+                        print(f'  actual:   {actual_hash}')
+                        content_errors += 1
+                    else:
+                        print(f'  OK: {sid} content hash verified.')
+
     git_ls = subprocess.check_output(['git', '-C', str(ROOT), 'ls-files'], text=True).splitlines()
     forbidden_prefixes = [
         'research/javascript-skills/upstream/',
@@ -67,6 +127,10 @@ def main() -> int:
             if f.startswith(prefix):
                 fail(f'Forbidden tracked file path: {f}')
 
+    if content_errors:
+        print(f'\n{content_errors} content hash mismatch(es) detected.')
+        raise SystemExit(1)
+
     print(f'Validation passed for {len(MANIFESTS)} manifests and {len(seen)} unique skills.')
     return 0
 
