|
8 | 8 | import { describe, it, expect } from 'vitest'; |
9 | 9 | import fs from 'node:fs'; |
10 | 10 | import path from 'node:path'; |
| 11 | +import { execFileSync } from 'node:child_process'; |
11 | 12 | import { fileURLToPath } from 'node:url'; |
12 | 13 | import { syncObjectStackDeps } from './pkg-utils.js'; |
13 | 14 |
|
14 | 15 | const pkgRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..'); |
| 16 | +const repoRoot = path.resolve(pkgRoot, '..', '..'); |
15 | 17 | const ownPkg = JSON.parse(fs.readFileSync(path.join(pkgRoot, 'package.json'), 'utf8')); |
16 | 18 | const ownMajor = Number(ownPkg.version.split('.')[0]); |
17 | 19 |
|
@@ -82,6 +84,96 @@ describe('README template table', () => { |
82 | 84 | }); |
83 | 85 | }); |
84 | 86 |
|
| 87 | +// Skills catalog boundary (15.1 third-party eval): scaffolded projects once |
| 88 | +// received the repo-internal `dogfood-verification` skill because the |
| 89 | +// scaffolder installed with a repo-wide `skills add … --all`, whose discovery |
| 90 | +// also walks `.claude/skills/`. The published catalog is exactly the root |
| 91 | +// `skills/` directory; everything else must stay repo-internal. |
| 92 | +describe('skills catalog boundary', () => { |
| 93 | + const frontmatterOf = (file: string): string => |
| 94 | + /^---\n([\s\S]*?)\n---/.exec(fs.readFileSync(file, 'utf8'))?.[1] ?? ''; |
| 95 | + const isMarkedInternal = (fm: string): boolean => |
| 96 | + /^metadata:\s*$/m.test(fm) && /^ +internal:\s*true\s*(#.*)?$/m.test(fm); |
| 97 | + |
| 98 | + const trackedSkillFiles = execFileSync('git', ['ls-files', '*SKILL.md'], { |
| 99 | + cwd: repoRoot, |
| 100 | + encoding: 'utf8', |
| 101 | + }) |
| 102 | + .split('\n') |
| 103 | + .filter(Boolean); |
| 104 | + |
| 105 | + it('finds the curated skills/ catalog (sanity)', () => { |
| 106 | + expect( |
| 107 | + trackedSkillFiles.filter((f) => f.startsWith('skills/')).length, |
| 108 | + ).toBeGreaterThan(0); |
| 109 | + }); |
| 110 | + |
| 111 | + it('marks every SKILL.md outside skills/ as metadata.internal', () => { |
| 112 | + for (const rel of trackedSkillFiles) { |
| 113 | + if (rel.startsWith('skills/')) continue; |
| 114 | + expect( |
| 115 | + isMarkedInternal(frontmatterOf(path.join(repoRoot, rel))), |
| 116 | + `${rel} is outside the published skills/ catalog but is not hidden from ` + |
| 117 | + 'the skills CLI. Add to its frontmatter:\n' + |
| 118 | + 'metadata:\n internal: true\n' + |
| 119 | + 'or move it into skills/ if it is meant for customers.', |
| 120 | + ).toBe(true); |
| 121 | + } |
| 122 | + }); |
| 123 | + |
| 124 | + it('never marks a curated skills/ entry internal', () => { |
| 125 | + for (const rel of trackedSkillFiles) { |
| 126 | + if (!rel.startsWith('skills/')) continue; |
| 127 | + expect( |
| 128 | + isMarkedInternal(frontmatterOf(path.join(repoRoot, rel))), |
| 129 | + `${rel} is in the published catalog but marked metadata.internal — ` + |
| 130 | + 'customers would silently stop receiving it.', |
| 131 | + ).toBe(false); |
| 132 | + } |
| 133 | + }); |
| 134 | + |
| 135 | + it('scaffolder installs from the curated skills/ subpath, not the repo root', () => { |
| 136 | + expect(REGISTRY_SOURCE).toContain( |
| 137 | + 'skills add objectstack-ai/framework/skills --all', |
| 138 | + ); |
| 139 | + expect(REGISTRY_SOURCE).not.toMatch( |
| 140 | + /skills add objectstack-ai\/framework(?!\/skills)/, |
| 141 | + ); |
| 142 | + }); |
| 143 | + |
| 144 | + // The /skills subpath is the hard boundary: the skills CLI's `--all` |
| 145 | + // implies `--skill '*'`, which INCLUDES metadata.internal skills — so any |
| 146 | + // customer-facing surface advertising a repo-root install would leak |
| 147 | + // internal skills again. |
| 148 | + it('no customer-facing surface advertises a repo-root skills install', () => { |
| 149 | + const surfaces = [ |
| 150 | + 'content/docs', |
| 151 | + 'skills', |
| 152 | + 'packages/create-objectstack', |
| 153 | + // this file mentions the bare form on purpose (needle + error message) |
| 154 | + ':(exclude)packages/create-objectstack/src/template-consistency.test.ts', |
| 155 | + ]; |
| 156 | + let candidates = ''; |
| 157 | + try { |
| 158 | + candidates = execFileSync( |
| 159 | + 'git', |
| 160 | + ['grep', '-nF', 'skills add objectstack-ai/framework', '--', ...surfaces], |
| 161 | + { cwd: repoRoot, encoding: 'utf8' }, |
| 162 | + ); |
| 163 | + } catch { |
| 164 | + // git grep exits 1 on no matches — nothing to check then. |
| 165 | + } |
| 166 | + const rootInstalls = candidates |
| 167 | + .split('\n') |
| 168 | + .filter((line) => /skills add objectstack-ai\/framework(?!\/skills)/.test(line)); |
| 169 | + expect( |
| 170 | + rootInstalls, |
| 171 | + 'these lines advertise `skills add objectstack-ai/framework` without ' + |
| 172 | + 'the /skills subpath — repo-root + --all installs internal skills', |
| 173 | + ).toEqual([]); |
| 174 | + }); |
| 175 | +}); |
| 176 | + |
85 | 177 | describe('syncObjectStackDeps', () => { |
86 | 178 | it('rewrites @objectstack/* ranges in deps and devDeps', () => { |
87 | 179 | const pkg = { |
|
0 commit comments