diff --git a/.aiox-core/data/entity-registry.yaml b/.aiox-core/data/entity-registry.yaml index 37826faa16..0a1da2d7d5 100644 --- a/.aiox-core/data/entity-registry.yaml +++ b/.aiox-core/data/entity-registry.yaml @@ -1,6 +1,6 @@ metadata: version: 1.0.0 - lastUpdated: '2026-05-07T15:02:59.792Z' + lastUpdated: '2026-05-07T21:02:02.654Z' entityCount: 746 checksumAlgorithm: sha256 resolutionRate: 100 @@ -16376,8 +16376,8 @@ entities: score: 0.7 constraints: [] extensionPoints: [] - checksum: sha256:0fbc1baff25f20e3a37d3e4be51d146a75254d5ed638b3438d9f1bf0e587c997 - lastVerified: '2026-03-11T00:48:55.955Z' + checksum: sha256:5b8cb91ddcfda2b6b66328c12129e71665e4e1cb6f13b1792746e9d761d1ebcf + lastVerified: '2026-05-07T21:02:02.645Z' brownfield-analyzer: path: .aiox-core/infrastructure/scripts/documentation-integrity/brownfield-analyzer.js layer: L2 diff --git a/.aiox-core/infrastructure/scripts/codex-skills-sync/README.md b/.aiox-core/infrastructure/scripts/codex-skills-sync/README.md index 2501effdce..6f1d4a4456 100644 --- a/.aiox-core/infrastructure/scripts/codex-skills-sync/README.md +++ b/.aiox-core/infrastructure/scripts/codex-skills-sync/README.md @@ -47,9 +47,11 @@ Used by CI and by the unified ide-sync pipeline. Does not generate squad-chief s ``` npm run validate:codex-skills # validate aiox-* skill / agent parity (strict) +npm run validate:codex-skills:self-test # parity + deterministic Skill tool self-test ``` Verifies that every core agent has a corresponding `.codex/skills/aiox-/SKILL.md`. Squad-chief skills are out of scope for this validator. +The self-test mode additionally simulates a Skill tool invocation for each generated skill and checks that the skill frontmatter, source-of-truth path, and greeting command can activate the target agent without requiring a live ping-pong tool call. ## Files generated diff --git a/.aiox-core/infrastructure/scripts/codex-skills-sync/validate.js b/.aiox-core/infrastructure/scripts/codex-skills-sync/validate.js index e9db0877c6..43f43184ab 100644 --- a/.aiox-core/infrastructure/scripts/codex-skills-sync/validate.js +++ b/.aiox-core/infrastructure/scripts/codex-skills-sync/validate.js @@ -3,6 +3,7 @@ const fs = require('fs'); const path = require('path'); +const yaml = require('js-yaml'); const { parseAllAgents } = require('../ide-sync/agent-parser'); const { getSkillId, getLegacySkillId } = require('./index'); @@ -17,6 +18,7 @@ function getDefaultOptions() { skillsDir: path.join(projectRoot, '.codex', 'skills'), strict: false, allowOrphaned: false, + selfTest: false, quiet: false, json: false, }; @@ -28,6 +30,7 @@ function parseArgs(argv = process.argv.slice(2)) { strict: args.has('--strict'), quiet: args.has('--quiet') || args.has('-q'), json: args.has('--json'), + selfTest: args.has('--self-test'), }; } @@ -62,6 +65,158 @@ function validateSkillContent(content, expected) { return issues; } +/** + * Parses a generated Codex skill frontmatter block without reading external files. + */ +function parseSkillFrontmatter(content) { + const match = String(content || '').match(/^---\r?\n([\s\S]*?)\r?\n---(?:\r?\n|$)/); + if (!match) { + return { data: null, error: 'missing YAML frontmatter' }; + } + + try { + const data = yaml.load(match[1]) || {}; + return { data, error: null }; + } catch (error) { + return { data: null, error: `invalid YAML frontmatter (${error.message})` }; + } +} + +/** + * Builds the deterministic Skill tool payload used by validator self-tests. + */ +function createSkillToolSelfTestPayload(skillId, prompt = 'AIOX skill self-test') { + return { + type: 'tool_use', + name: 'Skill', + input: { + skill: skillId, + prompt, + }, + }; +} + +/** + * Extracts a skill id from the payload shapes used by Skill tool invocations. + */ +function normalizeSkillToolTarget(payload) { + if (typeof payload === 'string') { + return payload.trim().replace(/^\$/, ''); + } + + if (!payload || typeof payload !== 'object') { + return ''; + } + + const input = payload.input && typeof payload.input === 'object' ? payload.input : {}; + const candidates = [ + input.skill, + input.skillId, + input.skill_id, + input.name, + input.id, + payload.skill, + payload.skillId, + payload.skill_id, + payload.id, + payload.name && payload.name !== 'Skill' ? payload.name : '', + ]; + + const target = candidates.find((candidate) => typeof candidate === 'string' && candidate.trim()); + return target ? target.trim().replace(/^\$/, '') : ''; +} + +/** + * Finds the canonical AIOX agent path declared in a generated skill stub. + */ +function extractCanonicalAgentPath(content) { + const match = String(content || '').match(/`(\.aiox-core\/development\/agents\/[^`]+\.md)`/); + return match ? match[1] : ''; +} + +/** + * Runs structural self-tests for generated Codex skills without invoking live tools. + */ +function runSkillSelfTests(options = {}) { + const projectRoot = options.projectRoot || process.cwd(); + const resolved = { + projectRoot, + skillsDir: options.skillsDir || path.join(projectRoot, '.codex', 'skills'), + expected: options.expected || [], + }; + const expectedIds = new Set(resolved.expected.map(item => item.skillId)); + const results = []; + + for (const item of resolved.expected) { + const skillPath = path.join(resolved.skillsDir, item.skillId, 'SKILL.md'); + const relativeSkillPath = path.relative(resolved.projectRoot, skillPath); + const errors = []; + + let content = ''; + try { + content = fs.readFileSync(skillPath, 'utf8'); + } catch (error) { + results.push({ + skillId: item.skillId, + ok: false, + errors: [`self-test unable to read ${relativeSkillPath} (${error.message})`], + }); + continue; + } + + const frontmatter = parseSkillFrontmatter(content); + let declaredSkillId = ''; + let canRoundtripSkillPayload = false; + if (frontmatter.error) { + errors.push(`self-test ${frontmatter.error}`); + } else { + declaredSkillId = String(frontmatter.data.name || '').trim(); + if (declaredSkillId !== item.skillId) { + errors.push(`self-test frontmatter name mismatch: expected "${item.skillId}"`); + } else { + canRoundtripSkillPayload = true; + } + if (!String(frontmatter.data.description || '').trim()) { + errors.push('self-test missing frontmatter description'); + } + } + + const canonicalAgentPath = extractCanonicalAgentPath(content); + if (!canonicalAgentPath) { + errors.push('self-test missing canonical source path'); + } else { + const expectedCanonicalPath = `.aiox-core/development/agents/${item.filename}`; + if (canonicalAgentPath !== expectedCanonicalPath) { + errors.push( + `self-test canonical source path mismatch: expected "${expectedCanonicalPath}", got "${canonicalAgentPath}"`, + ); + } + const absoluteAgentPath = path.join(resolved.projectRoot, canonicalAgentPath); + if (!fs.existsSync(absoluteAgentPath)) { + errors.push(`self-test source file not found: ${canonicalAgentPath}`); + } + } + + if (canRoundtripSkillPayload) { + const payload = createSkillToolSelfTestPayload(declaredSkillId); + const target = normalizeSkillToolTarget(payload); + if (!expectedIds.has(target)) { + errors.push(`self-test Skill payload target is not a generated skill: ${target || ''}`); + } else if (target !== item.skillId) { + errors.push(`self-test Skill payload target mismatch: expected "${item.skillId}", got "${target}"`); + } + } + + results.push({ + skillId: item.skillId, + ok: errors.length === 0, + errors, + }); + } + + return results; +} + function extractGeneratedSquadSource(content) { const value = String(content || ''); const patterns = [ @@ -92,13 +247,32 @@ function isGeneratedSquadSkill(content, projectRoot) { } function validateCodexSkills(options = {}) { - const resolved = { ...getDefaultOptions(), ...options }; + const defaults = getDefaultOptions(); + const projectRoot = options.projectRoot || defaults.projectRoot; + const resolved = { + ...defaults, + ...options, + projectRoot, + sourceDir: options.sourceDir || path.join(projectRoot, '.aiox-core', 'development', 'agents'), + skillsDir: options.skillsDir || path.join(projectRoot, '.codex', 'skills'), + }; const errors = []; const warnings = []; if (!fs.existsSync(resolved.skillsDir)) { errors.push(`Skills directory not found: ${resolved.skillsDir}`); - return { ok: false, checked: 0, expected: 0, errors, warnings, missing: [], orphaned: [], ignored: [] }; + return { + ok: false, + checked: 0, + expected: 0, + errors, + warnings, + missing: [], + orphaned: [], + legacy: [], + ignored: [], + selfTests: [], + }; } const agents = parseAllAgents(resolved.sourceDir).filter(isParsableAgent); @@ -173,6 +347,20 @@ function validateCodexSkills(options = {}) { warnings.push('No parseable agents found in sourceDir'); } + const selfTests = resolved.selfTest + ? runSkillSelfTests({ + projectRoot: resolved.projectRoot, + skillsDir: resolved.skillsDir, + expected, + }) + : []; + + for (const test of selfTests) { + for (const error of test.errors) { + errors.push(`${test.skillId}: ${error}`); + } + } + return { ok: errors.length === 0, checked: expected.length, @@ -183,12 +371,16 @@ function validateCodexSkills(options = {}) { orphaned, legacy, ignored, + selfTests, }; } function formatHumanReport(result) { if (result.ok) { - return `✅ Codex skills validation passed (${result.checked} skills checked)`; + const suffix = result.selfTests && result.selfTests.length > 0 + ? `, ${result.selfTests.length} self-test(s) passed` + : ''; + return `✅ Codex skills validation passed (${result.checked} skills checked${suffix})`; } const lines = [ @@ -226,6 +418,10 @@ if (require.main === module) { module.exports = { validateCodexSkills, validateSkillContent, + parseSkillFrontmatter, + createSkillToolSelfTestPayload, + normalizeSkillToolTarget, + runSkillSelfTests, extractGeneratedSquadSource, isGeneratedSquadSkill, parseArgs, diff --git a/.aiox-core/install-manifest.yaml b/.aiox-core/install-manifest.yaml index 73558be09b..1aa3a7b4b2 100644 --- a/.aiox-core/install-manifest.yaml +++ b/.aiox-core/install-manifest.yaml @@ -8,7 +8,7 @@ # - File types for categorization # version: 5.1.15 -generated_at: "2026-05-07T19:24:04.148Z" +generated_at: "2026-05-07T21:04:31.193Z" generator: scripts/generate-install-manifest.js file_count: 1103 files: @@ -1229,7 +1229,7 @@ files: type: data size: 9590 - path: data/entity-registry.yaml - hash: sha256:16fcaa219a4c1364337ae2d534494fe4f058570ba6f3db797e373d5cd0d7e672 + hash: sha256:416ffc477206f2b75700a800a35b6af51269f73b3bbc9952a155b1f350089030 type: data size: 522368 - path: data/learned-patterns.yaml @@ -3005,13 +3005,13 @@ files: type: script size: 5533 - path: infrastructure/scripts/codex-skills-sync/README.md - hash: sha256:908f02e6b36e2bd5ec706427992955b411bb70db7b1e34b00d4845fe0fc9337a + hash: sha256:2e4366ccc31b617a9be8a8b19c0379e1fd3ae64c4d5cfbb4d233860f9db0f949 type: script - size: 3737 + size: 4072 - path: infrastructure/scripts/codex-skills-sync/validate.js - hash: sha256:295ec61dab026ff087c685ad4244fbbca2b9a220c24703ea9d8b1e7a2ca23772 + hash: sha256:5b8cb91ddcfda2b6b66328c12129e71665e4e1cb6f13b1792746e9d761d1ebcf type: script - size: 6366 + size: 12185 - path: infrastructure/scripts/collect-tool-usage.js hash: sha256:8a739b79182dc41e28b7e02aeb9ec1dde5ec49f3ca534399acc59711b3b92bbf type: script diff --git a/docs/stories/epic-123/STORY-123.21-codex-skill-tool-self-test.md b/docs/stories/epic-123/STORY-123.21-codex-skill-tool-self-test.md new file mode 100644 index 0000000000..a6d874bf71 --- /dev/null +++ b/docs/stories/epic-123/STORY-123.21-codex-skill-tool-self-test.md @@ -0,0 +1,54 @@ +# STORY-123.21: Self-test determinístico para skills Codex via Skill tool + +## Status + +Done + +## Story + +Como mantenedor do AIOX Core, quero um modo de self-test para skills Codex geradas a partir dos agentes core, para validar skills acionadas pelo Skill tool sem depender de protocolo ping-pong ou execução manual. + +## Acceptance Criteria + +- [x] AC1. O validador de skills Codex expõe um modo `--self-test` para executar checks determinísticos além da paridade estática. +- [x] AC2. O self-test simula payloads do Skill tool e confirma que cada skill gerada resolve para o skill id esperado. +- [x] AC3. O self-test valida frontmatter, caminho source-of-truth e comando de greeting necessários para ativação do agente. +- [x] AC4. Falhas de self-test aparecem no resultado do validador com mensagens acionáveis. +- [x] AC5. Há cobertura automatizada para sucesso, falha de source path e normalização de payload Skill. +- [x] AC6. Validações locais antes de PR são concluídas. + +## Tasks + +- [x] Investigar o issue #622 e localizar a superfície funcional em `codex-skills-sync/validate.js`. +- [x] Adicionar helpers de frontmatter, payload Skill e execução de self-tests. +- [x] Integrar o modo `--self-test` ao validador e ao script npm. +- [x] Documentar o novo comando no README do sync de skills Codex. +- [x] Adicionar testes unitários do self-test. +- [x] Rodar testes focados e validadores relevantes. +- [x] Preparar artefatos para PR após validação. + +## Dev Notes + +- O relatório temporário citado no issue #622 não estava mais disponível em `/tmp`, então a correção foi guiada pela descrição do issue e pelas superfícies existentes de sync/validação de skills Codex. +- A correção não invoca um Skill tool real. Ela cria um harness determinístico para validar o contrato que esse tool consumiria: frontmatter, skill id, source-of-truth e greeting command. +- O escopo ficou no `aiox-core`; o submódulo `pro/` não foi alterado. + +## Validation + +- `node -c .aiox-core/infrastructure/scripts/codex-skills-sync/validate.js` -> PASS. +- `npm test -- tests/unit/codex-skills-validate.test.js --runInBand --forceExit` -> PASS, 1 suite / 10 tests. +- `npm run validate:codex-skills:self-test` -> PASS, 12 skills checked / 12 self-tests passed. +- `npm run generate:manifest` -> PASS, 1.103 files, manifest v5.1.15. +- `npm run validate:manifest` -> PASS. +- `npm run lint -- --quiet` -> PASS. +- `npm run typecheck` -> PASS. +- `npm run test:ci` -> PASS, 317 suites / 7.870 tests / 149 skipped. + +## File List + +- `.aiox-core/infrastructure/scripts/codex-skills-sync/validate.js` +- `.aiox-core/infrastructure/scripts/codex-skills-sync/README.md` +- `.aiox-core/install-manifest.yaml` +- `package.json` +- `tests/unit/codex-skills-validate.test.js` +- `docs/stories/epic-123/STORY-123.21-codex-skill-tool-self-test.md` diff --git a/package.json b/package.json index b3a49dfc9a..041de9bf9a 100644 --- a/package.json +++ b/package.json @@ -89,6 +89,7 @@ "setup:codex-skills": "node .aiox-core/infrastructure/scripts/codex-skills-sync/bootstrap.js", "setup:codex-skills:dry": "node .aiox-core/infrastructure/scripts/codex-skills-sync/bootstrap.js --dry-run", "validate:codex-skills": "node .aiox-core/infrastructure/scripts/codex-skills-sync/validate.js --strict", + "validate:codex-skills:self-test": "node .aiox-core/infrastructure/scripts/codex-skills-sync/validate.js --strict --self-test", "repair:agent-references": "node .aiox-core/infrastructure/scripts/repair-agent-references.js", "validate:paths": "node .aiox-core/infrastructure/scripts/validate-paths.js", "validate:parity": "node .aiox-core/infrastructure/scripts/validate-parity.js", diff --git a/tests/unit/codex-skills-validate.test.js b/tests/unit/codex-skills-validate.test.js index 996686c699..74538277af 100644 --- a/tests/unit/codex-skills-validate.test.js +++ b/tests/unit/codex-skills-validate.test.js @@ -4,8 +4,17 @@ const fs = require('fs'); const os = require('os'); const path = require('path'); -const { syncSkills } = require('../../.aiox-core/infrastructure/scripts/codex-skills-sync/index'); -const { validateCodexSkills } = require('../../.aiox-core/infrastructure/scripts/codex-skills-sync/validate'); +const { syncSkills } = require(path.join( + process.cwd(), + '.aiox-core/infrastructure/scripts/codex-skills-sync/index', +)); +const { + validateCodexSkills, + normalizeSkillToolTarget, +} = require(path.join( + process.cwd(), + '.aiox-core/infrastructure/scripts/codex-skills-sync/validate', +)); describe('Codex Skills Validator', () => { let tmpRoot; @@ -39,6 +48,98 @@ describe('Codex Skills Validator', () => { expect(result.errors).toEqual([]); }); + it('self-tests generated skills with simulated Skill tool payloads', () => { + syncSkills({ sourceDir, localSkillsDir: skillsDir, dryRun: false }); + + const result = validateCodexSkills({ + projectRoot: process.cwd(), + sourceDir, + skillsDir, + strict: true, + selfTest: true, + }); + + expect(result.ok).toBe(true); + expect(result.selfTests).toHaveLength(expectedAgentCount); + expect(result.selfTests.every(test => test.ok)).toBe(true); + }); + + it('derives default source and skills directories from a supplied project root', () => { + const tmpSourceDir = path.join(tmpRoot, '.aiox-core', 'development', 'agents'); + fs.mkdirSync(path.dirname(tmpSourceDir), { recursive: true }); + fs.cpSync(sourceDir, tmpSourceDir, { recursive: true }); + syncSkills({ sourceDir: tmpSourceDir, localSkillsDir: skillsDir, dryRun: false }); + + const result = validateCodexSkills({ + projectRoot: tmpRoot, + strict: true, + selfTest: true, + }); + + expect(result.ok).toBe(true); + expect(result.checked).toBe(expectedAgentCount); + expect(result.selfTests).toHaveLength(expectedAgentCount); + }); + + it('fails self-test when a skill source path cannot be resolved', () => { + syncSkills({ sourceDir, localSkillsDir: skillsDir, dryRun: false }); + const target = path.join(skillsDir, 'aiox-dev', 'SKILL.md'); + const original = fs.readFileSync(target, 'utf8'); + fs.writeFileSync( + target, + original.replace('.aiox-core/development/agents/dev.md', '.aiox-core/development/agents/missing-dev.md'), + 'utf8', + ); + + const result = validateCodexSkills({ + projectRoot: process.cwd(), + sourceDir, + skillsDir, + strict: true, + selfTest: true, + }); + + expect(result.ok).toBe(false); + const devSelfTest = result.selfTests.find(test => test.skillId === 'aiox-dev'); + expect(devSelfTest).toBeDefined(); + expect(devSelfTest.ok).toBe(false); + expect(result.errors.some(error => error.includes('self-test source file not found'))).toBe(true); + }); + + it('does not cascade payload errors when a skill frontmatter name mismatches', () => { + syncSkills({ sourceDir, localSkillsDir: skillsDir, dryRun: false }); + const target = path.join(skillsDir, 'aiox-dev', 'SKILL.md'); + const original = fs.readFileSync(target, 'utf8'); + fs.writeFileSync(target, original.replace('name: aiox-dev', 'name: aiox-not-dev'), 'utf8'); + + const result = validateCodexSkills({ + projectRoot: process.cwd(), + sourceDir, + skillsDir, + strict: true, + selfTest: true, + }); + + const devSelfTest = result.selfTests.find(test => test.skillId === 'aiox-dev'); + expect(result.ok).toBe(false); + expect(devSelfTest).toBeDefined(); + expect(devSelfTest.errors).toContain('self-test frontmatter name mismatch: expected "aiox-dev"'); + expect(devSelfTest.errors.some(error => error.includes('Skill payload target mismatch'))).toBe(false); + expect(devSelfTest.errors.some(error => error.includes('Skill payload target is not'))).toBe(false); + }); + + it('normalizes Skill tool invocation targets', () => { + expect(normalizeSkillToolTarget({ + type: 'tool_use', + name: 'Skill', + input: { + skill: 'aiox-dev', + prompt: 'self-test', + }, + })).toBe('aiox-dev'); + expect(normalizeSkillToolTarget('$aiox-qa')).toBe('aiox-qa'); + }); + it('fails when a generated skill is missing', () => { syncSkills({ sourceDir, localSkillsDir: skillsDir, dryRun: false }); fs.rmSync(path.join(skillsDir, 'aiox-architect', 'SKILL.md'), { force: true }); @@ -54,6 +155,23 @@ describe('Codex Skills Validator', () => { expect(result.errors.some(error => error.includes('Missing skill file'))).toBe(true); }); + it('returns a stable result shape when the skills directory is missing', () => { + const result = validateCodexSkills({ + projectRoot: tmpRoot, + sourceDir, + skillsDir: path.join(tmpRoot, '.codex', 'missing-skills'), + strict: true, + selfTest: true, + }); + + expect(result.ok).toBe(false); + expect(result.missing).toEqual([]); + expect(result.orphaned).toEqual([]); + expect(result.legacy).toEqual([]); + expect(result.ignored).toEqual([]); + expect(result.selfTests).toEqual([]); + }); + it('fails when greeting command is removed from a skill', () => { syncSkills({ sourceDir, localSkillsDir: skillsDir, dryRun: false }); const target = path.join(skillsDir, 'aiox-dev', 'SKILL.md');