From e9aed304e012f36fb8a23a15ff4c67d4f3f9ce4b Mon Sep 17 00:00:00 2001 From: rafaelscosta Date: Thu, 7 May 2026 17:04:59 -0300 Subject: [PATCH 1/5] fix: add codex skill self-test mode [Story 123.21] Closes #622 --- .aiox-core/data/entity-registry.yaml | 6 +- .../scripts/codex-skills-sync/README.md | 2 + .../scripts/codex-skills-sync/validate.js | 156 +++++++++++++++++- .aiox-core/install-manifest.yaml | 12 +- ...STORY-123.21-codex-skill-tool-self-test.md | 54 ++++++ package.json | 1 + tests/unit/codex-skills-validate.test.js | 56 ++++++- 7 files changed, 276 insertions(+), 11 deletions(-) create mode 100644 docs/stories/epic-123/STORY-123.21-codex-skill-tool-self-test.md diff --git a/.aiox-core/data/entity-registry.yaml b/.aiox-core/data/entity-registry.yaml index 37826faa16..fa31c5a8cf 100644 --- a/.aiox-core/data/entity-registry.yaml +++ b/.aiox-core/data/entity-registry.yaml @@ -1,6 +1,6 @@ metadata: version: 1.0.0 - lastUpdated: '2026-05-07T15:02:59.792Z' + lastUpdated: '2026-05-07T20:05:00.330Z' entityCount: 746 checksumAlgorithm: sha256 resolutionRate: 100 @@ -16376,8 +16376,8 @@ entities: score: 0.7 constraints: [] extensionPoints: [] - checksum: sha256:0fbc1baff25f20e3a37d3e4be51d146a75254d5ed638b3438d9f1bf0e587c997 - lastVerified: '2026-03-11T00:48:55.955Z' + checksum: sha256:0252101d64cadeb09f1b86a6e8a478c00f31aa79b64c88d0bfa11dabc38cc502 + lastVerified: '2026-05-07T20:05:00.327Z' brownfield-analyzer: path: .aiox-core/infrastructure/scripts/documentation-integrity/brownfield-analyzer.js layer: L2 diff --git a/.aiox-core/infrastructure/scripts/codex-skills-sync/README.md b/.aiox-core/infrastructure/scripts/codex-skills-sync/README.md index 2501effdce..6f1d4a4456 100644 --- a/.aiox-core/infrastructure/scripts/codex-skills-sync/README.md +++ b/.aiox-core/infrastructure/scripts/codex-skills-sync/README.md @@ -47,9 +47,11 @@ Used by CI and by the unified ide-sync pipeline. Does not generate squad-chief s ``` npm run validate:codex-skills # validate aiox-* skill / agent parity (strict) +npm run validate:codex-skills:self-test # parity + deterministic Skill tool self-test ``` Verifies that every core agent has a corresponding `.codex/skills/aiox-/SKILL.md`. Squad-chief skills are out of scope for this validator. +The self-test mode additionally simulates a Skill tool invocation for each generated skill and checks that the skill frontmatter, source-of-truth path, and greeting command can activate the target agent without requiring a live ping-pong tool call. ## Files generated diff --git a/.aiox-core/infrastructure/scripts/codex-skills-sync/validate.js b/.aiox-core/infrastructure/scripts/codex-skills-sync/validate.js index e9db0877c6..45e011d4ee 100644 --- a/.aiox-core/infrastructure/scripts/codex-skills-sync/validate.js +++ b/.aiox-core/infrastructure/scripts/codex-skills-sync/validate.js @@ -3,6 +3,7 @@ const fs = require('fs'); const path = require('path'); +const yaml = require('js-yaml'); const { parseAllAgents } = require('../ide-sync/agent-parser'); const { getSkillId, getLegacySkillId } = require('./index'); @@ -17,6 +18,7 @@ function getDefaultOptions() { skillsDir: path.join(projectRoot, '.codex', 'skills'), strict: false, allowOrphaned: false, + selfTest: false, quiet: false, json: false, }; @@ -28,6 +30,7 @@ function parseArgs(argv = process.argv.slice(2)) { strict: args.has('--strict'), quiet: args.has('--quiet') || args.has('-q'), json: args.has('--json'), + selfTest: args.has('--self-test'), }; } @@ -62,6 +65,135 @@ function validateSkillContent(content, expected) { return issues; } +function parseSkillFrontmatter(content) { + const match = String(content || '').match(/^---\r?\n([\s\S]*?)\r?\n---(?:\r?\n|$)/); + if (!match) { + return { data: null, error: 'missing YAML frontmatter' }; + } + + try { + const data = yaml.load(match[1]) || {}; + return { data, error: null }; + } catch (error) { + return { data: null, error: `invalid YAML frontmatter (${error.message})` }; + } +} + +function createSkillToolSelfTestPayload(skillId, prompt = 'AIOX skill self-test') { + return { + type: 'tool_use', + name: 'Skill', + input: { + skill: skillId, + prompt, + }, + }; +} + +function normalizeSkillToolTarget(payload) { + if (typeof payload === 'string') { + return payload.trim().replace(/^\$/, ''); + } + + if (!payload || typeof payload !== 'object') { + return ''; + } + + const input = payload.input && typeof payload.input === 'object' ? payload.input : {}; + const candidates = [ + input.skill, + input.skillId, + input.skill_id, + input.name, + input.id, + payload.skill, + payload.skillId, + payload.skill_id, + payload.id, + payload.name && payload.name !== 'Skill' ? payload.name : '', + ]; + + const target = candidates.find((candidate) => typeof candidate === 'string' && candidate.trim()); + return target ? target.trim().replace(/^\$/, '') : ''; +} + +function extractCanonicalAgentPath(content) { + const match = String(content || '').match(/`(\.aiox-core\/development\/agents\/[^`]+\.md)`/); + return match ? match[1] : ''; +} + +function runSkillSelfTests(options = {}) { + const resolved = { + projectRoot: process.cwd(), + skillsDir: path.join(process.cwd(), '.codex', 'skills'), + expected: [], + ...options, + }; + const expectedIds = new Set(resolved.expected.map(item => item.skillId)); + const results = []; + + for (const item of resolved.expected) { + const skillPath = path.join(resolved.skillsDir, item.skillId, 'SKILL.md'); + const relativeSkillPath = path.relative(resolved.projectRoot, skillPath); + const errors = []; + + let content = ''; + try { + content = fs.readFileSync(skillPath, 'utf8'); + } catch (error) { + results.push({ + skillId: item.skillId, + ok: false, + errors: [`self-test unable to read ${relativeSkillPath} (${error.message})`], + }); + continue; + } + + const frontmatter = parseSkillFrontmatter(content); + if (frontmatter.error) { + errors.push(`self-test ${frontmatter.error}`); + } else { + if (frontmatter.data.name !== item.skillId) { + errors.push(`self-test frontmatter name mismatch: expected "${item.skillId}"`); + } + if (!String(frontmatter.data.description || '').trim()) { + errors.push('self-test missing frontmatter description'); + } + } + + const canonicalAgentPath = extractCanonicalAgentPath(content); + if (!canonicalAgentPath) { + errors.push('self-test missing canonical source path'); + } else { + const absoluteAgentPath = path.join(resolved.projectRoot, canonicalAgentPath); + if (!fs.existsSync(absoluteAgentPath)) { + errors.push(`self-test source file not found: ${canonicalAgentPath}`); + } + } + + const payload = createSkillToolSelfTestPayload(item.skillId); + const target = normalizeSkillToolTarget(payload); + if (target !== item.skillId) { + errors.push(`self-test Skill payload target mismatch: expected "${item.skillId}", got "${target}"`); + } + if (!expectedIds.has(target)) { + errors.push(`self-test Skill payload target is not a generated skill: ${target || ''}`); + } + + if (!content.includes(`generate-greeting.js ${item.agentId}`)) { + errors.push(`self-test greeting command cannot activate "${item.agentId}"`); + } + + results.push({ + skillId: item.skillId, + ok: errors.length === 0, + errors, + }); + } + + return results; +} + function extractGeneratedSquadSource(content) { const value = String(content || ''); const patterns = [ @@ -173,6 +305,20 @@ function validateCodexSkills(options = {}) { warnings.push('No parseable agents found in sourceDir'); } + const selfTests = resolved.selfTest + ? runSkillSelfTests({ + projectRoot: resolved.projectRoot, + skillsDir: resolved.skillsDir, + expected, + }) + : []; + + for (const test of selfTests) { + for (const error of test.errors) { + errors.push(`${test.skillId}: ${error}`); + } + } + return { ok: errors.length === 0, checked: expected.length, @@ -183,12 +329,16 @@ function validateCodexSkills(options = {}) { orphaned, legacy, ignored, + selfTests, }; } function formatHumanReport(result) { if (result.ok) { - return `✅ Codex skills validation passed (${result.checked} skills checked)`; + const suffix = result.selfTests && result.selfTests.length > 0 + ? `, ${result.selfTests.length} self-test(s) passed` + : ''; + return `✅ Codex skills validation passed (${result.checked} skills checked${suffix})`; } const lines = [ @@ -226,6 +376,10 @@ if (require.main === module) { module.exports = { validateCodexSkills, validateSkillContent, + parseSkillFrontmatter, + createSkillToolSelfTestPayload, + normalizeSkillToolTarget, + runSkillSelfTests, extractGeneratedSquadSource, isGeneratedSquadSkill, parseArgs, diff --git a/.aiox-core/install-manifest.yaml b/.aiox-core/install-manifest.yaml index 73558be09b..adf4b6f25b 100644 --- a/.aiox-core/install-manifest.yaml +++ b/.aiox-core/install-manifest.yaml @@ -8,7 +8,7 @@ # - File types for categorization # version: 5.1.15 -generated_at: "2026-05-07T19:24:04.148Z" +generated_at: "2026-05-07T20:05:40.619Z" generator: scripts/generate-install-manifest.js file_count: 1103 files: @@ -1229,7 +1229,7 @@ files: type: data size: 9590 - path: data/entity-registry.yaml - hash: sha256:16fcaa219a4c1364337ae2d534494fe4f058570ba6f3db797e373d5cd0d7e672 + hash: sha256:731001c0e94b6d3c9166836995666df527e6f82dad6c0d9ee7ec81ceb42a687d type: data size: 522368 - path: data/learned-patterns.yaml @@ -3005,13 +3005,13 @@ files: type: script size: 5533 - path: infrastructure/scripts/codex-skills-sync/README.md - hash: sha256:908f02e6b36e2bd5ec706427992955b411bb70db7b1e34b00d4845fe0fc9337a + hash: sha256:2e4366ccc31b617a9be8a8b19c0379e1fd3ae64c4d5cfbb4d233860f9db0f949 type: script - size: 3737 + size: 4072 - path: infrastructure/scripts/codex-skills-sync/validate.js - hash: sha256:295ec61dab026ff087c685ad4244fbbca2b9a220c24703ea9d8b1e7a2ca23772 + hash: sha256:0252101d64cadeb09f1b86a6e8a478c00f31aa79b64c88d0bfa11dabc38cc502 type: script - size: 6366 + size: 10886 - path: infrastructure/scripts/collect-tool-usage.js hash: sha256:8a739b79182dc41e28b7e02aeb9ec1dde5ec49f3ca534399acc59711b3b92bbf type: script diff --git a/docs/stories/epic-123/STORY-123.21-codex-skill-tool-self-test.md b/docs/stories/epic-123/STORY-123.21-codex-skill-tool-self-test.md new file mode 100644 index 0000000000..a6d874bf71 --- /dev/null +++ b/docs/stories/epic-123/STORY-123.21-codex-skill-tool-self-test.md @@ -0,0 +1,54 @@ +# STORY-123.21: Self-test determinístico para skills Codex via Skill tool + +## Status + +Done + +## Story + +Como mantenedor do AIOX Core, quero um modo de self-test para skills Codex geradas a partir dos agentes core, para validar skills acionadas pelo Skill tool sem depender de protocolo ping-pong ou execução manual. + +## Acceptance Criteria + +- [x] AC1. O validador de skills Codex expõe um modo `--self-test` para executar checks determinísticos além da paridade estática. +- [x] AC2. O self-test simula payloads do Skill tool e confirma que cada skill gerada resolve para o skill id esperado. +- [x] AC3. O self-test valida frontmatter, caminho source-of-truth e comando de greeting necessários para ativação do agente. +- [x] AC4. Falhas de self-test aparecem no resultado do validador com mensagens acionáveis. +- [x] AC5. Há cobertura automatizada para sucesso, falha de source path e normalização de payload Skill. +- [x] AC6. Validações locais antes de PR são concluídas. + +## Tasks + +- [x] Investigar o issue #622 e localizar a superfície funcional em `codex-skills-sync/validate.js`. +- [x] Adicionar helpers de frontmatter, payload Skill e execução de self-tests. +- [x] Integrar o modo `--self-test` ao validador e ao script npm. +- [x] Documentar o novo comando no README do sync de skills Codex. +- [x] Adicionar testes unitários do self-test. +- [x] Rodar testes focados e validadores relevantes. +- [x] Preparar artefatos para PR após validação. + +## Dev Notes + +- O relatório temporário citado no issue #622 não estava mais disponível em `/tmp`, então a correção foi guiada pela descrição do issue e pelas superfícies existentes de sync/validação de skills Codex. +- A correção não invoca um Skill tool real. Ela cria um harness determinístico para validar o contrato que esse tool consumiria: frontmatter, skill id, source-of-truth e greeting command. +- O escopo ficou no `aiox-core`; o submódulo `pro/` não foi alterado. + +## Validation + +- `node -c .aiox-core/infrastructure/scripts/codex-skills-sync/validate.js` -> PASS. +- `npm test -- tests/unit/codex-skills-validate.test.js --runInBand --forceExit` -> PASS, 1 suite / 10 tests. +- `npm run validate:codex-skills:self-test` -> PASS, 12 skills checked / 12 self-tests passed. +- `npm run generate:manifest` -> PASS, 1.103 files, manifest v5.1.15. +- `npm run validate:manifest` -> PASS. +- `npm run lint -- --quiet` -> PASS. +- `npm run typecheck` -> PASS. +- `npm run test:ci` -> PASS, 317 suites / 7.870 tests / 149 skipped. + +## File List + +- `.aiox-core/infrastructure/scripts/codex-skills-sync/validate.js` +- `.aiox-core/infrastructure/scripts/codex-skills-sync/README.md` +- `.aiox-core/install-manifest.yaml` +- `package.json` +- `tests/unit/codex-skills-validate.test.js` +- `docs/stories/epic-123/STORY-123.21-codex-skill-tool-self-test.md` diff --git a/package.json b/package.json index b3a49dfc9a..041de9bf9a 100644 --- a/package.json +++ b/package.json @@ -89,6 +89,7 @@ "setup:codex-skills": "node .aiox-core/infrastructure/scripts/codex-skills-sync/bootstrap.js", "setup:codex-skills:dry": "node .aiox-core/infrastructure/scripts/codex-skills-sync/bootstrap.js --dry-run", "validate:codex-skills": "node .aiox-core/infrastructure/scripts/codex-skills-sync/validate.js --strict", + "validate:codex-skills:self-test": "node .aiox-core/infrastructure/scripts/codex-skills-sync/validate.js --strict --self-test", "repair:agent-references": "node .aiox-core/infrastructure/scripts/repair-agent-references.js", "validate:paths": "node .aiox-core/infrastructure/scripts/validate-paths.js", "validate:parity": "node .aiox-core/infrastructure/scripts/validate-parity.js", diff --git a/tests/unit/codex-skills-validate.test.js b/tests/unit/codex-skills-validate.test.js index 996686c699..4e8ff8aa0b 100644 --- a/tests/unit/codex-skills-validate.test.js +++ b/tests/unit/codex-skills-validate.test.js @@ -5,7 +5,10 @@ const os = require('os'); const path = require('path'); const { syncSkills } = require('../../.aiox-core/infrastructure/scripts/codex-skills-sync/index'); -const { validateCodexSkills } = require('../../.aiox-core/infrastructure/scripts/codex-skills-sync/validate'); +const { + validateCodexSkills, + normalizeSkillToolTarget, +} = require('../../.aiox-core/infrastructure/scripts/codex-skills-sync/validate'); describe('Codex Skills Validator', () => { let tmpRoot; @@ -39,6 +42,57 @@ describe('Codex Skills Validator', () => { expect(result.errors).toEqual([]); }); + it('self-tests generated skills with simulated Skill tool payloads', () => { + syncSkills({ sourceDir, localSkillsDir: skillsDir, dryRun: false }); + + const result = validateCodexSkills({ + projectRoot: process.cwd(), + sourceDir, + skillsDir, + strict: true, + selfTest: true, + }); + + expect(result.ok).toBe(true); + expect(result.selfTests).toHaveLength(expectedAgentCount); + expect(result.selfTests.every(test => test.ok)).toBe(true); + }); + + it('fails self-test when a skill source path cannot be resolved', () => { + syncSkills({ sourceDir, localSkillsDir: skillsDir, dryRun: false }); + const target = path.join(skillsDir, 'aiox-dev', 'SKILL.md'); + const original = fs.readFileSync(target, 'utf8'); + fs.writeFileSync( + target, + original.replace('.aiox-core/development/agents/dev.md', '.aiox-core/development/agents/missing-dev.md'), + 'utf8', + ); + + const result = validateCodexSkills({ + projectRoot: process.cwd(), + sourceDir, + skillsDir, + strict: true, + selfTest: true, + }); + + expect(result.ok).toBe(false); + expect(result.selfTests.find(test => test.skillId === 'aiox-dev').ok).toBe(false); + expect(result.errors.some(error => error.includes('self-test source file not found'))).toBe(true); + }); + + it('normalizes Skill tool invocation targets', () => { + expect(normalizeSkillToolTarget({ + type: 'tool_use', + name: 'Skill', + input: { + skill: 'aiox-dev', + prompt: 'self-test', + }, + })).toBe('aiox-dev'); + expect(normalizeSkillToolTarget('$aiox-qa')).toBe('aiox-qa'); + }); + it('fails when a generated skill is missing', () => { syncSkills({ sourceDir, localSkillsDir: skillsDir, dryRun: false }); fs.rmSync(path.join(skillsDir, 'aiox-architect', 'SKILL.md'), { force: true }); From 012a1fefd136bfd1bbf1e578704e4f0d23ff7d91 Mon Sep 17 00:00:00 2001 From: rafaelscosta Date: Thu, 7 May 2026 17:16:03 -0300 Subject: [PATCH 2/5] docs: add codex skill self-test docstrings [Story 123.21] --- .aiox-core/data/entity-registry.yaml | 6 +++--- .../scripts/codex-skills-sync/validate.js | 15 +++++++++++++++ .aiox-core/install-manifest.yaml | 8 ++++---- 3 files changed, 22 insertions(+), 7 deletions(-) diff --git a/.aiox-core/data/entity-registry.yaml b/.aiox-core/data/entity-registry.yaml index fa31c5a8cf..bae304aaa6 100644 --- a/.aiox-core/data/entity-registry.yaml +++ b/.aiox-core/data/entity-registry.yaml @@ -1,6 +1,6 @@ metadata: version: 1.0.0 - lastUpdated: '2026-05-07T20:05:00.330Z' + lastUpdated: '2026-05-07T20:16:04.723Z' entityCount: 746 checksumAlgorithm: sha256 resolutionRate: 100 @@ -16376,8 +16376,8 @@ entities: score: 0.7 constraints: [] extensionPoints: [] - checksum: sha256:0252101d64cadeb09f1b86a6e8a478c00f31aa79b64c88d0bfa11dabc38cc502 - lastVerified: '2026-05-07T20:05:00.327Z' + checksum: sha256:5af18ff8a20f17d3412441c873d245aa3411592805639e1c40f57000058da9e4 + lastVerified: '2026-05-07T20:16:04.719Z' brownfield-analyzer: path: .aiox-core/infrastructure/scripts/documentation-integrity/brownfield-analyzer.js layer: L2 diff --git a/.aiox-core/infrastructure/scripts/codex-skills-sync/validate.js b/.aiox-core/infrastructure/scripts/codex-skills-sync/validate.js index 45e011d4ee..61b0a72978 100644 --- a/.aiox-core/infrastructure/scripts/codex-skills-sync/validate.js +++ b/.aiox-core/infrastructure/scripts/codex-skills-sync/validate.js @@ -65,6 +65,9 @@ function validateSkillContent(content, expected) { return issues; } +/** + * Parses a generated Codex skill frontmatter block without reading external files. + */ function parseSkillFrontmatter(content) { const match = String(content || '').match(/^---\r?\n([\s\S]*?)\r?\n---(?:\r?\n|$)/); if (!match) { @@ -79,6 +82,9 @@ function parseSkillFrontmatter(content) { } } +/** + * Builds the deterministic Skill tool payload used by validator self-tests. + */ function createSkillToolSelfTestPayload(skillId, prompt = 'AIOX skill self-test') { return { type: 'tool_use', @@ -90,6 +96,9 @@ function createSkillToolSelfTestPayload(skillId, prompt = 'AIOX skill self-test' }; } +/** + * Extracts a skill id from the payload shapes used by Skill tool invocations. + */ function normalizeSkillToolTarget(payload) { if (typeof payload === 'string') { return payload.trim().replace(/^\$/, ''); @@ -117,11 +126,17 @@ function normalizeSkillToolTarget(payload) { return target ? target.trim().replace(/^\$/, '') : ''; } +/** + * Finds the canonical AIOX agent path declared in a generated skill stub. + */ function extractCanonicalAgentPath(content) { const match = String(content || '').match(/`(\.aiox-core\/development\/agents\/[^`]+\.md)`/); return match ? match[1] : ''; } +/** + * Runs structural self-tests for generated Codex skills without invoking live tools. + */ function runSkillSelfTests(options = {}) { const resolved = { projectRoot: process.cwd(), diff --git a/.aiox-core/install-manifest.yaml b/.aiox-core/install-manifest.yaml index adf4b6f25b..32d4afc8f8 100644 --- a/.aiox-core/install-manifest.yaml +++ b/.aiox-core/install-manifest.yaml @@ -8,7 +8,7 @@ # - File types for categorization # version: 5.1.15 -generated_at: "2026-05-07T20:05:40.619Z" +generated_at: "2026-05-07T20:16:31.655Z" generator: scripts/generate-install-manifest.js file_count: 1103 files: @@ -1229,7 +1229,7 @@ files: type: data size: 9590 - path: data/entity-registry.yaml - hash: sha256:731001c0e94b6d3c9166836995666df527e6f82dad6c0d9ee7ec81ceb42a687d + hash: sha256:8589e341574a88f4e3de31ac855bc44079a6f072938678dc6affca2af8fea037 type: data size: 522368 - path: data/learned-patterns.yaml @@ -3009,9 +3009,9 @@ files: type: script size: 4072 - path: infrastructure/scripts/codex-skills-sync/validate.js - hash: sha256:0252101d64cadeb09f1b86a6e8a478c00f31aa79b64c88d0bfa11dabc38cc502 + hash: sha256:5af18ff8a20f17d3412441c873d245aa3411592805639e1c40f57000058da9e4 type: script - size: 10886 + size: 11327 - path: infrastructure/scripts/collect-tool-usage.js hash: sha256:8a739b79182dc41e28b7e02aeb9ec1dde5ec49f3ca534399acc59711b3b92bbf type: script From a2ac8a720edbf252529d0e5c6c9b823a40cf0284 Mon Sep 17 00:00:00 2001 From: rafaelscosta Date: Thu, 7 May 2026 17:27:42 -0300 Subject: [PATCH 3/5] fix: address codex skill self-test review [Story 123.21] --- .aiox-core/data/entity-registry.yaml | 6 +++--- .../scripts/codex-skills-sync/validate.js | 10 +++++++++- .aiox-core/install-manifest.yaml | 8 ++++---- tests/unit/codex-skills-validate.test.js | 14 +++++++++++--- 4 files changed, 27 insertions(+), 11 deletions(-) diff --git a/.aiox-core/data/entity-registry.yaml b/.aiox-core/data/entity-registry.yaml index bae304aaa6..fea8925445 100644 --- a/.aiox-core/data/entity-registry.yaml +++ b/.aiox-core/data/entity-registry.yaml @@ -1,6 +1,6 @@ metadata: version: 1.0.0 - lastUpdated: '2026-05-07T20:16:04.723Z' + lastUpdated: '2026-05-07T20:27:43.806Z' entityCount: 746 checksumAlgorithm: sha256 resolutionRate: 100 @@ -16376,8 +16376,8 @@ entities: score: 0.7 constraints: [] extensionPoints: [] - checksum: sha256:5af18ff8a20f17d3412441c873d245aa3411592805639e1c40f57000058da9e4 - lastVerified: '2026-05-07T20:16:04.719Z' + checksum: sha256:83fcbbc19a52e5b95b491b06730b22787e2ae27416c27e1b5e2483a835e2f7f7 + lastVerified: '2026-05-07T20:27:43.803Z' brownfield-analyzer: path: .aiox-core/infrastructure/scripts/documentation-integrity/brownfield-analyzer.js layer: L2 diff --git a/.aiox-core/infrastructure/scripts/codex-skills-sync/validate.js b/.aiox-core/infrastructure/scripts/codex-skills-sync/validate.js index 61b0a72978..819a25307f 100644 --- a/.aiox-core/infrastructure/scripts/codex-skills-sync/validate.js +++ b/.aiox-core/infrastructure/scripts/codex-skills-sync/validate.js @@ -165,9 +165,11 @@ function runSkillSelfTests(options = {}) { } const frontmatter = parseSkillFrontmatter(content); + let declaredSkillId = item.skillId; if (frontmatter.error) { errors.push(`self-test ${frontmatter.error}`); } else { + declaredSkillId = String(frontmatter.data.name || '').trim(); if (frontmatter.data.name !== item.skillId) { errors.push(`self-test frontmatter name mismatch: expected "${item.skillId}"`); } @@ -180,13 +182,19 @@ function runSkillSelfTests(options = {}) { if (!canonicalAgentPath) { errors.push('self-test missing canonical source path'); } else { + const expectedCanonicalPath = `.aiox-core/development/agents/${item.filename}`; + if (canonicalAgentPath !== expectedCanonicalPath) { + errors.push( + `self-test canonical source path mismatch: expected "${expectedCanonicalPath}", got "${canonicalAgentPath}"`, + ); + } const absoluteAgentPath = path.join(resolved.projectRoot, canonicalAgentPath); if (!fs.existsSync(absoluteAgentPath)) { errors.push(`self-test source file not found: ${canonicalAgentPath}`); } } - const payload = createSkillToolSelfTestPayload(item.skillId); + const payload = createSkillToolSelfTestPayload(declaredSkillId); const target = normalizeSkillToolTarget(payload); if (target !== item.skillId) { errors.push(`self-test Skill payload target mismatch: expected "${item.skillId}", got "${target}"`); diff --git a/.aiox-core/install-manifest.yaml b/.aiox-core/install-manifest.yaml index 32d4afc8f8..efe7140c50 100644 --- a/.aiox-core/install-manifest.yaml +++ b/.aiox-core/install-manifest.yaml @@ -8,7 +8,7 @@ # - File types for categorization # version: 5.1.15 -generated_at: "2026-05-07T20:16:31.655Z" +generated_at: "2026-05-07T20:28:16.723Z" generator: scripts/generate-install-manifest.js file_count: 1103 files: @@ -1229,7 +1229,7 @@ files: type: data size: 9590 - path: data/entity-registry.yaml - hash: sha256:8589e341574a88f4e3de31ac855bc44079a6f072938678dc6affca2af8fea037 + hash: sha256:50015ae32f297d4e8bbb321a9efe4c9c78128eb246497f1690f355cf54f8ecd4 type: data size: 522368 - path: data/learned-patterns.yaml @@ -3009,9 +3009,9 @@ files: type: script size: 4072 - path: infrastructure/scripts/codex-skills-sync/validate.js - hash: sha256:5af18ff8a20f17d3412441c873d245aa3411592805639e1c40f57000058da9e4 + hash: sha256:83fcbbc19a52e5b95b491b06730b22787e2ae27416c27e1b5e2483a835e2f7f7 type: script - size: 11327 + size: 11742 - path: infrastructure/scripts/collect-tool-usage.js hash: sha256:8a739b79182dc41e28b7e02aeb9ec1dde5ec49f3ca534399acc59711b3b92bbf type: script diff --git a/tests/unit/codex-skills-validate.test.js b/tests/unit/codex-skills-validate.test.js index 4e8ff8aa0b..e65238381a 100644 --- a/tests/unit/codex-skills-validate.test.js +++ b/tests/unit/codex-skills-validate.test.js @@ -4,11 +4,17 @@ const fs = require('fs'); const os = require('os'); const path = require('path'); -const { syncSkills } = require('../../.aiox-core/infrastructure/scripts/codex-skills-sync/index'); +const { syncSkills } = require(path.join( + process.cwd(), + '.aiox-core/infrastructure/scripts/codex-skills-sync/index', +)); const { validateCodexSkills, normalizeSkillToolTarget, -} = require('../../.aiox-core/infrastructure/scripts/codex-skills-sync/validate'); +} = require(path.join( + process.cwd(), + '.aiox-core/infrastructure/scripts/codex-skills-sync/validate', +)); describe('Codex Skills Validator', () => { let tmpRoot; @@ -77,7 +83,9 @@ describe('Codex Skills Validator', () => { }); expect(result.ok).toBe(false); - expect(result.selfTests.find(test => test.skillId === 'aiox-dev').ok).toBe(false); + const devSelfTest = result.selfTests.find(test => test.skillId === 'aiox-dev'); + expect(devSelfTest).toBeDefined(); + expect(devSelfTest.ok).toBe(false); expect(result.errors.some(error => error.includes('self-test source file not found'))).toBe(true); }); From cdcf53677decbc0150b3c108d35ebc57b7788181 Mon Sep 17 00:00:00 2001 From: rafaelscosta Date: Thu, 7 May 2026 17:48:14 -0300 Subject: [PATCH 4/5] fix: harden codex skill self-test validation [Story 123.21] --- .aiox-core/data/entity-registry.yaml | 6 +- .../scripts/codex-skills-sync/validate.js | 57 ++++++++++++------- .aiox-core/install-manifest.yaml | 8 +-- tests/unit/codex-skills-validate.test.js | 56 ++++++++++++++++++ 4 files changed, 101 insertions(+), 26 deletions(-) diff --git a/.aiox-core/data/entity-registry.yaml b/.aiox-core/data/entity-registry.yaml index fea8925445..86fff0e7ff 100644 --- a/.aiox-core/data/entity-registry.yaml +++ b/.aiox-core/data/entity-registry.yaml @@ -1,6 +1,6 @@ metadata: version: 1.0.0 - lastUpdated: '2026-05-07T20:27:43.806Z' + lastUpdated: '2026-05-07T20:48:22.488Z' entityCount: 746 checksumAlgorithm: sha256 resolutionRate: 100 @@ -16376,8 +16376,8 @@ entities: score: 0.7 constraints: [] extensionPoints: [] - checksum: sha256:83fcbbc19a52e5b95b491b06730b22787e2ae27416c27e1b5e2483a835e2f7f7 - lastVerified: '2026-05-07T20:27:43.803Z' + checksum: sha256:377f52359075c546436f16c596c1e3ffdee61b4218269fc5013dc0e8f4f9da61 + lastVerified: '2026-05-07T20:48:22.477Z' brownfield-analyzer: path: .aiox-core/infrastructure/scripts/documentation-integrity/brownfield-analyzer.js layer: L2 diff --git a/.aiox-core/infrastructure/scripts/codex-skills-sync/validate.js b/.aiox-core/infrastructure/scripts/codex-skills-sync/validate.js index 819a25307f..ebff338db9 100644 --- a/.aiox-core/infrastructure/scripts/codex-skills-sync/validate.js +++ b/.aiox-core/infrastructure/scripts/codex-skills-sync/validate.js @@ -138,11 +138,11 @@ function extractCanonicalAgentPath(content) { * Runs structural self-tests for generated Codex skills without invoking live tools. */ function runSkillSelfTests(options = {}) { + const projectRoot = options.projectRoot || process.cwd(); const resolved = { - projectRoot: process.cwd(), - skillsDir: path.join(process.cwd(), '.codex', 'skills'), - expected: [], - ...options, + projectRoot, + skillsDir: options.skillsDir || path.join(projectRoot, '.codex', 'skills'), + expected: options.expected || [], }; const expectedIds = new Set(resolved.expected.map(item => item.skillId)); const results = []; @@ -165,13 +165,16 @@ function runSkillSelfTests(options = {}) { } const frontmatter = parseSkillFrontmatter(content); - let declaredSkillId = item.skillId; + let declaredSkillId = ''; + let canRoundtripSkillPayload = false; if (frontmatter.error) { errors.push(`self-test ${frontmatter.error}`); } else { declaredSkillId = String(frontmatter.data.name || '').trim(); - if (frontmatter.data.name !== item.skillId) { + if (declaredSkillId !== item.skillId) { errors.push(`self-test frontmatter name mismatch: expected "${item.skillId}"`); + } else { + canRoundtripSkillPayload = true; } if (!String(frontmatter.data.description || '').trim()) { errors.push('self-test missing frontmatter description'); @@ -194,17 +197,14 @@ function runSkillSelfTests(options = {}) { } } - const payload = createSkillToolSelfTestPayload(declaredSkillId); - const target = normalizeSkillToolTarget(payload); - if (target !== item.skillId) { - errors.push(`self-test Skill payload target mismatch: expected "${item.skillId}", got "${target}"`); - } - if (!expectedIds.has(target)) { - errors.push(`self-test Skill payload target is not a generated skill: ${target || ''}`); - } - - if (!content.includes(`generate-greeting.js ${item.agentId}`)) { - errors.push(`self-test greeting command cannot activate "${item.agentId}"`); + if (canRoundtripSkillPayload) { + const payload = createSkillToolSelfTestPayload(declaredSkillId); + const target = normalizeSkillToolTarget(payload); + if (target !== item.skillId) { + errors.push(`self-test Skill payload target mismatch: expected "${item.skillId}", got "${target}"`); + } else if (!expectedIds.has(target)) { + errors.push(`self-test Skill payload target is not a generated skill: ${target || ''}`); + } } results.push({ @@ -247,13 +247,32 @@ function isGeneratedSquadSkill(content, projectRoot) { } function validateCodexSkills(options = {}) { - const resolved = { ...getDefaultOptions(), ...options }; + const defaults = getDefaultOptions(); + const projectRoot = options.projectRoot || defaults.projectRoot; + const resolved = { + ...defaults, + ...options, + projectRoot, + sourceDir: options.sourceDir || path.join(projectRoot, '.aiox-core', 'development', 'agents'), + skillsDir: options.skillsDir || path.join(projectRoot, '.codex', 'skills'), + }; const errors = []; const warnings = []; if (!fs.existsSync(resolved.skillsDir)) { errors.push(`Skills directory not found: ${resolved.skillsDir}`); - return { ok: false, checked: 0, expected: 0, errors, warnings, missing: [], orphaned: [], ignored: [] }; + return { + ok: false, + checked: 0, + expected: 0, + errors, + warnings, + missing: [], + orphaned: [], + legacy: [], + ignored: [], + selfTests: [], + }; } const agents = parseAllAgents(resolved.sourceDir).filter(isParsableAgent); diff --git a/.aiox-core/install-manifest.yaml b/.aiox-core/install-manifest.yaml index efe7140c50..c416595aaf 100644 --- a/.aiox-core/install-manifest.yaml +++ b/.aiox-core/install-manifest.yaml @@ -8,7 +8,7 @@ # - File types for categorization # version: 5.1.15 -generated_at: "2026-05-07T20:28:16.723Z" +generated_at: "2026-05-07T20:49:07.348Z" generator: scripts/generate-install-manifest.js file_count: 1103 files: @@ -1229,7 +1229,7 @@ files: type: data size: 9590 - path: data/entity-registry.yaml - hash: sha256:50015ae32f297d4e8bbb321a9efe4c9c78128eb246497f1690f355cf54f8ecd4 + hash: sha256:62bec046e3142cee510223168b996798e1a358bc2ddef1bdc33954263f46b7e2 type: data size: 522368 - path: data/learned-patterns.yaml @@ -3009,9 +3009,9 @@ files: type: script size: 4072 - path: infrastructure/scripts/codex-skills-sync/validate.js - hash: sha256:83fcbbc19a52e5b95b491b06730b22787e2ae27416c27e1b5e2483a835e2f7f7 + hash: sha256:377f52359075c546436f16c596c1e3ffdee61b4218269fc5013dc0e8f4f9da61 type: script - size: 11742 + size: 12185 - path: infrastructure/scripts/collect-tool-usage.js hash: sha256:8a739b79182dc41e28b7e02aeb9ec1dde5ec49f3ca534399acc59711b3b92bbf type: script diff --git a/tests/unit/codex-skills-validate.test.js b/tests/unit/codex-skills-validate.test.js index e65238381a..74538277af 100644 --- a/tests/unit/codex-skills-validate.test.js +++ b/tests/unit/codex-skills-validate.test.js @@ -64,6 +64,23 @@ describe('Codex Skills Validator', () => { expect(result.selfTests.every(test => test.ok)).toBe(true); }); + it('derives default source and skills directories from a supplied project root', () => { + const tmpSourceDir = path.join(tmpRoot, '.aiox-core', 'development', 'agents'); + fs.mkdirSync(path.dirname(tmpSourceDir), { recursive: true }); + fs.cpSync(sourceDir, tmpSourceDir, { recursive: true }); + syncSkills({ sourceDir: tmpSourceDir, localSkillsDir: skillsDir, dryRun: false }); + + const result = validateCodexSkills({ + projectRoot: tmpRoot, + strict: true, + selfTest: true, + }); + + expect(result.ok).toBe(true); + expect(result.checked).toBe(expectedAgentCount); + expect(result.selfTests).toHaveLength(expectedAgentCount); + }); + it('fails self-test when a skill source path cannot be resolved', () => { syncSkills({ sourceDir, localSkillsDir: skillsDir, dryRun: false }); const target = path.join(skillsDir, 'aiox-dev', 'SKILL.md'); @@ -89,6 +106,28 @@ describe('Codex Skills Validator', () => { expect(result.errors.some(error => error.includes('self-test source file not found'))).toBe(true); }); + it('does not cascade payload errors when a skill frontmatter name mismatches', () => { + syncSkills({ sourceDir, localSkillsDir: skillsDir, dryRun: false }); + const target = path.join(skillsDir, 'aiox-dev', 'SKILL.md'); + const original = fs.readFileSync(target, 'utf8'); + fs.writeFileSync(target, original.replace('name: aiox-dev', 'name: aiox-not-dev'), 'utf8'); + + const result = validateCodexSkills({ + projectRoot: process.cwd(), + sourceDir, + skillsDir, + strict: true, + selfTest: true, + }); + + const devSelfTest = result.selfTests.find(test => test.skillId === 'aiox-dev'); + expect(result.ok).toBe(false); + expect(devSelfTest).toBeDefined(); + expect(devSelfTest.errors).toContain('self-test frontmatter name mismatch: expected "aiox-dev"'); + expect(devSelfTest.errors.some(error => error.includes('Skill payload target mismatch'))).toBe(false); + expect(devSelfTest.errors.some(error => error.includes('Skill payload target is not'))).toBe(false); + }); + it('normalizes Skill tool invocation targets', () => { expect(normalizeSkillToolTarget({ type: 'tool_use', @@ -116,6 +155,23 @@ describe('Codex Skills Validator', () => { expect(result.errors.some(error => error.includes('Missing skill file'))).toBe(true); }); + it('returns a stable result shape when the skills directory is missing', () => { + const result = validateCodexSkills({ + projectRoot: tmpRoot, + sourceDir, + skillsDir: path.join(tmpRoot, '.codex', 'missing-skills'), + strict: true, + selfTest: true, + }); + + expect(result.ok).toBe(false); + expect(result.missing).toEqual([]); + expect(result.orphaned).toEqual([]); + expect(result.legacy).toEqual([]); + expect(result.ignored).toEqual([]); + expect(result.selfTests).toEqual([]); + }); + it('fails when greeting command is removed from a skill', () => { syncSkills({ sourceDir, localSkillsDir: skillsDir, dryRun: false }); const target = path.join(skillsDir, 'aiox-dev', 'SKILL.md'); From e56353f7aac24ff961f51084386e94fde10a70e1 Mon Sep 17 00:00:00 2001 From: rafaelscosta Date: Thu, 7 May 2026 18:02:00 -0300 Subject: [PATCH 5/5] fix: simplify codex skill target validation [Story 123.21] --- .aiox-core/data/entity-registry.yaml | 6 +++--- .../infrastructure/scripts/codex-skills-sync/validate.js | 6 +++--- .aiox-core/install-manifest.yaml | 6 +++--- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.aiox-core/data/entity-registry.yaml b/.aiox-core/data/entity-registry.yaml index 86fff0e7ff..0a1da2d7d5 100644 --- a/.aiox-core/data/entity-registry.yaml +++ b/.aiox-core/data/entity-registry.yaml @@ -1,6 +1,6 @@ metadata: version: 1.0.0 - lastUpdated: '2026-05-07T20:48:22.488Z' + lastUpdated: '2026-05-07T21:02:02.654Z' entityCount: 746 checksumAlgorithm: sha256 resolutionRate: 100 @@ -16376,8 +16376,8 @@ entities: score: 0.7 constraints: [] extensionPoints: [] - checksum: sha256:377f52359075c546436f16c596c1e3ffdee61b4218269fc5013dc0e8f4f9da61 - lastVerified: '2026-05-07T20:48:22.477Z' + checksum: sha256:5b8cb91ddcfda2b6b66328c12129e71665e4e1cb6f13b1792746e9d761d1ebcf + lastVerified: '2026-05-07T21:02:02.645Z' brownfield-analyzer: path: .aiox-core/infrastructure/scripts/documentation-integrity/brownfield-analyzer.js layer: L2 diff --git a/.aiox-core/infrastructure/scripts/codex-skills-sync/validate.js b/.aiox-core/infrastructure/scripts/codex-skills-sync/validate.js index ebff338db9..43f43184ab 100644 --- a/.aiox-core/infrastructure/scripts/codex-skills-sync/validate.js +++ b/.aiox-core/infrastructure/scripts/codex-skills-sync/validate.js @@ -200,10 +200,10 @@ function runSkillSelfTests(options = {}) { if (canRoundtripSkillPayload) { const payload = createSkillToolSelfTestPayload(declaredSkillId); const target = normalizeSkillToolTarget(payload); - if (target !== item.skillId) { - errors.push(`self-test Skill payload target mismatch: expected "${item.skillId}", got "${target}"`); - } else if (!expectedIds.has(target)) { + if (!expectedIds.has(target)) { errors.push(`self-test Skill payload target is not a generated skill: ${target || ''}`); + } else if (target !== item.skillId) { + errors.push(`self-test Skill payload target mismatch: expected "${item.skillId}", got "${target}"`); } } diff --git a/.aiox-core/install-manifest.yaml b/.aiox-core/install-manifest.yaml index c416595aaf..1aa3a7b4b2 100644 --- a/.aiox-core/install-manifest.yaml +++ b/.aiox-core/install-manifest.yaml @@ -8,7 +8,7 @@ # - File types for categorization # version: 5.1.15 -generated_at: "2026-05-07T20:49:07.348Z" +generated_at: "2026-05-07T21:04:31.193Z" generator: scripts/generate-install-manifest.js file_count: 1103 files: @@ -1229,7 +1229,7 @@ files: type: data size: 9590 - path: data/entity-registry.yaml - hash: sha256:62bec046e3142cee510223168b996798e1a358bc2ddef1bdc33954263f46b7e2 + hash: sha256:416ffc477206f2b75700a800a35b6af51269f73b3bbc9952a155b1f350089030 type: data size: 522368 - path: data/learned-patterns.yaml @@ -3009,7 +3009,7 @@ files: type: script size: 4072 - path: infrastructure/scripts/codex-skills-sync/validate.js - hash: sha256:377f52359075c546436f16c596c1e3ffdee61b4218269fc5013dc0e8f4f9da61 + hash: sha256:5b8cb91ddcfda2b6b66328c12129e71665e4e1cb6f13b1792746e9d761d1ebcf type: script size: 12185 - path: infrastructure/scripts/collect-tool-usage.js