diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..c01e80f --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,53 @@ +name: CI + +on: + push: + branches: ['main', 'agent/**'] + pull_request: + +permissions: + contents: read + +jobs: + test: + strategy: + fail-fast: false + matrix: + include: + - os: ubuntu-latest + node-version: 22 + upload-source: false + - os: ubuntu-latest + node-version: 24 + upload-source: true + - os: windows-latest + node-version: 24 + upload-source: false + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: ${{ matrix.node-version }} + cache: npm + cache-dependency-path: global-template/docgen/package.json + - name: Syntax check + working-directory: global-template/docgen + run: npm run check + - name: Unit and integration tests + working-directory: global-template/docgen + run: npm test + - name: Installer dry run + run: node install.mjs --dry-run --no-link-cli + - name: Upload exact CI source + if: always() && matrix.upload-source + uses: actions/upload-artifact@v4 + with: + name: docgen-ci-source + if-no-files-found: error + retention-days: 3 + path: | + global-template/docgen + global-template/docgen/project-template + install.mjs + VERSION diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..6ef4e01 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,44 @@ +# Changelog + +## 2.0.0 — Token-efficient semantic index + +### Breaking + +- Removed the v1 monolithic `docgen.mjs` engine. +- Removed discovery, analyze, semantics, enterprise-pass, enrichment, fix, update-impact, and full re-audit commands/prompts. +- Removed all custom pipeline agents and parent-to-subagent delegation. +- Removed legacy auto-enrich, auto-fix, and re-audit configuration. +- Requires Node.js 22.5+ for built-in `node:sqlite`. +- Existing v1 repositories must run `docgen migrate`. + +### Added + +- SQLite/FTS5 semantic index with incremental file hashing. +- Overlapping source chunks plus deterministic symbol/interface/config/SQL extraction. +- Bounded content-addressed context compiler. +- Hard provider call/input/output/per-call budgets. +- Provider telemetry and stage usage reports. +- Two-call model synthesis: core and enterprise. +- Deterministic catalog/reference rendering. +- Item-level page invalidation. +- Claim-level traceability generated in the same page invocation. +- Deterministic FACT/evidence validation. +- Selective, risk-scored, hash-cached LLM audit. +- Breaking v1-to-v2 migration with backup and docs/ignore preservation. +- Node.js 22/24 CI and semantic-index regression suite. + +### Changed + +- Planning defaults to at most 30 pages unless explicitly configured otherwise. +- Provider prompts are direct and context-only. +- Skills remain available for manual Command Code work but are not automatically loaded by the pipeline. +- Source inventory moved from `.docgen/state` to `.docgen/index`. +- Publishing and P3 workspace aggregation remain deterministic. + +### Removed dead managed files during installation + +The installer backs up and removes v1 executable, prompts, agents, and obsolete slash commands from global and project-local installations. + +## 1.0.0 + +P3 system-of-systems workspace release built on the original provider-heavy repository pipeline. diff --git a/PACKAGE-MANIFEST.json b/PACKAGE-MANIFEST.json index 41952b3..9fc75bc 100644 --- a/PACKAGE-MANIFEST.json +++ b/PACKAGE-MANIFEST.json @@ -1,99 +1,105 @@ { "name": "commandcode-docgen-kit", - "version": "1.0.0", + "version": "2.0.0", + "releaseType": "breaking-token-efficiency-redesign", + "runtime": { + "node": ">=22.5.0", + "database": "node:sqlite with FTS5", + "provider": "Command Code CLI" + }, "defaultInstallationScope": "global", "globalComponents": [ - "agents", "skills", "commands", "hooks", - "orchestrator", - "prompts", + "modular-engine", + "bounded-prompts", "schemas", "project-template" ], "projectComponents": [ ".docgen/config", - ".docgen/evidence", + ".docgen/index", + ".docgen/context", ".docgen/model", ".docgen/plan", + ".docgen/traceability", ".docgen/audit", - ".docgen/state", + ".docgen/telemetry", + ".docgen/budget", ".docgen/runs", - ".docgen/traceability", + ".docgen/publish", + ".docgen/state", "docs", - ".docgen/quarantine", - ".docgenignore", - ".docgen/publish" + ".docgenignore" ], "counts": { - "agents": 8, + "agents": 0, "skills": 50, - "commands": 21, + "commands": 15, "schemas": 35, - "prompts": 14 + "providerPrompts": 5 }, - "contractFirewall": { - "boundaries": 19, - "deterministicTests": 21, - "transactionalStages": 6, - "zeroTokenCommand": "docgen contract-test" + "tokenEfficiency": { + "singlePassSourceIndex": true, + "incrementalFileHashing": true, + "sourceChunkIndex": true, + "sqliteFts5": true, + "boundedContextCompiler": true, + "contentAddressedContext": true, + "hardProviderBudgets": true, + "providerTelemetry": true, + "twoModelSynthesisCalls": true, + "deterministicReferenceRendering": true, + "itemLevelPageInvalidation": true, + "selectiveRiskAudit": true, + "auditInputCaching": true, + "parentAgentDelegation": false, + "autoEnrichLoop": false, + "autoFixLoop": false, + "fullReauditLoop": false, + "legacyPipeline": false }, - "p0Trustworthiness": { + "trustworthiness": { "typedSemanticModels": true, "claimLevelTraceability": true, - "evidenceCentricQuality": true, - "crossPageContradictions": true, - "duplicateDetection": true, - "sourceFreshness": true, - "wordCountGate": "advisory", - "zeroTokenCommand": "docgen traceability" + "factEvidenceValidation": true, + "ignoredEvidenceRejected": true, + "contextOnlyReadHook": true, + "mermaidOnly": true }, - "p1EnterpriseDepth": { - "security": true, - "operations": true, - "testing": true, - "dataGovernance": true, - "architectureDecisions": true, - "configuration": true, - "changeImpact": true, - "ownership": true - }, - "ignoreBoundary": { + "sourceBoundary": { "gitignore": true, + "nestedGitignoreFallback": true, "docgenignore": true, - "readHook": true, - "sourceInventory": true, - "fingerprintsRespectIgnore": true, - "traceabilityRejectsIgnoredEvidence": true, "binaryAndNonText": true, - "magicAndUtf8Probe": true + "magicAndUtf8Probe": true, + "oversizedTextGuard": true }, - "p2DocumentationExperience": { - "documentModes": true, + "publishing": { "frontmatter": true, "llmsTxt": true, + "llmsFullTxt": true, "searchIndex": true, - "backlinks": true, - "aliasesRedirects": true, - "versionDeprecationMigration": true, - "evidenceDerivedExamples": true + "navigationIndex": true }, - "p3SystemOfSystems": { + "systemOfSystems": { "workspaceRegistry": true, "multiRepositoryAggregation": true, - "globalDependencyGraph": true, - "sharedContractRegistry": true, - "domainCapabilityMap": true, + "dependencyGraph": true, + "contractRegistry": true, + "capabilityMap": true, "businessJourneys": true, - "crossServiceRequestFlows": true, - "crossServiceEventFlows": true, - "crossRepositoryDataLineage": true, - "sharedInfrastructure": true, - "ownershipAggregation": true, - "crossRepositoryChangeImpact": true, - "incrementalWorkspaceRebuild": true, - "mermaidOnly": true, - "zeroProviderBasePipeline": true + "crossServiceFlows": true, + "dataLineage": true, + "ownership": true, + "changeImpact": true, + "incrementalWorkspaceRebuild": true + }, + "migration": { + "from": "1.x", + "command": "docgen migrate", + "preserves": ["docs/**", ".docgenignore", "selected runtime/ignore configuration"], + "archivesLegacyArtifacts": true } } diff --git a/README.md b/README.md index c4385ff..5dd42cf 100644 --- a/README.md +++ b/README.md @@ -1,4089 +1,348 @@ -# Command Code DocGen Kit +# Command Code DocGen Kit 2.0 -**Global-first, evidence-grounded repository-to-Markdown documentation generation for Command Code CLI.** +**Token-efficient, evidence-grounded repository-to-Markdown documentation for Command Code CLI.** -Command Code DocGen Kit turns Command Code into a reusable documentation engineering runtime that can be installed once and used across many repositories. +DocGen 2 replaces the former provider-heavy eight-phase pipeline with a deterministic semantic index and bounded context compiler. Repository source is read once into local SQLite/FTS5. Every provider invocation receives only a content-addressed context pack selected for that stage or page. -The default installation model is intentionally split into two scopes: - -```text -GLOBAL USER SCOPE -~/.commandcode/ -├── agents/ reusable specialized agents -├── skills/ reusable procedures and technology/domain knowledge -├── commands/ global /docgen-* commands -├── settings.json conditional DocGen hooks merged with existing settings -└── docgen/ reusable engine - ├── bin/ - ├── hooks/ - ├── prompts/ - ├── schemas/ - └── project-template/ - -PROJECT SCOPE -/ -├── .docgen/ repository-specific config, evidence, models, plans, audit, state -├── docs/ generated Markdown + Mermaid -└── .commandcode/ optional project overrides only; not created by default -``` - -The key boundary is: - -```text -GLOBAL = how DocGen works -PROJECT = what this repository contains -``` - -Install the engine once. Initialize any number of repositories independently. - ---- - -## What's new in v1.0.0 - -v1.0.0 is the **P3 System-of-Systems release**. It retains the single-repository P0 trustworthiness, P1 enterprise-depth, P2 publishing experience, ignore policy, binary budget guard, contract firewall, batching, retry, and resume behavior. P3 adds a workspace layer that aggregates any number of already-initialized DocGen repositories into one cross-repository system knowledge base. - -P3 is intentionally model-first and token-efficient: - -```text -Repository A source → validated .docgen models ─┐ -Repository B source → validated .docgen models ─┼─► .docgen-workspace -Repository C source → validated .docgen models ─┘ - │ - ├── global repository/service map - ├── dependency graph - ├── shared contract registry - ├── domain capability map - ├── end-to-end business journeys - ├── cross-service request flows - ├── cross-service event flows - ├── cross-service data lineage - ├── ownership and infrastructure - └── cross-repository blast radius -``` - -The workspace stage does **not** blindly rescan every repository. It consumes repository-level artifacts that already passed the DocGen contract firewall, ignore boundary, binary classifier, traceability checks, and quality gates. This keeps P3 deterministic, resumable, and inexpensive. - -### P3 system workspace capabilities - -| Capability | Canonical output | -|---|---| -| Repository registry | `.docgen-workspace/repositories.json` | -| Global system map | `.docgen-workspace/model/system-map.json` | -| Directed dependency graph | `.docgen-workspace/model/dependency-graph.json` | -| Shared HTTP/message contract registry | `.docgen-workspace/model/contract-registry.json` and `contracts/registry.json` | -| Domain capability map | `.docgen-workspace/model/capability-map.json` | -| End-to-end business journeys | `.docgen-workspace/model/business-journeys.json` | -| Cross-service request paths | `.docgen-workspace/model/request-flows.json` | -| Producer/channel/consumer event paths | `.docgen-workspace/model/event-flows.json` | -| Cross-repository data lineage | `.docgen-workspace/model/data-lineage.json` | -| Shared infrastructure | `.docgen-workspace/model/shared-infrastructure.json` | -| Ownership and escalation aggregation | `.docgen-workspace/model/ownership.json` | -| Direct and transitive blast radius | `.docgen-workspace/model/change-impact.json` | -| Workspace traceability | `.docgen-workspace/traceability/index.json` | -| Workspace quality | `.docgen-workspace/audit/quality-summary.json` | -| Aggregate documentation | `docs/system/**` | - -### P3 quick start - -Each member repository must first have repository-level DocGen artifacts: - -```bash -cd ../quote-service - -docgen init # once -docgen resume # build or refresh repository-level models -``` - -Initialize the parent system workspace: - -```bash -cd ../platform-workspace - -docgen workspace init . --name "Quote and Order Platform" -docgen workspace add ../catalog-service --domain catalog --owner catalog-team -docgen workspace add ../quote-service --domain quoting --owner quote-team -docgen workspace add ../order-service --domain ordering --owner order-team -docgen workspace add ../integration-service --domain integration - -docgen workspace all -``` - -The generated system documentation includes: - -```text -docs/system/ -├── index.md -├── SUMMARY.md -├── repository-map.md -├── service-catalog.md -├── dependency-graph.md -├── capability-map.md -├── contract-catalog.md -├── business-journeys/index.md -├── request-flows/index.md -├── event-flows/index.md -├── data-lineage/index.md -├── shared-infrastructure.md -├── ownership.md -├── change-impact.md -├── llms.txt -└── llms-full.txt -``` - -All diagrams are Mermaid. - -### P3 command reference - -```bash -docgen workspace init [directory] [--name NAME] [--id ID] -docgen workspace add [--id ID] [--domain DOMAIN] [--owner OWNER] [--criticality LEVEL] -docgen workspace remove -docgen workspace list -docgen workspace validate -docgen workspace collect -docgen workspace analyze [--force] -docgen workspace generate [--force] -docgen workspace impact -docgen workspace changed -docgen workspace snapshot -docgen workspace quality -docgen workspace status -docgen workspace resume -docgen workspace all [--force] -``` - -`workspace resume` and `workspace all` are deterministic. Analysis and publishing are skipped when repository commit/source/model hashes have not changed. - -### P3 resolution rules - -Cross-repository links are accepted only when they can be grounded by one of these signals: - -1. an explicit registered repository ID or alias; -2. an external-dependency target matching a registered repository; -3. producer and consumer contracts sharing an evidenced topic, queue, exchange, or channel; -4. a repository-local request/data flow explicitly naming another registered repository; -5. a repository model reference that names another registered repository. - -Ambiguous targets are preserved as unresolved edges. P3 does not invent a connection merely because two class or field names look similar. - -### P2 documentation experience retained - -P3 retains the P2 documentation-product layer: - -- `tutorial`, `how-to`, `explanation`, `reference`, `runbook`, `decision-record`, `migration-guide`, and `troubleshooting` modes; -- deterministic frontmatter; -- `llms.txt` and `llms-full.txt`; -- navigation/search indexes; -- backlinks, aliases, redirects, and orphan reports; -- version, deprecation, replacement, and migration metadata; -- evidence-derived examples; -- binary/non-text token-budget guard. - -The full pipeline is now: - -```text -repository source - → P0/P1/P2 repository knowledge base - → validated repository models and traceability - → P3 workspace collection - → cross-repository graph/contracts/journeys/lineage/impact - → Mermaid-only system knowledge base -``` - -### P1 enterprise-depth models - -The enterprise stage is split into four bounded passes so non-frontier models do not have to synthesize every enterprise concern in one context: - -| Pass | Canonical outputs | Primary concerns | -|---|---|---| -| `governance` | `security.json`, `ownership.json` | trust boundaries, AuthN/AuthZ, permissions, secrets, sensitive data, threats, controls, owners, RACI, approvals, escalation | -| `operability` | `operations.json`, `testing.json` | health, observability, SLI/SLO, alerts, capacity, scaling, failures, recovery, backups, deployment, runbooks, test strategy and quality gates | -| `data-and-configuration` | `data-governance.json`, `configuration.json` | source of truth, ownership, classification, retention, transactions, consistency, concurrency, reconciliation, lineage, migration, environment matrix, flags and tuning | -| `evolution` | `decisions.json`, `change-impact.json` | ADRs, inferred rationale, alternatives, trade-offs, consequences, change surfaces, blast radius, compatibility and safe extension points | - -Canonical project artifacts: - -```text -.docgen/model/ -├── security.json -├── operations.json -├── testing.json -├── data-governance.json -├── decisions.json -├── configuration.json -├── change-impact.json -└── ownership.json -``` - -Each item uses the same P0 typed semantic contract: stable ID, kind, statement, `FACT/INFERENCE/UNKNOWN`, confidence, evidence references, model relationships, unknowns, and tags. A `FACT` in any P1 model must resolve to an included repository source. - -Run only the enterprise stage: - -```bash -docgen enterprise -``` - -### Ignore-aware source boundary - -DocGen constructs a canonical source inventory before discovery: - -```text -.docgen/state/source-inventory.json -.docgen/state/source-files.txt -.docgen/state/ignore-report.json -``` - -The effective exclusion order is: - -```text -DocGen hard exclusions - ↓ -config.exclude - ↓ -.gitignore - ↓ -.docgenignore -``` - -`.docgenignore` uses Gitignore-style patterns, including comments, directory patterns, `*`, `**`, `?`, anchoring, and `!` negation. A `.docgenignore` negation can re-include a path excluded by an earlier `.docgenignore` rule, but it deliberately **cannot re-include a path ignored by `.gitignore`**. This keeps the repository's Git privacy/build boundary authoritative. Root `.docgenignore` is user-owned policy: `docgen init --force` and project-local `install.mjs --force` preserve an existing file rather than replacing it. - -Example `.docgenignore`: - -```gitignore -# Additional DocGen-only exclusions -private-notes/** -generated-clients/** -fixtures/large/** - -# Re-include a public explanation from a DocGen-excluded tree -!private-notes/public-architecture.md -``` - -Inspect the effective boundary without consuming LLM tokens: - -```bash -docgen ignore -docgen ignore path/to/file -docgen source-list -docgen source-list jersey -docgen source-grep "@Path" -docgen source-grep --regex "Kafka(Listen|Handler)" -``` - -During `DOCGEN_MODE=1`, hooks block broad source reads/searches that could bypass the inventory. Agents must read `.docgen/state/source-files.txt`, use explicit included paths, or use `docgen source-grep`. Ignored files are also rejected as evidence for typed `FACT` items and page claims. - -The same inventory powers: - -- discovery scope; -- repository source fingerprints; -- `docgen snapshot`; -- `docgen changed`; -- incremental impact analysis; -- source freshness detection; -- claim and semantic-model evidence validation. - -This prevents a path from being ignored during discovery but unexpectedly appearing later in change detection or traceability. - -### Eight-phase full pipeline - -```text -Phase 1/8 source inventory + evidence discovery -Phase 2/8 technical architecture analysis -Phase 3/8 business, flow, and catalog semantics -Phase 4/8 P1 enterprise depth (four bounded passes) -Phase 5/8 preflighted multi-page planning -Phase 6/8 batched generation + targeted enrichment -Phase 7/8 batched independent audit + targeted repair -Phase 8/8 semantic quality summary + source snapshot -``` - -### Upgrade from v0.7.0 - -```powershell -# Install the global v1.0.0 engine -.\install.ps1 -Force - -cd C:\path\to\repository - -docgen migrate -docgen contract-test -docgen ignore -docgen validate -docgen resume -``` - -Migration is additive: existing project values, evidence, models, pages, traceability, and checkpoints are preserved. The new P1 defaults and ignore policy are merged without resetting user configuration. - -## P0 trustworthiness retained from v0.7.0 - -v1.0.0 retains strongly typed semantic items, claim-level traceability, evidence-centric quality metrics, cross-page contradiction and duplicate detection, source freshness metadata, transactional contract boundaries, and advisory rather than primary word-count gates. - -### Strongly typed semantic items - -Every normalized model item is committed as an object with a stable identity and epistemic contract: - -```json -{ - "id": "rule-submit-draft-only", - "kind": "business-rule", - "name": "Draft-only submission", - "statement": "A quote can be submitted only from DRAFT", - "classification": "FACT", - "confidence": 1, - "evidence": [ - { - "id": "quote-service-submit", - "path": "src/main/java/example/QuoteService.java", - "symbol": "QuoteService#submit", - "startLine": 120, - "endLine": 146, - "note": null - } - ], - "sourceModelRefs": [], - "unknowns": [], - "tags": [] -} -``` - -Typed contracts apply to components, relationships, workflows, actors, capabilities, concepts, rules, decisions, branches, lifecycles, use cases, six flow types, endpoints, message handlers, dependencies, data stores, and scheduled jobs. - -`FACT` items without resolvable direct evidence are rejected before downstream generation. Unsupported statements must be classified as `INFERENCE` or `UNKNOWN`. - -### Claim-level page traceability - -Every generated page now has a companion artifact: - -```text -.docgen/traceability/pages/.json -``` - -The sidecar records material claims, their page section, epistemic classification, source evidence, normalized model/catalog references, and coverage information: - -```json -{ - "pageId": "quote-lifecycle", - "pagePath": "docs/business/quote-lifecycle.md", - "claims": [ - { - "id": "quote-lifecycle-claim-1", - "kind": "claim", - "section": "Submission", - "statement": "Only DRAFT quotes may be submitted", - "classification": "FACT", - "confidence": 1, - "subject": "quote", - "predicate": "submission-allowed-from", - "object": "DRAFT", - "polarity": "positive", - "evidence": [ - { - "path": "src/main/java/example/QuoteService.java", - "symbol": "QuoteService#submit", - "startLine": 120, - "endLine": 146 - } - ], - "sourceModelRefs": ["rule-submit-draft-only"], - "intentionalDuplicate": false - } - ], - "coverage": { - "evidenceRefsUsed": ["quote-service"], - "modelItemRefs": ["rule-submit-draft-only"], - "catalogItemRefs": [], - "branchItemRefs": ["branch-submit-status"] - } -} -``` - -Writers create the Markdown page and traceability sidecar in the same bounded Command Code run. The orchestrator fills page/input hashes after generation. - -### Evidence-centric quality gates - -`docgen quality` now prioritizes: - -- claim grounding ratio; -- declared evidence coverage ratio; -- model-item coverage; -- exhaustive catalog coverage; -- decision/branch coverage; -- unsupported claim count; -- stale page/input/source fingerprints; -- cross-page contradictions; -- audit severity. - -Default P0 thresholds: - -```json -{ - "semanticMetrics": { - "minStructuredClaimRatio": 0.7, - "minClaimGroundingRatio": 0.9, - "minEvidenceCoverageRatio": 0.8, - "minModelCoverageRatio": 0.9, - "minCatalogCoverageRatio": 1.0, - "minBranchCoverageRatio": 0.9, - "minEvidenceClaimsPer1000Words": 1.5, - "evidenceClaimDensityGate": "hard", - "maxUnsupportedClaims": 0, - "maxContradictions": 0, - "maxClaimIdCollisions": 0, - "maxStalePages": 0, - "duplicateClaimsAsWarning": true - } -} -``` - -Word count remains visible but is advisory. A short grounded reference page can pass; a long generic page with unsupported claims fails. - -### Cross-page consistency index - -Run without consuming LLM tokens: - -```powershell -docgen traceability -``` - -It creates: - -```text -.docgen/traceability/index.json -.docgen/traceability/contradictions.json -.docgen/traceability/duplicates.json -.docgen/traceability/freshness.json -``` - -Contradictions are detected when grounded claims use the same normalized subject/predicate but disagree on value or polarity. Exact semantic duplicates are grouped separately; claims explicitly marked `intentionalDuplicate` are excluded from duplicate failures. - -### Source freshness - -Traceability artifacts capture: - -```text -git commit -branch -dirty state -repository source fingerprint -page hash -evidence/model input hash -``` - -A page becomes stale when its Markdown changes, declared evidence/model content changes, or the repository source fingerprint changes without regeneration/revalidation. - -### Upgrade from v0.6.0 - -```powershell -# Install the global v1.0.0 engine -.\install.ps1 -Force - -cd C:\path o -epository - -docgen migrate -docgen contract-test -docgen validate -docgen resume -``` - -Existing valid Markdown is preserved. Pages without a v0.7 traceability sidecar are treated as legacy-unmapped and receive targeted enrichment rather than full regeneration. - -### Trustworthiness commands - -```powershell -docgen traceability # rebuild claim, contradiction, duplicate and freshness indexes -docgen quality # run semantic/evidence quality gates -docgen contract-test # zero-token contract regression suite -docgen validate # contract + static + generated artifact validation -``` - -## Contract firewall retained from v0.6.0 - -v1.0.0 retains the **contract firewall** introduced in v0.6.0. It addresses the root class behind both expensive failures reported in earlier versions: - -```text -LLM producer emits a reasonable representation - │ - ▼ -orchestrator assumes a different literal representation - │ - ▼ -failure appears only after a costly stage has completed -``` - -The two observed examples were: - -- discovery wrote an evidence index using `files[]`, while the validator required `artifacts[]`; -- planning/writing used `orientation/overview` versus `docs/orientation/overview.md`. - -Those are no longer handled as isolated special cases. Every LLM-output boundary now passes through the same protocol: - -```text -snapshot previous canonical artifact - │ - ▼ -run bounded Command Code stage - │ - ▼ -accept known semantic aliases - │ - ▼ -normalize to one canonical representation - │ - ▼ -validate identity, paths, references and invariants - │ - ├── PASS → atomically commit canonical artifact - │ - └── FAIL → quarantine raw output + restore previous artifact + stop -``` - -### Contract-firewall coverage - -| LLM stage | Canonical committed artifact | Examples of accepted variants | -|---|---|---| -| Discovery | `.docgen/evidence/index.json` with `artifacts[]` | `files`, `entries`, `documents`, `items` | -| Architecture analysis | `.docgen/model/system.json` | `services→components`, `dependencies→relationships`, `processes→workflows` | -| Business semantics | `.docgen/model/business.json` | `roles→actors`, `rules→businessRules`, `conditions→branchConditions` | -| Flow semantics | `.docgen/model/flows.json` | generic typed `flows`, `httpFlows`, `networkFlows`, `messageFlows` | -| Catalog semantics | `.docgen/model/catalogs.json` | `routes→endpoints`, producers/consumers/listeners→`messageHandlers`, integrations→`externalDependencies` | -| Planning | `.docgen/plan/manifest.json` | `documents→pages`, `categories→navigation`, `outputPath→path` | -| Page writing | `docs/**/*.md` | missing `docs/`, missing `.md`, uniquely reconcilable misplaced page | -| Audit | `.docgen/audit/pages/.json` | `issues→findings`, `id→pageId`, `path→pagePath`, `hash→pageHash` | -| Incremental update | `.docgen/plan/update-plan.json` | `changedFiles`, `scopes`, `models`, `pages`, `reasons` | - -Canonical writeback removes ambiguous aliases after normalization. Downstream agents therefore never receive both `outputPath` and `path`, or both `files` and `artifacts`, as competing sources of truth. - -### New zero-token regression command - -```powershell -docgen contract-test -``` - -This does not call an LLM provider. It tests: - -- alias normalization; -- canonical page paths; -- normalizer idempotence: `normalize(normalize(x)) == normalize(x)`; -- catalog losslessness across producers, consumers and listeners; -- audit identity normalization; -- evidence-path safety; -- canonical update-plan normalization. - -The result is stored in: - -```text -.docgen/state/contract-report.json -``` - -`docgen doctor` and `docgen validate` run the same contract suite automatically. - -### Transactional stage recovery - -Discovery, analysis, semantics, planning and incremental-impact analysis are transactional. If Command Code exits unsuccessfully or exits successfully with malformed/incompatible output: - -1. partial output is copied to `.docgen/quarantine/-/`; -2. the last valid canonical artifact is restored; -3. downstream stages are not started; -4. the terminal reports the quarantine path and exact contract cause. - -Before an automatic provider retry, stage output is reset to the original snapshot. A second attempt cannot inherit half-written JSON from the first attempt. - -### Dependency-aware resume - -A completed status is no longer enough to skip a stage. DocGen normalizes and validates the checkpoint before reuse. If an upstream checkpoint must rerun, all dependent semantic/planning stages rerun as well. - -Generated pages now record a hash of: - -```text -page manifest contract -+ declared evidence content -+ declared model content -``` - -A page is skipped only while that input fingerprint remains current. Existing valid pages from v0.4/v0.5 are adopted once without regeneration, preserving already-spent tokens; future evidence/model changes invalidate only affected page checkpoints. Audit reuse additionally requires both the current page hash and the current input fingerprint. - -### Safe recovery from the reported failure - -```powershell -# From the extracted v1.0.0 package -.\install.ps1 -Force - -cd C:\path\to\your\repository - -docgen migrate -docgen contract-test -docgen validate -docgen resume -``` - -Do not delete `.docgen` or `docs`. A page already generated as `docs/orientation/overview.md` is canonicalized/adopted and will not be regenerated solely because an older manifest omitted `docs/` or `.md`. - -## Table of contents - -1. [What DocGen is](#what-docgen-is) -2. [Why the global-first architecture matters](#why-the-global-first-architecture-matters) -3. [Capabilities](#capabilities) -4. [Requirements](#requirements) -5. [Install globally](#install-globally) -6. [Initialize a repository](#initialize-a-repository) -7. [Quick start](#quick-start) -8. [Live progress, heartbeat, logs, and error visibility](#live-progress-heartbeat-logs-and-error-visibility) -9. [Comprehensive quality profile](#comprehensive-quality-profile) -10. [P0 trustworthiness and traceability](#p0-trustworthiness-and-traceability) -10. [How it works](#how-it-works) -9. [Execution flow](#execution-flow) -10. [State machine](#state-machine) -11. [Global versus project-local files](#global-versus-project-local-files) -12. [Agents](#agents) -13. [Skills](#skills) -14. [Global slash commands](#global-slash-commands) -15. [CLI command reference](#cli-command-reference) -16. [Fail-fast preflight and canonical paths](#fail-fast-preflight-and-canonical-paths) -17. [Contract firewall and transactional artifacts](#contract-firewall-and-transactional-artifacts) -17. [Resumability, batching, and checkpoints](#resumability-batching-and-checkpoints) -18. [Rate limits, retries, and provider failures](#rate-limits-retries-and-provider-failures) -16. [Evidence model](#evidence-model) -17. [FACT, INFERENCE, and UNKNOWN](#fact-inference-and-unknown) -18. [Documentation manifest and bounded generation](#documentation-manifest-and-bounded-generation) -19. [Audit and repair](#audit-and-repair) -20. [Incremental regeneration](#incremental-regeneration) -21. [Configuration](#configuration) -22. [Model and turn-budget configuration](#model-and-turn-budget-configuration) -23. [Safety model and hooks](#safety-model-and-hooks) -24. [Project overrides and precedence](#project-overrides-and-precedence) -25. [Large repositories and monorepos](#large-repositories-and-monorepos) -26. [Git and team workflow](#git-and-team-workflow) -27. [Upgrade](#upgrade) -28. [Uninstall](#uninstall) -29. [Self-contained project-local installation](#self-contained-project-local-installation) -30. [Troubleshooting](#troubleshooting) -31. [Extending DocGen](#extending-docgen) -32. [Included technology and domain coverage](#included-technology-and-domain-coverage) -33. [Known limitations](#known-limitations) -34. [Compatibility notes](#compatibility-notes) - ---- - -# What DocGen is - -DocGen is a documentation compiler workflow built around Command Code. - -It does **not** treat the LLM as the source of truth. The repository remains authoritative. - -The intended transformation is: - -```text -source repository - │ - ▼ -source-grounded evidence - │ - ▼ -normalized technical architecture - │ - ▼ -business + flow + catalog semantics - │ - ▼ -documentation information architecture - │ - ▼ -bounded page generation - │ - ▼ -independent audit - │ - ▼ -Markdown + Mermaid -``` - -The primary output is plain files under: - -```text -docs/ -``` - -Those files can be rendered by any Markdown-capable system, for example: - -- Mintlify -- Docusaurus -- VitePress -- MkDocs -- GitHub -- GitLab -- an internal documentation portal - -DocGen does not require a particular renderer. - ---- - -# Why the global-first architecture matters - -Earlier versions of the kit copied the entire engine into every target repository. That model is useful for a fully self-contained team-owned repository, but it is inefficient when one user wants to use the same documentation system across many repositories. - -The default architecture in v1.0.0 is therefore: - -```text -install once globally - │ - ├──────────────► repository A ──► .docgen/ + docs/ - │ - ├──────────────► repository B ──► .docgen/ + docs/ - │ - └──────────────► repository C ──► .docgen/ + docs/ -``` - -Benefits: - -- one engine installation; -- one place to upgrade agents, skills, hooks, prompts, and schemas; -- independent repository state; -- smaller repository footprint; -- project-specific overrides remain possible; -- normal Command Code use remains unaffected because DocGen hooks are conditional. - -The global installation uses the Command Code user-level extension locations: - -```text -~/.commandcode/agents/ -~/.commandcode/skills/ -~/.commandcode/commands/ -~/.commandcode/settings.json -``` - -The engine itself is stored under: - -```text -~/.commandcode/docgen/ -``` - -On native Windows, `~` means the current user's home directory, typically: - -```text -C:\Users\ -``` - -So the default engine path becomes: - -```text -C:\Users\\.commandcode\docgen -``` - ---- - -# Capabilities - -The kit currently provides: - -- **6 specialized custom agents** -- **28 reusable skills** -- **15 global slash commands** -- **conditional global hooks** -- **12 JSON artifact schemas** -- **12 bounded stage prompts** -- **a global cross-platform Node.js orchestrator** -- **per-repository state and configuration** -- **runtime compatibility diagnostics** -- **batched bounded generation with per-page validation and fallback** -- **independent factual audit** -- **audit-backed repair** -- **source fingerprinting** -- **incremental impact analysis and regeneration** -- **Markdown + Mermaid output** -- **optional project-level overrides** -- **optional self-contained project-local installation mode** - -The core design goals are: - -1. source code and explicit source evidence are authoritative; -2. discovery and writing are separate stages; -3. important conclusions preserve epistemic classification; -4. generation is bounded rather than one giant repository prompt; -5. state is explicit on disk; -6. audit is separated from writing; -7. the reusable engine is separate from repository-specific knowledge. - ---- - -# Requirements - -## Required - -- Node.js 20 or newer recommended; -- Command Code CLI installed; -- Command Code authenticated before LLM-backed stages are executed. - -Install Command Code: - -```bash -npm i -g command-code@latest -``` - -Authenticate: - -```bash -cmd login -``` - -On native Windows, Command Code may use `cmdc`: - -```powershell -cmdc login -``` - -Verify: - -```bash -cmd --version -``` - -or: - -```powershell -cmdc --version -``` - -## Recommended - -- Git repository; -- a clean working tree before the first large generation; -- review of generated documentation diffs before commit. - ---- - -# Install globally - -Extract the release ZIP first. - -## Windows PowerShell - -```powershell -Expand-Archive ` - .\commandcode-docgen-kit-1.0.0.zip ` - -DestinationPath .\commandcode-docgen-kit - -cd .\commandcode-docgen-kit\commandcode-docgen-kit-1.0.0 - -.\install.ps1 -``` - -## macOS / Linux - -```bash -unzip commandcode-docgen-kit-1.0.0.zip -cd commandcode-docgen-kit-1.0.0 -./install.sh -``` - -## Cross-platform Node.js - -```bash -node install.mjs -``` - -The default target is: - -```text -~/.commandcode/ -``` - -The installer places reusable components at: - -```text -~/.commandcode/ -├── agents/ -├── skills/ -├── commands/ -├── settings.json -└── docgen/ -``` - -The installer attempts to expose the global `docgen` CLI through: - -```bash -npm link -``` - -The package contains no runtime npm dependencies; the link is only used to make the `docgen` executable available on PATH. - -To skip that step: - -```bash -node install.mjs --no-link-cli -``` - -Then invoke the engine directly: - -```bash -node ~/.commandcode/docgen/bin/docgen.mjs -``` - -On native Windows: - -```powershell -node "$env:USERPROFILE\.commandcode\docgen\bin\docgen.mjs" -``` - -## Installer options - -```text ---force - overwrite conflicting DocGen-owned global files after backing them up - ---dry-run - print the installation plan without writing - ---no-hooks - install the engine without merging conditional DocGen hooks - ---no-link-cli - do not run npm link - ---commandcode-home - use a non-default Command Code home; useful for testing - ---project-local - use the optional self-contained installation model instead of global-first -``` - -Examples: - -```bash -node install.mjs --dry-run -node install.mjs --force -node install.mjs --no-link-cli -node install.mjs --commandcode-home /tmp/test-commandcode-home -``` - -## What global installation changes - -The installer: - -1. installs five user-level custom agents; -2. installs twenty-two user-level skills; -3. installs thirteen user-level `/docgen-*` commands; -4. installs the reusable engine under `~/.commandcode/docgen/`; -5. merges conditional DocGen hooks into the existing user `settings.json`; -6. preserves unrelated settings and hooks; -7. records the installation under `~/.commandcode/docgen/installation.json`; -8. optionally exposes the `docgen` executable through `npm link`. - -The installer does **not** initialize any repository by default. - ---- - -# Initialize a repository - -After the global engine is installed, enter any repository: - -```bash -cd /path/to/repository -``` - -Initialize it: - -```bash -docgen init -``` - -Or explicitly: - -```bash -docgen init /path/to/repository -``` - -Without the linked CLI: - -```bash -node ~/.commandcode/docgen/bin/docgen.mjs init -``` - -The default init creates only repository-specific state: - -```text -repository/ -├── .docgen/ -│ ├── project.json -│ ├── README.md -│ ├── config/ -│ │ ├── documentation.json -│ │ ├── glossary.md -│ │ └── style-guide.md -│ ├── evidence/ -│ ├── model/ -│ │ ├── components/ -│ │ ├── relationships/ -│ │ ├── workflows/ -│ │ └── unknowns/ -│ ├── plan/ -│ ├── audit/ -│ │ └── pages/ -│ ├── state/ -│ └── runs/ -└── docs/ -``` - -Default init does **not** copy the global engine into the project and does **not** create `.commandcode/`. - -That keeps the repository clean and makes the engine reusable. - -Init is designed to be idempotent. Existing project configuration is preserved unless `--force` is explicitly used: - -```bash -docgen init --force -``` - -Use `--force` carefully because it can replace project-template-owned files such as default configuration. - ---- - -# Quick start - -## One-time machine setup - -```bash -npm i -g command-code@latest -cmd login - -# Extract DocGen release, then: -node install.mjs -``` - -## Per repository - -```bash -cd /path/to/repository - -docgen init -docgen doctor -docgen all -``` - -The first full run executes: - -```text -build ignore-aware source inventory - ↓ -discover repository evidence - ↓ -analyze technical architecture - ↓ -semantics: business + six flow types + catalogs - ↓ -enterprise depth: security, operations, testing, data governance, -decisions, configuration, change impact, ownership - ↓ -preflight and plan a category-rich multi-page knowledge base - ↓ -batched generation + targeted enrichment - ↓ -batched audit + targeted repair - ↓ -semantic quality gate + source snapshot -``` - -For a large repository, prefer an explicit staged workflow: - -```bash -docgen ignore -docgen discover src/main/java -docgen analyze -docgen semantics -docgen enterprise -docgen plan -docgen preflight -docgen generate --all -docgen audit --all -docgen snapshot -``` - ---- - -# Live progress, heartbeat, logs, and error visibility - -DocGen v1.0.0 does not run Command Code as a silent blocking child process. Every LLM-backed stage is monitored as a live asynchronous process. - -A run now looks like: - -```text -Phase 1/8 — source inventory + evidence discovery - -==> discover: . | phase 1/8 - cmdc -p --trust --skip-onboarding --yolo --max-turns 30 --verbose - logs: .docgen/runs/.stdout.log | .docgen/runs/.stderr.log - -session: - -[docgen] discover:. RUNNING | elapsed 0m 10s | pid 18420 | changed artifacts 2 -[docgen] discover:. RUNNING | elapsed 0m 20s | pid 18420 | changed artifacts 5 -... -[docgen] discover:. COMPLETED | 2m 14s | exit 0 (success) -``` - -For page collections, DocGen also prints page-level progress: - -```text -[========................] 33% generate 4/12 — quote-lifecycle -``` - -## What the heartbeat means - -Command Code headless mode may remain quiet while the model is reasoning or executing tools. DocGen therefore prints a heartbeat even when Command Code has not emitted new stdout/stderr. - -The heartbeat reports: - -- current stage and target; -- elapsed time; -- child process PID; -- number of `.docgen/**` or `docs/**` artifacts changed since the run started; -- how long the Command Code process has been quiet. - -A quiet heartbeat is **not** treated as failure. It means the process is still alive. - -Default configuration: - -```json -{ - "progress": { - "heartbeatSeconds": 10, - "noOutputWarningSeconds": 45, - "showCommandOutput": true, - "verboseCommandCode": true - } -} -``` - -You can make the heartbeat more frequent: - -```json -{ - "progress": { - "heartbeatSeconds": 5 - } -} -``` - -## Per-run logs - -Every LLM-backed run writes three files under `.docgen/runs/`: - -```text -.json -.stdout.log -.stderr.log -``` - -The metadata JSON records: - -- stage; -- target; -- start and finish timestamps; -- elapsed duration; -- PID; -- exact Command Code arguments; -- exit code; -- signal when interrupted; -- normalized error classification; -- stdout/stderr log paths. - -This means a failed run is diagnosable after the terminal session ends. - -## API and runtime error classification - -DocGen maps Command Code's documented headless exit codes into explicit categories: - -| Exit | Classification | Meaning | -|---:|---|---| -| `0` | `success` | completed | -| `1` | `general-error` | generic Command Code failure | -| `3` | `not-authenticated` | login required | -| `4` | `permission-denied` | permission/hook denial | -| `5` | `rate-limited` | provider/API rate limit | -| `6` | `network-failure` | network/provider connectivity failure | -| `7` | `api-server-error` | provider/API 5xx | -| `8` | `max-turns` | configured headless turn limit reached | -| `130` | `interrupted` | SIGINT/SIGTERM | - -On failure, DocGen prints the classification, a remediation hint, and the tail of stderr. Full stderr remains in `.docgen/runs/*.stderr.log`. - -This is especially useful for distinguishing: - -```text -model is still working -``` - -from: - -```text -provider rate limited the request -``` - -or: - -```text -network/API server failed -``` - -or: - -```text -Command Code reached --max-turns -``` - ---- - -# Comprehensive quality profile - -The default v1.0.0 profile is: - -```json -{ - "quality": { - "profile": "comprehensive" - } -} -``` - -The goal is not merely to produce valid Markdown. The goal is to produce a curated developer-documentation set with the depth expected from a high-quality developer portal while remaining grounded in repository evidence. - -No orchestration layer can guarantee frontier-model reasoning from a weak model. DocGen instead improves cheap-model reliability through decomposition, explicit contracts, deterministic quality gates, and multiple bounded passes. - -## Comprehensive pipeline - -```text -ignore-aware source inventory - │ - ▼ -evidence discovery - │ - ▼ -normalized technical architecture/workflow model - │ - ▼ -business, flow, endpoint, messaging, integration semantics - │ - ▼ -security, operations, testing, data governance, decisions, -configuration, change impact, and ownership models - │ - ▼ -coverage-driven documentation manifest + preflight - │ - ▼ -batched page generation + claim traceability - │ - ▼ -targeted enrichment only for failed local gates - │ - ▼ -batched independent audit + targeted repair/re-audit - │ - ▼ -evidence-centric quality gate + source snapshot -``` - -The important design choice is that a cheap model is **not** asked to understand an entire repository and produce perfect documentation in one response. - -## Coverage-driven planning - -The planner must consider whether evidence supports documentation for: - -- overview and architecture at a glance; -- components/modules and ownership boundaries; -- domain concepts and terminology; -- important request/event/workflow/state lifecycles; -- API, messaging, persistence, configuration, security, and external integrations; -- trust boundaries, authentication, authorization, secrets, sensitive data, and threats; -- ownership, RACI, approval authority, escalation, and on-call responsibility; -- transactions, consistency, concurrency, retention, reconciliation, lineage, and migration; -- health, observability, SLI/SLO, capacity, scaling, failure modes, recovery, backup, deployment, and runbooks; -- unit, integration, contract, end-to-end, failure-injection, test-data, and CI quality-gate strategy; -- architecture decisions, alternatives, trade-offs, change impact, compatibility, and safe extension points; -- local development and common engineering tasks. - -The planner must not invent a category with no evidence, but it also must not omit a material system surface simply because that surface is complex. - -Each manifest page now declares: - -```json -{ - "id": "quote-lifecycle", - "path": "docs/concepts/quote-lifecycle.md", - "title": "Quote Lifecycle", - "type": "concept", - "purpose": "Explain the lifecycle and invariants of a quote", - "audience": ["engineer", "architect"], - "evidence": ["..."], - "models": ["..."], - "requiredSections": [ - "Purpose and Scope", - "Mental Model", - "Lifecycle", - "Invariants", - "Failure Behavior" - ], - "diagramIntents": ["state lifecycle", "submission sequence"], - "relatedPages": ["pricing", "order-conversion"], - "qualityHints": ["explain optimistic locking if evidenced"] -} -``` - -The manifest is therefore a **content contract**, not merely a filename list. - -## Automatic enrichment - -With `quality.profile = "comprehensive"` and `quality.autoEnrich = true`, DocGen evaluates every generated or legacy-adopted page, but runs a second writer pass only for pages that fail deterministic local trust/depth gates. - -The enrichment pass looks specifically for shallow areas and strengthens supported detail such as: - -- mental models and scope boundaries; -- end-to-end flows; -- state transitions; -- invariants and assumptions; -- dependency and data-ownership implications; -- failure modes and recovery behavior; -- operational/troubleshooting guidance; -- practical examples and decision tables; -- planned Mermaid diagrams; -- navigation to related pages. - -It preserves useful material rather than replacing the page with generic prose. - -## Quality gates - -Default local gates: - -```json -{ - "quality": { - "profile": "comprehensive", - "autoEnrich": true, - "autoFix": true, - "reAuditAfterFix": true, - "minWordsByType": { - "overview": 900, - "architecture": 1200, - "concept": 900, - "guide": 1000, - "reference": 700, - "operations": 1000 - }, - "minHeadings": 4, - "requireDeclaredSections": true, - "requirePlannedDiagrams": true, - "maxCriticalFindings": 0, - "maxHighFindings": 0 - } -} -``` - -Word counts are **minimum anti-shallowness signals**, not a target to pad prose. The writer is explicitly instructed not to add generic filler. - -Run the gate directly: - -```bash -docgen quality -``` - -Example: - -```text -PASS overview 1840 words | 8 headings | 2 mermaid -PASS quote-lifecycle 2315 words | 11 headings | 3 mermaid -PASS configuration-reference 1460 words | 9 headings | 0 mermaid - -Quality profile: comprehensive -Local gate failures: 0 -Audit findings: {"critical":0,"high":0,"medium":2,"low":3} -Quality gate: PASS -``` - -The machine-readable summary is written to: - -```text -.docgen/audit/quality-summary.json -``` - -## `docgen all` in comprehensive mode - -The full pipeline now performs: - -```text -Phase 1/8 build source inventory + discover -Phase 2/8 analyze technical architecture -Phase 3/8 extract business/flow/catalog semantics -Phase 4/8 extract P1 enterprise-depth models -Phase 5/8 plan + deterministic preflight -Phase 6/8 batch-generate + targeted enrichment -Phase 7/8 batch-audit + targeted fix/re-audit -Phase 8/8 traceability/quality summary + source snapshot -``` - -For faster or cheaper operation, set another profile and disable automatic passes: - -```json -{ - "quality": { - "profile": "balanced", - "autoEnrich": false, - "autoFix": false, - "reAuditAfterFix": false - } -} -``` - ---- - -# How it works - -DocGen separates responsibilities rather than asking one LLM run to understand and document everything at once. - -```text -┌──────────────────────────┐ -│ SOURCE REPOSITORY │ -└────────────┬─────────────┘ - │ - ▼ -┌──────────────────────────┐ -│ DISCOVERY STAGE │ -│ doc-discoverer │ -└────────────┬─────────────┘ - │ - ▼ - .docgen/evidence/** - │ - ▼ -┌──────────────────────────┐ -│ ANALYSIS STAGE │ -│ doc-architect │ -└────────────┬─────────────┘ - │ - ▼ - .docgen/model/** - │ - ▼ -┌──────────────────────────┐ -│ PLANNING STAGE │ -│ doc-planner │ -└────────────┬─────────────┘ - │ - ▼ - .docgen/plan/manifest.json - │ - ▼ - one page per run - │ - ▼ -┌──────────────────────────┐ -│ GENERATION STAGE │ -│ doc-writer │ -└────────────┬─────────────┘ - │ - ▼ - docs/** - │ - ▼ -┌──────────────────────────┐ -│ AUDIT STAGE │ -│ doc-auditor │ -└────────────┬─────────────┘ - │ - ▼ - .docgen/audit/** -``` - -The LLM is used as a bounded reasoning component inside this workflow. - -The orchestrator controls: - -- stage ordering; -- current repository root; -- global versus project asset resolution; -- Command Code headless arguments; -- `DOCGEN_MODE` activation; -- per-stage turn budgets; -- run metadata; -- validation; -- state updates; -- page-by-page generation; -- page-by-page audit. - ---- - -# Execution flow - -A typical full run is: - -```text -User - │ - │ docgen all - ▼ -Global DocGen Orchestrator - │ - ├─ resolves initialized project root - ├─ loads project config - ├─ loads global prompts/schemas - ├─ starts Command Code headless run - │ DOCGEN_MODE=1 - │ DOCGEN_STAGE=discover - │ - ▼ -Command Code - │ - ├─ global SessionStart hook injects DocGen context - ├─ global write guard becomes active - ├─ global shell guard becomes active - └─ prompt delegates to global custom agent - │ - ▼ - doc-discoverer - │ - ├─ uses global core skills - ├─ uses relevant technology skills - └─ writes project-local evidence - -... subsequent stages repeat with different bounded roles ... -``` - -All LLM-backed runs execute with the current repository as the working directory. - -Global reusable assets remain outside the repository. - ---- - -# State machine - -Conceptually: - -```text -UNINITIALIZED - │ - │ docgen init - ▼ -INITIALIZED - │ - │ discover - ▼ -DISCOVERED - │ - │ analyze - ▼ -MODELLED - │ - │ plan - ▼ -PLANNED - │ - │ generate - ▼ -GENERATED - │ - │ audit - ▼ -AUDITED - │ - ├─────────────► VERIFIED - │ - └─────────────► NEEDS_FIX ──► fix ──► audit -``` - -Current stage state is recorded under: - -```text -.docgen/state/state.json -``` - -Every headless run also creates metadata under: - -```text -.docgen/runs/ -``` - ---- - -# Global versus project-local files - -## Global reusable engine - -```text -~/.commandcode/ -├── agents/ -│ ├── doc-discoverer.md -│ ├── doc-architect.md -│ ├── doc-planner.md -│ ├── doc-writer.md -│ └── doc-auditor.md -│ -├── skills/ -│ ├── doc-*/ -│ ├── tech-*/ -│ └── domain-*/ -│ -├── commands/ -│ └── docgen-*.md -│ -├── settings.json -│ -└── docgen/ - ├── VERSION - ├── package.json - ├── installation.json - ├── bin/ - │ └── docgen.mjs - ├── hooks/ - ├── prompts/ - ├── schemas/ - └── project-template/ -``` - -## Project-specific workspace - -```text -/.docgen/ -├── project.json -├── config/ -├── evidence/ -├── model/ -├── plan/ -├── audit/ -├── state/ -└── runs/ -``` - -Published output: - -```text -/docs/ -``` - -## Optional project overrides - -A project may intentionally add: - -```text -/.commandcode/ -├── agents/ -├── skills/ -└── commands/ -``` - -or project-specific prompt/schema overrides: - -```text -/.docgen/prompts/ -/.docgen/schemas/ -``` - -Those override or supplement global behavior without copying the complete engine. - ---- - -# Agents - -## `doc-discoverer` - -Purpose: - -```text -source → evidence -``` - -Responsibilities include identifying: - -- repository structure; -- build system; -- modules; -- application entry points; -- HTTP endpoints; -- persistence access; -- database objects when visible in source; -- messaging producers and consumers; -- external integrations; -- configuration; -- scheduled/background jobs; -- security boundaries. - -It should not produce polished user-facing documentation. - -Primary output: - -```text -.docgen/evidence/** -``` - -## `doc-architect` - -Purpose: - -```text -evidence → normalized system model -``` - -Responsibilities: - -- components; -- responsibilities; -- relationships; -- dependencies; -- data ownership; -- workflows; -- state transitions; -- failure boundaries; -- unresolved behavior. - -Primary output: - -```text -.docgen/model/** -``` - -## `doc-domain-analyst` - -Purpose: - -```text -technical evidence + architecture → business semantics + flows + catalogs -``` - -Responsibilities: - -- actors and business capabilities; -- domain concepts; -- business rules and validations; -- decisions and branch conditions; -- lifecycles and invariants; -- business, control, request, traffic, data and event flows; -- complete endpoint inventory; -- Kafka/RabbitMQ/queue/stream handler and producer inventory; -- external/internal/cloud service and dependency inventory; -- data stores and scheduled jobs. - -Primary outputs: - -```text -.docgen/model/business.json -.docgen/model/flows.json -.docgen/model/catalogs.json -``` - -## `doc-planner` - -Purpose: - -```text -system model → documentation information architecture -``` - -It decides: - -- which pages should exist; -- page type; -- target audience; -- stable page IDs; -- page paths; -- evidence/model inputs; -- required sections; -- useful diagrams. - -Primary output: - -```text -.docgen/plan/manifest.json -``` - -## `doc-writer` - -Purpose: - -```text -one manifest entry + bounded evidence → one page -``` - -The writer is intentionally scoped to exactly one page per generation run. - -Primary output: - -```text -docs/**/*.md -``` - -It is also used by the repair stage, where its scope is constrained by audit findings. - -## `doc-auditor` - -Purpose: - -```text -document claims → evidence verification -``` - -It checks for: - -- unsupported claims; -- incorrect claims; -- inference presented as fact; -- stale or broken references; -- terminology inconsistencies; -- contradictions; -- missing caveats; -- misleading diagrams. - -Primary output: - -```text -.docgen/audit/** -``` - ---- - -# Skills - -Skills encode repeatable procedures and technology/domain knowledge. - -The intended relationship is: - -```text -AGENT = who performs a role -SKILL = how a capability should be performed -ORCHESTRATOR = when and in what order work runs -``` - -## Core documentation skills - -```text -doc-evidence-contract -doc-repository-discovery -doc-architecture-analysis -doc-workflow-analysis -doc-business-analysis -doc-flow-analysis -doc-data-model-analysis -doc-api-catalog -doc-messaging-catalog -doc-integration-catalog -doc-page-planning -doc-concept-writing -doc-guide-writing -doc-reference-writing -doc-mermaid -doc-claim-verification -``` - -## Technology skills - -```text -tech-java -tech-maven -tech-jakarta-rest-jersey -tech-hk2 -tech-mybatis -tech-jpa -tech-postgresql -tech-kafka -tech-rabbitmq -tech-camunda -tech-kubernetes -``` - -## Domain skill - -```text -domain-cpq-order -``` - -Technology and domain skills are interpretation aids, not replacement sources of truth. - -For example, a domain skill may recognize that several states resemble a quote lifecycle, but it must not claim an industry-standard lifecycle exists unless repository evidence supports that conclusion. - ---- - -# Global slash commands - -The global installer provides: - -```text -/docgen-init -/docgen-doctor -/docgen-discover -/docgen-analyze -/docgen-semantics -/docgen-plan -/docgen-generate -/docgen-audit -/docgen-fix -/docgen-update -/docgen-status -``` - -These commands are wrappers around the global `docgen` orchestrator. - -That is deliberate. - -Interactive commands should not bypass: - -- the workflow state machine; -- `DOCGEN_MODE`; -- conditional hooks; -- configured turn budgets; -- run metadata; -- validation; -- bounded generation. - -Examples: - -```text -/docgen-init -``` - -```text -/docgen-discover src/main/java -``` - -```text -/docgen-analyze -``` - -```text -/docgen-semantics -``` - -```text -/docgen-plan -``` - -```text -/docgen-preflight -``` - -```text -/docgen-resume -``` - -```text -/docgen-generate quote-lifecycle -``` - -```text -/docgen-audit quote-lifecycle -``` - -```text -/docgen-fix quote-lifecycle -``` - -```text -/docgen-update src/main/java/com/example/QuoteService.java -``` - -```text -/docgen-status -``` - -For deterministic automation and CI, use the `docgen` CLI directly. - ---- - -# CLI command reference - -## `docgen init [repository]` - -Initialize project-local DocGen state. - -```bash -docgen init -``` - -```bash -docgen init /path/to/repository -``` - -Optional: - -```bash -docgen init --force -``` - -## `docgen doctor` - -Check: - -- project initialization; -- global engine structure; -- Command Code executable discovery; -- Command Code version invocation; -- required headless flags; -- global DocGen skill loading; -- authentication readiness. - -```bash -docgen doctor -``` - -Machine-readable report: - -```text -.docgen/state/compatibility.json -``` - -## `docgen doctor --global` - -Check global engine installation without requiring an initialized repository. - -```bash -docgen doctor --global -``` - -## `docgen version` - -```bash -docgen version -``` - -## `docgen where` - -Print: - -- engine home; -- Command Code home; -- detected project root. - -```bash -docgen where -``` - -## `docgen status` - -Show stage state, generated page counts, and audit summary when available. - -```bash -docgen status -``` - -## `docgen migrate` - -Add newly introduced default configuration keys while preserving existing project-specific values. This is useful after upgrading the global engine. - -```bash -docgen migrate -``` - -## `docgen validate` - -Validate global/project structure and generated artifacts available in the current repository. - -```bash -docgen validate -``` - -## `docgen discover [scope]` - -Extract source-grounded evidence. - -```bash -docgen discover -``` - -```bash -docgen discover src/main/java -``` - -```bash -docgen discover "quote and pricing modules" -``` - -Output: - -```text -.docgen/evidence/** -``` - -## `docgen analyze [scope]` - -Build or reconcile the normalized system model. - -```bash -docgen analyze -``` - -```bash -docgen analyze "quote lifecycle" -``` - -Output: - -```text -.docgen/model/system.json -``` - -## `docgen semantics` - -Extract repository-specific business semantics, distinct flow models, and exhaustive interface/dependency catalogs. - -```bash -docgen semantics -``` - -Outputs: - -```text -.docgen/model/business.json -.docgen/model/flows.json -.docgen/model/catalogs.json -``` - -`business.json` contains actors, capabilities, concepts, business rules, decisions, branch conditions, lifecycles, invariants and use cases. - -`flows.json` separates business, control, request, traffic, data and event flows. - -`catalogs.json` inventories endpoints, message handlers/producers/consumers, external dependencies, data stores and scheduled jobs. - -## `docgen plan` - -Create or reconcile the documentation manifest. - -```bash -docgen plan -``` - -Output: - -```text -.docgen/plan/manifest.json -``` - -## `docgen preflight` - -Normalize and validate the entire manifest before page generation: - -```powershell -docgen preflight -``` - -This is the recommended command immediately after `docgen plan` when running stages manually. - -## `docgen resume` - -Continue a failed or interrupted full pipeline from existing checkpoints: - -```powershell -docgen resume -``` - -It skips completed stages, valid pages, and current audits. - -## `docgen generate ` - -Generate exactly one page. - -```bash -docgen generate quote-lifecycle -``` - -## `docgen generate --all` - -Generate all pages using bounded batches. Every page is validated independently, and only missing/invalid pages fall back to individual generation. - -```bash -docgen generate --all -``` - - -## `docgen enrich ` - -Run the explicit depth-and-completeness pass for one existing generated page. - -```bash -docgen enrich quote-lifecycle -``` - -This is normally automatic only for pages that fail deterministic local quality gates under the `comprehensive` profile. - -## `docgen enrich --all` - -Run targeted enrichment across all manifest pages that currently fail local quality gates, using bounded batches. - -```bash -docgen enrich --all -``` - -## `docgen quality` - -Evaluate generated pages against local structural/depth gates and the configured audit severity thresholds. - -```bash -docgen quality -``` - -The command also writes: - -```text -.docgen/audit/quality-summary.json -``` - - -## `docgen audit ` - -Audit exactly one page. - -```bash -docgen audit quote-lifecycle -``` - -## `docgen audit --all` - -Audit all generated pages in bounded batches while preserving one independent report per page. Current reports are skipped when their `pageHash` matches. - -```bash -docgen audit --all -``` - -## `docgen fix ` - -Repair one page from its current audit findings. - -```bash -docgen fix quote-lifecycle -``` - -Recommended sequence: - -```bash -docgen audit quote-lifecycle -docgen fix quote-lifecycle -docgen audit quote-lifecycle -``` - -## `docgen snapshot` - -Create source fingerprints for incremental update detection. - -```bash -docgen snapshot -``` - -## `docgen changed` - -List paths changed since the last snapshot. - -```bash -docgen changed -``` - -## `docgen update [paths...]` - -Perform impact analysis and bounded regeneration. - -Automatic changed-path detection: - -```bash -docgen update -``` - -Explicit paths: - -```bash -docgen update src/main/java/com/acme/quote/QuoteService.java -``` - -## `docgen all [--fresh]` - -The default is resumable. Use `--fresh` only to deliberately regenerate all stage/page outputs. - - -Run the complete initial pipeline: - -```bash -docgen all -``` - -Equivalent conceptually to: - -```text -discover → analyze → semantics → preflight → batched generation → targeted enrichment → batched audit → repair/re-audit → quality → snapshot -``` - ---- - -# Fail-fast preflight and canonical paths - -`docgen preflight` validates the complete documentation plan before any page-generation LLM request is sent. - -It performs these deterministic checks: - -- canonicalizes every target to `docs/**/*.md`; -- adds a missing `docs/` prefix; -- adds a missing `.md` extension; -- rejects traversal and targets outside `docs/`; -- resolves evidence artifact IDs through `.docgen/evidence/index.json`; -- resolves normalized model names such as `system` to `.docgen/model/system.json`; -- verifies every evidence/model input exists; -- detects duplicate page ids and output paths; -- verifies navigation and related-page references; -- evaluates conditional coverage requirements. - -The result is written to: - -```text -.docgen/plan/preflight.json -``` - -A failed preflight stops immediately. It does not begin page 1 of 59 and discover a path/input mismatch hours later. - -Example normalization: - -```text -manifest input: orientation/overview -canonical path: docs/orientation/overview.md -``` - -Run it explicitly after planning: - -```powershell -docgen plan -docgen preflight -docgen generate --all -``` - -`docgen all` and `docgen resume` invoke the same preflight automatically. - -# Contract firewall and transactional artifacts - -Prompt instructions are soft constraints. v1.0.0 therefore does not trust an LLM to reproduce an exact JSON spelling or output-path notation. - -## Single representation principle - -LLM output may contain aliases during the uncommitted stage, but the committed artifact contains only canonical fields. Examples: - -```text -UNCOMMITTED COMMITTED -files[] artifacts[] -services[] components[] -rules[] businessRules[] -handlers/consumers messageHandlers[] -outputPath path -issues[] findings[] -``` - -This prevents a downstream cheap model from choosing a stale alias over the authoritative value. - -## Idempotence invariant - -Every normalizer must satisfy: - -```text -normalize(normalize(x)) == normalize(x) -``` - -Without this invariant, repeatedly validating a flow model could duplicate request/data/event flows. The built-in contract suite tests this behavior. - -## Stage transaction - -The following stages use snapshot/normalize/validate/commit semantics: - -- `discover`; -- `analyze`; -- `semantics`; -- `plan` and coverage repair; -- `update-impact`. - -Provider failure, malformed JSON and incompatible semantic shapes all follow the same rollback path. - -## Quarantine - -Rejected raw or partial output is retained for diagnosis: - -```text -.docgen/quarantine/ -└── -/ - ├── - └── error.json -``` - -The previous valid artifact is restored before the command exits. - -## Checkpoint validation and dependency invalidation - -`docgen resume` performs real artifact validation, not only status inspection. If `system.json` is invalid, analysis reruns and forces semantics and planning to rerun. It does not continue with stale downstream models. - -## Page input fingerprints - -`.docgen/state/pages.json` records `generateInputHash` for each page. The hash covers the normalized page contract and every declared evidence/model file. A structurally valid Markdown file is not considered current when its inputs have changed. - -For migration, a valid page without an old input hash is adopted once. This avoids repaying the generation cost merely to create the new checkpoint metadata. - -## Audit input fingerprints - -An audit report is current only when both are equal: - -```text -report.pageHash == current Markdown hash -report.inputHash == current page/evidence/model input hash -``` - -Thus a page whose text is unchanged but whose underlying architecture model changed is audited again. - -## Contract commands - -```powershell -docgen contract-test # zero-token deterministic regression suite -docgen validate # contract suite + static/generated artifact validation -docgen doctor # runtime compatibility + contract suite -``` - -# Resumability, batching, and checkpoints - -v1.0.0 is resumable by default. - -```powershell -docgen resume -``` - -is equivalent to continuing the full pipeline while reusing valid checkpoints. `docgen all` uses the same behavior unless `--fresh` is supplied. - -The orchestrator reuses: - -- completed evidence discovery when `index.json` exists; -- completed technical analysis when `system.json` exists; -- completed semantics when `business.json`, `flows.json`, and `catalogs.json` exist; -- a completed plan when manifest preflight passes; -- generated Markdown pages that pass structural validation; -- audit reports whose `pageHash` matches the current page content. - -Per-page state is stored in: - -```text -.docgen/state/pages.json -``` - -A provider failure in generation batch 7 does not invalidate batches 1-6. Rerunning `docgen resume` skips their valid pages. - -## Batched request strategy - -Defaults: - -```json -{ - "execution": { - "generateBatchSize": 4, - "enrichBatchSize": 4, - "auditBatchSize": 6, - "resumeByDefault": true, - "skipValidPages": true - } -} -``` - -For a 59-page plan, the base request count changes approximately from: - -```text -v0.4.x worst-case baseline -59 generate + 59 enrich + 59 audit = 177 LLM runs -``` - -into: - -```text -v1.0.0 default baseline -ceil(59 / 4) generation batches = 15 -ceil(59 / 6) audit batches = 10 -enrichment = only pages failing local quality gates -``` - -A batch that produces only three of four pages does not restart all four. DocGen validates every target and retries only the missing/invalid page individually. - -Use a clean rerun only when intentionally discarding checkpoints: - -```powershell -docgen all --fresh -``` - -# Rate limits, retries, and provider failures - -Command Code documents rate-limit failures as exit code `5`; connection failures use `6`, API 5xx failures use `7`, and max-turn exhaustion uses `8`. v1.0.0 handles `5`, `6`, and `7` as retryable by default. It also detects common provider text such as `429`, `rate limit exceeded`, `too many requests`, and `quota exceeded` when a provider reports a generic exit code. - -Default policy: - -```json -{ - "retry": { - "enabled": true, - "maxAttempts": 4, - "retryableExitCodes": [5, 6, 7], - "initialDelaySeconds": 15, - "rateLimitDelaySeconds": 30, - "maxDelaySeconds": 120, - "multiplier": 2, - "jitterRatio": 0.2, - "countdownSeconds": 10, - "interRequestDelaySeconds": 3 - } -} -``` - -During cooldown the terminal remains explicit: - -```text -[docgen] retryable rate-limited on generate:overview; retry 2/4 after ~30s. -[docgen] retry cooldown (rate-limited): 30s remaining -[docgen] retry cooldown (rate-limited): 20s remaining -[docgen] retry cooldown (rate-limited): 10s remaining -``` - -Each attempt has separate metadata, stdout, and stderr logs under `.docgen/runs/`. - -DocGen remains serial by default. This aligns with Command Code guidance to reduce concurrent sessions when rate limited. Batching reduces request count without introducing parallel provider pressure. - -Max-turn exhaustion (`8`) is not blindly retried because a fresh retry may duplicate partial writes. Increase the stage turn budget or resume after inspecting the generated artifacts. - -## Stage timeouts - -A living child process is not allowed to run forever. Default timeouts are configurable: - -```json -{ - "execution": { - "stageTimeoutMinutes": { - "default": 20, - "discover": 35, - "analyze": 35, - "semantics": 35, - "plan": 25, - "generate": 20, - "enrich": 15, - "audit": 15, - "fix": 15 - } - } -} -``` - -A timeout terminates the child process, records exit classification `stage-timeout`, preserves completed files/checkpoints, and allows `docgen resume`. - -# Evidence model - -The evidence stage exists to prevent polished prose from becoming detached from source reality. - -A useful evidence artifact records: - -- stable fact identifier; -- fact kind; -- classification; -- source path; -- source symbol when available; -- source location when available; -- structured data; -- confidence or uncertainty. - -Conceptual example: - -```json -{ - "factId": "http.quote.create", - "kind": "http_endpoint", - "classification": "FACT", - "source": { - "path": "src/main/java/com/acme/quote/QuoteResource.java", - "symbol": "QuoteResource#createQuote" - }, - "data": { - "method": "POST", - "path": "/quotes" - } -} -``` - -Published documentation should be synthesized from evidence and model artifacts, not from memory alone. - ---- - -# FACT, INFERENCE, and UNKNOWN - -DocGen uses three epistemic categories. - -## FACT - -Directly supported by source evidence. - -Example: - -```text -POST /quotes is declared by a Jakarta REST resource. -``` - -## INFERENCE - -A reasonable architectural interpretation derived from multiple facts but not explicitly declared as a formal rule. - -Example: - -```text -The Quote component appears to own quote lifecycle coordination. -``` - -An inference should retain supporting evidence. - -## UNKNOWN - -Evidence is insufficient. - -Example: - -```text -It is unknown whether submitted quotes are immutable because no enforcement rule was found in the inspected scope. -``` - -The important invariant is: - -```text -UNKNOWN must not be converted into a confident claim merely to make documentation sound complete. -``` - ---- - -# Documentation manifest and bounded generation - -The planner writes: - -```text -.docgen/plan/manifest.json -``` - -A manifest entry defines a page contract, conceptually: - -```json -{ - "id": "quote-lifecycle", - "path": "docs/concepts/quote-lifecycle.md", - "type": "concept", - "inputs": { - "evidence": [ - ".docgen/evidence/quote/**" - ], - "models": [ - ".docgen/model/workflows/quote-lifecycle.json" - ] - } -} -``` - -Generation is page-bounded: - -```text -manifest page A → Command Code run A → docs/page-a.md -manifest page B → Command Code run B → docs/page-b.md -manifest page C → Command Code run C → docs/page-c.md -``` - -This improves: - -- context focus; -- failure isolation; -- reproducibility; -- auditability; -- incremental regeneration; -- review quality. - ---- - -# Audit and repair - -The writer is not assumed to be correct merely because it produced fluent documentation. - -Audit is a separate stage: - -```text -page - │ - ▼ -claims - │ - ▼ -evidence/model verification - │ - ├─ supported - ├─ unsupported - ├─ contradicted - ├─ overstated inference - └─ unresolved -``` - -Per-page audit files are stored under: - -```text -.docgen/audit/pages/ -``` - -The repair flow is: - -```text -audit - ↓ -findings - ↓ -fix exact page - ↓ -re-audit -``` - -Audit should not silently modify the page it is evaluating. - ---- - -# Incremental regeneration - -After a stable generation: - -```bash -docgen snapshot -``` - -Later: - -```bash -docgen changed -``` - -Then: - -```bash -docgen update -``` - -Flow: - -```text -source changes - │ - ▼ -fingerprint comparison - │ - ▼ -changed paths - │ - ▼ -impact analysis - │ - ▼ -.docgen/plan/update-plan.json - │ - ├─ affected evidence scopes - ├─ affected models - └─ affected page IDs - │ - ▼ - bounded rediscovery - │ - ▼ - model reconciliation - │ - ▼ - plan reconciliation - │ - ▼ - regenerate affected pages - │ - ▼ - re-audit - │ - ▼ - new snapshot -``` - -Incremental update is intentionally impact-driven rather than regenerating every page after every source change. - ---- - -# Configuration - -Project configuration is stored at: - -```text -.docgen/config/documentation.json -``` - -This is repository-specific. - -Typical concerns include: - -- project name; -- documentation audience; -- output root; -- preferred page categories; -- Command Code executable override; -- trust behavior; -- onboarding behavior; -- permission mode; -- max turns per stage; -- global/default model; -- per-stage models. - -The repository also owns: - -```text -.docgen/config/glossary.md -.docgen/config/style-guide.md -``` - -Use the glossary for project/domain terminology that should not be assumed globally. - -Use the style guide for project-specific output conventions. - ---- - -# Model and turn-budget configuration - -The orchestrator invokes Command Code headlessly. - -Default effective arguments are built from project config and typically include: - -```text --p ---trust ---skip-onboarding ---yolo ---max-turns -``` - -A typical configuration is: - -```json -{ - "commandCode": { - "trust": true, - "skipOnboarding": true, - "yolo": true, - "maxTurns": { - "default": 20, - "discover": 30, - "analyze": 30, - "plan": 20, - "generate": 20, - "audit": 20, - "fix": 20, - "update-impact": 20 - } - } -} -``` - -## Global model for all stages - -```json -{ - "commandCode": { - "model": "provider/model-id" - } -} -``` - -## Per-stage models - -```json -{ - "commandCode": { - "stageModels": { - "discover": "provider/model-a", - "analyze": "provider/model-b", - "generate": "provider/model-c", - "audit": "provider/model-d" - } - } -} -``` - -## Environment overrides - -Executable: - -```bash -DOCGEN_COMMAND_CODE_BIN=/custom/path/to/cmd docgen doctor -``` - -Model: - -```bash -DOCGEN_MODEL=provider/model-id docgen generate quote-lifecycle -``` - -Turn budget: - -```bash -DOCGEN_MAX_TURNS=50 docgen analyze -``` - -Environment overrides are useful for temporary execution changes without editing repository config. - ---- - -# Safety model and hooks - -DocGen installs hooks globally but they are intentionally inert during normal Command Code use. - -The activation condition is: - -```text -DOCGEN_MODE=1 -``` - -Normal Command Code session: - -```text -DOCGEN_MODE absent - │ - ▼ -DocGen hooks do nothing -``` - -Orchestrated DocGen stage: - -```text -docgen command - │ - ▼ -orchestrator starts Command Code - │ - ├─ DOCGEN_MODE=1 - ├─ DOCGEN_STAGE= - └─ DOCGEN_TARGET= - │ - ▼ -conditional global hooks activate -``` - -## Write guard - -During DocGen mode, writes are allowed only under: - -```text -docs/** -.docgen/** -``` - -Writes to application source or unrelated repository files are denied. - -## Shell guard - -During DocGen mode, shell use is conservative and inspection-oriented. - -Examples of expected read-only usage include: - -```text -rg -grep -find -fd -ls -dir -tree -cat -head -tail -git status -git log -git show -git diff -git rev-parse -git ls-files -java -version -mvn -version -node --version -``` - -Known mutating commands and shell chaining/redirection are blocked in DocGen mode. - -## Artifact validation hook - -After writes, basic validation checks include: - -- JSON parse validity; -- non-empty Markdown; -- balanced Markdown code fences; -- H1 presence for published pages. - -These checks are structural, not a replacement for semantic audit. - ---- - -# Project overrides and precedence - -The default installation is global, but Command Code supports project-level extensions. - -This is useful when one repository needs special behavior. - -## Skill override - -Global: - -```text -~/.commandcode/skills/tech-mybatis/ -``` - -Project-specific override: - -```text -/.commandcode/skills/tech-mybatis/ -``` - -A project-level skill can encode repository-specific conventions such as: - -- custom mapper locations; -- internal base mapper patterns; -- tenant interceptors; -- stored procedure conventions. - -## Custom project skill - -```text -/.commandcode/skills/domain-company-order/ -└── SKILL.md -``` - -## Project agent override - -```text -/.commandcode/agents/doc-architect.md -``` - -Use this only when the project genuinely needs a different role definition. - -## Prompt override - -Global default: - -```text -~/.commandcode/docgen/prompts/generate.md -``` - -Project override: - -```text -/.docgen/prompts/generate.md -``` - -The orchestrator resolves the project override first when it exists. - -## Schema override - -Global default: - -```text -~/.commandcode/docgen/schemas/manifest.schema.json -``` - -Project override: - -```text -/.docgen/schemas/manifest.schema.json -``` - -Schema overrides should be used carefully because they can change engine contracts. - -## Configuration precedence - -Conceptually: - -```text -engine defaults - │ - ▼ -project .docgen/config - │ - ▼ -environment overrides -``` - -For Command Code user/project extension files, Command Code's own scope precedence applies. - ---- - -# Large repositories and monorepos - -Do not immediately run the broadest possible discovery on a very large repository. - -Prefer bounded scopes: - -```bash -docgen discover services/quote-service -docgen discover services/pricing-service -docgen discover libs/contracts -``` - -Then reconcile: - -```bash -docgen analyze "quote and pricing domain" -docgen plan -``` - -For very large systems, use hierarchical documentation: - -```text -repository/module evidence - │ - ▼ -service models - │ - ▼ -domain synthesis - │ - ▼ -system-level documentation -``` - -A useful repository hierarchy is: - -```text -docs/ -├── system/ -├── domains/ -├── services/ -├── integrations/ -├── concepts/ -├── guides/ -├── operations/ -└── reference/ -``` - -The planner should derive the useful information architecture from the actual system rather than mechanically generating one page per file or class. - ---- - -# Git and team workflow - -There are several valid choices for `.docgen/` artifacts. - -## Option A: commit everything - -Useful when evidence and model artifacts are part of an auditable engineering process. - -```text -commit: -.docgen/config/** -.docgen/evidence/** -.docgen/model/** -.docgen/plan/** -.docgen/audit/** -docs/** -``` - -## Option B: commit config, manifest, audit, and docs - -Treat evidence/model as rebuildable intermediate artifacts. - -## Option C: commit only docs and project configuration - -Best when intermediate artifacts are too noisy or large. - -The reusable engine itself is global and is not copied into the repository by default. - -For a team that needs exact engine reproducibility inside the repository, use the explicit self-contained project-local mode described later. - ---- - - -## Automatic additive migration from v0.3.x project config - -When v1.0.0 runs inside a repository initialized by an older global-first release, it additively merges new defaults into `.docgen/config/documentation.json`. Existing custom scalar values and existing array entries are preserved; new page types, audiences, semantics turn-budget defaults, Mermaid-only quality settings, and knowledge-base settings are added. The project marker is updated to the current kit version. - -You can run the migration explicitly: - -```bash -docgen migrate -``` - -This avoids `docgen init --force`, which could overwrite project-owned configuration. - -# Upgrade - -## Migrating from v0.1.x project-local installs - -Version 0.1.x installed the complete engine inside each repository. Version 0.6.0 defaults to a global engine. - -Recommended migration: - -```bash -# 1. Install v1.0.0 globally once -node install.mjs --force - -# 2. Enter an existing v0.1.x repository -cd /path/to/repository - -# 3. Add the v1.0.0 project marker/template without replacing existing config -docgen init - -# 4. Verify the global runtime -docgen doctor -``` - -The old project-local `.commandcode/`, `.docgen/prompts/`, `.docgen/schemas/`, and `scripts/docgen.*` files are not automatically deleted because they may contain local modifications. After verification, remove or archive the old copied engine files deliberately if you want the repository to use only the global engine. - -Be aware that project-level Command Code agents/skills/commands can override user-level global definitions. Therefore, leaving old v0.1.x `.commandcode/` copies in place may intentionally or unintentionally keep the old behavior for that repository. - -### Clean global-first target after migration - -Usually keep: - -```text -.docgen/config/** -.docgen/evidence/** -.docgen/model/** -.docgen/plan/** -.docgen/audit/** -.docgen/state/** -.docgen/runs/** -docs/** -``` - -Usually remove only after review if they are unmodified v0.1.x engine copies: - -```text -.commandcode/agents/doc-* -.commandcode/skills/doc-* -.commandcode/skills/tech-* -.commandcode/skills/domain-* -.commandcode/commands/docgen-* -.commandcode/hooks/docgen-* -.docgen/prompts/** -.docgen/schemas/** -scripts/docgen.* -``` - -Do not blindly delete project `.commandcode/` content that belongs to the repository rather than DocGen. - -## Upgrade the global engine - -Extract a newer release and run: - -```bash -node install.mjs --force -``` - -This updates the global reusable components. - -Conflicting global files are backed up under: - -```text -~/.commandcode/docgen-backup// -``` - -Then verify: - -```bash -docgen doctor --global -``` - -For each important repository: - -```bash -cd /path/to/repository -docgen init -``` - -Running `init` again adds missing project-template files without replacing existing project configuration by default. - -Then: - -```bash -docgen doctor -``` - -## Why upgrade is now simpler - -Global-first: - -```text -one global engine upgrade - │ - ├─ repo A keeps its state - ├─ repo B keeps its state - └─ repo C keeps its state -``` - -The reusable engine changes once; repository evidence and documentation state remain independent. - ---- - -# Uninstall - -Remove the global engine and DocGen-owned global extension files: - -```bash -node uninstall.mjs -``` - -Dry run: - -```bash -node uninstall.mjs --dry-run -``` - -The uninstaller removes: - -- DocGen global agents; -- DocGen global commands; -- installed DocGen skill directories; -- DocGen hook entries from global `settings.json`; -- `~/.commandcode/docgen/`. - -It does **not** delete repository-local: - -```text -.docgen/ -docs/ -``` - -That is intentional because those directories contain repository-specific state and generated artifacts. - -The uninstaller also attempts `npm unlink -g commandcode-docgen-kit` unless `--no-unlink-cli` is supplied. It removes only files recorded by the DocGen installation record and does not delete unrelated user skills merely because they share a naming prefix. - ---- - -# Self-contained project-local installation - -Global-first is the default and recommended mode for one user working across many repositories. - -A fully self-contained repository mode is still available: - -```bash -node install.mjs --project-local /path/to/repository -``` - -PowerShell: - -```powershell -.\install.ps1 -ProjectLocal "C:\path\to\repository" -``` - -This mode copies: - -```text -AGENTS.md -.commandcode/ -.docgen/ -scripts/ -docs/ -``` - -into the target repository. - -Use project-local mode when: - -- the repository must carry its exact DocGen engine configuration; -- the whole team should get identical agents/skills/hooks from Git; -- a CI environment cannot rely on a preinstalled global engine; -- reproducibility is more important than repository footprint. - -Use global-first mode when: - -- one user runs DocGen across many repositories; -- you want centralized upgrades; -- you want project repositories to contain only state/config/output; -- you want global reusable technology/domain skills. - -Do not mix both modes casually in the same repository because duplicate global and project-level names can create precedence differences. - ---- - -# Troubleshooting - -## Writer created the page but DocGen says `Missing generated page` - -This was a v0.4.x manifest-path normalization defect. A manifest target such as `orientation/overview` was validated literally even though the writer correctly created `docs/orientation/overview.md`. - -Upgrade and resume: - -```powershell -.\install.ps1 -Force -cd C:\path\to\repository -docgen migrate -docgen preflight -docgen resume -``` - -Do not delete the generated page or rerun discovery. The canonical path is repaired in the manifest and the existing valid page is skipped. - -## Provider rate limit or HTTP 429 - -v1.0.0 retries automatically with visible exponential backoff. Review the attempt logs in `.docgen/runs/`. Reduce other concurrent Command Code sessions and lower batch sizes only when the provider still rejects batched requests. - -To make the policy more conservative: - -```json -{ - "execution": { - "generateBatchSize": 2, - "auditBatchSize": 3 - }, - "retry": { - "rateLimitDelaySeconds": 60, - "maxAttempts": 5, - "interRequestDelaySeconds": 5 - } -} -``` +> Version 2 is a breaking redesign. The legacy discovery/analyze/semantics/enterprise/enrich/fix/re-audit execution path is not included. +## Why version 2 exists +On large repositories, the previous design could repeatedly expose the same source, evidence, and broad model JSON to dozens of agentic calls. A documentation run could consume millions of tokens even when most work involved extracting catalogs or re-reading unchanged knowledge. -## `docgen all` appears to hang or stays quiet - -v1.0.0 prints a heartbeat while every Command Code child process is alive. You should see output similar to: - -```text -[docgen] discover:. RUNNING | elapsed 1m 20s | pid 18420 | changed artifacts 7 -``` - -Check the live run metadata and logs: +Version 2 changes the cost model: ```text -.docgen/runs/.json -.docgen/runs/.stdout.log -.docgen/runs/.stderr.log -``` - -A process that remains alive but emits no Command Code output will trigger a warning after `progress.noOutputWarningSeconds`. This warning is informational; headless model/tool execution can legitimately be quiet. - -To increase heartbeat frequency: - -```json -{ - "progress": { - "heartbeatSeconds": 5, - "noOutputWarningSeconds": 30 - } -} +repository text source + │ + ▼ +deterministic inventory and binary guard + │ + ▼ +SQLite / FTS5 semantic index + files, source chunks, symbols, endpoints, + messages, SQL/config facts, model items + │ + ▼ +bounded context compiler + relevance retrieval + deduplication + hard token cap + │ + ├──────────────► deterministic reference renderer + │ + └──────────────► selective provider reasoning/writing + │ + ▼ + deterministic traceability and risk audit ``` -When the child exits, DocGen reports a normalized classification such as `rate-limited`, `network-failure`, `api-server-error`, or `max-turns`. - - -## `docgen: command not found` - -The installer could not create or expose the npm link, or your global npm bin directory is not on PATH. +The pipeline no longer asks a parent session to delegate to custom agents. Provider prompts execute directly, cannot broadly scan source or model directories, and are constrained by read hooks. -Use the direct engine: +## Requirements -```bash -node ~/.commandcode/docgen/bin/docgen.mjs version -``` +- Node.js **22.5+** for `node:sqlite`; +- Git; +- Command Code CLI, authenticated for provider-backed stages; +- npm only for the global CLI link performed by the installer. -Windows: +DocGen has no runtime npm dependencies. SQLite and FTS5 use Node's built-in `node:sqlite` module. -```powershell -node "$env:USERPROFILE\.commandcode\docgen\bin\docgen.mjs" version -``` +## Install -Or reinstall without hiding npm output and check `npm link`: +### Global-first ```bash node install.mjs --force ``` -## Repository not initialized - -Error conceptually: +The installer copies the modular engine to `~/.commandcode/docgen`, installs slash-command definitions and hooks, links the `docgen` executable, and removes backed-up managed v1 files. -```text -This repository is not initialized for DocGen. -``` - -Run: +Initialize a new repository: ```bash +cd /path/to/repository docgen init -``` - -## Command Code executable not found - -Verify: - -```bash -cmd --version -``` - -Native Windows: - -```powershell -cmdc --version -``` - -Or configure: - -```json -{ - "commandCode": { - "executable": "/custom/path/to/cmd" - } -} -``` - -Temporary override: - -```bash -DOCGEN_COMMAND_CODE_BIN=/custom/path/to/cmd docgen doctor -``` - -## Not authenticated - -Run: - -```bash -cmd login -``` - -or: - -```powershell -cmdc login -``` - -Then: - -```bash docgen doctor +docgen all ``` -## Skill loading failure - -Run: +### Project-local ```bash -cmd skills list --debug +node install.mjs --project-local /path/to/repository --force ``` -DocGen doctor also checks that expected global DocGen skills are visible. - -## Max-turn exhaustion - -Increase the relevant stage budget: - -```json -{ - "commandCode": { - "maxTurns": { - "analyze": 50 - } - } -} -``` +Project-local installation keeps the engine under `.commandcode/docgen` and preserves an existing `.docgenignore`. -Or temporarily: +## Migrate a v1 repository ```bash -DOCGEN_MAX_TURNS=50 docgen analyze -``` - -## Hooks affect normal coding - -DocGen hooks are designed to be inert unless: - -```text -DOCGEN_MODE=1 +node install.mjs --force +cd /path/to/existing/repository +docgen migrate +docgen doctor +docgen all ``` -If a custom modification accidentally removed that condition, restore the official hook files or reinstall the kit. - -## A project needs a different technology convention +`docgen migrate`: -Create a project-level skill override under: +- preserves `docs/**`; +- preserves `.docgenignore`; +- preserves selected project name, Command Code runtime/model, and ignore settings; +- archives old `.docgen` workflow artifacts under `.docgen/migration-backup//`; +- installs the v2 config and state contract. -```text -.commandcode/skills//SKILL.md -``` - -or create a new project-specific skill with a unique name. - -## Generated docs contain unsupported claims +It does not attempt to run v1 evidence/checkpoints through the v2 engine. -Run: +## Repository pipeline ```bash -docgen audit --all -``` - -Then inspect: - -```text -.docgen/audit/ +docgen index +docgen model +docgen plan +docgen generate +docgen audit +docgen publish ``` -Repair specific pages: +Or: ```bash -docgen fix -docgen audit -``` - ---- - -# Extending DocGen - -## Add a global skill - -Create: - -```text -~/.commandcode/skills/my-docgen-skill/ -└── SKILL.md -``` - -A skill should have required frontmatter: - -```markdown ---- -name: my-docgen-skill -description: Explain what this capability does and when it should be used. ---- -``` - -Keep skills focused. - -Good examples: - -```text -tech-grpc -tech-opentelemetry -tech-liquibase -tech-flyway -tech-redis -domain-billing -``` - -## Add a project-only skill - -Create: - -```text -/.commandcode/skills/company-order-conventions/ -└── SKILL.md -``` - -This is appropriate for internal conventions that should not apply to every repository. - -## Add a global custom agent - -Create: - -```text -~/.commandcode/agents/.md -``` - -Use a new agent only when there is a genuinely separate role with a distinct context/tool boundary. - -Avoid creating one agent per trivial task. - -## Add a custom page type - -Typical steps: - -1. add or refine a writing skill; -2. update planner guidance; -3. optionally add a project-specific prompt override; -4. ensure the manifest can describe the required inputs; -5. keep generation page-bounded; -6. audit the new page type. - -## Add a technology pack - -A technology pack may contain one or more skills: - -```text -tech-grpc -tech-protobuf -tech-openapi -tech-opentelemetry +docgen all +# content-hash resumable alias +docgen resume ``` -The discovery and architecture agents can then use those skills when repository evidence indicates the technology is present. +### 1. Index -## Add a domain pack - -Domain packs should help interpretation without becoming a false source of truth. - -For example: +`docgen index` constructs: ```text -domain-product-catalog -domain-quote -domain-order -domain-pricing -domain-fulfillment +.docgen/index/ +├── inventory.json +├── source-files.txt +└── semantic.db ``` -Domain knowledge may suggest questions and relationships to inspect. It must not invent system behavior. +The SQLite database contains: ---- +- included files and content hashes; +- overlapping source chunks; +- discovered symbols/functions; +- JAX-RS and Spring HTTP annotations; +- Kafka/RabbitMQ channels and listeners; +- configuration keys; +- SQL table references; +- scheduled/security annotations; +- typed model items; +- content-addressed context metadata. -# Included technology and domain coverage +Indexing is incremental at file-hash granularity. -The initial kit includes focused knowledge for: +### 2. Model -## Java and build +`docgen model` uses two bounded provider calls: -```text -Java -Maven -``` +1. core models: system, business, flows, catalogs; +2. enterprise models: security, operations, testing, data governance, decisions, configuration, change impact, ownership. -## Jakarta/Jersey application structure +Core models are ingested into SQLite before enterprise synthesis, so the second call can retrieve relevant typed items without reading broad model files. -```text -Jakarta REST -Jersey -HK2 -``` +### 3. Plan -## Persistence and data +`docgen plan` receives one bounded context pack and creates `.docgen/plan/manifest.json`. -```text -MyBatis -JPA -PostgreSQL -``` +The default maximum is 30 pages. The planner is instructed to split by distinct user intent or ownership boundary, not to maximize page count. Increase `execution.maxPlannedPages` explicitly when a larger information architecture is justified. -## Messaging and workflow +### 4. Generate -```text -Kafka -RabbitMQ -Camunda 7/8 concepts -``` +`docgen generate` uses two paths: -## Platform +- deterministic rendering for endpoint, message, dependency, data-store, scheduled-job, configuration, ownership, and change-impact references; +- bounded provider writing for narrative/business/architecture/runbook/migration pages. -```text -Kubernetes -``` +Each narrative page receives its own `.docgen/context/generate/.json`. Pages are regenerated only when their page contract or selected item hashes change. -## Domain +Every page has a companion: ```text -CPQ / Quote / Order interpretation +.docgen/traceability/pages/.json ``` -The engine is not limited to these technologies. Missing technologies can be added as skills without changing the core workflow. - ---- - -# Known limitations - -## LLM correctness is not guaranteed - -The evidence/audit architecture reduces risk but cannot guarantee perfect documentation. - -Human review remains appropriate for: - -- critical architecture claims; -- security behavior; -- regulatory or compliance assertions; -- business rules; -- operational runbooks; -- irreversible decisions. - -## Static source inspection may not reveal runtime reality - -Examples: - -- dynamic configuration; -- reflection; -- runtime dependency injection; -- generated code; -- environment-specific routing; -- external infrastructure behavior; -- database-side logic not present in the repository. - -Such gaps should remain UNKNOWN unless additional evidence is supplied. - -## Incremental impact analysis is heuristic +Provider pages write the sidecar in the same invocation. Deterministic reference pages derive claims directly from typed items. -A source change can have indirect impact that is not obvious from path-level fingerprinting. +### 5. Audit -For high-risk changes, run broader discovery/analyze/audit stages. +`docgen audit` always runs deterministic checks: -## Global upgrade can change future behavior +- Markdown/frontmatter/H1 validity; +- Mermaid-only diagrams; +- traceability identity and hashes; +- duplicate claim IDs; +- FACT claims without evidence; +- evidence paths outside the canonical inventory; +- unresolved placeholders and risky absolute wording. -Because the engine is global, upgrading it affects subsequent runs across all repositories. +Only pages above `audit.llmRiskThreshold` receive a bounded LLM semantic audit. The risk audit is content-hash cached and is not repeated when page/context inputs are unchanged. There is no automatic enrich/fix/re-audit loop. -For a repository that requires an exact frozen engine version, use the self-contained project-local installation mode or version the release artifact externally. +### 6. Publish ---- - -# Compatibility notes - -The v1.0.0 architecture intentionally aligns with Command Code user-level and project-level extension scopes: +`docgen publish` is deterministic and produces: ```text -User-level reusable components: -~/.commandcode/agents/ -~/.commandcode/skills/ -~/.commandcode/commands/ -~/.commandcode/settings.json - -Project-level optional overrides: -.commandcode/agents/ -.commandcode/skills/ -.commandcode/commands/ -.commandcode/settings.json +docs/llms.txt +docs/llms-full.txt +.docgen/publish/navigation.json +.docgen/publish/search-index.json +.docgen/traceability/index.json ``` -The engine uses Command Code headless mode for automated stages. - -`docgen doctor` checks the currently installed Command Code executable rather than assuming a forever-stable CLI version. +## Token budget and telemetry -The intended compatibility states are: +Every provider run appends terminal telemetry to: ```text -GLOBAL ENGINE STRUCTURE - │ - ├─ fail → DocGen installation problem - │ - ▼ -COMMAND CODE EXECUTABLE - │ - ├─ fail → CLI installation/path problem - │ - ▼ -REQUIRED HEADLESS FLAGS - │ - ├─ fail → CLI compatibility problem - │ - ▼ -GLOBAL DOCGEN SKILLS LOAD - │ - ├─ fail → skill format/discovery problem - │ - ▼ -AUTHENTICATION - │ - ├─ not ready → login required - │ - ▼ -PROJECT INITIALIZATION - │ - ├─ missing → run docgen init - │ - ▼ -READY +.docgen/telemetry/provider-runs.jsonl ``` -For current runtime truth on your machine, run: +Inspect the budget: ```bash -docgen doctor --global +docgen budget ``` -then inside an initialized repository: +Default limits: -```bash -docgen doctor +```json +{ + "maxProviderCalls": 24, + "maxEstimatedInputTokens": 2500000, + "maxEstimatedOutputTokens": 500000, + "maxContextTokensPerCall": 80000 +} ``` ---- - -# Recommended first adoption sequence +DocGen stops before a provider call that would exceed a hard limit. -For a real repository, do not begin by generating everything and accepting it blindly. - -Recommended: +Compile and inspect a context without calling a provider: ```bash -# 1. install once -node install.mjs - -# 2. initialize repository -cd /path/to/repository -docgen init - -# 3. verify runtime -docgen doctor - -# 4. tune repository-specific terminology -edit .docgen/config/glossary.md -edit .docgen/config/style-guide.md -edit .docgen/config/documentation.json - -# 5. bounded discovery -docgen discover src/main/java - -# 6. inspect evidence -review .docgen/evidence/ - -# 7. synthesize architecture -docgen analyze - -# 8. inspect model -review .docgen/model/ - -# 9. create documentation plan -docgen plan - -# 10. inspect manifest -review .docgen/plan/manifest.json - -# 11. generate pages -docgen generate --all - -# 12. audit -docgen audit --all - -# 13. repair important findings -docgen fix -docgen audit - -# 14. baseline incremental tracking -docgen snapshot +docgen context generate "quote lifecycle submission rules" \ + --target quote-lifecycle \ + --max-tokens 30000 ``` -This workflow preserves the most important principle of the system: +The result reports selected tokens plus omitted fact/model-item counts. -> Documentation quality comes from evidence preservation, bounded synthesis, explicit contracts, and independent verification—not from one very large prompt. +## Source and privacy boundary +DocGen applies, before indexing: -# Deep system knowledge-base target +1. hard workflow exclusions such as `.git`, `.docgen`, `.commandcode`, `docs`, build outputs, and dependencies; +2. native Git ignore rules when inside a Git repository; +3. nested `.gitignore` fallback outside Git; +4. root `.docgenignore`; +5. binary extension and magic-signature detection; +6. NUL/invalid-UTF-8/control-character checks; +7. maximum text-file size. -DocGen v1.0.0 does **not** treat the two benchmark home pages as the complete target. A Mintlify-style site is a hierarchy of categories, pages, and deep sections. DocGen therefore optimizes for **breadth × depth**: +Inspect the boundary: -```text -Repository - │ - ├─ Orientation / Getting Started - ├─ Business & Domain - │ ├─ actors and capabilities - │ ├─ concepts and glossary - │ ├─ business rules and validations - │ ├─ decisions and branch conditions - │ └─ lifecycles and invariants - ├─ Architecture - │ ├─ system context - │ ├─ components/modules - │ ├─ dependencies - │ └─ deployment/runtime - ├─ Flows - │ ├─ business flows - │ ├─ control/execution flows - │ ├─ request flows - │ ├─ traffic flows - │ ├─ data flows - │ └─ event/message flows - ├─ Interfaces & Integrations - │ ├─ endpoint catalog - │ ├─ endpoint deep dives - │ ├─ message-handler catalog - │ └─ external/cloud/internal dependencies - ├─ Data & Persistence - ├─ Security & Configuration - ├─ Development Guides - ├─ Operations & Observability - └─ Troubleshooting / Reference +```bash +docgen ignore +docgen ignore path/to/file +docgen source-list +docgen source-grep "@Path" ``` -The planner creates only evidence-backed categories, but it is explicitly allowed to produce many focused pages. There is no fixed small page-count target. Complex repositories should produce substantially richer navigation than simple libraries. - -## Business and logic extraction - -The semantics stage writes `.docgen/model/business.json` containing actors, capabilities, concepts, business rules, decisions, branch conditions, lifecycles, invariants, use cases and unresolved unknowns. A source-level branch is promoted to a business rule only when it changes a domain outcome, eligibility, state, monetary result, permission, obligation or externally visible behavior. - -## Six separate flow models - -`.docgen/model/flows.json` keeps these views separate: +Provider sessions run with `DOCGEN_CONTEXT_ONLY=1`. Hooks allow only their declared context pack and stage output paths. They cannot read repository source, SQLite, broad model directories, unrelated pages, agents, or skill files. -| Flow | Answers | -|---|---| -| Business flow | What business goal, decisions and outcomes occur? | -| Control flow | What code/components execute, branch, loop or retry? | -| Request flow | How does an inbound request travel from entry point to response? | -| Traffic flow | What network/protocol/trust-boundary hops occur? | -| Data flow | Where does data originate, transform, persist and propagate? | -| Event flow | How do producers, channels and consumers interact asynchronously? | +## Configuration -## Exhaustive catalogs - -`.docgen/model/catalogs.json` contains: - -- `endpoints`: all evidenced HTTP/RPC endpoints; -- `messageHandlers`: Kafka/RabbitMQ/queue/stream producers, consumers, listeners, processors, retry and DLQ handlers; -- `externalDependencies`: internal services, third-party APIs, cloud services, identity systems, storage, databases, brokers, caches and other integrations; -- `dataStores`; -- `scheduledJobs`. - -When these arrays are non-empty, the manifest quality gate requires pages with matching coverage tags such as `endpoint-catalog`, `message-handler-catalog`, and `external-dependency-catalog`. - -## Mermaid-only diagrams - -Every generated diagram must be a fenced `mermaid` block. PlantUML, Graphviz/DOT and other diagram fences fail validation. Use focused Mermaid views rather than one unreadable mega-diagram. - -## Evidence-index compatibility fix - -The discovery contract requires: +Project configuration is stored in `.docgen/config/documentation.json`. ```json { - "schemaVersion": "1.0", - "generatedAt": "...", - "artifacts": [] + "schemaVersion": "2.0", + "budget": { + "maxProviderCalls": 24, + "maxEstimatedInputTokens": 2500000, + "maxContextTokensPerCall": 80000 + }, + "context": { + "maxTokens": { + "modelCore": 80000, + "modelEnterprise": 80000, + "plan": 50000, + "generate": 30000, + "audit": 18000 + } + }, + "execution": { + "generationBatchSize": 4, + "maxPlannedPages": 30 + }, + "audit": { + "llmEnabled": true, + "llmRiskThreshold": 50 + } } ``` -However, cheap models can still emit semantically equivalent shapes such as `files` or `entries`. v1.0.0 normalizes these variants after discovery. If no list exists, it scans `.docgen/evidence/**` and constructs canonical `artifacts[]` deterministically. The exact v0.3.0 failure: +Models can be routed by stage through `commandCode.stageModels`. + +## Commands ```text -Error: .docgen/evidence/index.json missing required key: artifacts +docgen init [directory] +docgen migrate +docgen doctor +docgen index [--force] +docgen model +docgen plan +docgen generate +docgen audit +docgen publish +docgen all +docgen resume +docgen status +docgen budget [report|reset] +docgen context [query] [--target ID] [--max-tokens N] +docgen ignore [path] +docgen source-list [substring] +docgen source-grep +docgen stats +docgen workspace ``` -is therefore handled by the orchestrator before validation. - - - -# v0.9.0 — P2 Documentation Experience and Binary Budget Guard +## Multi-repository workspace -## P2 outputs +P3 workspace support remains deterministic. It consumes validated repository models, hashes, catalogs, and ownership rather than rescanning source. -DocGen now assigns every page a primary document mode: `tutorial`, `how-to`, `explanation`, `reference`, `runbook`, `decision-record`, `migration-guide`, or `troubleshooting`. The planner records aliases, status/version/deprecation/replacement metadata, search keywords, and evidence-derived example intents. - -After generation, `docgen publish` deterministically creates or refreshes: - -- canonical YAML frontmatter on every page; -- `docs/llms.txt`; -- `docs/llms-full.txt` (bounded by configuration); -- `.docgen/publish/navigation.json`; -- `.docgen/publish/search-index.json`; -- `.docgen/publish/backlinks.json`; -- `.docgen/publish/redirects.json`; -- `.docgen/publish/orphans.json`; -- `.docgen/publish/examples.json`; -- `.docgen/publish/report.json`. +```bash +mkdir platform-docs && cd platform-docs +docgen workspace init . --name "Platform" +docgen workspace add ../catalog-service +docgen workspace add ../quote-service +docgen workspace add ../order-service +docgen workspace all +``` -`docgen all` and `docgen resume` run publishing metadata generation in the final deterministic phase. `docgen publish` itself consumes no LLM tokens. +Workspace outputs include dependency graphs, shared contracts, capability maps, journeys, request/event/data flows, ownership, and transitive change impact. -## Binary/non-text budget guard +## Skills -The canonical source inventory now excludes images, audio, video, PDFs, Office documents, archives, compiled objects, fonts, databases, keystores, executables, WebAssembly, and other binary/non-text content. Detection uses: known extensions, magic signatures, NUL-byte detection, strict UTF-8 decoding, control-character ratio, and a configurable maximum text file size. +The repository still ships reusable documentation and technology knowledge skills for manual Command Code use. The v2 pipeline does **not** automatically load them into provider contexts. This prevents repeated skill text from consuming generation tokens. -Configuration: +## Development -```json -{ - "ignore": { - "binary": { - "enabled": true, - "probeBytes": 16384, - "maxTextFileBytes": 4194304, - "controlCharacterRatio": 0.08, - "allowExtensions": [], - "denyExtensions": [] - } - } -} +```bash +cd global-template/docgen +npm run check +npm test ``` -Binary/non-text files never enter `.docgen/state/source-files.txt`, source grep, source fingerprints, change detection, traceability evidence, or LLM source reads. The read guard rejects explicit attempts to read them during `DOCGEN_MODE=1`. `.gitignore` remains authoritative; `.docgenignore` remains the user-controlled documentation exclusion layer; binary classification is an additional hard budget boundary. +CI runs syntax and regression tests on Node.js 22 and 24, followed by installer dry-run validation. -Inspect the result with: +## License -```bash -docgen ignore -docgen ignore path/to/image.png -docgen source-list -docgen publish -``` +MIT. diff --git a/VERSION b/VERSION index 3eefcb9..227cea2 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.0.0 +2.0.0 diff --git a/global-template/agents/doc-architect.md b/global-template/agents/doc-architect.md deleted file mode 100644 index 22ba8b1..0000000 --- a/global-template/agents/doc-architect.md +++ /dev/null @@ -1,25 +0,0 @@ ---- -name: "doc-architect" -description: "Use to synthesize normalized components, relationships, workflows, state transitions, ownership, and failure boundaries from existing evidence." -tools: "read_file, read_multiple_files, read_directory, glob, grep, write_file, edit_file, todo_write" ---- -You are the architecture synthesis worker. - -Before working, apply these installed Command Code skills by capability name: - -- `doc-evidence-contract` -- `doc-architecture-analysis` -- `doc-workflow-analysis` -- relevant `tech-*` and `domain-*` skills discovered for this repository - -Evidence is authoritative over prior prose. Produce normalized JSON under `.docgen/model/**`. Every non-obvious conclusion must carry evidence references and an epistemic classification. Verify uncertain claims against source only when needed; do not broaden scope unnecessarily. - -Do not write published documentation and never modify application source. - -## P0 Trustworthiness - -Apply `doc-traceability` and `doc-semantic-quality`. Produce typed semantic objects, claim-level evidence mappings, and explicit UNKNOWNs. Never promote unsupported prose to FACT. -## Ignore boundary - -Before reading repository source, read `.docgen/state/source-files.txt`. Do not read, search, cite, or derive facts from repository paths absent from that inventory. Existing `.docgen/**` and `docs/**` workflow artifacts remain available. - diff --git a/global-template/agents/doc-auditor.md b/global-template/agents/doc-auditor.md deleted file mode 100644 index 733964d..0000000 --- a/global-template/agents/doc-auditor.md +++ /dev/null @@ -1,20 +0,0 @@ ---- -name: "doc-auditor" -description: "Use to independently audit one generated page for factual grounding, coverage completeness, catalog completeness, flow depth, and structural quality." -tools: "read_file, read_multiple_files, read_directory, glob, grep, write_file, edit_file, todo_write" ---- -You are the independent documentation auditor. - -Apply `doc-claim-verification` and relevant analysis/catalog skills. Compare the page to its manifest, evidence, and normalized technical/business/flow/catalog models. - -Find unsupported claims, omissions, shallow treatment, missing branches, missing catalog entries, missing failure semantics, contradictions, broken links, and any non-Mermaid diagram. Audit independently from the writer. - -Write only `.docgen/audit/**`. Never modify published docs or application source. - -## P0 Trustworthiness - -Apply `doc-traceability` and `doc-semantic-quality`. Produce typed semantic objects, claim-level evidence mappings, and explicit UNKNOWNs. Never promote unsupported prose to FACT. -## Ignore boundary - -Before reading repository source, read `.docgen/state/source-files.txt`. Do not read, search, cite, or derive facts from repository paths absent from that inventory. Existing `.docgen/**` and `docs/**` workflow artifacts remain available. - diff --git a/global-template/agents/doc-discoverer.md b/global-template/agents/doc-discoverer.md deleted file mode 100644 index fb3a0fd..0000000 --- a/global-template/agents/doc-discoverer.md +++ /dev/null @@ -1,20 +0,0 @@ ---- -name: "doc-discoverer" -description: "Use to inspect repository source and produce factual evidence artifacts without writing published documentation." -tools: "read_file, read_multiple_files, read_directory, glob, grep, shell_command, write_file, edit_file, todo_write" ---- -You are the repository evidence extraction worker. - -Apply these installed skills by capability name: - -- `doc-evidence-contract` -- `doc-repository-discovery` -- relevant `tech-*` and `domain-*` skills discovered for this repository - -Discover not only structure but also evidence required for deep system documentation: entry points, endpoints, handlers, business/domain clues, guards and branch conditions, states, persistence, messages, integrations, jobs, configuration, security, runtime/deployment and tests/examples. - -Write only under `.docgen/evidence/**`. Maintain canonical `.docgen/evidence/index.json` with top-level `artifacts[]`. Each important fact must be source-grounded. Preserve unknowns. Do not write published documentation and never modify application source. -## Ignore boundary - -Before reading repository source, read `.docgen/state/source-files.txt`. Do not read, search, cite, or derive facts from repository paths absent from that inventory. Existing `.docgen/**` and `docs/**` workflow artifacts remain available. - diff --git a/global-template/agents/doc-domain-analyst.md b/global-template/agents/doc-domain-analyst.md deleted file mode 100644 index 4154c47..0000000 --- a/global-template/agents/doc-domain-analyst.md +++ /dev/null @@ -1,45 +0,0 @@ ---- -name: "doc-domain-analyst" -description: "Use to extract business semantics, rules, decisions, branch conditions, lifecycles, data semantics, flow models, API/message catalogs, and external dependency catalogs from evidence." -tools: "read_file, read_multiple_files, read_directory, glob, grep, write_file, edit_file, todo_write" ---- -You are the business-and-system semantics synthesis worker. - -Apply these installed Command Code skills by capability name: - -- `doc-evidence-contract` -- `doc-business-analysis` -- `doc-flow-analysis` -- `doc-data-model-analysis` -- `doc-api-catalog` -- `doc-messaging-catalog` -- `doc-integration-catalog` -- relevant `tech-*` and `domain-*` skills discovered for this repository - -Primary input is `.docgen/evidence/**` plus `.docgen/model/system.json`. Inspect source only for targeted verification. - -Produce and reconcile: - -- `.docgen/model/business.json` -- `.docgen/model/flows.json` -- `.docgen/model/catalogs.json` - -Rules: - -- distinguish business semantics from implementation mechanics; -- extract explicit rules, validations, guards, eligibility checks, decisions, branch conditions, state transitions, invariants, actors, outcomes, and data semantics; -- model business flow, control flow, request flow, traffic flow, data flow, and event flow separately; -- inventory all evidenced HTTP endpoints and message handlers/consumers/producers/listeners; -- inventory external systems, cloud services, databases, brokers, caches, identity providers, downstream/upstream services, scheduled jobs, and other runtime dependencies; -- preserve FACT / INFERENCE / UNKNOWN classification and source evidence; -- use empty arrays when a category has no evidence; never invent missing behavior. - -Do not write published documentation and never modify application source. - -## P0 Trustworthiness - -Apply `doc-traceability` and `doc-semantic-quality`. Produce typed semantic objects, claim-level evidence mappings, and explicit UNKNOWNs. Never promote unsupported prose to FACT. -## Ignore boundary - -Before reading repository source, read `.docgen/state/source-files.txt`. Do not read, search, cite, or derive facts from repository paths absent from that inventory. Existing `.docgen/**` and `docs/**` workflow artifacts remain available. - diff --git a/global-template/agents/doc-enterprise-analyst.md b/global-template/agents/doc-enterprise-analyst.md deleted file mode 100644 index 6c4cda4..0000000 --- a/global-template/agents/doc-enterprise-analyst.md +++ /dev/null @@ -1,28 +0,0 @@ ---- -name: "doc-enterprise-analyst" -description: "Use to extract enterprise-depth security, operations, testing, data governance, architecture decisions, configuration, ownership, and change-impact models from grounded repository evidence." -tools: "read_file, read_multiple_files, read_directory, glob, grep, write_file, edit_file, todo_write" ---- -You are the enterprise-depth documentation analysis worker. - -Apply these installed skills by capability name as relevant: - -- `doc-security-analysis` -- `doc-operations-analysis` -- `doc-testing-analysis` -- `doc-data-governance-analysis` -- `doc-decision-analysis` -- `doc-configuration-analysis` -- `doc-change-impact-analysis` -- `doc-ownership-analysis` -- `doc-evidence-contract` -- `doc-traceability` -- `doc-semantic-quality` - -Read `.docgen/state/source-files.txt` first. Never read a repository source path that is absent from that inventory. `.gitignore`, `.docgenignore`, and configured exclusions are mandatory boundaries. - -Primary inputs are existing evidence and normalized technical/business models. Inspect source only for targeted verification. - -Produce only the output files named by the invoking pass. Every semantic array item must be a typed object with stable ID, FACT/INFERENCE/UNKNOWN classification, confidence, resolvable evidence, model references, and explicit unknowns. FACT without direct non-ignored evidence is invalid. - -Do not write published documentation and do not modify application source. diff --git a/global-template/agents/doc-planner.md b/global-template/agents/doc-planner.md deleted file mode 100644 index 718785e..0000000 --- a/global-template/agents/doc-planner.md +++ /dev/null @@ -1,21 +0,0 @@ ---- -name: "doc-planner" -description: "Use to design a deep multi-page documentation information architecture and coverage-driven manifest from evidence and normalized models." -tools: "read_file, read_multiple_files, read_directory, glob, grep, write_file, edit_file, todo_write" ---- -You are the documentation information-architecture planner. - -Apply the installed `doc-page-planning` skill by capability name. - -Read project documentation config/style/glossary plus all normalized model surfaces. Plan a navigable, category-rich system knowledge base. Do not mechanically create one page per class/file, but also do not collapse a complex repository into a handful of oversized pages. - -The plan must make exhaustive catalogs discoverable and give important concepts/flows their own deep-dive pages. Preserve stable page ids and paths when reconciling unless evidence supports restructuring. - -Produce `.docgen/plan/manifest.json` conforming to its schema. Do not write published pages and never modify application source. -## Ignore boundary - -Before reading repository source, read `.docgen/state/source-files.txt`. Do not read, search, cite, or derive facts from repository paths absent from that inventory. Existing `.docgen/**` and `docs/**` workflow artifacts remain available. - - - -Apply `doc-documentation-experience` and `doc-migration-versioning`. Every page contract must choose a primary document mode, search keywords, aliases when relevant, lifecycle/version metadata, and evidence-derived example intents. diff --git a/global-template/agents/doc-system-analyst.md b/global-template/agents/doc-system-analyst.md deleted file mode 100644 index 0cd6f5d..0000000 --- a/global-template/agents/doc-system-analyst.md +++ /dev/null @@ -1,25 +0,0 @@ ---- -name: "doc-system-analyst" -description: "Use for system-of-systems analysis across multiple DocGen repositories: dependency graphs, shared contracts, business journeys, cross-service flows, data lineage, ownership, and blast radius." -tools: "read_file, read_multiple_files, read_directory, glob, grep, write_file, edit_file, todo_write" ---- -You are the P3 system-of-systems documentation analyst. - -Consume only validated repository-level DocGen artifacts and workspace registries. Do not rescan arbitrary repository source unless a workspace command explicitly asks for targeted verification. - -Apply these installed skills by capability name: - -- `doc-workspace-aggregation` -- `doc-cross-repo-dependency` -- `doc-contract-registry` -- `doc-business-journey` -- `doc-cross-repo-flow` -- `doc-data-lineage-workspace` -- `doc-workspace-change-impact` -- `doc-workspace-publishing` -- `doc-traceability` -- `doc-semantic-quality` - -Preserve repository identity, contract ownership, evidence provenance, FACT/INFERENCE/UNKNOWN classification, and unresolved edges. Never invent a cross-service connection merely because two names look related. Ambiguous matches remain unresolved. - -All published diagrams must use Mermaid. diff --git a/global-template/agents/doc-writer.md b/global-template/agents/doc-writer.md deleted file mode 100644 index a189390..0000000 --- a/global-template/agents/doc-writer.md +++ /dev/null @@ -1,25 +0,0 @@ ---- -name: "doc-writer" -description: "Use to write or enrich exactly one evidence-grounded Markdown page with deep repository-specific explanation and Mermaid diagrams." -tools: "read_file, read_multiple_files, read_directory, glob, grep, write_file, edit_file, todo_write" ---- -You are the bounded documentation page writer. - -Apply the writing skill matching the page type plus `doc-mermaid` when diagrams are planned. Use the page manifest as a strict content contract. - -Write for multiple reading depths: orient a newcomer, give a maintainer a working model, and preserve deep technical/reference details for expert use. Prefer repository-specific facts, tables, decision matrices, step-by-step flows and explicit caveats over generic prose. - -For catalog pages, be exhaustive over the normalized catalog in scope. For flow pages, preserve branches and alternate/failure paths. For business pages, distinguish rules and inferred semantics. All diagrams must be Mermaid. - -Modify exactly one target page under `docs/**`. Never modify application source. - -## P0 Trustworthiness - -Apply `doc-traceability` and `doc-semantic-quality`. Produce typed semantic objects, claim-level evidence mappings, and explicit UNKNOWNs. Never promote unsupported prose to FACT. -## Ignore boundary - -Before reading repository source, read `.docgen/state/source-files.txt`. Do not read, search, cite, or derive facts from repository paths absent from that inventory. Existing `.docgen/**` and `docs/**` workflow artifacts remain available. - - - -Apply `doc-documentation-experience`, `doc-example-scenario-writing`, `doc-migration-versioning`, and `doc-publishing-metadata`. Respect the page mode: tutorials teach progressively, how-to pages are goal-oriented, references are exhaustive, runbooks are executable under pressure, decision records explain rationale/trade-offs, and migration guides include verification/rollback. diff --git a/global-template/commands/docgen-analyze.md b/global-template/commands/docgen-analyze.md deleted file mode 100644 index 6933475..0000000 --- a/global-template/commands/docgen-analyze.md +++ /dev/null @@ -1 +0,0 @@ -Run the globally installed DocGen orchestrator command `docgen analyze` for scope `$ARGUMENTS` (use all current evidence when empty). Report completion or the exact failure. diff --git a/global-template/commands/docgen-audit.md b/global-template/commands/docgen-audit.md index b7c3d90..f818a04 100644 --- a/global-template/commands/docgen-audit.md +++ b/global-template/commands/docgen-audit.md @@ -1 +1 @@ -Run the globally installed DocGen orchestrator command `docgen audit $1`. Report the audit result for exactly that page. +Run `docgen audit` in the current repository. Execute deterministic structural/traceability checks for every page and invoke the LLM auditor only for pages above the configured risk threshold whose audit input hash changed. Report deterministic failures, high-risk findings, audited-page count, and budget usage. diff --git a/global-template/commands/docgen-budget.md b/global-template/commands/docgen-budget.md new file mode 100644 index 0000000..826ab59 --- /dev/null +++ b/global-template/commands/docgen-budget.md @@ -0,0 +1 @@ +Run `docgen budget $ARGUMENTS`. With no argument, report provider calls, estimated input/output tokens, stage totals, remaining limits, and whether the hard budget is exceeded. Use `reset` only when the user explicitly wants telemetry cleared. diff --git a/global-template/commands/docgen-context.md b/global-template/commands/docgen-context.md new file mode 100644 index 0000000..7a9fc5b --- /dev/null +++ b/global-template/commands/docgen-context.md @@ -0,0 +1 @@ +Run `docgen context $ARGUMENTS`. Compile one bounded context pack from SQLite/FTS5 for the requested stage/query, then report the output path, estimated tokens, and omitted fact/model-item counts. Do not invoke a provider. diff --git a/global-template/commands/docgen-contract-test.md b/global-template/commands/docgen-contract-test.md deleted file mode 100644 index d63624a..0000000 --- a/global-template/commands/docgen-contract-test.md +++ /dev/null @@ -1,5 +0,0 @@ -Run the global DocGen contract firewall regression suite without calling an LLM provider: - -`docgen contract-test` - -Report the command output and stop if any producer/consumer contract test fails. diff --git a/global-template/commands/docgen-discover.md b/global-template/commands/docgen-discover.md deleted file mode 100644 index 2a460e0..0000000 --- a/global-template/commands/docgen-discover.md +++ /dev/null @@ -1 +0,0 @@ -Run the globally installed DocGen orchestrator command `docgen discover` for scope `$ARGUMENTS` (use `.` when empty). The orchestrator owns bounded headless execution, safety mode, state updates, and validation. Report completion or the exact failure. diff --git a/global-template/commands/docgen-enrich.md b/global-template/commands/docgen-enrich.md deleted file mode 100644 index 5b67adb..0000000 --- a/global-template/commands/docgen-enrich.md +++ /dev/null @@ -1 +0,0 @@ -Run the globally installed DocGen orchestrator command `docgen enrich $1`. Perform the bounded depth-and-completeness pass for exactly the requested manifest page. diff --git a/global-template/commands/docgen-enterprise.md b/global-template/commands/docgen-enterprise.md deleted file mode 100644 index 16a7572..0000000 --- a/global-template/commands/docgen-enterprise.md +++ /dev/null @@ -1,11 +0,0 @@ -Run the global DocGen enterprise-depth stage from the current repository: - -```bash -node ~/.commandcode/docgen/bin/docgen.mjs enterprise -``` - -On native Windows use: - -```powershell -node "$env:USERPROFILE\.commandcode\docgen\bin\docgen.mjs" enterprise -``` diff --git a/global-template/commands/docgen-fix.md b/global-template/commands/docgen-fix.md deleted file mode 100644 index 9409f98..0000000 --- a/global-template/commands/docgen-fix.md +++ /dev/null @@ -1 +0,0 @@ -Run the globally installed DocGen orchestrator command `docgen fix $1`, then report the repaired page. Re-audit separately when requested. diff --git a/global-template/commands/docgen-generate.md b/global-template/commands/docgen-generate.md index a142773..a154869 100644 --- a/global-template/commands/docgen-generate.md +++ b/global-template/commands/docgen-generate.md @@ -1 +1 @@ -Run the globally installed DocGen orchestrator command `docgen generate $1`. Generate exactly the requested manifest page through the bounded orchestrated workflow. +Run `docgen generate` in the current repository. Render low-risk reference/catalog pages deterministically, compile bounded item-level context packs for narrative pages, batch only pages with compatible context, generate traceability sidecars in the same invocation, and skip pages whose input hash is unchanged. diff --git a/global-template/commands/docgen-index.md b/global-template/commands/docgen-index.md new file mode 100644 index 0000000..81f9d2e --- /dev/null +++ b/global-template/commands/docgen-index.md @@ -0,0 +1 @@ +Run `docgen index $ARGUMENTS` in the current repository. Build or incrementally refresh the deterministic SQLite/FTS5 semantic index from the canonical text-source inventory, then report changed files, extracted facts, source chunks, and totals. diff --git a/global-template/commands/docgen-migrate.md b/global-template/commands/docgen-migrate.md new file mode 100644 index 0000000..0602ed8 --- /dev/null +++ b/global-template/commands/docgen-migrate.md @@ -0,0 +1 @@ +Run `docgen migrate` in the current initialized repository. This is a breaking v1-to-v2 migration: preserve `docs/**` and `.docgenignore`, archive legacy `.docgen` workflow artifacts, install the v2 config/state contract, and report the backup path. diff --git a/global-template/commands/docgen-model.md b/global-template/commands/docgen-model.md new file mode 100644 index 0000000..00c49f1 --- /dev/null +++ b/global-template/commands/docgen-model.md @@ -0,0 +1 @@ +Run `docgen model` in the current repository. Synthesize the core and enterprise model bundles from bounded semantic-index context packs, ingest the resulting typed model items back into SQLite, and report model/item counts and budget usage. diff --git a/global-template/commands/docgen-preflight.md b/global-template/commands/docgen-preflight.md deleted file mode 100644 index 62e4643..0000000 --- a/global-template/commands/docgen-preflight.md +++ /dev/null @@ -1 +0,0 @@ -Run `docgen preflight` from the current repository using the shell. Report whether manifest paths and all evidence/model references are valid before generation. diff --git a/global-template/commands/docgen-quality.md b/global-template/commands/docgen-quality.md deleted file mode 100644 index cf81ef5..0000000 --- a/global-template/commands/docgen-quality.md +++ /dev/null @@ -1 +0,0 @@ -Run `docgen quality` for the current initialized repository. Report local page-quality metrics, audit severity totals, and whether the configured quality gate passes. diff --git a/global-template/commands/docgen-semantics.md b/global-template/commands/docgen-semantics.md deleted file mode 100644 index cd58c99..0000000 --- a/global-template/commands/docgen-semantics.md +++ /dev/null @@ -1,7 +0,0 @@ -Run the global DocGen orchestrator semantics stage in the current initialized repository: - -```bash -docgen semantics -``` - -This extracts business rules/decisions/branches, six flow types, endpoints, message handlers, external dependencies, data stores and scheduled jobs into normalized model files. diff --git a/global-template/commands/docgen-traceability.md b/global-template/commands/docgen-traceability.md deleted file mode 100644 index e9c261b..0000000 --- a/global-template/commands/docgen-traceability.md +++ /dev/null @@ -1,9 +0,0 @@ -Run the global DocGen traceability and consistency index for the current repository. - -Use the shell tool: - -```bash -docgen traceability -``` - -This is deterministic and does not call an LLM provider. diff --git a/global-template/commands/docgen-update.md b/global-template/commands/docgen-update.md deleted file mode 100644 index 16ffd4a..0000000 --- a/global-template/commands/docgen-update.md +++ /dev/null @@ -1 +0,0 @@ -Run the globally installed DocGen orchestrator command `docgen update $ARGUMENTS`. When no paths are supplied, let DocGen use its fingerprint snapshot to detect changes. diff --git a/global-template/docgen/CONTRACTS.md b/global-template/docgen/CONTRACTS.md index 82036d4..a0c992f 100644 --- a/global-template/docgen/CONTRACTS.md +++ b/global-template/docgen/CONTRACTS.md @@ -1,85 +1,120 @@ -# DocGen Contract Firewall +# DocGen 2 contracts -DocGen treats every LLM-produced artifact as untrusted, uncommitted output until it has passed canonicalization and invariant validation. +DocGen 2 is a language-, framework-, and architecture-neutral documentation pipeline. It treats provider output as untrusted. Deterministic code owns source eligibility, indexing, retrieval, budgets, checkpoints, artifact validation, low-risk rendering, quality gates, and publishing. Providers receive bounded context packs rather than unrestricted repository access. ## Boundary matrix -| Stage | Canonical artifact | Canonical arrays/fields | +| Boundary | Canonical artifact | Owner | |---|---|---| -| discover | `.docgen/evidence/index.json` | `artifacts` | -| analyze | `.docgen/model/system.json` | `components`, `relationships`, `workflows`, `unknowns` | -| semantics/business | `.docgen/model/business.json` | `actors`, `capabilities`, `concepts`, `businessRules`, `decisions`, `branchConditions`, `lifecycles`, `invariants`, `useCases`, `unknowns` | -| semantics/flows | `.docgen/model/flows.json` | `businessFlows`, `controlFlows`, `requestFlows`, `trafficFlows`, `dataFlows`, `eventFlows` | -| semantics/catalogs | `.docgen/model/catalogs.json` | `endpoints`, `messageHandlers`, `externalDependencies`, `dataStores`, `scheduledJobs` | -| enterprise/security | `.docgen/model/security.json` | `trustBoundaries`, `principals`, `authenticationFlows`, `authorizationRules`, `permissions`, `serviceIdentities`, `secrets`, `sensitiveData`, `threats`, `controls`, `unknowns` | -| enterprise/operations | `.docgen/model/operations.json` | runtime, health, observability, SLI/SLO, alerts, capacity, scaling, failures, recovery, backup, deployment, runbooks | -| enterprise/testing | `.docgen/model/testing.json` | suites, test types, fixtures, test data, environments, commands, gaps, contract tests, failure injection, quality gates | -| enterprise/data governance | `.docgen/model/data-governance.json` | entities, ownership, sources of truth, classification, retention, transactions, consistency, concurrency, idempotency, reconciliation, lineage, migration, auditability | -| enterprise/decisions | `.docgen/model/decisions.json` | recorded/inferred decisions, alternatives, trade-offs, constraints, consequences, superseded decisions | -| enterprise/configuration | `.docgen/model/configuration.json` | settings, environments, flags, secrets, tuning, validation, reload behavior, deprecations | -| enterprise/change impact | `.docgen/model/change-impact.json` | change surfaces, impact edges, compatibility, extension points, risks, tests, operations and contracts | -| enterprise/ownership | `.docgen/model/ownership.json` | teams, responsibilities, RACI, component/data/operational ownership, approvals and escalation | -| plan | `.docgen/plan/manifest.json` | `navigation`, `pages`, canonical `docs/**/*.md` paths | -| generate/enrich/fix | `docs/**/*.md` | exact manifest target, valid Markdown, Mermaid-only diagrams | -| page traceability | `.docgen/traceability/pages/.json` | typed claims, evidence refs, model/catalog/branch coverage, source snapshot | -| cross-page consistency | `.docgen/traceability/{index,contradictions,duplicates,freshness}.json` | unique claim IDs, contradiction groups, duplicate groups, freshness | -| audit | `.docgen/audit/pages/.json` | `pageId`, `pagePath`, `pageHash`, `inputHash`, `findings` | -| update-impact | `.docgen/plan/update-plan.json` | `changedPaths`, `affectedEvidenceScopes`, `affectedModels`, `affectedPageIds`, `rationale` | -| publishing | `.docgen/publish/*.json`, `docs/llms*.txt`, page frontmatter | mode, aliases, lifecycle/version metadata, search/navigation/backlink/redirect/example indexes | - -## Invariants - -1. **Single canonical representation** — aliases are removed from committed artifacts. -2. **Idempotence** — normalizing canonical output again cannot change or duplicate it. -3. **Losslessness for known split aliases** — producers, consumers and listeners are merged into the complete message-handler catalog. -4. **Path safety** — evidence remains under `.docgen/evidence/**`; published pages remain under `docs/**/*.md`. -5. **Identity consistency** — audit page ID/path/hash must match the current manifest/page. -6. **Input consistency** — generated pages and audits are fingerprinted against their declared evidence/model inputs. -7. **Transactional stages** — partial output is quarantined and the previous valid artifact restored. -8. **Dependency invalidation** — rerunning an upstream stage invalidates dependent stage skips. -9. **Typed semantic items** — each model item has stable ID, kind, epistemic classification, confidence, evidence, and source references. -10. **Direct evidence for FACT** — FACT items and claims cannot commit without resolvable repository evidence. -11. **Claim-level traceability** — material page claims map to source and normalized semantic/catalog items. -12. **Cross-page consistency** — claim ID collisions and subject/predicate contradictions fail quality gates. -13. **Freshness** — page, input, Git/source fingerprint changes make traceability stale. -14. **Evidence-centric quality** — grounding and coverage are hard gates; word count is advisory. -15. **Ignore-aware source boundary** — `.gitignore`, `.docgenignore`, hard exclusions, and project `config.exclude` produce one canonical source inventory used by every stage. -16. **Ignored evidence rejection** — an ignored path cannot support a typed FACT or page claim. -17. **Enterprise-depth coverage** — security, operations, testing, data governance, decisions, configuration, change impact, and ownership use typed, evidence-backed item contracts. -18. **Binary/non-text budget boundary** — binary extensions, magic signatures, NUL bytes, invalid UTF-8, excessive control characters, and oversized text are excluded before reads, fingerprints, change detection, or evidence validation. -19. **Mode-aware documentation** — every planned page has a canonical document mode and mode-specific structural contract. -20. **Deterministic publishing** — frontmatter, navigation, search, backlinks, redirects, orphan reports, examples indexes, and `llms*.txt` are produced without a provider call. -21. **Evidence-derived examples** — publishable examples must map to page claims and source/model references; generic unsupported examples do not satisfy the quality contract. - -Run the zero-token suite with: - -```bash -docgen contract-test +| source eligibility | `.docgen/index/inventory.json` | deterministic inventory | +| searchable knowledge | `.docgen/index/semantic.db` | deterministic indexer | +| provider input | `.docgen/context//*.json` | bounded context compiler | +| core synthesis | `.docgen/model/{system,business,flows,catalogs}.json` | bounded provider call + orchestrator validation | +| enterprise synthesis | `.docgen/model/{security,operations,testing,data-governance,decisions,configuration,change-impact,ownership}.json` | bounded provider call + orchestrator validation | +| page plan | `.docgen/plan/manifest.json` | bounded planner + deterministic canonicalization | +| generated pages | `docs/**/*.md` | deterministic renderer or bounded writer | +| page claims | `.docgen/traceability/pages/.json` | page generation + deterministic normalization | +| runtime checkpoints | `.docgen/state/state.json` | orchestrator | +| structural and grounding audit | `.docgen/audit/deterministic.json` | deterministic auditor | +| selective semantic-risk audit | `.docgen/audit/llm-risk.json` | hash-cached bounded provider call | +| quality summary | `.docgen/audit/quality-summary.json` | deterministic aggregator | +| publishing | `.docgen/publish/*.json`, `docs/llms*.txt` | deterministic publisher | +| provider usage | `.docgen/telemetry/provider-runs.jsonl`, `.docgen/budget/report.json` | orchestrator | +| provider diagnostics | `.docgen/runs/*.stdout.log`, `.docgen/runs/*.stderr.log` | orchestrator | + +## Hard invariants + +1. **Technology neutrality** — no language, framework, protocol, datastore, messaging system, deployment model, or repository shape is assumed. Applications, libraries, CLIs, jobs, plugins, infrastructure, data pipelines, embedded systems, monoliths, services, and mixed workspaces are all valid inputs. +2. **Canonical source boundary** — Git-aware discovery or filesystem fallback, `.gitignore`, `.docgenignore`, binary signatures, UTF-8 checks, and size limits are applied before indexing. +3. **One index phase per full run** — `docgen all` indexes once, then all later phases consume that inventory and semantic database. +4. **Context-only provider** — provider work is bounded by declared context packs and explicit output contracts; prompts must not perform broad repository scans. +5. **Bounded context** — every pack has a token budget and records omitted facts and model items. +6. **Content addressing** — contexts, stages, pages, traces, audits, and publishing are reusable only while their source, model, input, and artifact hashes remain current. +7. **Crash-safe page checkpoints** — page state records running, completed, and failed work. A failed generation batch preserves valid pages and retries only missing or invalid pages. +8. **Fresh-artifact recovery** — a provider non-zero exit may be recovered only when the current invocation produced artifacts that pass the complete output contract. Pre-existing stale artifacts are never accepted as recovery proof. +9. **Effective provider configuration** — every call records and prints the executable, model, effective `maxTurns`, timeout, context size, and log files. The supported minimum conversation-turn budget is 30. +10. **Hard provider budget** — a call is refused before execution when configured call, token, or per-call limits would be exceeded. +11. **Typed semantic claims** — model items and page claims use `FACT`, `INFERENCE`, `ASSUMPTION`, or `UNKNOWN`, with confidence, evidence, and stable qualified identity. +12. **Grounded facts** — a `FACT` requires repository-relative evidence inside the canonical inventory. By default it also requires valid line evidence whose source hash still matches the index. +13. **Context-bound generation** — generated claim evidence and model references must come from the page's supplied context unless explicitly disabled. +14. **Qualified model identity** — references use `:` to prevent collisions and permit deterministic validation. +15. **Deterministic references where possible** — generic components, interfaces, dependencies, data assets, automation, configuration, ownership, and change-impact catalogs can be rendered without provider calls. +16. **Quality before publishing** — publishing requires a current passing audit and revalidates source, model, page, trace, links, evidence, and audit hashes to reject stale output. +17. **Selective semantic-risk audit** — provider audit is limited to risk-scored pages after deterministic validation and is reused only while its inputs remain unchanged. +18. **No hidden repair loop** — there is no unbounded enrich/fix/re-audit cycle. Recovery is bounded and checkpointed. +19. **Mermaid only** — generated diagrams may not depend on PlantUML, Graphviz, or image-only formats. +20. **Breaking migration boundary** — legacy workflow artifacts are archived rather than interpreted as current checkpoints. + +## Framework-neutral semantic surfaces + +The shape is extensible. Technology-specific arrays are optional signals, not requirements. + +- `system.json`: components, modules, packages, runtimes, deployment units, relationships, workflows, and unknowns. +- `business.json`: actors, capabilities, concepts, rules, decisions, branches, lifecycles, invariants, use cases, and unknowns. +- `flows.json`: execution, control, request, traffic, data, event, batch, and other evidenced flows. +- `catalogs.json`: interfaces, contracts, dependencies, data assets, automations, build artifacts, configuration surfaces, plus optional protocol-specific catalogs when present. +- enterprise models: security, operations, testing, data governance, decisions, configuration, change impact, and ownership. + +The deterministic index always provides generic file artifacts and source chunks. It may additionally recognize common symbols, functions, imports/modules, manifests, configuration keys, runtime declarations, infrastructure resources, interfaces, data entities, scheduled automation, or security boundaries. Absence of a recognizer never makes a technology unsupported because bounded source chunks remain the fallback evidence surface. + +## Runtime and resume contract + +A full run is: + +```text +index -> modelCore -> modelEnterprise -> plan -> generate -> audit -> publish ``` -The machine-readable report is written to `.docgen/state/contract-report.json`. +`docgen resume` runs the same state-aware pipeline. Completed stages and pages are reused only when their input hashes and required outputs remain valid. During generation: + +1. each page is marked `running` with batch, context, and input identity; +2. provider output is validated page-by-page; +3. valid pages are checkpointed immediately; +4. failed or missing pages are marked with their error; +5. bounded recovery retries only the unresolved subset; +6. a page becomes `completed` only after Markdown and traceability validation succeeds. + +## Correctness gate -## Trustworthiness reports +The deterministic audit validates, at minimum: -Run `docgen traceability` to rebuild deterministic claim, contradiction, duplicate, and freshness reports. Run `docgen quality` to apply semantic thresholds. Neither command requires an LLM call. +- source inventory membership, live-source hash, and evidence line ranges; +- model JSON validity, qualified identities, classifications, confidence, and FACT evidence; +- page frontmatter, H1, required sections, Mermaid policy, and local links; +- page/trace identity, page hash, input hash, and generation context identity; +- claim classification, confidence, evidence, context-bound grounding, and model references; +- missing, duplicate, conflicting, stale, orphaned, or substantially duplicated artifacts; +- model-reference coverage and configurable warning/failure policy. -# P3 workspace contracts +The detailed report is `.docgen/audit/deterministic.json`. `.docgen/audit/quality-summary.json` is the publish gate and contains aggregate claim, evidence, model-reference, failure, warning, and selective-risk metrics. -A system workspace is rooted by `.docgen-workspace/workspace.json` and references repositories by stable ID plus absolute path. Repository-local `.docgen/model/**` artifacts remain authoritative for repository facts. +## Page traceability + +Each page sidecar contains: + +```json +{ + "schemaVersion": "2.0", + "pageId": "component-lifecycle", + "pagePath": "docs/architecture/component-lifecycle.md", + "pageHash": "...", + "inputHash": "...", + "contextId": "...", + "claims": [ + { + "id": "component-lifecycle:transition", + "statement": "The component transitions from pending to active.", + "classification": "FACT", + "confidence": 1, + "evidence": [{"path": "src/component.ext", "startLine": 120, "endLine": 146}], + "sourceModelRefs": ["business:transition-pending-active"] + } + ] +} +``` -P3 canonical workspace outputs: +The orchestrator normalizes and verifies page, input, and context hashes after generation. -- `repositories.json` -- `model/system-map.json` -- `model/dependency-graph.json` -- `model/contract-registry.json` -- `model/capability-map.json` -- `model/business-journeys.json` -- `model/request-flows.json` -- `model/event-flows.json` -- `model/data-lineage.json` -- `model/shared-infrastructure.json` -- `model/ownership.json` -- `model/change-impact.json` +## Workspace boundary -Cross-repository edges require explicit registered repository identity, dependency target, shared producer/consumer channel, or repository model reference. Ambiguous relationships remain unresolved. Workspace publishing is Mermaid-only and deterministic. +Cross-repository synthesis remains deterministic and requires explicit repository identity plus evidenced relationships, declared dependencies, shared producer/consumer contracts, or qualified model references. Ambiguous relationships remain unresolved instead of being inferred from a preferred stack. diff --git a/global-template/docgen/VERSION b/global-template/docgen/VERSION index 3eefcb9..227cea2 100644 --- a/global-template/docgen/VERSION +++ b/global-template/docgen/VERSION @@ -1 +1 @@ -1.0.0 +2.0.0 diff --git a/global-template/docgen/bin/docgen-launcher.mjs b/global-template/docgen/bin/docgen-launcher.mjs new file mode 100644 index 0000000..e42b713 --- /dev/null +++ b/global-template/docgen/bin/docgen-launcher.mjs @@ -0,0 +1,22 @@ +#!/usr/bin/env node +import path from 'node:path'; +import { spawn } from 'node:child_process'; +import { fileURLToPath } from 'node:url'; + +const here = path.dirname(fileURLToPath(import.meta.url)); +const main = path.join(here, 'docgen-v2.mjs'); +const child = spawn(process.execPath, ['--disable-warning=ExperimentalWarning', main, ...process.argv.slice(2)], { + stdio: 'inherit', + shell: false, + windowsHide: true, + env: process.env +}); + +child.on('error', (error) => { + console.error(`ERROR: failed to start DocGen: ${error.message}`); + process.exitCode = 1; +}); +child.on('exit', (code, signal) => { + if (signal) process.kill(process.pid, signal); + else process.exitCode = code ?? 1; +}); diff --git a/global-template/docgen/bin/docgen-v2.mjs b/global-template/docgen/bin/docgen-v2.mjs new file mode 100644 index 0000000..56c989d --- /dev/null +++ b/global-template/docgen/bin/docgen-v2.mjs @@ -0,0 +1,164 @@ +#!/usr/bin/env node +import fs from 'node:fs'; +import path from 'node:path'; +import { buildInventory } from '../lib/inventory.mjs'; +import { compileContext } from '../lib/context.mjs'; +import { databaseStats } from '../lib/indexer.mjs'; +import { all, audit, generate, index, model, plan, publish, status } from '../lib/pipeline.mjs'; +import { budgetReport, resetTelemetry } from '../lib/provider.mjs'; +import { commandExists, engineHome, ensureDir, kitVersion, loadConfig, parseArgs, projectPaths, readJson, requireProjectRoot, writeJson } from '../lib/core.mjs'; +import { runWorkspace } from './workspace.mjs'; + +const MIN_PROVIDER_TURNS = 30; +const TURN_KEYS = ['default', 'modelCore', 'modelEnterprise', 'plan', 'generate', 'audit']; + +function normalizedTurns(value) { + const parsed = Number(value); + return Number.isFinite(parsed) ? Math.max(MIN_PROVIDER_TURNS, Math.trunc(parsed)) : MIN_PROVIDER_TURNS; +} + +function normalizeTurnEnvironment() { + if (process.env.DOCGEN_MAX_TURNS !== undefined) { + process.env.DOCGEN_MAX_TURNS = String(normalizedTurns(process.env.DOCGEN_MAX_TURNS)); + } +} + +function enforceMinimumTurns(root) { + const paths = projectPaths(root); + const config = loadConfig(root); + config.commandCode ??= {}; + config.commandCode.maxTurns ??= {}; + let changed = false; + for (const key of TURN_KEYS) { + const current = config.commandCode.maxTurns[key]; + const next = normalizedTurns(current); + if (current !== next) { + config.commandCode.maxTurns[key] = next; + changed = true; + } + } + if (changed) writeJson(paths.config, config); + return config.commandCode.maxTurns; +} + +normalizeTurnEnvironment(); + +function usage() { + console.log(`Command Code DocGen ${kitVersion} — token-efficient semantic-index pipeline + +Repository commands: + docgen init [directory] + docgen migrate + docgen doctor + docgen index [--force] + docgen model + docgen plan + docgen generate + docgen audit + docgen publish + docgen all | resume + docgen status + +Budget and context: + docgen budget [report|reset] + docgen context [query] [--target ID] [--max-tokens N] + docgen ignore [path] + docgen source-list [substring] + docgen source-grep + +System workspace: + docgen workspace +`); +} + +function copyTree(source, target) { + for (const entry of fs.readdirSync(source, { withFileTypes: true })) { + const from = path.join(source, entry.name); const to = path.join(target, entry.name); + if (entry.isDirectory()) { ensureDir(to); copyTree(from, to); } + else if (!fs.existsSync(to)) { ensureDir(path.dirname(to)); fs.copyFileSync(from, to); } + } +} + +function defaultConfig() { return readJson(path.join(engineHome, 'project-template', 'config', 'documentation.json')); } +function defaultState() { return readJson(path.join(engineHome, 'project-template', 'state', 'state.json')); } +function init(target = '.') { + const root = path.resolve(target); ensureDir(root); const paths = projectPaths(root); const template = path.join(engineHome, 'project-template'); + if (fs.existsSync(template)) copyTree(template, paths.base); + for (const dir of [paths.context, paths.telemetry, path.dirname(paths.budget), paths.model, path.dirname(paths.plan), paths.audit, paths.publish, paths.traceability, paths.runs, path.dirname(paths.inventory)]) ensureDir(dir); + const marker = readJson(paths.project, {}); writeJson(paths.project, { ...marker, schemaVersion: '2.0', kitVersion, initializedAt: marker.initializedAt ?? new Date().toISOString(), engineScope: marker.engineScope ?? 'global', projectRoot: root.replaceAll('\\', '/') }); + if (!fs.existsSync(paths.config)) writeJson(paths.config, defaultConfig()); + if (!fs.existsSync(paths.state)) writeJson(paths.state, defaultState()); + console.log(`Initialized DocGen ${kitVersion} at ${root}`); +} + +function migrate(root) { + const paths = projectPaths(root); const current = readJson(paths.config, {}); const marker = readJson(paths.project, {}); + if (current.schemaVersion === '2.0' && marker.kitVersion === kitVersion) { console.log(`DocGen project is already on ${kitVersion}.`); return; } + const timestamp = new Date().toISOString().replace(/[:.]/g, '-'); const backup = path.join(paths.base, 'migration-backup', timestamp); ensureDir(backup); + const moveNames = ['evidence','model','plan','audit','traceability','state','publish','index','context','telemetry','budget','runs','prompts']; + for (const name of moveNames) { + const source = path.join(paths.base, name); if (!fs.existsSync(source)) continue; + const target = path.join(backup, name); ensureDir(path.dirname(target)); fs.renameSync(source, target); + } + const configDir = path.join(paths.base, 'config'); + if (fs.existsSync(configDir)) { const target = path.join(backup, 'config'); ensureDir(path.dirname(target)); fs.renameSync(configDir, target); } + const next = defaultConfig(); + next.projectName = current.projectName ?? next.projectName; + next.outputRoot = current.outputRoot ?? next.outputRoot; + next.commandCode = { ...next.commandCode, executable: current.commandCode?.executable ?? '', trust: current.commandCode?.trust ?? next.commandCode.trust, skipOnboarding: current.commandCode?.skipOnboarding ?? next.commandCode.skipOnboarding, yolo: current.commandCode?.yolo ?? next.commandCode.yolo, model: current.commandCode?.model ?? '', stageModels: { ...next.commandCode.stageModels, ...(current.commandCode?.stageModels ?? {}) } }; + next.ignore = { ...next.ignore, ...(current.ignore ?? {}), binary: { ...next.ignore.binary, ...(current.ignore?.binary ?? {}) } }; + writeJson(paths.config, next); writeJson(paths.state, defaultState()); + for (const dir of [paths.context, paths.telemetry, path.dirname(paths.budget), paths.model, path.dirname(paths.plan), paths.audit, paths.publish, paths.traceability, paths.runs, path.dirname(paths.inventory)]) ensureDir(dir); + writeJson(paths.project, { ...marker, schemaVersion: '2.0', kitVersion, migratedAt: new Date().toISOString(), migrationBackup: path.relative(root, backup).replaceAll('\\', '/') }); + console.log(`Migrated project to DocGen ${kitVersion}.`); + console.log(`Legacy .docgen artifacts were archived at ${path.relative(root, backup).replaceAll('\\', '/')}.`); + console.log('Generated docs and .docgenignore were preserved. Run `docgen index`, then `docgen all`.'); +} + +function doctor(root) { + const errors = []; const warnings = []; + if (!commandExists('git')) errors.push('git executable not found'); + try { const config = loadConfig(root); if (!config || typeof config !== 'object') errors.push('invalid documentation config'); if (config.schemaVersion !== '2.0') errors.push('legacy configuration detected; run `docgen migrate`'); } catch (error) { errors.push(error.message); } + if (!process.versions.node || Number(process.versions.node.split('.')[0]) < 22) errors.push('Node.js 22+ is required for node:sqlite'); + if (!commandExists(process.platform === 'win32' ? 'cmdc' : 'cmd') && !loadConfig(root).commandCode?.executable) warnings.push('Command Code executable was not found using default names; configure commandCode.executable if needed.'); + try { const inventory = buildInventory(root); if (!inventory.files.length) warnings.push('source inventory is empty'); } catch (error) { errors.push(error.message); } + for (const warning of warnings) console.warn(`WARNING: ${warning}`); for (const error of errors) console.error(`ERROR: ${error}`); + if (errors.length) process.exitCode = 1; else console.log('Doctor checks passed.'); +} + +function printStatus(root) { console.log(JSON.stringify(status(root), null, 2)); } +function printBudget(root) { console.log(JSON.stringify(budgetReport(root), null, 2)); } +function sourceList(root, filter = '') { const inv = buildInventory(root); for (const file of inv.files) if (!filter || file.path.toLowerCase().includes(filter.toLowerCase())) console.log(file.path); } +function sourceGrep(root, query) { if (!query) throw new Error('Usage: docgen source-grep '); const inv = buildInventory(root); const needle = query.toLowerCase(); for (const item of inv.files) { const text = fs.readFileSync(path.join(root, item.path), 'utf8'); for (const [lineIndex, line] of text.split(/\r?\n/).entries()) if (line.toLowerCase().includes(needle)) console.log(`${item.path}:${lineIndex + 1}:${line.trim()}`); } } +function ignore(root, target) { const inv = buildInventory(root); if (target) { const found = inv.files.find((item) => item.path === target); const excluded = inv.excluded.find((item) => item.path === target); console.log(JSON.stringify(found ? { included: true, ...found } : { included: false, ...(excluded ?? { path: target, reason: 'not-found' }) }, null, 2)); } else console.log(JSON.stringify(inv.metrics, null, 2)); } + +async function main() { + const [command, ...rest] = process.argv.slice(2); const { positional, options } = parseArgs(rest); + if (!command || ['help','--help','-h'].includes(command)) { usage(); return; } + if (command === 'init') { init(positional[0] ?? '.'); return; } + if (command === 'workspace') { await runWorkspace(rest, { kitVersion }); return; } + const root = requireProjectRoot(); + enforceMinimumTurns(root); + switch (command) { + case 'migrate': migrate(root); break; + case 'doctor': doctor(root); break; + case 'index': console.log(JSON.stringify(index(root, { force: Boolean(options.force) }), null, 2)); break; + case 'model': console.log(JSON.stringify(await model(root), null, 2)); break; + case 'plan': console.log(JSON.stringify(await plan(root), null, 2)); break; + case 'generate': console.log(JSON.stringify(await generate(root), null, 2)); break; + case 'audit': console.log(JSON.stringify(await audit(root), null, 2)); break; + case 'publish': console.log(JSON.stringify(publish(root), null, 2)); break; + case 'all': + case 'resume': console.log(JSON.stringify(await all(root), null, 2)); break; + case 'status': printStatus(root); break; + case 'budget': if (positional[0] === 'reset') { resetTelemetry(root); console.log('Budget telemetry reset.'); } else printBudget(root); break; + case 'context': { const stage = positional[0]; if (!stage) throw new Error('Usage: docgen context [query]'); const result = compileContext(root, { stage, query: positional.slice(1).join(' '), target: options.target ?? '', maxTokens: options.maxTokens ? Number(options.maxTokens) : undefined }); console.log(JSON.stringify({ file: path.relative(root, result.file).replaceAll('\\', '/'), estimatedTokens: result.payload.estimatedTokens, omissions: result.payload.omissions }, null, 2)); break; } + case 'ignore': ignore(root, positional[0]); break; + case 'source-list': sourceList(root, positional.join(' ')); break; + case 'source-grep': sourceGrep(root, positional.join(' ')); break; + case 'stats': console.log(JSON.stringify(databaseStats(root), null, 2)); break; + default: usage(); throw new Error(`Unknown command: ${command}`); + } +} + +main().catch((error) => { console.error(`ERROR: ${error.message}`); process.exitCode = error.exitCode ?? 1; }); diff --git a/global-template/docgen/bin/docgen.mjs b/global-template/docgen/bin/docgen.mjs deleted file mode 100644 index 8b9c8b1..0000000 --- a/global-template/docgen/bin/docgen.mjs +++ /dev/null @@ -1,2612 +0,0 @@ -#!/usr/bin/env node -import fs from 'node:fs'; -import path from 'node:path'; -import crypto from 'node:crypto'; -import { spawn, spawnSync } from 'node:child_process'; -import { fileURLToPath } from 'node:url'; - -const self = fileURLToPath(import.meta.url); -const engineHome = path.resolve(path.dirname(self), '..'); -const commandCodeHome = path.resolve(engineHome, '..'); -const kitVersion = fs.readFileSync(path.join(engineHome, 'VERSION'), 'utf8').trim(); - -function findProjectRoot(start = process.cwd()) { - let current = path.resolve(start); - while (true) { - if (fs.existsSync(path.join(current, '.docgen', 'project.json'))) return current; - const parent = path.dirname(current); - if (parent === current) return null; - current = parent; - } -} - -let root = findProjectRoot() ?? path.resolve(process.cwd()); -function setRoot(nextRoot) { root = path.resolve(nextRoot); sourceSnapshotCache=null; sourceInventoryCache=null; gitRepositoryAvailabilityCache=null; gitIgnoreSingleCache.clear(); ignoreRulesCache.clear(); } - -const statePath = path.join(root, '.docgen', 'state', 'state.json'); -const manifestPath = path.join(root, '.docgen', 'plan', 'manifest.json'); -const evidenceIndexPath = path.join(root, '.docgen', 'evidence', 'index.json'); -const systemPath = path.join(root, '.docgen', 'model', 'system.json'); -const businessPath = path.join(root, '.docgen', 'model', 'business.json'); -const flowsPath = path.join(root, '.docgen', 'model', 'flows.json'); -const catalogsPath = path.join(root, '.docgen', 'model', 'catalogs.json'); -const securityPath = path.join(root, '.docgen', 'model', 'security.json'); -const operationsPath = path.join(root, '.docgen', 'model', 'operations.json'); -const testingPath = path.join(root, '.docgen', 'model', 'testing.json'); -const dataGovernancePath = path.join(root, '.docgen', 'model', 'data-governance.json'); -const decisionsPath = path.join(root, '.docgen', 'model', 'decisions.json'); -const configurationPath = path.join(root, '.docgen', 'model', 'configuration.json'); -const changeImpactPath = path.join(root, '.docgen', 'model', 'change-impact.json'); -const ownershipPath = path.join(root, '.docgen', 'model', 'ownership.json'); -const auditIndexPath = path.join(root, '.docgen', 'audit', 'index.json'); -const configPath = path.join(root, '.docgen', 'config', 'documentation.json'); -const fingerprintsPath = path.join(root, '.docgen', 'state', 'fingerprints.json'); -const pageStatePath = path.join(root, '.docgen', 'state', 'pages.json'); -const preflightPath = path.join(root, '.docgen', 'plan', 'preflight.json'); -const traceabilityRoot = path.join(root, '.docgen', 'traceability'); -const traceabilityPagesRoot = path.join(traceabilityRoot, 'pages'); -const traceabilityIndexPath = path.join(traceabilityRoot, 'index.json'); -const contradictionsPath = path.join(traceabilityRoot, 'contradictions.json'); -const duplicatesPath = path.join(traceabilityRoot, 'duplicates.json'); -const freshnessPath = path.join(traceabilityRoot, 'freshness.json'); -const sourceInventoryPath = path.join(root, '.docgen', 'state', 'source-inventory.json'); -const sourceFilesPath = path.join(root, '.docgen', 'state', 'source-files.txt'); -const ignoreReportPath = path.join(root, '.docgen', 'state', 'ignore-report.json'); -const publishRoot = path.join(root, '.docgen', 'publish'); -const navigationIndexPath = path.join(publishRoot, 'navigation.json'); -const searchIndexPath = path.join(publishRoot, 'search-index.json'); -const backlinksPath = path.join(publishRoot, 'backlinks.json'); -const redirectsPath = path.join(publishRoot, 'redirects.json'); -const orphansPath = path.join(publishRoot, 'orphans.json'); -const examplesIndexPath = path.join(publishRoot, 'examples.json'); -const publishingReportPath = path.join(publishRoot, 'report.json'); -let sourceSnapshotCache = null; -let sourceInventoryCache = null; -let gitRepositoryAvailabilityCache = null; -const gitIgnoreSingleCache = new Map(); -const ignoreRulesCache = new Map(); - -function fail(message, code = 1) { console.error(`ERROR: ${message}`); process.exit(code); } -function exists(relOrAbs) { return fs.existsSync(path.isAbsolute(relOrAbs) ? relOrAbs : path.join(root, relOrAbs)); } -function readJson(file) { return JSON.parse(fs.readFileSync(file, 'utf8')); } -function writeJson(file, value) { fs.mkdirSync(path.dirname(file), { recursive: true }); fs.writeFileSync(file, JSON.stringify(value, null, 2) + '\n'); } -function now() { return new Date().toISOString(); } -function rel(file) { return path.relative(root, file).replaceAll('\\', '/'); } -function sleep(ms) { return new Promise((resolve) => setTimeout(resolve, ms)); } -function terminateProcessTree(child) { if (!child?.pid) return; try { if (process.platform === 'win32') spawnSync('taskkill', ['/PID', String(child.pid), '/T', '/F'], { stdio: 'ignore', shell: true }); else child.kill('SIGTERM'); } catch {} } -function sha256Text(text) { return crypto.createHash('sha256').update(text).digest('hex'); } -function fileSha256(file) { return fs.existsSync(file) ? crypto.createHash('sha256').update(fs.readFileSync(file)).digest('hex') : null; } -function loadPageState() { return fs.existsSync(pageStatePath) ? readJson(pageStatePath) : { schemaVersion: '1.0', kitVersion, pages: {} }; } -function updatePageState(pageId, patch) { const s = loadPageState(); s.schemaVersion = '1.0'; s.kitVersion = kitVersion; s.updatedAt = now(); s.pages ??= {}; s.pages[pageId] = { ...(s.pages[pageId] ?? {}), ...patch, updatedAt: now() }; writeJson(pageStatePath, s); } - - -function loadState() { - if (!fs.existsSync(statePath)) return { schemaVersion: '1.0', kitVersion, stages: {} }; - return readJson(statePath); -} -function updateStage(stage, status, details = {}) { - const state = loadState(); - state.schemaVersion = '1.0'; state.kitVersion = kitVersion; state.updatedAt = now(); - state.stages ??= {}; state.stages[stage] = { status, updatedAt: now(), ...details }; - if (status === 'completed') { - const downstream = { discover:['analyze','semantics','enterprise','plan','generate','audit'], analyze:['semantics','enterprise','plan','generate','audit'], semantics:['enterprise','plan','generate','audit'], enterprise:['plan','generate','audit'], plan:['generate','audit'] }[stage] ?? []; - for (const next of downstream) state.stages[next] = { ...(state.stages[next] ?? {}), status:'pending', invalidatedBy:stage, updatedAt:now() }; - } - writeJson(statePath, state); -} - -function commandExists(command) { - const checker = process.platform === 'win32' ? 'where' : 'which'; - const r = spawnSync(checker, [command], { stdio: 'ignore', shell: process.platform === 'win32' }); - return r.status === 0; -} -function loadConfig() { - if (!fs.existsSync(configPath)) return {}; - return readJson(configPath); -} -function mergeAdditiveDefaults(current, defaults, keyPath = '') { - if (Array.isArray(defaults)) { - if (!Array.isArray(current)) return defaults; - const unionPaths = new Set(['audiences', 'pageTypes', 'sourceExtensions', 'exclude', 'quality.requiredCoverageTagsWhenEvidenceExists', 'enterpriseDepth.requiredCoverageTagsWhenEvidenceExists', 'enterpriseDepth.passes', 'enterpriseDepth.models']); - if (unionPaths.has(keyPath)) return [...new Set([...current, ...defaults])]; - return current; - } - if (defaults && typeof defaults === 'object') { - const out = current && typeof current === 'object' && !Array.isArray(current) ? { ...current } : {}; - for (const [k, v] of Object.entries(defaults)) out[k] = mergeAdditiveDefaults(out[k], v, keyPath ? `${keyPath}.${k}` : k); - return out; - } - return current === undefined ? defaults : current; -} -function migrateProjectConfig(silent = false) { - if (!fs.existsSync(configPath)) return false; - const defaultsPath = path.join(engineHome, 'project-template', 'config', 'documentation.json'); - if (!fs.existsSync(defaultsPath)) return false; - const current = readJson(configPath); - const defaults = readJson(defaultsPath); - const merged = mergeAdditiveDefaults(current, defaults); - merged.schemaVersion = defaults.schemaVersion ?? merged.schemaVersion; - const changed = JSON.stringify(current) !== JSON.stringify(merged); - if (changed) { - writeJson(configPath, merged); - if (!silent) console.log(`[docgen] migrated project configuration additively to DocGen ${kitVersion}. Existing custom values were preserved.`); - } else if (!silent) { - console.log(`[docgen] project configuration is already compatible with DocGen ${kitVersion}.`); - } - const markerPath = path.join(root, '.docgen', 'project.json'); - if (fs.existsSync(markerPath)) { - const marker = readJson(markerPath); - if (marker.kitVersion !== kitVersion) { marker.kitVersion = kitVersion; marker.migratedAt = now(); writeJson(markerPath, marker); } - } - return changed; -} -function commandCodeBin() { - if (process.env.DOCGEN_COMMAND_CODE_BIN) return process.env.DOCGEN_COMMAND_CODE_BIN; - const configured = loadConfig().commandCode?.executable; - if (configured) return configured; - const candidates = process.platform === 'win32' ? ['cmdc', 'command-code'] : ['cmd', 'command-code', 'cmdc']; - for (const c of candidates) if (commandExists(c)) return c; - return null; -} -function commandCodeArgs(stage) { - const cc = loadConfig().commandCode ?? {}; - const args = ['-p']; - if (cc.trust !== false) args.push('--trust'); - if (cc.skipOnboarding !== false) args.push('--skip-onboarding'); - if (cc.yolo !== false) args.push('--yolo'); - - const envTurns = Number.parseInt(process.env.DOCGEN_MAX_TURNS ?? '', 10); - const configuredTurns = cc.maxTurns?.[stage] ?? cc.maxTurns?.default; - const maxTurns = Number.isFinite(envTurns) && envTurns > 0 ? envTurns : configuredTurns; - if (Number.isInteger(maxTurns) && maxTurns > 0) args.push('--max-turns', String(maxTurns)); - - const model = process.env.DOCGEN_MODEL || cc.stageModels?.[stage] || cc.model; - if (model) args.push('--model', String(model)); - const progress = loadConfig().progress ?? {}; - if (progress.verboseCommandCode !== false) args.push('--verbose'); - return args; -} -function assetFile(kind, name) { - const projectOverride = path.join(root, '.docgen', kind, name); - if (fs.existsSync(projectOverride)) return projectOverride; - return path.join(engineHome, kind, name); -} -function renderPrompt(name, vars = {}) { - const file = assetFile('prompts', name); - let text = fs.readFileSync(file, 'utf8'); - for (const [k, v] of Object.entries(vars)) text = text.replaceAll(`{{${k}}}`, String(v)); - return text; -} -const EXIT_CLASSIFICATION = { - 0: ['success', 'Command Code completed successfully.'], - 1: ['general-error', 'Command Code returned a general error.'], - 3: ['not-authenticated', 'Command Code is not authenticated. Run `cmd login` (`cmdc login` on native Windows).'], - 4: ['permission-denied', 'Command Code denied a required permission or a DocGen hook blocked an operation.'], - 5: ['rate-limited', 'The LLM/API provider rate limit was exceeded. DocGen exhausted its configured retry/backoff policy.'], - 6: ['network-failure', 'Command Code could not reach the API/provider. Check network, proxy, DNS, or provider availability.'], - 7: ['api-server-error', 'The LLM/API provider returned a server-side 5xx error.'], - 8: ['max-turns', 'The headless max-turn limit was reached before completion. Increase commandCode.maxTurns for this stage.'], - 124: ['stage-timeout', 'The configured DocGen stage timeout was reached and the Command Code process was terminated.'], - 130: ['interrupted', 'The process was interrupted by SIGINT/SIGTERM.'] -}; - -function duration(ms) { - const total = Math.max(0, Math.floor(ms / 1000)); - const h = Math.floor(total / 3600); - const m = Math.floor((total % 3600) / 60); - const s = total % 60; - return h ? `${h}h ${String(m).padStart(2, '0')}m ${String(s).padStart(2, '0')}s` : `${m}m ${String(s).padStart(2, '0')}s`; -} -function bar(current, total, width = 24) { - if (!total || total < 1) return '[????????????????????????]'; - const ratio = Math.min(1, Math.max(0, current / total)); - const filled = Math.round(ratio * width); - return `[${'='.repeat(filled)}${'.'.repeat(width - filled)}] ${(ratio * 100).toFixed(0).padStart(3)}%`; -} -function pageWordCount(text) { - return (text.replace(/```[\s\S]*?```/g, ' ').match(/\b[\p{L}\p{N}_'-]+\b/gu) ?? []).length; -} -function headingNames(text) { - return [...text.matchAll(/^#{2,6}\s+(.+)$/gm)].map((m) => m[1].trim().toLowerCase()); -} -function normalizeHeading(s) { return String(s).toLowerCase().replace(/[`*_]/g, '').replace(/[^a-z0-9]+/g, ' ').trim(); } -function qualityConfig() { return loadConfig().quality ?? {}; } -function progressConfig() { return loadConfig().progress ?? {}; } -function qualityProfile() { return process.env.DOCGEN_QUALITY_PROFILE || qualityConfig().profile || 'balanced'; } -function isComprehensive() { return qualityProfile() === 'comprehensive'; } -function printItemProgress(action, current, total, id = '') { - console.log(`\n${bar(current, total)} ${action} ${current}/${total}${id ? ` — ${id}` : ''}`); -} -function recentArtifactActivity(sinceMs) { - let count = 0; - for (const relDir of ['.docgen', 'docs']) { - const base = path.join(root, relDir); - if (!fs.existsSync(base)) continue; - const stack = [base]; - while (stack.length) { - const dir = stack.pop(); - for (const e of fs.readdirSync(dir, { withFileTypes: true })) { - const full = path.join(dir, e.name); - if (e.isDirectory()) stack.push(full); - else { - try { if (fs.statSync(full).mtimeMs >= sinceMs) count++; } catch {} - } - } - } - } - return count; -} - -function retryConfig() { - const cfg = loadConfig().retry ?? {}; - return { - enabled: cfg.enabled !== false, - maxAttempts: Math.max(1, Number(cfg.maxAttempts ?? 5)), - retryableExitCodes: Array.isArray(cfg.retryableExitCodes) ? cfg.retryableExitCodes.map(Number) : [5, 6, 7], - initialDelaySeconds: Math.max(1, Number(cfg.initialDelaySeconds ?? 30)), - rateLimitDelaySeconds: Math.max(1, Number(cfg.rateLimitDelaySeconds ?? 60)), - maxDelaySeconds: Math.max(1, Number(cfg.maxDelaySeconds ?? 300)), - multiplier: Math.max(1, Number(cfg.multiplier ?? 2)), - jitterRatio: Math.min(1, Math.max(0, Number(cfg.jitterRatio ?? 0.2))), - countdownSeconds: Math.max(1, Number(cfg.countdownSeconds ?? 10)), - interRequestDelaySeconds: Math.max(0, Number(cfg.interRequestDelaySeconds ?? 2)) - }; -} -function classifyCommandFailure(exitCode, stderrText = '', stdoutText = '') { - const combined = `${stderrText}\n${stdoutText}`; - if (exitCode === 5 || /(?:rate[ -]?limit|too many requests|\b429\b|quota exceeded|usage exceeded)/i.test(combined)) { - return ['rate-limited', EXIT_CLASSIFICATION[5][1]]; - } - if (exitCode === 6 || /(?:connection error|network failure|ECONNRESET|ETIMEDOUT|ENOTFOUND|socket hang up)/i.test(combined)) { - return ['network-failure', EXIT_CLASSIFICATION[6][1]]; - } - if (exitCode === 7 || /(?:\b5\d\d\b|internal server error|bad gateway|service unavailable|gateway timeout)/i.test(combined)) { - return ['api-server-error', EXIT_CLASSIFICATION[7][1]]; - } - return EXIT_CLASSIFICATION[exitCode] ?? ['unknown-error', `Command Code exited with code ${exitCode}.`]; -} -async function cooldown(seconds, reason) { - let remaining = Math.ceil(seconds); - const step = retryConfig().countdownSeconds; - while (remaining > 0) { - console.log(`[docgen] retry cooldown (${reason}): ${remaining}s remaining`); - const wait = Math.min(step, remaining); - await sleep(wait * 1000); - remaining -= wait; - } -} -async function runCommandCodeOnce(stage, prompt, target = '', progressLabel = '', attempt = 1, maxAttempts = 1) { - const bin = commandCodeBin(); - if (!bin) throw Object.assign(new Error('Command Code executable not found. Install it or set DOCGEN_COMMAND_CODE_BIN.'), { exitCode: 1 }); - const args = commandCodeArgs(stage); - const runId = `${new Date().toISOString().replace(/[:.]/g, '-')}-${stage}-attempt-${String(attempt).padStart(2, '0')}`; - const startedMs = Date.now(); - const meta = { - schemaVersion: '1.2', runId, stage, target, progressLabel, attempt, maxAttempts, startedAt: now(), - commandCodeBin: bin, commandCodeArgs: args, status: 'running' - }; - const runDir = path.join(root, '.docgen', 'runs'); - fs.mkdirSync(runDir, { recursive: true }); - const metaPath = path.join(runDir, `${runId}.json`); - const stdoutLogPath = path.join(runDir, `${runId}.stdout.log`); - const stderrLogPath = path.join(runDir, `${runId}.stderr.log`); - writeJson(metaPath, meta); - fs.writeFileSync(stdoutLogPath, ''); fs.writeFileSync(stderrLogPath, ''); - - const env = { ...process.env, DOCGEN_MODE: '1', DOCGEN_STAGE: stage, DOCGEN_TARGET: target }; - console.log(`\n==> ${stage}${target ? `: ${target}` : ''}${progressLabel ? ` | ${progressLabel}` : ''} | attempt ${attempt}/${maxAttempts}`); - console.log(` ${bin} ${args.join(' ')}`); - console.log(` logs: ${rel(stdoutLogPath)} | ${rel(stderrLogPath)}`); - - const pc = progressConfig(); - const heartbeatMs = Math.max(2, Number(pc.heartbeatSeconds ?? 10)) * 1000; - const noOutputWarnMs = Math.max(5, Number(pc.noOutputWarningSeconds ?? 45)) * 1000; - let lastOutputMs = Date.now(); - let warnedNoOutput = false; - let stderrTail = ''; - let stdoutTail = ''; - - return await new Promise((resolve, reject) => { - let settled = false; - const child = spawn(bin, args, { - cwd: root, env, shell: process.platform === 'win32', stdio: ['pipe', 'pipe', 'pipe'], windowsHide: true - }); - meta.pid = child.pid ?? null; writeJson(metaPath, meta); - - const onChunk = (stream, chunk, logPath) => { - lastOutputMs = Date.now(); warnedNoOutput = false; - const chunkText = chunk.toString(); - fs.appendFileSync(logPath, chunkText); - if (stream === 'stderr') stderrTail = (stderrTail + chunkText).slice(-16000); - else stdoutTail = (stdoutTail + chunkText).slice(-16000); - if (pc.showCommandOutput !== false) (stream === 'stdout' ? process.stdout : process.stderr).write(chunkText); - }; - child.stdout.on('data', (c) => onChunk('stdout', c, stdoutLogPath)); - child.stderr.on('data', (c) => onChunk('stderr', c, stderrLogPath)); - - const timeoutMs = stageTimeoutMs(stage); - const stageTimer = timeoutMs > 0 ? setTimeout(() => { - if (settled) return; - console.error(`[docgen] ${stage}${target ? `:${target}` : ''} exceeded stage timeout ${duration(timeoutMs)}; terminating process ${child.pid ?? '?'}.`); - terminateProcessTree(child); - setTimeout(() => { try { if (!settled && process.platform !== 'win32') child.kill('SIGKILL'); } catch {} }, 5000).unref?.(); - }, timeoutMs) : null; - stageTimer?.unref?.(); - - const heartbeat = setInterval(() => { - const nowMs = Date.now(); - const quiet = nowMs - lastOutputMs; - const artifacts = recentArtifactActivity(startedMs); - const quietMsg = quiet >= noOutputWarnMs ? ` | no CLI output for ${duration(quiet)}` : ''; - console.log(`[docgen] ${stage}${target ? `:${target}` : ''} RUNNING | attempt ${attempt}/${maxAttempts} | elapsed ${duration(nowMs - startedMs)} | pid ${child.pid ?? '?'} | changed artifacts ${artifacts}${quietMsg}`); - if (quiet >= noOutputWarnMs && !warnedNoOutput) { - warnedNoOutput = true; - console.log('[docgen] Process is alive. Provider/API failures will trigger retry/backoff when the process exits.'); - } - }, heartbeatMs); - - child.on('error', (err) => { - if (settled) return; settled = true; - clearInterval(heartbeat); if (stageTimer) clearTimeout(stageTimer); - meta.finishedAt = now(); meta.status = 'failed-to-launch'; meta.error = err.message; - writeJson(metaPath, meta); - reject(Object.assign(new Error(`${stage} failed to launch: ${err.message}`), { exitCode: 1, classification: 'launch-error' })); - }); - child.on('close', (code, signal) => { - if (settled) return; settled = true; - clearInterval(heartbeat); if (stageTimer) clearTimeout(stageTimer); - const elapsed = Date.now() - startedMs; - const timedOut = stageTimeoutMs(stage) > 0 && elapsed >= stageTimeoutMs(stage) - 1000 && signal; - const exitCode = timedOut ? 124 : (code ?? (signal ? 130 : 1)); - const [classification, explanation] = classifyCommandFailure(exitCode, stderrTail, stdoutTail); - meta.finishedAt = now(); meta.durationMs = Date.now() - startedMs; meta.exitCode = exitCode; - meta.signal = signal ?? null; meta.status = exitCode === 0 ? 'completed' : 'failed'; - meta.errorClassification = classification; meta.stdoutLog = rel(stdoutLogPath); meta.stderrLog = rel(stderrLogPath); - writeJson(metaPath, meta); - console.log(`[docgen] ${stage}${target ? `:${target}` : ''} ${exitCode === 0 ? 'COMPLETED' : 'FAILED'} | attempt ${attempt}/${maxAttempts} | ${duration(meta.durationMs)} | exit ${exitCode} (${classification})`); - if (exitCode !== 0) { - const tail = (stderrTail || stdoutTail).trim(); - const detail = tail ? `\n\nLast provider/CLI output:\n${tail}` : ''; - reject(Object.assign(new Error(`${stage} failed: ${explanation}${detail}`), { exitCode, classification, stderrTail, stdoutTail })); - } else resolve(meta); - }); - - child.stdin.on('error', () => {}); - child.stdin.end(prompt); - }); -} -async function runCommandCode(stage, prompt, target = '', progressLabel = '', lifecycle = {}) { - const cfg = retryConfig(); - const maxAttempts = cfg.enabled ? cfg.maxAttempts : 1; - let lastError; - for (let attempt = 1; attempt <= maxAttempts; attempt++) { - try { - const result = await runCommandCodeOnce(stage, prompt, target, progressLabel, attempt, maxAttempts); - if (cfg.interRequestDelaySeconds > 0) await sleep(cfg.interRequestDelaySeconds * 1000); - return result; - } catch (err) { - lastError = err; - const retryable = cfg.enabled && attempt < maxAttempts && (cfg.retryableExitCodes.includes(Number(err.exitCode)) || ['rate-limited', 'network-failure', 'api-server-error'].includes(err.classification)); - if (!retryable) throw err; - const base = err.classification === 'rate-limited' ? cfg.rateLimitDelaySeconds : cfg.initialDelaySeconds; - const exponential = Math.min(cfg.maxDelaySeconds, base * (cfg.multiplier ** (attempt - 1))); - const jitter = exponential * cfg.jitterRatio * ((Math.random() * 2) - 1); - const delay = Math.max(1, Math.round(exponential + jitter)); - console.warn(`[docgen] retryable ${err.classification ?? 'provider error'} on ${stage}${target ? `:${target}` : ''}; retry ${attempt + 1}/${maxAttempts} after ~${delay}s.`); - try { await lifecycle.beforeRetry?.(err, attempt); } catch (resetError) { throw Object.assign(new Error(`Failed to reset stage outputs before retry: ${resetError.message}`), { exitCode: 1, classification: 'contract-reset-error' }); } - await cooldown(delay, err.classification ?? 'provider-error'); - } - } - throw lastError; -} - - -function arrayValue(obj, keys, fallback = []) { - for (const key of keys) { - const value = obj?.[key]; - if (Array.isArray(value)) return value; - if (value && typeof value === 'object') return Object.values(value); - if (value !== undefined && value !== null && value !== '') return [value]; - } - return fallback; -} -function scalarValue(obj, keys, fallback = undefined) { - for (const key of keys) { - const value = obj?.[key]; - if (value !== undefined && value !== null && value !== '') return value; - } - return fallback; -} -function itemIdentity(value) { - if (value && typeof value === 'object') { - const identity = value.id ?? value.key ?? value.name ?? value.path ?? value.operationId ?? value.handler ?? value.topic ?? value.queue; - if (identity !== undefined) return `id:${String(identity)}`; - const ordered = Object.fromEntries(Object.entries(value).sort(([a], [b]) => a.localeCompare(b))); - return `json:${JSON.stringify(ordered)}`; - } - return `scalar:${String(value)}`; -} -function uniqueArray(values) { - const seen = new Set(); const out = []; - for (const value of values) { const key = itemIdentity(value); if (seen.has(key)) continue; seen.add(key); out.push(value); } - return out; -} -function canonicalArray(obj, canonicalKey, aliases = []) { - const values = []; - for (const key of [canonicalKey, ...aliases]) { - const value = obj?.[key]; - if (Array.isArray(value)) values.push(...value); - else if (value && typeof value === 'object') values.push(...Object.values(value)); - else if (value !== undefined && value !== null && value !== '') values.push(value); - } - return uniqueArray(values); -} -function canonicalModelBase(obj = {}) { - return { - ...obj, - schemaVersion: '1.0', - generatedAt: obj.generatedAt ?? obj.createdAt ?? obj.updatedAt ?? obj.timestamp ?? now() - }; -} -function normalizeClassification(value, evidence = []) { - const raw = String(value ?? '').trim().toUpperCase(); - if (['FACT', 'INFERENCE', 'UNKNOWN'].includes(raw)) return raw; - return evidence.length ? 'FACT' : 'UNKNOWN'; -} -function normalizeConfidence(value, classification = 'UNKNOWN') { - const n = Number(value); - if (Number.isFinite(n)) return Math.max(0, Math.min(1, n)); - return classification === 'FACT' ? 1 : classification === 'INFERENCE' ? 0.65 : 0; -} -function parseEvidenceLocation(raw) { - let value = String(raw ?? '').replaceAll('\\', '/'); let startLine = null, endLine = null; - let m = value.match(/^(.*)#L(\d+)(?:-L?(\d+))?$/i); - if (!m) m = value.match(/^(.*?):(\d+)(?:-(\d+))?$/); - if (m && m[1] && !/^[A-Za-z]$/.test(m[1])) { value = m[1]; startLine = Number(m[2]); endLine = Number(m[3] ?? m[2]); } - return { path: value, startLine, endLine }; -} -function normalizeEvidenceRef(value, index = 0) { - if (typeof value === 'string') { const loc=parseEvidenceLocation(value); return { id:`evidence-${index+1}`, path:loc.path, symbol:null, startLine:loc.startLine, endLine:loc.endLine, note:null }; } - const obj = value && typeof value === 'object' ? value : {}; - const loc = parseEvidenceLocation(scalarValue(obj, ['path','file','sourcePath','source','location'], '')); - const line = scalarValue(obj, ['line','lineNumber'], null); - return { - id: slug(scalarValue(obj, ['id','key','name'], `evidence-${index+1}`)), path: loc.path, - symbol: scalarValue(obj, ['symbol','method','class','function','member'], null), - startLine: Number(scalarValue(obj, ['startLine','lineStart'], loc.startLine ?? line)) || null, - endLine: Number(scalarValue(obj, ['endLine','lineEnd'], loc.endLine ?? line)) || null, - note: scalarValue(obj, ['note','reason','description'], null) - }; -} -function normalizeEvidenceRefs(value) { - const arr = Array.isArray(value) ? value : value ? [value] : []; - return uniqueArray(arr.map(normalizeEvidenceRef).filter((x) => x.path || x.symbol)); -} -function normalizeStringRefs(value) { - const arr = Array.isArray(value) ? value : value ? [value] : []; - return [...new Set(arr.map((x) => typeof x === 'string' ? x : x?.id ?? x?.path ?? x?.name).filter(Boolean).map(String))]; -} -function typedBase(value, kind, index, options = {}) { - const obj = typeof value === 'string' ? { statement: value, name: value } : value && typeof value === 'object' ? { ...value } : { statement: String(value ?? '') }; - const evidence = normalizeEvidenceRefs(canonicalArray(obj, 'evidence', ['sources', 'sourceRefs', 'references', 'proof'])); - const name = String(scalarValue(obj, ['name', 'title', 'label', 'operationId', 'handler', 'statement', 'description'], `${kind}-${index + 1}`)); - const statement = String(scalarValue(obj, ['statement', 'rule', 'summary', 'description', 'name', 'title'], name)); - const classification = normalizeClassification(scalarValue(obj, ['classification', 'epistemicStatus', 'status'], null), evidence); - const out = { - id: slug(scalarValue(obj, ['id', 'key', 'code', 'operationId', 'name', 'title'], `${kind}-${index + 1}`)), - kind, - name, - statement, - description: scalarValue(obj, ['description', 'details', 'explanation'], null), - classification, - confidence: normalizeConfidence(scalarValue(obj, ['confidence', 'score'], null), classification), - evidence, - sourceModelRefs: normalizeStringRefs(canonicalArray(obj, 'sourceModelRefs', ['modelRefs', 'relatedModelItems', 'semanticRefs'])), - unknowns: canonicalArray(obj, 'unknowns', ['openQuestions', 'gaps']).map(String), - tags: normalizeStringRefs(canonicalArray(obj, 'tags', ['labels', 'categories'])) - }; - for (const [key, aliases, fallback] of options.fields ?? []) out[key] = scalarValue(obj, [key, ...aliases], fallback); - for (const [key, aliases] of options.arrays ?? []) out[key] = canonicalArray(obj, key, aliases); - return out; -} -function normalizeStep(value, index = 0) { - const x = typedBase(value, 'flow-step', index, { fields: [ - ['actor', ['role', 'principal'], null], ['component', ['service', 'module', 'system'], null], - ['action', ['operation', 'activity', 'statement'], null], ['input', ['request', 'sourceData'], null], - ['output', ['response', 'result', 'targetData'], null], ['condition', ['guard', 'when'], null] - ]}); - const obj = value && typeof value === 'object' ? value : {}; - x.order = Number(scalarValue(obj, ['order', 'sequence', 'index'], index + 1)) || index + 1; - return x; -} - -function normalizeBranch(value, index = 0) { - return typedBase(value, 'branch', index, { fields: [ - ['condition', ['guard', 'when', 'expression'], null], ['outcome', ['result', 'then'], null], - ['elseOutcome', ['otherwise', 'else'], null] - ], arrays: [['nextSteps', ['targets', 'transitions']]] }); -} -function collectArrayValues(obj, canonicalKey, aliases = []) { - const values=[]; for(const key of [canonicalKey,...aliases]){ const value=obj?.[key]; if(Array.isArray(value)) values.push(...value); else if(value&&typeof value==='object') values.push(...Object.values(value)); else if(value!==undefined&&value!==null&&value!=='') values.push(value); } return values; -} -function normalizeTypedArray(obj, canonicalKey, aliases, kind, options = {}) { - return collectArrayValues(obj, canonicalKey, aliases).map((x, i) => { - const item = typedBase(x, kind, i, options); - if (options.steps) item.steps = canonicalArray(x && typeof x === 'object' ? x : {}, 'steps', ['activities', 'sequence']).map(normalizeStep); - if (options.branches) item.branches = canonicalArray(x && typeof x === 'object' ? x : {}, 'branches', ['conditions', 'decisionBranches']).map(normalizeBranch); - return item; - }); -} -function normalizeSystemObject(input = {}) { - const obj = input && typeof input === 'object' && !Array.isArray(input) ? input : {}; - return { - schemaVersion: '1.0', generatedAt: obj.generatedAt ?? obj.createdAt ?? obj.updatedAt ?? obj.timestamp ?? now(), - components: normalizeTypedArray(obj, 'components', ['services', 'modules', 'subsystems', 'applications', 'nodes'], 'component', { arrays: [['responsibilities', ['capabilities', 'functions']], ['interfaces', ['ports', 'endpoints']], ['dependencies', ['dependsOn']]] }), - relationships: normalizeTypedArray(obj, 'relationships', ['dependencies', 'links', 'interactions', 'connections', 'edges'], 'relationship', { fields: [['from', ['source', 'origin'], null], ['to', ['target', 'destination'], null], ['mechanism', ['protocol', 'channel', 'type'], null]] }), - workflows: normalizeTypedArray(obj, 'workflows', ['processes', 'executionFlows', 'systemFlows', 'scenarios'], 'workflow', { fields: [['trigger', ['start', 'initiator'], null], ['outcome', ['result', 'end'], null]], arrays: [['failurePaths', ['errors', 'exceptions']]], steps: true, branches: true }), - unknowns: normalizeTypedArray(obj, 'unknowns', ['openQuestions', 'unresolved', 'gaps', 'uncertainties'], 'unknown'), - metadata: obj.metadata ?? {} - }; -} -function normalizeBusinessObject(input = {}) { - const obj = input && typeof input === 'object' && !Array.isArray(input) ? input : {}; - return { - schemaVersion: '1.0', generatedAt: obj.generatedAt ?? obj.createdAt ?? obj.updatedAt ?? obj.timestamp ?? now(), - actors: normalizeTypedArray(obj, 'actors', ['roles', 'personas', 'participants', 'users'], 'actor', { arrays: [['responsibilities', ['duties']], ['goals', ['outcomes']]] }), - capabilities: normalizeTypedArray(obj, 'capabilities', ['businessCapabilities', 'functions', 'features'], 'capability', { arrays: [['owners', ['actors']], ['outcomes', ['results']]] }), - concepts: normalizeTypedArray(obj, 'concepts', ['domainConcepts', 'entities', 'terms', 'vocabulary'], 'concept', { fields: [['definition', ['meaning', 'description'], null]], arrays: [['aliases', ['synonyms']], ['relationships', ['relatedConcepts']]] }), - businessRules: normalizeTypedArray(obj, 'businessRules', ['rules', 'policies', 'businessLogic'], 'business-rule', { fields: [['trigger', ['when'], null], ['outcome', ['result', 'then'], null], ['failureOutcome', ['otherwise', 'error'], null]], arrays: [['conditions', ['guards', 'preconditions']], ['exceptions', ['overrides']]] }), - decisions: normalizeTypedArray(obj, 'decisions', ['decisionPoints', 'decisionRules'], 'decision', { fields: [['question', ['decision', 'statement'], null]], branches: true }), - branchConditions: normalizeTypedArray(obj, 'branchConditions', ['branches', 'conditions', 'guards'], 'branch-condition', { fields: [['expression', ['condition', 'guard', 'when'], null], ['outcome', ['result', 'then'], null], ['elseOutcome', ['otherwise', 'else'], null]] }), - lifecycles: normalizeTypedArray(obj, 'lifecycles', ['lifeCycles', 'stateMachines', 'stateLifecycles'], 'lifecycle', { arrays: [['states', ['statuses']], ['transitions', ['stateTransitions']]], branches: true }), - invariants: normalizeTypedArray(obj, 'invariants', ['domainInvariants', 'constraints'], 'invariant', { fields: [['scope', ['appliesTo'], null], ['violationOutcome', ['failureOutcome'], null]] }), - useCases: normalizeTypedArray(obj, 'useCases', ['usecases', 'scenarios', 'businessScenarios'], 'use-case', { fields: [['actor', ['primaryActor'], null], ['trigger', ['precondition'], null], ['outcome', ['postcondition'], null]], steps: true, branches: true }), - unknowns: normalizeTypedArray(obj, 'unknowns', ['openQuestions', 'unresolved', 'gaps', 'uncertainties'], 'unknown'), - metadata: obj.metadata ?? {} - }; -} -function normalizeFlowsObject(input = {}) { - const obj = input && typeof input === 'object' && !Array.isArray(input) ? input : {}; - const make = (key, aliases, kind) => normalizeTypedArray(obj, key, aliases, kind, { fields: [['trigger', ['start', 'initiator'], null], ['outcome', ['result', 'end'], null]], arrays: [['failurePaths', ['errors', 'exceptions']], ['trustBoundaries', ['boundaries']], ['protocols', ['protocol']]], steps: true, branches: true }); - const result = { - schemaVersion: '1.0', generatedAt: obj.generatedAt ?? obj.createdAt ?? obj.updatedAt ?? obj.timestamp ?? now(), - businessFlows: make('businessFlows', ['businessProcesses', 'businessWorkflows'], 'business-flow'), - controlFlows: make('controlFlows', ['executionFlows', 'codeFlows', 'callFlows'], 'control-flow'), - requestFlows: make('requestFlows', ['httpFlows', 'apiFlows', 'inboundFlows'], 'request-flow'), - trafficFlows: make('trafficFlows', ['networkFlows', 'runtimeTrafficFlows'], 'traffic-flow'), - dataFlows: make('dataFlows', ['dataPipelines', 'informationFlows'], 'data-flow'), - eventFlows: make('eventFlows', ['messageFlows', 'messagingFlows', 'asyncFlows'], 'event-flow'), - metadata: obj.metadata ?? {} - }; - const generic = arrayValue(obj, ['flows'], []); - for (const flow of generic) { - const type = String(flow?.type ?? flow?.kind ?? flow?.category ?? '').toLowerCase(); - let key = null, kind = null; - if (/business/.test(type)) [key, kind] = ['businessFlows', 'business-flow']; - else if (/control|execution|call/.test(type)) [key, kind] = ['controlFlows', 'control-flow']; - else if (/request|http|api/.test(type)) [key, kind] = ['requestFlows', 'request-flow']; - else if (/traffic|network|runtime/.test(type)) [key, kind] = ['trafficFlows', 'traffic-flow']; - else if (/data|information/.test(type)) [key, kind] = ['dataFlows', 'data-flow']; - else if (/event|message|async|kafka|rabbit/.test(type)) [key, kind] = ['eventFlows', 'event-flow']; - if (key) result[key].push(normalizeTypedArray({ x: [flow] }, 'x', [], kind, { fields: [['trigger', ['start', 'initiator'], null], ['outcome', ['result', 'end'], null]], arrays: [['failurePaths', ['errors', 'exceptions']], ['trustBoundaries', ['boundaries']], ['protocols', ['protocol']]], steps: true, branches: true })[0]); - } - return result; -} -function normalizeCatalogsObject(input = {}) { - const obj = input && typeof input === 'object' && !Array.isArray(input) ? input : {}; - return { - schemaVersion: '1.0', generatedAt: obj.generatedAt ?? obj.createdAt ?? obj.updatedAt ?? obj.timestamp ?? now(), - endpoints: normalizeTypedArray(obj, 'endpoints', ['apis', 'routes', 'httpEndpoints', 'apiEndpoints', 'restEndpoints', 'grpcEndpoints', 'websocketEndpoints', 'sseEndpoints'], 'endpoint', { fields: [['protocol', ['transport'], 'HTTP'], ['method', ['httpMethod', 'verb'], null], ['path', ['route', 'uri'], null], ['handler', ['operation', 'symbol'], null], ['authentication', ['authn'], null], ['authorization', ['authz'], null]], arrays: [['statusCodes', ['responses']], ['validations', ['validation']], ['sideEffects', ['effects']], ['emittedEvents', ['events']]] }), - messageHandlers: normalizeTypedArray(obj, 'messageHandlers', ['handlers', 'consumers', 'listeners', 'producers', 'messageConsumers', 'messageProducers', 'publishers', 'kafkaHandlers', 'rabbitHandlers', 'queueHandlers', 'streamHandlers'], 'message-handler', { fields: [['role', ['direction', 'type'], null], ['technology', ['broker'], null], ['channel', ['topic', 'queue', 'exchange', 'stream'], null], ['handler', ['symbol', 'method'], null], ['deliverySemantics', ['delivery'], null]], arrays: [['retry', ['retries']], ['deadLetter', ['dlq', 'deadLetterQueue']], ['sideEffects', ['effects']]] }), - externalDependencies: normalizeTypedArray(obj, 'externalDependencies', ['dependencies', 'integrations', 'externalServices', 'cloudServices', 'services', 'internalServices', 'upstreamServices', 'downstreamServices', 'thirdPartyServices', 'cloudResources'], 'external-dependency', { fields: [['direction', ['relationship'], null], ['protocol', ['mechanism', 'transport'], null], ['authentication', ['auth'], null], ['criticality', ['importance'], null]], arrays: [['dataExchanged', ['data']], ['failureBehavior', ['failures']], ['callSites', ['sources']]] }), - dataStores: normalizeTypedArray(obj, 'dataStores', ['datastores', 'databases', 'storage', 'stores', 'caches'], 'data-store', { fields: [['technology', ['engine', 'type'], null], ['ownership', ['owner'], null], ['consistency', ['consistencyModel'], null]], arrays: [['entities', ['tables', 'collections']], ['accessPaths', ['clients', 'repositories']]] }), - scheduledJobs: normalizeTypedArray(obj, 'scheduledJobs', ['jobs', 'schedulers', 'cronJobs', 'scheduledTasks'], 'scheduled-job', { fields: [['schedule', ['cron', 'interval'], null], ['handler', ['symbol', 'method'], null], ['purpose', ['description'], null]], arrays: [['sideEffects', ['effects']], ['failureBehavior', ['failures']]] }), - metadata: obj.metadata ?? {} - }; -} - -const ENTERPRISE_MODEL_SPECS = { - security: { - arrays: { - trustBoundaries: ['boundaries', 'securityBoundaries'], principals: ['actors', 'identities', 'subjects'], - authenticationFlows: ['authnFlows', 'loginFlows'], authorizationRules: ['authzRules', 'accessRules', 'permissionsRules'], - permissions: ['rolesPermissions', 'accessMatrix'], serviceIdentities: ['serviceAccounts', 'machineIdentities'], - secrets: ['credentials', 'keys', 'certificates'], sensitiveData: ['pii', 'protectedData', 'classifiedData'], - threats: ['risks', 'abuseCases'], controls: ['securityControls', 'mitigations'], unknowns: ['gaps', 'openQuestions'] - }, - kinds: { trustBoundaries:'trust-boundary', principals:'principal', authenticationFlows:'authentication-flow', authorizationRules:'authorization-rule', permissions:'permission', serviceIdentities:'service-identity', secrets:'secret', sensitiveData:'sensitive-data', threats:'threat', controls:'security-control', unknowns:'unknown' }, - flowKeys: new Set(['authenticationFlows']) - }, - operations: { - arrays: { - runtimeComponents:['runtimes','runtimeServices','runtimeNodes'], healthChecks:['health','readiness','liveness'], observabilitySignals:['signals','logsMetricsTraces'], - slis:['serviceLevelIndicators'], slos:['serviceLevelObjectives'], alerts:['alarms'], capacityLimits:['limits','quotas'], scalingSignals:['autoscalingSignals'], - failureModes:['failures'], recoveryProcedures:['recovery','remediation'], backups:['backupRestore'], deployments:['deploymentStrategies','rollouts'], - runbooks:['operationalRunbooks'], unknowns:['gaps','openQuestions'] - }, - kinds: { runtimeComponents:'runtime-component', healthChecks:'health-check', observabilitySignals:'observability-signal', slis:'sli', slos:'slo', alerts:'alert', capacityLimits:'capacity-limit', scalingSignals:'scaling-signal', failureModes:'failure-mode', recoveryProcedures:'recovery-procedure', backups:'backup', deployments:'deployment', runbooks:'runbook', unknowns:'unknown' }, - flowKeys: new Set(['recoveryProcedures','deployments','runbooks']) - }, - testing: { - arrays: { - testSuites:['suites'], testTypes:['testingTypes','levels'], fixtures:['testFixtures'], testData:['testDatasets'], environments:['testEnvironments'], - commands:['testCommands'], coverageGaps:['gaps','untestedAreas'], contractTests:['contracts'], failureInjection:['chaosTests','faultInjection'], - qualityGates:['ciGates','testGates'], unknowns:['openQuestions'] - }, - kinds: { testSuites:'test-suite', testTypes:'test-type', fixtures:'test-fixture', testData:'test-data', environments:'test-environment', commands:'test-command', coverageGaps:'coverage-gap', contractTests:'contract-test', failureInjection:'failure-injection', qualityGates:'test-quality-gate', unknowns:'unknown' }, - flowKeys: new Set() - }, - dataGovernance: { - arrays: { - dataEntities:['entities','records'], ownership:['dataOwners'], sourcesOfTruth:['authoritativeSources'], classifications:['dataClassifications'], - retentionPolicies:['retention','deletionPolicies'], transactionBoundaries:['transactions'], consistencyModels:['consistency'], concurrencyControls:['locking','concurrency'], - idempotencyRules:['idempotency'], reconciliationProcesses:['reconciliation'], lineageFlows:['lineage','dataLineage'], migrationPolicies:['schemaEvolution','migrations'], - auditRequirements:['auditability'], unknowns:['gaps','openQuestions'] - }, - kinds: { dataEntities:'data-entity', ownership:'data-ownership', sourcesOfTruth:'source-of-truth', classifications:'data-classification', retentionPolicies:'retention-policy', transactionBoundaries:'transaction-boundary', consistencyModels:'consistency-model', concurrencyControls:'concurrency-control', idempotencyRules:'idempotency-rule', reconciliationProcesses:'reconciliation-process', lineageFlows:'data-lineage-flow', migrationPolicies:'migration-policy', auditRequirements:'data-audit-requirement', unknowns:'unknown' }, - flowKeys: new Set(['reconciliationProcesses','lineageFlows','migrationPolicies']) - }, - decisions: { - arrays: { - recordedDecisions:['adrs','architectureDecisions'], inferredDecisions:['inferences'], alternatives:['options'], tradeoffs:['tradeOffs'], constraints:['decisionConstraints'], - consequences:['implications'], supersededDecisions:['deprecatedDecisions'], unknowns:['unknownRationale','gaps','openQuestions'] - }, - kinds: { recordedDecisions:'recorded-decision', inferredDecisions:'inferred-decision', alternatives:'decision-alternative', tradeoffs:'tradeoff', constraints:'decision-constraint', consequences:'decision-consequence', supersededDecisions:'superseded-decision', unknowns:'unknown' }, - flowKeys: new Set() - }, - configuration: { - arrays: { - settings:['properties','configurations'], environments:['environmentMatrix'], featureFlags:['flags'], secrets:['secretSettings'], runtimeTuning:['tuning'], - validationRules:['configValidation'], reloadBehavior:['reload','restartRequirements'], deprecations:['deprecatedSettings'], unknowns:['gaps','openQuestions'] - }, - kinds: { settings:'configuration-setting', environments:'environment-configuration', featureFlags:'feature-flag', secrets:'configuration-secret', runtimeTuning:'runtime-tuning', validationRules:'configuration-validation', reloadBehavior:'reload-behavior', deprecations:'configuration-deprecation', unknowns:'unknown' }, - flowKeys: new Set() - }, - changeImpact: { - arrays: { - changeSurfaces:['changePoints','symbols'], impactEdges:['dependencies','blastRadius'], compatibilityBoundaries:['compatibility'], extensionPoints:['safeExtensionPoints'], - migrationRisks:['risks'], affectedTests:['tests'], affectedOperations:['operationalImpact'], affectedContracts:['contracts'], unknowns:['gaps','openQuestions'] - }, - kinds: { changeSurfaces:'change-surface', impactEdges:'impact-edge', compatibilityBoundaries:'compatibility-boundary', extensionPoints:'extension-point', migrationRisks:'migration-risk', affectedTests:'affected-test', affectedOperations:'operational-impact', affectedContracts:'affected-contract', unknowns:'unknown' }, - flowKeys: new Set() - }, - ownership: { - arrays: { - teams:['owners','groups'], responsibilities:['duties'], raciAssignments:['raci'], componentOwners:['serviceOwners','moduleOwners'], dataOwners:['informationOwners'], - operationalOwners:['onCallOwners','runbookOwners'], approvalAuthorities:['approvers'], escalationPaths:['escalations'], unknowns:['gaps','openQuestions'] - }, - kinds: { teams:'team', responsibilities:'responsibility', raciAssignments:'raci-assignment', componentOwners:'component-owner', dataOwners:'data-owner', operationalOwners:'operational-owner', approvalAuthorities:'approval-authority', escalationPaths:'escalation-path', unknowns:'unknown' }, - flowKeys: new Set(['escalationPaths']) - } -}; -function normalizeEnterpriseObject(input = {}, specName) { - const obj = input && typeof input === 'object' && !Array.isArray(input) ? input : {}; - const spec = ENTERPRISE_MODEL_SPECS[specName]; - if (!spec) throw new Error(`Unknown enterprise model spec: ${specName}`); - const out = { schemaVersion:'1.0', generatedAt:obj.generatedAt ?? obj.createdAt ?? obj.updatedAt ?? obj.timestamp ?? now() }; - for (const [key, aliases] of Object.entries(spec.arrays)) { - out[key] = normalizeTypedArray(obj, key, aliases, spec.kinds[key], spec.flowKeys.has(key) ? { steps:true, branches:true } : {}); - } - out.metadata = obj.metadata ?? {}; - return out; -} -function assertEnterpriseModel(fileName, obj, specName) { - const spec = ENTERPRISE_MODEL_SPECS[specName]; - const keys = Object.keys(spec.arrays); - assertCanonicalModel(fileName, obj, keys); - const groups = {}; - for (const key of keys) { assertTypedItems(`${fileName}.${key}`, obj[key], spec.kinds[key]); groups[key] = obj[key]; } - assertGlobalUniqueIds(fileName, groups); - return obj; -} -function normalizeSecurityObject(x={}) { return normalizeEnterpriseObject(x,'security'); } -function normalizeOperationsObject(x={}) { return normalizeEnterpriseObject(x,'operations'); } -function normalizeTestingObject(x={}) { return normalizeEnterpriseObject(x,'testing'); } -function normalizeDataGovernanceObject(x={}) { return normalizeEnterpriseObject(x,'dataGovernance'); } -function normalizeDecisionsObject(x={}) { return normalizeEnterpriseObject(x,'decisions'); } -function normalizeConfigurationObject(x={}) { return normalizeEnterpriseObject(x,'configuration'); } -function normalizeChangeImpactObject(x={}) { return normalizeEnterpriseObject(x,'changeImpact'); } -function normalizeOwnershipObject(x={}) { return normalizeEnterpriseObject(x,'ownership'); } -function normalizeUpdatePlanObject(input = {}, changedPaths = []) { - const obj = input && typeof input === 'object' && !Array.isArray(input) ? input : {}; - const normalizedChanges = canonicalArray(obj, 'changedPaths', ['changedFiles', 'paths', 'changes']); - return { - schemaVersion: '1.0', generatedAt: obj.generatedAt ?? obj.createdAt ?? obj.updatedAt ?? obj.timestamp ?? now(), - changedPaths: normalizedChanges.length ? normalizedChanges.map(String) : changedPaths.map(String), - affectedEvidenceScopes: canonicalArray(obj, 'affectedEvidenceScopes', ['evidenceScopes', 'affectedScopes', 'scopes']).map(String), - affectedModels: canonicalArray(obj, 'affectedModels', ['models', 'modelArtifacts', 'affectedModelArtifacts']).map((x) => typeof x === 'string' ? x : x?.path ?? x?.id ?? x?.name).filter(Boolean), - affectedPageIds: canonicalArray(obj, 'affectedPageIds', ['pageIds', 'pages', 'affectedPages']).map((x) => typeof x === 'string' ? slug(x) : slug(x?.id ?? x?.pageId ?? x?.name)), - rationale: canonicalArray(obj, 'rationale', ['reasons', 'reasoning', 'explanations']).map((x) => typeof x === 'string' ? x : x?.summary ?? x?.reason ?? JSON.stringify(x)), - metadata: obj.metadata ?? {} - }; -} -function normalizeAuditReportObject(input = {}, page, options = {}) { - const obj = input && typeof input === 'object' && !Array.isArray(input) ? input : {}; - const findings = canonicalArray(obj, 'findings', ['issues', 'problems', 'results', 'violations']).map((f, i) => { - if (typeof f === 'string') return { id: `finding-${i + 1}`, kind: 'audit-finding', severity: 'medium', summary: f, evidence: [] }; - const item = f && typeof f === 'object' ? { ...f } : { summary: String(f) }; - item.id = slug(item.id ?? item.code ?? `finding-${i + 1}`); - item.kind = 'audit-finding'; - item.severity = String(item.severity ?? item.level ?? item.priority ?? 'medium').toLowerCase(); - if (!['critical', 'high', 'medium', 'low'].includes(item.severity)) item.severity = 'medium'; - item.summary ??= item.message ?? item.description ?? item.title ?? 'Unspecified finding'; - item.evidence = normalizeEvidenceRefs(canonicalArray(item, 'evidence', ['sources', 'references'])); - item.claimIds = normalizeStringRefs(canonicalArray(item, 'claimIds', ['claims'])); - delete item.level; delete item.priority; delete item.message; - return item; - }); - return { - schemaVersion: '1.0', auditedAt: scalarValue(obj, ['auditedAt', 'generatedAt', 'createdAt', 'timestamp'], now()), - pageId: slug(scalarValue(obj, ['pageId', 'pageID', 'id', 'page'], page.id)), - pagePath: canonicalPagePath(scalarValue(obj, ['pagePath', 'path', 'file', 'documentPath'], page.path)), - pageHash: scalarValue(obj, ['pageHash', 'hash', 'contentHash'], options.defaultHashes ? pageCurrentHash(page) : null), - inputHash: scalarValue(obj, ['inputHash', 'pageInputHash', 'evidenceHash', 'contractHash'], options.defaultHashes ? pageInputHash(page) : null), - findings, - metadata: obj.metadata ?? {} - }; -} -function assertCanonicalModel(name, obj, arrayKeys) { - if (!obj || typeof obj !== 'object' || Array.isArray(obj)) throw new Error(`${name} must be a JSON object.`); - if (obj.schemaVersion !== '1.0') throw new Error(`${name} schemaVersion must normalize to 1.0.`); - for (const key of arrayKeys) if (!Array.isArray(obj[key])) throw new Error(`${name}.${key} must be an array after normalization.`); - return obj; -} -function assertTypedItems(name, items, expectedKind) { - const ids = new Set(); - for (let i = 0; i < items.length; i++) { - const item = items[i]; const p = `${name}[${i}]`; - if (!item || typeof item !== 'object' || Array.isArray(item)) throw new Error(`${p} must be an object.`); - if (!item.id || typeof item.id !== 'string') throw new Error(`${p}.id must be a non-empty string.`); - if (ids.has(item.id)) throw new Error(`${name} contains duplicate id: ${item.id}`); ids.add(item.id); - if (item.kind !== expectedKind) throw new Error(`${p}.kind must be ${expectedKind}.`); - if (!['FACT', 'INFERENCE', 'UNKNOWN'].includes(item.classification)) throw new Error(`${p}.classification must be FACT, INFERENCE, or UNKNOWN.`); - if (!Number.isFinite(item.confidence) || item.confidence < 0 || item.confidence > 1) throw new Error(`${p}.confidence must be between 0 and 1.`); - if (!Array.isArray(item.evidence) || !Array.isArray(item.sourceModelRefs) || !Array.isArray(item.unknowns)) throw new Error(`${p} evidence/sourceModelRefs/unknowns must be arrays.`); - if (item.classification === 'FACT' && item.evidence.length === 0) throw new Error(`${p} is FACT but has no direct evidence.`); - const aliases = buildReferenceAliases(); - for (const [j, ev] of item.evidence.entries()) { - if (!ev || typeof ev !== 'object' || (!ev.path && !ev.symbol)) throw new Error(`${p}.evidence[${j}] requires path or symbol.`); - if (ev.path && path.isAbsolute(ev.path)) throw new Error(`${p}.evidence[${j}].path must be repository-relative.`); - if (ev.path) { const resolved = normalizeReference(ev.path, aliases); if (!exists(resolved)) throw new Error(`${p}.evidence[${j}] cannot resolve: ${ev.path}`); const ignored = ignoreDecision(resolved, false); if (loadConfig().ignore?.rejectIgnoredEvidence !== false && ignored.ignored) throw new Error(`${p}.evidence[${j}] references ignored source ${ev.path} (${ignored.reason}).`); } - } - } -} -function assertGlobalUniqueIds(name, groups) { - const seen=new Map(); for(const [group,items] of Object.entries(groups)) for(const item of items){ if(seen.has(item.id)) throw new Error(`${name} duplicate semantic id ${item.id} in ${seen.get(item.id)} and ${group}`); seen.set(item.id,group); } -} -function assertSystemModel(obj) { - assertCanonicalModel('system.json', obj, ['components', 'relationships', 'workflows', 'unknowns']); - assertTypedItems('system.components', obj.components, 'component'); assertTypedItems('system.relationships', obj.relationships, 'relationship'); assertTypedItems('system.workflows', obj.workflows, 'workflow'); assertTypedItems('system.unknowns', obj.unknowns, 'unknown'); assertGlobalUniqueIds('system.json',{components:obj.components,relationships:obj.relationships,workflows:obj.workflows,unknowns:obj.unknowns}); return obj; -} -function assertBusinessModel(obj) { - assertCanonicalModel('business.json', obj, ['actors','capabilities','concepts','businessRules','decisions','branchConditions','lifecycles','invariants','useCases','unknowns']); - for (const [key, kind] of Object.entries({actors:'actor',capabilities:'capability',concepts:'concept',businessRules:'business-rule',decisions:'decision',branchConditions:'branch-condition',lifecycles:'lifecycle',invariants:'invariant',useCases:'use-case',unknowns:'unknown'})) assertTypedItems(`business.${key}`, obj[key], kind); assertGlobalUniqueIds('business.json',{actors:obj.actors,capabilities:obj.capabilities,concepts:obj.concepts,businessRules:obj.businessRules,decisions:obj.decisions,branchConditions:obj.branchConditions,lifecycles:obj.lifecycles,invariants:obj.invariants,useCases:obj.useCases,unknowns:obj.unknowns}); return obj; -} -function assertFlowsModel(obj) { - assertCanonicalModel('flows.json', obj, ['businessFlows','controlFlows','requestFlows','trafficFlows','dataFlows','eventFlows']); - for (const [key, kind] of Object.entries({businessFlows:'business-flow',controlFlows:'control-flow',requestFlows:'request-flow',trafficFlows:'traffic-flow',dataFlows:'data-flow',eventFlows:'event-flow'})) assertTypedItems(`flows.${key}`, obj[key], kind); assertGlobalUniqueIds('flows.json',{businessFlows:obj.businessFlows,controlFlows:obj.controlFlows,requestFlows:obj.requestFlows,trafficFlows:obj.trafficFlows,dataFlows:obj.dataFlows,eventFlows:obj.eventFlows}); return obj; -} -function assertCatalogsModel(obj) { - assertCanonicalModel('catalogs.json', obj, ['endpoints','messageHandlers','externalDependencies','dataStores','scheduledJobs']); - for (const [key, kind] of Object.entries({endpoints:'endpoint',messageHandlers:'message-handler',externalDependencies:'external-dependency',dataStores:'data-store',scheduledJobs:'scheduled-job'})) assertTypedItems(`catalogs.${key}`, obj[key], kind); assertGlobalUniqueIds('catalogs.json',{endpoints:obj.endpoints,messageHandlers:obj.messageHandlers,externalDependencies:obj.externalDependencies,dataStores:obj.dataStores,scheduledJobs:obj.scheduledJobs}); return obj; -} -function gitValue(args) { - const r = spawnSync('git', args, { cwd: root, encoding: 'utf8', shell: process.platform === 'win32' }); - return r.status === 0 ? String(r.stdout ?? '').trim() : null; -} -function currentSourceSnapshot(force = false) { - if (!force && sourceSnapshotCache) return { ...sourceSnapshotCache }; - const commit = gitValue(['rev-parse', 'HEAD']); - const branch = gitValue(['rev-parse', '--abbrev-ref', 'HEAD']); - const dirtyOutput = gitValue(['status', '--porcelain']); - let sourceFingerprint = null; - try { const snapshot=makeSnapshot(); sourceFingerprint = sha256Text(JSON.stringify({files:snapshot.files,ignorePolicyHash:snapshot.ignorePolicyHash})); } catch {} - sourceSnapshotCache = { capturedAt: now(), commit, branch, dirty: dirtyOutput === null ? null : dirtyOutput.length > 0, sourceFingerprint }; - return { ...sourceSnapshotCache }; -} -function traceabilityPath(page) { return path.join(traceabilityPagesRoot, `${page.id}.json`); } -function traceabilityRelPath(page) { return rel(traceabilityPath(page)); } -function normalizeClaim(value, page, index = 0) { - const obj = typeof value === 'string' ? { statement: value } : value && typeof value === 'object' ? value : { statement: String(value ?? '') }; - const evidence = normalizeEvidenceRefs(canonicalArray(obj, 'evidence', ['sources', 'references', 'sourceRefs'])); - const modelRefs = normalizeStringRefs(canonicalArray(obj, 'sourceModelRefs', ['modelRefs', 'semanticRefs', 'catalogRefs'])); - const classification = normalizeClassification(scalarValue(obj, ['classification', 'epistemicStatus', 'status'], null), evidence); - return { - id: slug(scalarValue(obj, ['id', 'claimId', 'key'], `${page.id}-claim-${index + 1}`)), - kind: 'claim', - pageId: page.id, - section: String(scalarValue(obj, ['section', 'heading'], 'Unmapped')), - statement: String(scalarValue(obj, ['statement', 'claim', 'text', 'summary'], 'Unspecified claim')), - classification, - confidence: normalizeConfidence(scalarValue(obj, ['confidence', 'score'], null), classification), - subject: scalarValue(obj, ['subject', 'entity'], null), - predicate: scalarValue(obj, ['predicate', 'property', 'relation'], null), - object: scalarValue(obj, ['object', 'value', 'target'], null), - polarity: String(scalarValue(obj, ['polarity'], 'positive')).toLowerCase() === 'negative' ? 'negative' : 'positive', - evidence, - sourceModelRefs: modelRefs, - intentionalDuplicate: Boolean(scalarValue(obj, ['intentionalDuplicate', 'repeatedForOrientation'], false)), - exclusivePredicate: Boolean(scalarValue(obj, ['exclusivePredicate', 'singleValued', 'exclusive'], false)) || String(scalarValue(obj, ['cardinality'], '')).toLowerCase() === 'one', - notes: scalarValue(obj, ['notes', 'note'], null), - unknowns: canonicalArray(obj, 'unknowns', ['gaps']).map(String), - tags: normalizeStringRefs(canonicalArray(obj, 'tags', ['labels'])) - }; -} -function normalizeTraceabilityObject(input = {}, page, options = {}) { - const obj = input && typeof input === 'object' && !Array.isArray(input) ? input : {}; - const coverageObj = obj.coverage && typeof obj.coverage === 'object' ? obj.coverage : {}; - return { - schemaVersion: '1.0', generatedAt: scalarValue(obj, ['generatedAt', 'createdAt', 'updatedAt'], now()), - pageId: page.id, pagePath: page.path, - pageHash: scalarValue(obj, ['pageHash', 'hash'], options.defaultHashes ? pageCurrentHash(page) : null), - inputHash: scalarValue(obj, ['inputHash', 'pageInputHash'], options.defaultHashes ? pageInputHash(page) : null), - sourceSnapshot: obj.sourceSnapshot && typeof obj.sourceSnapshot === 'object' ? { ...currentSourceSnapshot(), ...obj.sourceSnapshot } : currentSourceSnapshot(), - claims: canonicalArray(obj, 'claims', ['statements', 'assertions', 'facts']).map((x, i) => normalizeClaim(x, page, i)), - coverage: { - evidenceRefsUsed: normalizeStringRefs(canonicalArray(coverageObj, 'evidenceRefsUsed', ['evidence', 'sources'])), - modelItemRefs: normalizeStringRefs(canonicalArray(coverageObj, 'modelItemRefs', ['modelRefs', 'semanticRefs'])), - catalogItemRefs: normalizeStringRefs(canonicalArray(coverageObj, 'catalogItemRefs', ['catalogRefs'])), - branchItemRefs: normalizeStringRefs(canonicalArray(coverageObj, 'branchItemRefs', ['branchRefs'])) - }, - unknowns: canonicalArray(obj, 'unknowns', ['openQuestions', 'gaps']).map(String), - legacyUnmapped: Boolean(obj.legacyUnmapped) - }; -} -function assertTraceabilityObject(obj, page) { - if (obj.schemaVersion !== '1.0' || obj.pageId !== page.id || obj.pagePath !== page.path) throw new Error(`Traceability identity mismatch for ${page.id}.`); - if (!Array.isArray(obj.claims) || !obj.coverage || typeof obj.coverage !== 'object') throw new Error(`Traceability for ${page.id} requires claims[] and coverage.`); - assertTypedItems(`traceability.${page.id}.claims`, obj.claims, 'claim'); - for (const claim of obj.claims) if (claim.pageId !== page.id) throw new Error(`Traceability claim ${claim.id} pageId mismatch.`); - return obj; -} -function ensurePageTraceability(page, options = {}) { - const file = traceabilityPath(page); - if (!fs.existsSync(file)) writeJson(file, normalizeTraceabilityObject({ legacyUnmapped: true, claims: [], coverage: {} }, page, { defaultHashes: true })); - return normalizeJsonFile(file, (obj) => normalizeTraceabilityObject(obj, page, { defaultHashes: true }), (obj) => assertTraceabilityObject(obj, page)); -} -function refreshPageTraceabilityHashes(page) { - const trace=ensurePageTraceability(page); trace.pageHash=pageCurrentHash(page); trace.inputHash=pageInputHash(page); trace.sourceSnapshot=currentSourceSnapshot(); trace.generatedAt=now(); writeJson(traceabilityPath(page),trace); return trace; -} -function pageRuntimeContract(page) { - return { ...page, traceabilityPath: traceabilityRelPath(page), traceabilityContract: { required: true, claimClassifications: ['FACT','INFERENCE','UNKNOWN'], evidenceRequiredForFacts: true }, sourceSnapshot: currentSourceSnapshot() }; -} -function snapshotOutputs(paths) { - const files = new Map(); - const directories = []; - for (const target of paths) { - if (fs.existsSync(target) && fs.statSync(target).isDirectory()) { - directories.push(target); - for (const file of listFilesRecursive(target)) files.set(file, fs.readFileSync(file)); - } else files.set(target, fs.existsSync(target) ? fs.readFileSync(target) : null); - } - return { files, directories }; -} -function restoreOutputs(snapshot) { - for (const dir of snapshot.directories) if (fs.existsSync(dir)) { - for (const file of listFilesRecursive(dir)) if (!snapshot.files.has(file)) { try { fs.rmSync(file, { force: true }); } catch {} } - } - for (const [file, content] of snapshot.files.entries()) { - if (content === null) { try { fs.rmSync(file, { force: true }); } catch {} } - else { fs.mkdirSync(path.dirname(file), { recursive: true }); fs.writeFileSync(file, content); } - } -} -function copyPathToQuarantine(source, dir) { - if (!fs.existsSync(source)) return; - if (fs.statSync(source).isDirectory()) { - for (const file of listFilesRecursive(source)) { - const target = path.join(dir, rel(file).replaceAll('/', '__')); - try { fs.copyFileSync(file, target); } catch {} - } - } else { - const target = path.join(dir, rel(source).replaceAll('/', '__')); - try { fs.copyFileSync(source, target); } catch {} - } -} -function quarantineOutputs(stage, paths, error) { - const stamp = new Date().toISOString().replace(/[:.]/g, '-'); - const dir = path.join(root, '.docgen', 'quarantine', `${stamp}-${stage}`); - fs.mkdirSync(dir, { recursive: true }); - for (const target of paths) copyPathToQuarantine(target, dir); - writeJson(path.join(dir, 'error.json'), { schemaVersion: '1.0', stage, capturedAt: now(), error: error?.message ?? String(error), paths: paths.map(rel) }); - return rel(dir); -} -async function runContractStage(stage, paths, operation, normalizeAndValidate) { - const snapshot = snapshotOutputs(paths); - try { - await operation(() => restoreOutputs(snapshot)); - return normalizeAndValidate(); - } catch (error) { - const quarantine = quarantineOutputs(stage, paths, error); - restoreOutputs(snapshot); - const wrapped = new Error(`${stage} did not commit canonical output. Previous valid artifacts were restored. Raw/partial output: ${quarantine}. Cause: ${error.message}`); - wrapped.exitCode = error?.exitCode; - wrapped.classification = error?.classification ?? 'contract-failure'; - throw wrapped; - } -} -function normalizeJsonFile(file, normalizer, validator) { - if (!fs.existsSync(file)) throw new Error(`Missing ${rel(file)}`); - let raw; - try { raw = readJson(file); } catch (e) { throw new Error(`Invalid JSON in ${rel(file)}: ${e.message}`); } - const canonical = normalizer(raw); - validator(canonical); - writeJson(file, canonical); - return canonical; -} - -function canonicalPagePath(rawPath) { - let p = String(rawPath ?? '').trim().replaceAll('\\', '/'); - p = p.replace(/^\.\//, '').replace(/^\/+/, ''); - if (!p) throw new Error('Manifest page has an empty path.'); - if (!p.startsWith('docs/')) p = `docs/${p}`; - if (!/\.md$/i.test(p)) p += '.md'; - p = path.posix.normalize(p); - if (p === 'docs.md' || !p.startsWith('docs/') || p.includes('/../') || p.startsWith('../')) throw new Error(`Unsafe page path: ${rawPath}`); - return p; -} -function normalizeReference(ref, aliases) { - if (typeof ref !== 'string') return ref; - const value = ref.trim().replaceAll('\\', '/'); - if (!value) return value; - if (exists(value)) return path.isAbsolute(value) ? rel(value) : value; - const keys = [value, value.replace(/^\.\//, ''), path.basename(value), path.basename(value, path.extname(value)), slug(value)]; - for (const key of keys) if (aliases.has(key)) return aliases.get(key); - return value; -} -function buildReferenceAliases() { - const aliases = new Map(); - const add = (key, value) => { if (key && !aliases.has(String(key))) aliases.set(String(key), value); }; - if (fs.existsSync(evidenceIndexPath)) { - const idx = normalizeEvidenceIndex(); - for (const a of idx.artifacts ?? []) { - add(a.id, a.path); add(a.path, a.path); add(path.basename(a.path), a.path); add(path.basename(a.path, path.extname(a.path)), a.path); add(slug(a.id), a.path); - } - } - for (const p of [systemPath, businessPath, flowsPath, catalogsPath]) if (fs.existsSync(p)) { - const rp = rel(p); add(rp, rp); add(path.basename(rp), rp); add(path.basename(rp, path.extname(rp)), rp); add(slug(path.basename(rp, path.extname(rp))), rp); - } - for (const dir of [path.join(root, '.docgen', 'model'), path.join(root, '.docgen', 'evidence')]) for (const f of listFilesRecursive(dir)) { - if (f.endsWith('.gitkeep')) continue; const rp = rel(f); add(rp, rp); add(path.basename(rp), rp); add(path.basename(rp, path.extname(rp)), rp); - } - return aliases; -} -function normalizeManifest(write = true) { - if (!fs.existsSync(manifestPath)) throw new Error('Missing .docgen/plan/manifest.json. Run plan first.'); - let raw; - try { raw = readJson(manifestPath); } catch (e) { throw new Error(`Invalid ${rel(manifestPath)}: ${e.message}`); } - const sourcePages = arrayValue(raw, ['pages', 'documents', 'entries', 'articles'], []); - if (!sourcePages.length) throw new Error('Manifest contains no pages/documents/entries array.'); - const aliases = buildReferenceAliases(); - const pages = sourcePages.map((value, index) => { - const page = value && typeof value === 'object' ? value : { path: String(value) }; - const title = String(scalarValue(page, ['title', 'name', 'label'], `Page ${index + 1}`)); - const id = slug(scalarValue(page, ['id', 'pageId', 'key', 'slug'], title || page.path)); - return { - id, - path: canonicalPagePath(scalarValue(page, ['path', 'file', 'outputPath', 'targetPath', 'documentPath'], id)), - title, - type: String(scalarValue(page, ['type', 'pageType', 'kind'], 'concept')).toLowerCase(), - category: String(scalarValue(page, ['category', 'group', 'sectionGroup'], 'Documentation')), - section: scalarValue(page, ['section', 'subsection'], page.section), - purpose: String(scalarValue(page, ['purpose', 'objective', 'goal'], title)), - summary: String(scalarValue(page, ['summary', 'description', 'overview'], scalarValue(page, ['purpose', 'objective'], title))), - evidence: arrayValue(page, ['evidence', 'sources', 'evidenceIds', 'sourceArtifacts'], []).map((x) => normalizeReference(typeof x === 'string' ? x : x?.path ?? x?.id ?? x?.name, aliases)).filter(Boolean), - models: arrayValue(page, ['models', 'modelInputs', 'modelArtifacts'], []).map((x) => normalizeReference(typeof x === 'string' ? x : x?.path ?? x?.id ?? x?.name, aliases)).filter(Boolean), - audience: arrayValue(page, ['audience', 'audiences', 'readers'], ['engineer']).map(String), - requiredSections: arrayValue(page, ['requiredSections', 'sections', 'headings'], ['Overview']).map((x) => typeof x === 'string' ? x : x?.title ?? x?.name).filter(Boolean), - diagramIntents: arrayValue(page, ['diagramIntents', 'diagrams', 'diagramRequirements'], []).map((x) => typeof x === 'string' ? x : x?.intent ?? x?.title ?? x?.type).filter(Boolean), - coverageTags: arrayValue(page, ['coverageTags', 'coverage', 'tags'], []).map(String), - requiredTables: arrayValue(page, ['requiredTables', 'tables', 'tableRequirements'], []).map((x) => typeof x === 'string' ? x : x?.title ?? x?.name).filter(Boolean), - relatedPages: arrayValue(page, ['relatedPages', 'related', 'links'], []).map((x) => slug(typeof x === 'string' ? x : x?.id ?? x?.pageId ?? x?.title)).filter(Boolean), - qualityHints: arrayValue(page, ['qualityHints', 'hints', 'qualityRequirements'], []).map(String), - mode: String(scalarValue(page, ['mode','documentMode','intent'], page.type === 'reference' ? 'reference' : page.type === 'troubleshooting' ? 'troubleshooting' : page.type === 'guide' ? 'how-to' : 'explanation')).toLowerCase(), - aliases: arrayValue(page, ['aliases','redirectFrom','legacyPaths'], []).map((x)=>canonicalPagePath(x)).filter((x)=>x!==canonicalPagePath(scalarValue(page, ['path','file','outputPath','targetPath','documentPath'], id))), - status: String(scalarValue(page, ['status','lifecycleStatus','publicationStatus'], 'active')).toLowerCase(), - version: scalarValue(page, ['version','docVersion','since'], null), - deprecatedSince: scalarValue(page, ['deprecatedSince','deprecated','sunsetVersion'], null), - replacementPage: scalarValue(page, ['replacementPage','replacement','supersededBy'], null) ? slug(scalarValue(page, ['replacementPage','replacement','supersededBy'], null)) : null, - migrationFrom: arrayValue(page, ['migrationFrom','fromVersions','sourceVersions'], []).map(String), - migrationTo: scalarValue(page, ['migrationTo','targetVersion'], null), - exampleIntents: arrayValue(page, ['exampleIntents','examples','scenarios'], []).map((x)=>typeof x==='string'?x:x?.title??x?.name??x?.intent).filter(Boolean), - searchKeywords: arrayValue(page, ['searchKeywords','keywords','searchTerms'], []).map(String), - frontmatter: page.frontmatter && typeof page.frontmatter === 'object' ? page.frontmatter : {} - }; - }); - const pageIdByAlias = new Map(); - for (const p of pages) { - for (const alias of [p.id, p.title, p.path, p.path.replace(/^docs\//, '').replace(/\.md$/i, ''), path.posix.basename(p.path, '.md')]) pageIdByAlias.set(slug(alias), p.id); - } - for (const p of pages) { - p.replacementPage = p.replacementPage ? (pageIdByAlias.get(slug(p.replacementPage)) ?? slug(p.replacementPage)) : null; - p.aliases = [...new Set((p.aliases ?? []).filter((x)=>x!==p.path))]; - } - const rawNavigation = arrayValue(raw, ['navigation', 'categories', 'groups', 'sections'], []); - const navigation = rawNavigation.map((value, index) => { - const group = value && typeof value === 'object' ? value : { title: String(value) }; - const title = String(scalarValue(group, ['title', 'name', 'label'], `Section ${index + 1}`)); - const groupPages = arrayValue(group, ['pages', 'pageIds', 'items', 'documents'], []).map((x) => { - const rawId = typeof x === 'string' ? x : x?.id ?? x?.pageId ?? x?.path ?? x?.title; - return pageIdByAlias.get(slug(rawId)) ?? slug(rawId); - }).filter(Boolean); - return { id: slug(scalarValue(group, ['id', 'key', 'slug'], title)), title, description: group.description ?? group.summary, pages: groupPages }; - }); - const manifest = { - schemaVersion: '1.0', - generatedAt: raw.generatedAt ?? raw.createdAt ?? raw.updatedAt ?? now(), - navigation, - pages, - metadata: raw.metadata ?? {} - }; - if (write) writeJson(manifestPath, manifest); - return manifest; -} -function manifestPreflight(manifest = normalizeManifest(), options = {}) { - const errors = []; const warnings = []; const ids = new Set(); const paths = new Set(); - const cfg=loadConfig(); - const allowedTypes = new Set(cfg.pageTypes ?? ['overview','architecture','business','concept','flow','guide','reference','data','integration','operations','troubleshooting']); - const allowedModes = new Set(cfg.documentationExperience?.modes ?? ['tutorial','how-to','explanation','reference','runbook','decision-record','migration-guide','troubleshooting']); - const aliasOwners=new Map(); - for (const page of manifest.pages) { - if (ids.has(page.id)) errors.push(`duplicate page id: ${page.id}`); ids.add(page.id); - if (paths.has(page.path)) errors.push(`duplicate page path: ${page.path}`); - if (aliasOwners.has(page.path)) errors.push(`${page.id}: canonical path collides with alias owned by ${aliasOwners.get(page.path)}: ${page.path}`); - paths.add(page.path); - if (!page.title?.trim()) errors.push(`${page.id}: title is required`); - if (!page.category?.trim()) errors.push(`${page.id}: category is required`); - if (!allowedTypes.has(page.type)) errors.push(`${page.id}: unsupported page type ${page.type}`); - if (!allowedModes.has(page.mode)) errors.push(`${page.id}: unsupported document mode ${page.mode}`); - if (!Array.isArray(page.requiredSections) || !page.requiredSections.length) errors.push(`${page.id}: requiredSections must not be empty`); - for (const alias of page.aliases ?? []) { if(aliasOwners.has(alias)) errors.push(`${page.id}: duplicate alias ${alias} also owned by ${aliasOwners.get(alias)}`); else aliasOwners.set(alias,page.id); if(paths.has(alias)) errors.push(`${page.id}: alias collides with canonical page path ${alias}`); } - if(page.replacementPage && !manifest.pages.some((p)=>p.id===page.replacementPage)) warnings.push(`${page.id}: replacementPage does not exist: ${page.replacementPage}`); - if(page.status==='deprecated' && !page.deprecatedSince) warnings.push(`${page.id}: deprecated page should declare deprecatedSince`); - for (const ref of [...(page.evidence ?? []), ...(page.models ?? [])]) if (typeof ref === 'string' && !exists(ref)) errors.push(`${page.id}: unresolved input reference: ${ref}`); - } - const navigationCounts = new Map(); - for (const group of manifest.navigation ?? []) for (const id of group.pages ?? []) { - const pageId = typeof id === 'string' ? id : id?.id; - if (pageId && !ids.has(pageId)) errors.push(`navigation ${group.id ?? group.title}: unknown page id ${pageId}`); - else if (pageId) navigationCounts.set(pageId, (navigationCounts.get(pageId) ?? 0) + 1); - } - if ((manifest.navigation ?? []).length) for (const id of ids) { - const count = navigationCounts.get(id) ?? 0; - if (count === 0) errors.push(`page is missing from navigation: ${id}`); - if (count > 1) errors.push(`page appears in multiple navigation groups: ${id}`); - } - for (const page of manifest.pages) for (const related of page.relatedPages ?? []) if (!ids.has(related)) warnings.push(`${page.id}: related page does not exist: ${related}`); - const coverageGaps = options.includeCoverage === false ? [] : manifestCoverageGaps(manifest); - if (coverageGaps.length) errors.push(`manifest coverage gaps: ${coverageGaps.join(', ')}`); - const result = { schemaVersion: '1.0', checkedAt: now(), valid: errors.length === 0, pageCount: manifest.pages.length, errors, warnings }; - writeJson(preflightPath, result); - return result; -} -function requireManifestPreflight() { - const manifest = normalizeManifest(); - const result = manifestPreflight(manifest); - if (!result.valid) throw new Error(`Manifest preflight failed before generation:\n- ${result.errors.join('\n- ')}\nReport: ${rel(preflightPath)}`); - if (result.warnings.length) for (const w of result.warnings) console.warn(`WARNING: ${w}`); - return manifest; -} -function loadManifest() { - try { return normalizeManifest(); } catch (e) { fail(e.message); } -} -function findPage(id) { - const manifest = loadManifest(); - const page = manifest.pages.find((p) => p.id === id); - if (!page) fail(`Unknown page id: ${id}`); - return page; -} - -function validateJsonFile(file, required = []) { - if (!fs.existsSync(file)) throw new Error(`Missing ${rel(file)}`); - const obj = readJson(file); - for (const key of required) if (!(key in obj)) throw new Error(`${rel(file)} missing required key: ${key}`); - return obj; -} -function slug(value) { - return String(value || 'artifact').toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/^-+|-+$/g, '') || 'artifact'; -} -function listFilesRecursive(base) { - if (!fs.existsSync(base)) return []; - const out = []; const stack = [base]; - while (stack.length) { - const dir = stack.pop(); - for (const entry of fs.readdirSync(dir, { withFileTypes: true })) { - const full = path.join(dir, entry.name); - if (entry.isDirectory()) stack.push(full); else out.push(full); - } - } - return out; -} -function canonicalEvidencePath(rawPath, evidenceDir) { - let value = String(rawPath ?? '').trim().replaceAll('\\', '/'); - if (!value) return ''; - if (path.isAbsolute(value)) value = rel(value); - value = value.replace(/^\.\//, '').replace(/^\/+/, ''); - if (!value.startsWith('.docgen/evidence/')) { - const direct = path.join(root, value); - const underEvidence = path.join(evidenceDir, value.replace(/^evidence\//, '')); - if (fs.existsSync(underEvidence) || !fs.existsSync(direct)) value = rel(underEvidence); - } - value = path.posix.normalize(value); - if (!value.startsWith('.docgen/evidence/') || value.includes('/../')) throw new Error(`Unsafe evidence artifact path: ${rawPath}`); - return value; -} - -function normalizeEvidenceIndex() { - const evidenceDir = path.dirname(evidenceIndexPath); - let obj = {}; - if (fs.existsSync(evidenceIndexPath)) { - try { obj = readJson(evidenceIndexPath); } catch (e) { throw new Error(`Invalid ${rel(evidenceIndexPath)}: ${e.message}`); } - } - const candidates = [obj.artifacts, obj.files, obj.evidenceFiles, obj.entries, obj.documents, obj.items].find(Array.isArray) ?? []; - const normalizeEntry = (item, i) => { - if (typeof item === 'string') { - const p = canonicalEvidencePath(item, evidenceDir); - return { id: slug(path.basename(p, path.extname(p))), path: p, kind: 'evidence', scope: '.' }; - } - const x = item && typeof item === 'object' ? item : {}; - let p = canonicalEvidencePath(x.path ?? x.file ?? x.filePath ?? x.relativePath ?? x.artifactPath ?? '', evidenceDir); - return { - ...x, - id: String(x.id ?? x.name ?? x.key ?? slug(p || `artifact-${i + 1}`)), - path: p, - kind: String(x.kind ?? x.type ?? x.category ?? 'evidence'), - scope: String(x.scope ?? x.module ?? x.area ?? x.boundedContext ?? '.') - }; - }; - let artifacts = candidates.map(normalizeEntry).filter((x) => x.path); - if (!artifacts.length) { - artifacts = listFilesRecursive(evidenceDir) - .filter((p) => p !== evidenceIndexPath && !p.endsWith('.gitkeep')) - .map((p, i) => normalizeEntry(rel(p), i)); - } - const seen = new Set(); - artifacts = artifacts.filter((a) => { const key = a.path; if (!key || seen.has(key)) return false; seen.add(key); return true; }); - const ids = new Set(); - for (const artifact of artifacts) { - artifact.id = slug(artifact.id); - if (ids.has(artifact.id)) throw new Error(`Duplicate evidence artifact id: ${artifact.id}`); - ids.add(artifact.id); - const artifactFile = path.join(root, artifact.path); - if (!fs.existsSync(artifactFile) || !fs.statSync(artifactFile).isFile()) throw new Error(`Evidence artifact does not exist: ${artifact.path}`); - if (artifactFile.endsWith('.json')) { try { readJson(artifactFile); } catch (e) { throw new Error(`Invalid evidence JSON ${artifact.path}: ${e.message}`); } } - } - const canonical = { - schemaVersion: '1.0', - generatedAt: obj.generatedAt ?? obj.createdAt ?? obj.updatedAt ?? now(), - repository: obj.repository ?? {}, - artifacts: artifacts.map((a) => ({ id: a.id, path: a.path, kind: a.kind, scope: a.scope, summary: a.summary, factCount: a.factCount })).map((a) => Object.fromEntries(Object.entries(a).filter(([,v]) => v !== undefined))), - metadata: obj.metadata ?? {} - }; - const changed = !Array.isArray(obj.artifacts) || JSON.stringify(obj.artifacts) !== JSON.stringify(artifacts); - writeJson(evidenceIndexPath, canonical); - if (changed) console.log(`[docgen] normalized evidence index to canonical artifacts[] (${artifacts.length} artifacts).`); - if (!artifacts.length) throw new Error(`${rel(evidenceIndexPath)} contains no artifacts and no evidence files were found.`); - return canonical; -} -function validateMermaidOnly(text, pagePath) { - const forbidden = [...text.matchAll(/```\s*(plantuml|puml|dot|graphviz)\b/gi)].map((m) => m[1]); - if (forbidden.length) throw new Error(`${pagePath} contains non-Mermaid diagram fences: ${[...new Set(forbidden)].join(', ')}`); -} -function loadOptionalJson(file, fallback) { return fs.existsSync(file) ? readJson(file) : fallback; } -function manifestCoverageGaps(manifest) { - const tags = new Set((manifest.pages ?? []).flatMap((p) => p.coverageTags ?? [])); - // Normalize in memory as well as at stage commit boundaries. This keeps manual - // preflight/validate resilient when a repository still contains pre-v0.6 aliases. - const system = normalizeSystemObject(loadOptionalJson(systemPath, {})); - const business = normalizeBusinessObject(loadOptionalJson(businessPath, {})); - const flows = normalizeFlowsObject(loadOptionalJson(flowsPath, {})); - const catalogs = normalizeCatalogsObject(loadOptionalJson(catalogsPath, {})); - const security = normalizeSecurityObject(loadOptionalJson(securityPath, {})); - const operations = normalizeOperationsObject(loadOptionalJson(operationsPath, {})); - const testing = normalizeTestingObject(loadOptionalJson(testingPath, {})); - const dataGovernance = normalizeDataGovernanceObject(loadOptionalJson(dataGovernancePath, {})); - const decisions = normalizeDecisionsObject(loadOptionalJson(decisionsPath, {})); - const configuration = normalizeConfigurationObject(loadOptionalJson(configurationPath, {})); - const changeImpact = normalizeChangeImpactObject(loadOptionalJson(changeImpactPath, {})); - const ownership = normalizeOwnershipObject(loadOptionalJson(ownershipPath, {})); - const required = [['system-overview', true], ['architecture', true]]; - if ((business.capabilities ?? []).length) required.push(['business-domain', true]); - if ((business.businessRules ?? []).length) required.push(['business-rules', true]); - if ((business.branchConditions ?? []).length) required.push(['branch-conditions', true]); - if ((business.lifecycles ?? []).length) required.push(['state-lifecycle', true]); - for (const [key, tag] of [['businessFlows','business-flow'],['controlFlows','control-flow'],['requestFlows','request-flow'],['trafficFlows','traffic-flow'],['dataFlows','data-flow'],['eventFlows','event-flow']]) if ((flows[key] ?? []).length) required.push([tag, true]); - if ((catalogs.endpoints ?? []).length) required.push(['endpoint-catalog', true]); - if ((catalogs.messageHandlers ?? []).length) required.push(['message-handler-catalog', true]); - if ((catalogs.externalDependencies ?? []).length) required.push(['external-dependency-catalog', true]); - if (security.trustBoundaries.length || security.authenticationFlows.length) required.push(['security-trust-boundaries', true]); - if (security.authorizationRules.length || security.permissions.length) required.push(['authorization-model', true]); - if (Object.values(dataGovernance).some((v)=>Array.isArray(v)&&v.length)) required.push(['data-governance', true]); - if (dataGovernance.transactionBoundaries.length || dataGovernance.consistencyModels.length || dataGovernance.concurrencyControls.length || dataGovernance.idempotencyRules.length) required.push(['consistency-transactions', true]); - if (operations.observabilitySignals.length || operations.healthChecks.length || operations.alerts.length || operations.slis.length || operations.slos.length) required.push(['operations-observability', true]); - if (operations.failureModes.length || operations.recoveryProcedures.length || operations.backups.length) required.push(['failure-recovery', true]); - if (Object.values(testing).some((v)=>Array.isArray(v)&&v.length)) required.push(['testing-strategy', true]); - if (configuration.settings.length || configuration.environments.length || configuration.featureFlags.length) required.push(['configuration-matrix', true]); - if (decisions.recordedDecisions.length || decisions.inferredDecisions.length) required.push(['architecture-decisions', true]); - if (changeImpact.changeSurfaces.length || changeImpact.impactEdges.length || changeImpact.compatibilityBoundaries.length) required.push(['change-impact', true]); - if (ownership.teams.length || ownership.responsibilities.length || ownership.componentOwners.length || ownership.dataOwners.length || ownership.operationalOwners.length) required.push(['ownership-responsibilities', true]); - return required.filter(([tag]) => !tags.has(tag)).map(([tag]) => tag); -} -function writeNavigationSummary(manifest) { - const lines = ['# Documentation Map', '', 'Generated from `.docgen/plan/manifest.json`.', '']; - const byId = new Map((manifest.pages ?? []).map((p) => [p.id, p])); - const navigation = Array.isArray(manifest.navigation) && manifest.navigation.length ? manifest.navigation : []; - const groups = navigation.length ? navigation : Object.entries(Object.groupBy ? Object.groupBy(manifest.pages ?? [], (p) => p.category ?? 'Documentation') : (manifest.pages ?? []).reduce((a,p)=>{(a[p.category??'Documentation']??=[]).push(p);return a;},{})).map(([title,pages])=>({id:slug(title),title,pages:pages.map(p=>p.id)})); - for (const group of groups) { - lines.push(`## ${group.title}`, ''); - if (group.description) lines.push(group.description, ''); - for (const id of group.pages ?? []) { - const p = byId.get(typeof id === 'string' ? id : id.id); - if (!p) continue; - const target = path.relative(path.join(root, 'docs'), path.join(root, p.path)).replaceAll('\\', '/').replace(/\.md$/, ''); - lines.push(`- [${p.title}](${target}.md) — ${p.summary ?? p.purpose ?? ''}`); - } - lines.push(''); - } - fs.mkdirSync(path.join(root, 'docs'), { recursive: true }); - fs.writeFileSync(path.join(root, 'docs', 'SUMMARY.md'), lines.join('\n').trimEnd() + '\n'); -} - -function yamlScalar(value) { - if (value === null || value === undefined) return 'null'; - if (typeof value === 'boolean' || typeof value === 'number') return String(value); - return JSON.stringify(String(value)); -} -function pageFrontmatter(page) { - let trace=null; try{if(fs.existsSync(traceabilityPath(page)))trace=readJson(traceabilityPath(page));}catch{} - const snapshot=trace?.sourceSnapshot??currentSourceSnapshot(); - const verifiedAt=snapshot?.capturedAt??loadPageState().pages?.[page.id]?.generatedAt??now(); - return { - title:page.title, description:page.summary||page.purpose, pageId:page.id, category:page.category, mode:page.mode, - type:page.type, order:Number(page.order??0), audience:page.audience??[], status:page.status??'active', - version:page.version??null, deprecatedSince:page.deprecatedSince??null, replacementPage:page.replacementPage??null, - aliases:page.aliases??[], sourceCommit:snapshot?.commit??null, lastVerified:String(verifiedAt).slice(0,10), - coverage:page.coverageTags??[], ...(page.frontmatter??{}) - }; -} -function renderYamlFrontmatter(obj) { - const lines=['---']; - for(const [key,value] of Object.entries(obj)){ - if(value===null||value===undefined||value==='')continue; - if(Array.isArray(value)){lines.push(`${key}:`);for(const x of value)lines.push(` - ${yamlScalar(x)}`);} else lines.push(`${key}: ${yamlScalar(value)}`); - } - lines.push('---',''); return lines.join('\n'); -} -function ensurePageFrontmatter(page) { - const file=pageFile(page); if(!fs.existsSync(file))return; - let body=fs.readFileSync(file,'utf8'); - if(/^---\s*\n[\s\S]*?\n---\s*\n/.test(body)) body=body.replace(/^---\s*\n[\s\S]*?\n---\s*\n/,''); - const next=renderYamlFrontmatter(pageFrontmatter(page))+body.replace(/^\s+/,''); - if(fs.readFileSync(file,'utf8')!==next)fs.writeFileSync(file,next); -} -function stripFrontmatter(text){return String(text).replace(/^---\s*\n[\s\S]*?\n---\s*\n/,'');} -function markdownExcerpt(text,max=320){return stripFrontmatter(text).replace(/```[\s\S]*?```/g,' ').replace(/[#>*_`\[\]()]/g,' ').replace(/\s+/g,' ').trim().slice(0,max);} -function extractHeadings(text){return [...stripFrontmatter(text).matchAll(/^(#{2,6})\s+(.+)$/gm)].map((m)=>({level:m[1].length,title:m[2].trim(),anchor:slug(m[2])}));} -function extractExamples(page,text){ - const lines=stripFrontmatter(text).split(/\r?\n/);const out=[];let trace={claims:[]};try{if(fs.existsSync(traceabilityPath(page)))trace=readJson(traceabilityPath(page));}catch{} - for(let i=0;inormalizeHeading(c.section).includes(normalizeHeading(title))||normalizeHeading(title).includes(normalizeHeading(c.section)));const evidenceRefs=[...new Set(matching.flatMap((c)=>(c.evidence??[]).map((e)=>e.path)).filter(Boolean))];const modelRefs=[...new Set(matching.flatMap((c)=>c.sourceModelRefs??[]).filter(Boolean))];out.push({id:`${page.id}-${slug(title)}`,pageId:page.id,title,summary:markdownExcerpt(buf.join('\n'),500),evidenceDerived:evidenceRefs.length+modelRefs.length>0,evidenceRefs,modelRefs});} - return out; -} -function doPublish() { - const manifest=requireManifestPreflight(); const cfg=loadConfig().documentationExperience??{}; fs.mkdirSync(publishRoot,{recursive:true}); - const byId=new Map(manifest.pages.map((p)=>[p.id,p])); const navigation=[]; const search=[]; const backlinks={}; const redirects=[]; const examples=[]; - for(const group of manifest.navigation??[]) navigation.push({...group,pages:(group.pages??[]).map((id)=>{const p=byId.get(typeof id==='string'?id:id.id);return p?{id:p.id,title:p.title,path:p.path,mode:p.mode,status:p.status,summary:p.summary}:null;}).filter(Boolean)}); - for(const page of manifest.pages){ - const file=pageFile(page); if(!fs.existsSync(file))continue; ensurePageFrontmatter(page); const md=fs.readFileSync(file,'utf8'); - search.push({id:page.id,title:page.title,path:page.path,category:page.category,mode:page.mode,type:page.type,status:page.status,summary:page.summary,keywords:[...(page.searchKeywords??[]),...(page.coverageTags??[])],headings:extractHeadings(md),excerpt:markdownExcerpt(md),updatedAt:fs.statSync(file).mtime.toISOString()}); - backlinks[page.id]??=[]; for(const target of page.relatedPages??[]){backlinks[target]??=[];backlinks[target].push({pageId:page.id,path:page.path,title:page.title,relation:'related'});} - for(const alias of page.aliases??[])redirects.push({from:alias,to:page.path,pageId:page.id,status:page.status}); - examples.push(...extractExamples(page,md)); - } - const linked=new Set((manifest.navigation??[]).flatMap((g)=>g.pages??[]).map((x)=>typeof x==='string'?x:x.id)); const orphans=manifest.pages.filter((p)=>!linked.has(p.id)).map((p)=>({id:p.id,path:p.path,title:p.title})); - writeJson(navigationIndexPath,{schemaVersion:'1.0',generatedAt:now(),navigation}); writeJson(searchIndexPath,{schemaVersion:'1.0',generatedAt:now(),pages:search}); writeJson(backlinksPath,{schemaVersion:'1.0',generatedAt:now(),backlinks}); writeJson(redirectsPath,{schemaVersion:'1.0',generatedAt:now(),redirects}); writeJson(orphansPath,{schemaVersion:'1.0',generatedAt:now(),orphans}); writeJson(examplesIndexPath,{schemaVersion:'1.0',generatedAt:now(),examples}); - const llms=['# '+(loadConfig().projectName||path.basename(root)),'','> '+(manifest.metadata?.description||'Generated system knowledge base.'),'']; - for(const group of navigation){llms.push(`## ${group.title}`,'');for(const p of group.pages)llms.push(`- [${p.title}](${p.path.replace(/^docs\//,'')}) — ${p.summary??''}`);llms.push('');} - fs.writeFileSync(path.join(root,'docs','llms.txt'),llms.join('\n').trimEnd()+'\n'); - if(cfg.generateLlmsFull!==false){const max=Number(cfg.llmsFullMaxBytes??5*1024*1024);let full=llms.join('\n')+'\n';for(const p of manifest.pages){const f=pageFile(p);if(!fs.existsSync(f))continue;const chunk=`\n\n# ${p.title}\n\nSource: ${p.path}\n\n${stripFrontmatter(fs.readFileSync(f,'utf8'))}`;if(Buffer.byteLength(full+chunk)>max)break;full+=chunk;}fs.writeFileSync(path.join(root,'docs','llms-full.txt'),full.trimEnd()+'\n');} - const report={schemaVersion:'1.0',generatedAt:now(),pages:search.length,navigationGroups:navigation.length,redirects:redirects.length,backlinkTargets:Object.keys(backlinks).length,orphans:orphans.length,examples:examples.length,artifacts:[rel(navigationIndexPath),rel(searchIndexPath),rel(backlinksPath),rel(redirectsPath),rel(orphansPath),rel(examplesIndexPath),'docs/llms.txt',...(cfg.generateLlmsFull===false?[]:['docs/llms-full.txt'])]};writeJson(publishingReportPath,report); - console.log(`Publishing metadata generated: ${search.length} pages, ${navigation.length} groups, ${redirects.length} redirects, ${examples.length} examples, ${orphans.length} orphans.`); return report; -} - -function reconcileGeneratedPage(page) { - const expected = pageFile(page); - if (fs.existsSync(expected)) return expected; - const docsRoot = path.join(root, 'docs'); - if (!fs.existsSync(docsRoot)) return expected; - const expectedKey = canonicalPagePath(page.path).replace(/^docs\//, '').replace(/\.md$/i, '').toLowerCase(); - const candidates = listFilesRecursive(docsRoot).filter((f) => /(?:\.md|\.markdown)?$/i.test(f)).filter((f) => { - const rp = rel(f).replace(/^docs\//, '').replace(/(?:\.md|\.markdown)$/i, '').toLowerCase(); - const base = slug(path.basename(rp)); - return rp === expectedKey || base === slug(page.id) || base === slug(page.title); - }); - const manifest = fs.existsSync(manifestPath) ? normalizeManifest(false) : { pages: [] }; - const owned = new Set((manifest.pages ?? []).filter((p) => p.id !== page.id).map((p) => canonicalPagePath(p.path))); - const unique = candidates.filter((f) => !owned.has(rel(f))); - if (unique.length === 1) { - fs.mkdirSync(path.dirname(expected), { recursive: true }); - fs.renameSync(unique[0], expected); - console.log(`[docgen] reconciled generated page path: ${rel(unique[0])} -> ${page.path}`); - } - return expected; -} - -function validatePageFile(page) { - const file = reconcileGeneratedPage(page); - if (!fs.existsSync(file)) throw new Error(`Missing generated page: ${page.path}`); - ensurePageFrontmatter(page); - const text = fs.readFileSync(file, 'utf8'); - if (!/^---\s*\n[\s\S]*?\n---\s*\n/.test(text)) throw new Error(`${page.path} has no publishing frontmatter`); - if (!/^#\s+\S/m.test(stripFrontmatter(text))) throw new Error(`${page.path} has no H1 heading`); - if ((text.match(/```/g) ?? []).length % 2 !== 0) throw new Error(`${page.path} has an unclosed fenced code block`); - if (/[A-Za-z]:\\Users\\|\/home\/[^/]+\//.test(text)) throw new Error(`${page.path} appears to contain an absolute local user path`); - if (qualityConfig().requireMermaidOnly !== false) validateMermaidOnly(text, page.path); -} - -function validateSkills(errors) { - const skillRoot = path.join(commandCodeHome, 'skills'); - if (!fs.existsSync(skillRoot)) { errors.push(`Missing global skills directory: ${skillRoot}`); return; } - for (const entry of fs.readdirSync(skillRoot, { withFileTypes: true })) { - if (!entry.isDirectory() || !entry.name.startsWith('doc-') && !entry.name.startsWith('tech-') && !entry.name.startsWith('domain-')) continue; - const file = path.join(skillRoot, entry.name, 'SKILL.md'); - if (!fs.existsSync(file)) { errors.push(`Missing global skill file: ${file}`); continue; } - const text = fs.readFileSync(file, 'utf8'); - const m = text.match(/^---\s*\n([\s\S]*?)\n---/); - if (!m) { errors.push(`${file} missing YAML frontmatter`); continue; } - const nameLine = m[1].split(/\r?\n/).find((x) => x.startsWith('name:')); - const name = nameLine?.slice(5).trim().replace(/^[\'"]|[\'"]$/g, ''); - if (name !== entry.name) errors.push(`${file} name must equal directory: ${entry.name}`); - } -} -function validateStatic() { - const errors = []; - const projectMarker = path.join(root, '.docgen', 'project.json'); - if (!fs.existsSync(projectMarker)) errors.push(`Current repository is not initialized: missing ${rel(projectMarker)}. Run \`docgen init\`.`); - for (const f of [configPath, statePath]) { - try { validateJsonFile(f); } catch (e) { errors.push(e.message); } - } - validateSkills(errors); - const requiredAgents = ['doc-discoverer', 'doc-architect', 'doc-domain-analyst', 'doc-enterprise-analyst', 'doc-system-analyst', 'doc-planner', 'doc-writer', 'doc-auditor']; - for (const a of requiredAgents) if (!fs.existsSync(path.join(commandCodeHome, 'agents', `${a}.md`))) errors.push(`Missing global agent: ${a}`); - const requiredCommands = ['docgen-init', 'docgen-doctor', 'docgen-discover', 'docgen-analyze', 'docgen-plan', 'docgen-generate', 'docgen-audit', 'docgen-fix', 'docgen-update', 'docgen-status', 'docgen-enrich', 'docgen-quality', 'docgen-semantics', 'docgen-preflight', 'docgen-resume', 'docgen-contract-test', 'docgen-traceability', 'docgen-enterprise', 'docgen-ignore', 'docgen-publish', 'docgen-workspace']; - for (const c of requiredCommands) if (!fs.existsSync(path.join(commandCodeHome, 'commands', `${c}.md`))) errors.push(`Missing global command: ${c}`); - for (const prompt of ['discover.md', 'analyze.md', 'semantics.md', 'enterprise.md', 'plan.md', 'generate.md', 'enrich.md', 'audit.md', 'fix.md', 'update-impact.md', 'generate-batch.md', 'enrich-batch.md', 'audit-batch.md', 'workspace-synthesis.md']) if (!fs.existsSync(assetFile('prompts', prompt))) errors.push(`Missing prompt: ${prompt}`); - for (const schema of ['evidence-artifact.schema.json', 'evidence-index.schema.json', 'component.schema.json', 'workflow.schema.json', 'system.schema.json', 'business.schema.json', 'flows.schema.json', 'catalogs.schema.json', 'manifest.schema.json', 'audit-page.schema.json', 'audit-index.schema.json', 'update-plan.schema.json', 'semantic-item.schema.json', 'traceability.schema.json', 'quality-summary.schema.json', 'security.schema.json', 'operations.schema.json', 'testing.schema.json', 'data-governance.schema.json', 'decisions.schema.json', 'configuration.schema.json', 'change-impact.schema.json', 'ownership.schema.json', 'publishing-index.schema.json', 'workspace.schema.json', 'workspace-repositories.schema.json', 'system-map.schema.json', 'dependency-graph.schema.json', 'contract-registry.schema.json', 'capability-map.schema.json', 'business-journeys.schema.json', 'workspace-flows.schema.json', 'workspace-ownership.schema.json', 'workspace-change-impact.schema.json', 'workspace-quality.schema.json']) { - try { validateJsonFile(assetFile('schemas', schema)); } catch (e) { errors.push(e.message); } - } - if (errors.length) { - console.error('Static validation failed:'); - for (const e of errors) console.error(`- ${e}`); - return false; - } - console.log('Static validation passed.'); - return true; -} -function validateGenerated() { - const errors = []; - try { if (fs.existsSync(evidenceIndexPath)) normalizeEvidenceIndex(); } catch (e) { errors.push(e.message); } - try { if (fs.existsSync(systemPath)) normalizeJsonFile(systemPath, normalizeSystemObject, assertSystemModel); } catch (e) { errors.push(e.message); } - try { if (fs.existsSync(businessPath)) normalizeJsonFile(businessPath, normalizeBusinessObject, assertBusinessModel); } catch (e) { errors.push(e.message); } - try { if (fs.existsSync(flowsPath)) normalizeJsonFile(flowsPath, normalizeFlowsObject, assertFlowsModel); } catch (e) { errors.push(e.message); } - try { if (fs.existsSync(catalogsPath)) normalizeJsonFile(catalogsPath, normalizeCatalogsObject, assertCatalogsModel); } catch (e) { errors.push(e.message); } - for (const file of ENTERPRISE_PASSES.flatMap((p)=>p.outputs)) { try { if (fs.existsSync(file)) normalizeEnterpriseFile(file); } catch (e) { errors.push(e.message); } } - if (fs.existsSync(manifestPath)) { - try { - const manifest = normalizeManifest(); - const ids = new Set(); const paths = new Set(); - const coverageGaps = manifestCoverageGaps(manifest); - if (coverageGaps.length) errors.push(`Manifest coverage gaps: ${coverageGaps.join(', ')}`); - for (const page of manifest.pages) { - if (!page.id || !page.path) errors.push('Manifest page missing id/path'); - if (ids.has(page.id)) errors.push(`Duplicate page id: ${page.id}`); ids.add(page.id); - if (paths.has(page.path)) errors.push(`Duplicate page path: ${page.path}`); paths.add(page.path); - if (!page.path.startsWith('docs/') || !page.path.endsWith('.md')) errors.push(`Invalid page path: ${page.path}`); - if (fs.existsSync(path.join(root, page.path))) { try { validatePageFile(page); ensurePageTraceability(page); pageQualityReport(page); } catch (e) { errors.push(e.message); } } - } - } catch (e) { errors.push(e.message); } - } - if (errors.length) { - console.error('Generated artifact validation failed:'); - for (const e of errors) console.error(`- ${e}`); - return false; - } - console.log('Generated artifact validation passed.'); - return true; -} - -async function doDiscover(scope = '.', progressLabel = '') { - const normalizedScope = normalizeRepoPath(scope); - if (normalizedScope && normalizedScope !== '.' && fs.existsSync(path.join(root, normalizedScope))) { const d=ignoreDecision(normalizedScope, fs.statSync(path.join(root, normalizedScope)).isDirectory()); if (d.ignored) fail(`Discovery scope is ignored: ${scope} (${d.reason}).`); } - const inventory = writeSourceInventory(); - updateStage('discover', 'running', { scope, includedFiles: inventory.includedCount }); - try { - const evidenceIndex = await runContractStage('discover', [path.dirname(evidenceIndexPath)], - (reset) => runCommandCode('discover', renderPrompt('discover.md', { SCOPE: scope, SOURCE_INVENTORY: rel(sourceFilesPath), IGNORE_REPORT: rel(ignoreReportPath) }), scope, progressLabel, { beforeRetry: reset }), - () => normalizeEvidenceIndex()); - updateStage('discover', 'completed', { scope, artifactCount: evidenceIndex.artifacts.length, includedFiles: inventory.includedCount }); - } catch (e) { updateStage('discover', 'failed', { scope, error: e.message }); throw e; } -} -async function doAnalyze(scope = 'all current evidence', progressLabel = '') { - if (!fs.existsSync(evidenceIndexPath)) fail('Run discover first.'); - normalizeEvidenceIndex(); - updateStage('analyze', 'running', { scope }); - try { - const system = await runContractStage('analyze', [systemPath], - (reset) => runCommandCode('analyze', renderPrompt('analyze.md', { SCOPE: scope }), scope, progressLabel, { beforeRetry: reset }), - () => normalizeJsonFile(systemPath, normalizeSystemObject, assertSystemModel)); - updateStage('analyze', 'completed', { scope, components: system.components.length, relationships: system.relationships.length, workflows: system.workflows.length }); - } catch (e) { updateStage('analyze', 'failed', { scope, error: e.message }); throw e; } -} -async function doSemantics(progressLabel = '') { - if (!fs.existsSync(systemPath)) fail('Run analyze first.'); - normalizeJsonFile(systemPath, normalizeSystemObject, assertSystemModel); - updateStage('semantics', 'running'); - try { - const [business, flows, catalogs] = await runContractStage('semantics', [businessPath, flowsPath, catalogsPath], - (reset) => runCommandCode('semantics', renderPrompt('semantics.md'), '', progressLabel, { beforeRetry: reset }), - () => [ - normalizeJsonFile(businessPath, normalizeBusinessObject, assertBusinessModel), - normalizeJsonFile(flowsPath, normalizeFlowsObject, assertFlowsModel), - normalizeJsonFile(catalogsPath, normalizeCatalogsObject, assertCatalogsModel) - ]); - updateStage('semantics', 'completed', { endpoints: catalogs.endpoints.length, messageHandlers: catalogs.messageHandlers.length, externalDependencies: catalogs.externalDependencies.length, businessRules: business.businessRules.length, flows: Object.values(flows).filter(Array.isArray).reduce((n, x) => n + x.length, 0) }); - } catch (e) { updateStage('semantics', 'failed', { error: e.message }); throw e; } -} - -const ENTERPRISE_PASSES = [ - { id:'governance', outputs:[securityPath, ownershipPath], prompt:'enterprise.md' }, - { id:'operability', outputs:[operationsPath, testingPath], prompt:'enterprise.md' }, - { id:'data-and-configuration', outputs:[dataGovernancePath, configurationPath], prompt:'enterprise.md' }, - { id:'evolution', outputs:[decisionsPath, changeImpactPath], prompt:'enterprise.md' } -]; -function normalizeEnterpriseFile(file) { - const base = path.basename(file, '.json'); - const map = { - security:[normalizeSecurityObject, (x)=>assertEnterpriseModel('security.json',x,'security')], - operations:[normalizeOperationsObject, (x)=>assertEnterpriseModel('operations.json',x,'operations')], - testing:[normalizeTestingObject, (x)=>assertEnterpriseModel('testing.json',x,'testing')], - 'data-governance':[normalizeDataGovernanceObject, (x)=>assertEnterpriseModel('data-governance.json',x,'dataGovernance')], - decisions:[normalizeDecisionsObject, (x)=>assertEnterpriseModel('decisions.json',x,'decisions')], - configuration:[normalizeConfigurationObject, (x)=>assertEnterpriseModel('configuration.json',x,'configuration')], - 'change-impact':[normalizeChangeImpactObject, (x)=>assertEnterpriseModel('change-impact.json',x,'changeImpact')], - ownership:[normalizeOwnershipObject, (x)=>assertEnterpriseModel('ownership.json',x,'ownership')] - }; - if (!map[base]) throw new Error(`Unknown enterprise artifact: ${file}`); - return normalizeJsonFile(file, map[base][0], map[base][1]); -} -async function doEnterprise(progressLabel = '') { - if (!fs.existsSync(systemPath) || !fs.existsSync(businessPath) || !fs.existsSync(catalogsPath)) fail('Run analyze and semantics first.'); - updateStage('enterprise', 'running', { passCount: ENTERPRISE_PASSES.length }); - try { - for (let i=0;irel(f)); - await runContractStage(`enterprise-${pass.id}`, pass.outputs, - (reset)=>runCommandCode('enterprise', renderPrompt(pass.prompt, { ENTERPRISE_PASS:pass.id, OUTPUT_PATHS_JSON:JSON.stringify(outputContracts,null,2) }), pass.id, progressLabel || `enterprise ${i+1}/${ENTERPRISE_PASSES.length}`, { beforeRetry:reset }), - ()=>pass.outputs.map(normalizeEnterpriseFile)); - } - const counts={}; - for (const file of ENTERPRISE_PASSES.flatMap((p)=>p.outputs)) { const obj=normalizeEnterpriseFile(file); counts[path.basename(file,'.json')]=Object.values(obj).filter(Array.isArray).reduce((n,a)=>n+a.length,0); } - updateStage('enterprise','completed',{passCount:ENTERPRISE_PASSES.length,models:counts}); - } catch(e) { updateStage('enterprise','failed',{error:e.message}); throw e; } -} -async function doPlan(progressLabel = '') { - if (!fs.existsSync(systemPath)) fail('Run analyze first.'); - normalizeJsonFile(systemPath, normalizeSystemObject, assertSystemModel); - if (fs.existsSync(businessPath)) normalizeJsonFile(businessPath, normalizeBusinessObject, assertBusinessModel); - if (fs.existsSync(flowsPath)) normalizeJsonFile(flowsPath, normalizeFlowsObject, assertFlowsModel); - if (fs.existsSync(catalogsPath)) normalizeJsonFile(catalogsPath, normalizeCatalogsObject, assertCatalogsModel); - if (loadConfig().enterpriseDepth?.enabled !== false) { - for (const file of ENTERPRISE_PASSES.flatMap((p)=>p.outputs)) { if (!fs.existsSync(file)) fail(`Missing ${rel(file)}. Run enterprise first.`); normalizeEnterpriseFile(file); } - } - updateStage('plan', 'running'); - try { - await runContractStage('plan', [manifestPath], - (reset) => runCommandCode('plan', renderPrompt('plan.md', { MISSING_COVERAGE: '' }), '', progressLabel, { beforeRetry: reset }), - () => { const manifest = normalizeManifest(); const preflight = manifestPreflight(manifest, { includeCoverage: false }); if (!preflight.valid) throw new Error(`Manifest preflight failed:\n- ${preflight.errors.join('\n- ')}`); return manifest; }); - let manifest = normalizeManifest(); - let gaps = manifestCoverageGaps(manifest); - if (gaps.length && isComprehensive()) { - console.log(`[docgen] manifest coverage gaps detected: ${gaps.join(', ')}. Running one bounded coverage-repair planning pass.`); - await runContractStage('plan-coverage-repair', [manifestPath], - (reset) => runCommandCode('plan', renderPrompt('plan.md', { MISSING_COVERAGE: `The current manifest is missing these required evidence-backed coverage tags: ${gaps.join(', ')}. Reconcile the manifest so each is owned by an appropriate page; do not add unsupported content.` }), 'coverage-repair', progressLabel, { beforeRetry: reset }), - () => normalizeManifest()); - manifest = normalizeManifest(); gaps = manifestCoverageGaps(manifest); - } - if (gaps.length) throw new Error(`Manifest coverage gaps remain: ${gaps.join(', ')}`); - const preflight = manifestPreflight(manifest); - if (!preflight.valid) throw new Error(`Manifest preflight failed immediately after planning:\n- ${preflight.errors.join('\n- ')}\nReport: ${rel(preflightPath)}`); - writeNavigationSummary(manifest); - updateStage('plan', 'completed', { pageCount: manifest.pages.length, navigationGroups: manifest.navigation.length, preflight: 'passed' }); - } catch (e) { updateStage('plan', 'failed', { error: e.message }); throw e; } -} -function pageFile(page) { return path.join(root, canonicalPagePath(page.path)); } -function pageIsValid(page) { try { validatePageFile(page); return true; } catch { return false; } } -function pageCurrentHash(page) { const f=pageFile(page); return fs.existsSync(f)?sha256Text(stripFrontmatter(fs.readFileSync(f,'utf8'))):null; } -function pageInputHash(page) { - const hash = crypto.createHash('sha256'); - hash.update(JSON.stringify({ ...page, evidence: [...(page.evidence ?? [])].sort(), models: [...(page.models ?? [])].sort() })); - for (const ref of [...(page.evidence ?? []), ...(page.models ?? [])].sort()) { - const file = path.join(root, ref); - hash.update(`\n${ref}:`); - if (fs.existsSync(file) && fs.statSync(file).isFile()) hash.update(fs.readFileSync(file)); - else hash.update('MISSING'); - } - return hash.digest('hex'); -} -function pageIsReusable(page) { - if (!pageIsValid(page)) return false; - const currentInputHash = pageInputHash(page); - const state = loadPageState().pages?.[page.id]; - if (state?.generateInputHash === currentInputHash) return true; - const pageState=loadPageState(); - const preP2=String(pageState.kitVersion??'0.0.0').localeCompare('0.9.0',undefined,{numeric:true,sensitivity:'base'})<0; - if ((!state?.generateInputHash || (preP2 && loadConfig().documentationExperience?.adoptPreP2Pages !== false)) && executionConfig().adoptLegacyValidPages) { - updatePageState(page.id, { generateStatus: 'completed', generatedAt: state?.generatedAt ?? now(), pageHash: pageCurrentHash(page), generateInputHash: currentInputHash, targetPath: page.path, adoptedLegacyValidPage: true, adoptedP2Metadata:preP2 }); - console.log(`[docgen] adopted legacy valid page checkpoint: ${page.id}${preP2?' (P2 metadata/frontmatter migration)':''}`); - return true; - } - return false; -} -function pageNeedsEnrichment(page) { try { return pageQualityReport(page).errors.length > 0; } catch { return true; } } -function executionConfig() { - const e = loadConfig().execution ?? {}; - return { - generateBatchSize: Math.max(1, Number(e.generateBatchSize ?? 4)), - auditBatchSize: Math.max(1, Number(e.auditBatchSize ?? 6)), - enrichBatchSize: Math.max(1, Number(e.enrichBatchSize ?? 4)), - resumeByDefault: e.resumeByDefault !== false, - skipValidPages: e.skipValidPages !== false, - adoptLegacyValidPages: e.adoptLegacyValidPages !== false, - stageTimeoutMinutes: e.stageTimeoutMinutes ?? {} - }; -} -function stageTimeoutMs(stage) { - const configured = executionConfig().stageTimeoutMinutes; - const minutes = Number(configured?.[stage] ?? configured?.default ?? 20); - return Number.isFinite(minutes) && minutes > 0 ? minutes * 60 * 1000 : 0; -} -async function doGenerate(id, progressLabel = '', allowEnrich = true, force = false) { - const page = findPage(id); - if (!force && executionConfig().skipValidPages && pageIsReusable(page)) { - console.log(`[docgen] SKIP generate:${id} — valid page already exists at ${page.path}`); - } else { - updatePageState(id, { generateStatus: 'running', targetPath: page.path }); - await runCommandCode('generate', renderPrompt('generate.md', { PAGE_JSON: JSON.stringify(pageRuntimeContract(page), null, 2) }), id, progressLabel); - validatePageFile(page); refreshPageTraceabilityHashes(page); - updatePageState(id, { generateStatus: 'completed', generatedAt: now(), pageHash: pageCurrentHash(page), generateInputHash: pageInputHash(page), targetPath: page.path }); - } - if (allowEnrich && qualityConfig().autoEnrich !== false && isComprehensive() && pageNeedsEnrichment(page)) await doEnrich(id, progressLabel, force); -} -async function doGenerateBatch(pages, progressLabel = '') { - const pending = pages.filter((p) => !(executionConfig().skipValidPages && pageIsReusable(p))); - for (const p of pages.filter((p) => !pending.includes(p))) console.log(`[docgen] SKIP generate:${p.id} — valid page already exists.`); - if (!pending.length) { - if (qualityConfig().autoEnrich !== false && isComprehensive()) { const thin=pages.filter(pageNeedsEnrichment); if(thin.length) await doEnrichBatch(thin, `${progressLabel} | quality-repair`); } - return; - } - for (const p of pending) updatePageState(p.id, { generateStatus: 'running', targetPath: p.path }); - await runCommandCode('generate', renderPrompt('generate-batch.md', { PAGES_JSON: JSON.stringify(pending.map(pageRuntimeContract), null, 2) }), pending.map((p) => p.id).join(','), progressLabel); - const failures = []; - for (const page of pending) { - try { validatePageFile(page); refreshPageTraceabilityHashes(page); updatePageState(page.id, { generateStatus: 'completed', generatedAt: now(), pageHash: pageCurrentHash(page), generateInputHash: pageInputHash(page), targetPath: page.path }); } - catch (e) { failures.push({ page, error: e.message }); updatePageState(page.id, { generateStatus: 'failed', error: e.message }); } - } - if (failures.length) { - console.warn(`[docgen] batch generated ${pending.length - failures.length}/${pending.length} valid pages; retrying ${failures.length} failed page(s) individually.`); - for (const f of failures) await doGenerate(f.page.id, `individual fallback after batch`, false, true); - } - if (qualityConfig().autoEnrich !== false && isComprehensive()) { - const thin = pages.filter(pageNeedsEnrichment); - if (thin.length) await doEnrichBatch(thin, `${progressLabel} | quality-repair`); - } -} -function normalizeRefKey(value) { return String(value ?? '').replaceAll('\\','/').replace(/^\.\//,'').toLowerCase(); } -function itemRefsForPage(page) { - const system = normalizeSystemObject(loadOptionalJson(systemPath, {})); - const business = normalizeBusinessObject(loadOptionalJson(businessPath, {})); - const flows = normalizeFlowsObject(loadOptionalJson(flowsPath, {})); - const catalogs = normalizeCatalogsObject(loadOptionalJson(catalogsPath, {})); - const security = normalizeSecurityObject(loadOptionalJson(securityPath, {})); const operations = normalizeOperationsObject(loadOptionalJson(operationsPath, {})); - const testing = normalizeTestingObject(loadOptionalJson(testingPath, {})); const dataGovernance = normalizeDataGovernanceObject(loadOptionalJson(dataGovernancePath, {})); - const decisions = normalizeDecisionsObject(loadOptionalJson(decisionsPath, {})); const configuration = normalizeConfigurationObject(loadOptionalJson(configurationPath, {})); - const changeImpact = normalizeChangeImpactObject(loadOptionalJson(changeImpactPath, {})); const ownership = normalizeOwnershipObject(loadOptionalJson(ownershipPath, {})); - const tags = new Set(page.coverageTags ?? []); const expected = { model: [], catalog: [], branch: [] }; - const add = (target, items) => target.push(...(items ?? []).map((x) => x.id)); - if (tags.has('system-overview') || tags.has('architecture')) add(expected.model, [...system.components, ...system.relationships, ...system.workflows]); - if (tags.has('business-domain')) add(expected.model, [...business.actors, ...business.capabilities, ...business.concepts]); - if (tags.has('business-rules')) add(expected.model, business.businessRules); - if (tags.has('branch-conditions')) { add(expected.model, business.branchConditions); add(expected.branch, business.branchConditions); } - if (tags.has('state-lifecycle')) add(expected.model, business.lifecycles); - for (const [tag, key] of [['business-flow','businessFlows'],['control-flow','controlFlows'],['request-flow','requestFlows'],['traffic-flow','trafficFlows'],['data-flow','dataFlows'],['event-flow','eventFlows']]) if (tags.has(tag)) { add(expected.model, flows[key]); for (const f of flows[key]) add(expected.branch, f.branches ?? []); } - if (tags.has('endpoint-catalog')) add(expected.catalog, catalogs.endpoints); - if (tags.has('message-handler-catalog')) add(expected.catalog, catalogs.messageHandlers); - if (tags.has('external-dependency-catalog')) add(expected.catalog, catalogs.externalDependencies); - if (tags.has('security-trust-boundaries')) add(expected.model, [...security.trustBoundaries,...security.principals,...security.authenticationFlows,...security.serviceIdentities,...security.threats,...security.controls]); - if (tags.has('authorization-model')) add(expected.model, [...security.authorizationRules,...security.permissions]); - if (tags.has('data-governance')) add(expected.model, Object.values(dataGovernance).filter(Array.isArray).flat()); - if (tags.has('consistency-transactions')) add(expected.model, [...dataGovernance.transactionBoundaries,...dataGovernance.consistencyModels,...dataGovernance.concurrencyControls,...dataGovernance.idempotencyRules,...dataGovernance.reconciliationProcesses]); - if (tags.has('operations-observability')) add(expected.model, [...operations.runtimeComponents,...operations.healthChecks,...operations.observabilitySignals,...operations.slis,...operations.slos,...operations.alerts,...operations.capacityLimits,...operations.scalingSignals]); - if (tags.has('failure-recovery')) add(expected.model, [...operations.failureModes,...operations.recoveryProcedures,...operations.backups,...operations.runbooks]); - if (tags.has('testing-strategy')) add(expected.model, Object.values(testing).filter(Array.isArray).flat()); - if (tags.has('configuration-matrix')) add(expected.model, Object.values(configuration).filter(Array.isArray).flat()); - if (tags.has('architecture-decisions')) add(expected.model, Object.values(decisions).filter(Array.isArray).flat()); - if (tags.has('change-impact')) add(expected.model, Object.values(changeImpact).filter(Array.isArray).flat()); - if (tags.has('ownership-responsibilities')) add(expected.model, Object.values(ownership).filter(Array.isArray).flat()); - return { model: [...new Set(expected.model)], catalog: [...new Set(expected.catalog)], branch: [...new Set(expected.branch)] }; -} -function ratio(covered, expected) { return expected <= 0 ? 1 : Math.min(1, covered / expected); } -function pageSemanticMetrics(page, text) { - const trace = ensurePageTraceability(page); - const claims = trace.claims ?? []; - const knownIds = new Set(); - for (const model of [normalizeSystemObject(loadOptionalJson(systemPath, {})), normalizeBusinessObject(loadOptionalJson(businessPath, {})), normalizeFlowsObject(loadOptionalJson(flowsPath, {})), normalizeCatalogsObject(loadOptionalJson(catalogsPath, {})), normalizeSecurityObject(loadOptionalJson(securityPath, {})), normalizeOperationsObject(loadOptionalJson(operationsPath, {})), normalizeTestingObject(loadOptionalJson(testingPath, {})), normalizeDataGovernanceObject(loadOptionalJson(dataGovernancePath, {})), normalizeDecisionsObject(loadOptionalJson(decisionsPath, {})), normalizeConfigurationObject(loadOptionalJson(configurationPath, {})), normalizeChangeImpactObject(loadOptionalJson(changeImpactPath, {})), normalizeOwnershipObject(loadOptionalJson(ownershipPath, {}))]) for (const value of Object.values(model)) if (Array.isArray(value)) for (const item of value) if (item?.id) knownIds.add(normalizeRefKey(item.id)); - const aliases = buildReferenceAliases(); - const evidenceValid = (ev) => { const ref=normalizeRefKey(ev?.path); if(!ref) return Boolean(ev?.symbol); const resolved=normalizeReference(ref, aliases); if(!exists(resolved)) return false; if(ev?.startLine){ try{const lines=fs.readFileSync(path.join(root,resolved),'utf8').split(/\r?\n/).length; if(ev.startLine>lines || (ev.endLine&&ev.endLine>lines)) return false;}catch{return false;} } return true; }; - const modelRefValid = (ref) => knownIds.has(normalizeRefKey(ref)); - const groundable = claims.filter((c) => c.classification !== 'UNKNOWN'); - const grounded = groundable.filter((c) => (c.evidence ?? []).some(evidenceValid) || (c.sourceModelRefs ?? []).some(modelRefValid)); - const unsupported = groundable.filter((c) => !(c.evidence ?? []).some(evidenceValid) && !(c.sourceModelRefs ?? []).some(modelRefValid)); - const aliasesForCoverage = buildReferenceAliases(); - const resolveCoverageRef = (ref) => normalizeRefKey(normalizeReference(ref, aliasesForCoverage) ?? ref); - const declared = [...(page.evidence ?? [])].map(resolveCoverageRef); - const usedEvidence = new Set([...(trace.coverage?.evidenceRefsUsed ?? []), ...claims.flatMap((c) => (c.evidence ?? []).map((e) => e.path))].map(resolveCoverageRef)); - const declaredCovered = declared.filter((d) => usedEvidence.has(d)).length; - const expected = itemRefsForPage(page); - const modelRefs = new Set([...(trace.coverage?.modelItemRefs ?? []), ...claims.flatMap((c) => c.sourceModelRefs ?? [])].map(normalizeRefKey)); - const catalogRefs = new Set([...(trace.coverage?.catalogItemRefs ?? []), ...claims.flatMap((c) => c.sourceModelRefs ?? [])].map(normalizeRefKey)); - const branchRefs = new Set([...(trace.coverage?.branchItemRefs ?? []), ...claims.flatMap((c) => c.sourceModelRefs ?? [])].map(normalizeRefKey)); - const countCovered = (ids, refs) => ids.filter((id) => refs.has(normalizeRefKey(id))).length; - const words = pageWordCount(text); - return { - claimCount: claims.length, groundableClaimCount: groundable.length, groundedClaimCount: grounded.length, structuredClaimCount: claims.filter((c)=>c.subject&&c.predicate).length, - unsupportedClaims: unsupported.map((c) => c.id), - claimGroundingRatio: ratio(grounded.length, groundable.length), - structuredClaimRatio: ratio(claims.filter((c)=>c.subject&&c.predicate).length, claims.length), - evidenceCoverageRatio: ratio(declaredCovered, declared.length), - modelCoverageRatio: ratio(countCovered(expected.model, modelRefs), expected.model.length), - catalogCoverageRatio: ratio(countCovered(expected.catalog, catalogRefs), expected.catalog.length), - branchCoverageRatio: ratio(countCovered(expected.branch, branchRefs), expected.branch.length), - evidenceClaimDensityPer1000Words: words ? grounded.length / words * 1000 : 0, - traceabilityPath: traceabilityRelPath(page), legacyUnmapped: trace.legacyUnmapped, - stale: trace.pageHash !== pageCurrentHash(page) || trace.inputHash !== pageInputHash(page) || Boolean(trace.sourceSnapshot?.sourceFingerprint && currentSourceSnapshot().sourceFingerprint && trace.sourceSnapshot.sourceFingerprint !== currentSourceSnapshot().sourceFingerprint), - sourceStale: Boolean(trace.sourceSnapshot?.sourceFingerprint && currentSourceSnapshot().sourceFingerprint && trace.sourceSnapshot.sourceFingerprint !== currentSourceSnapshot().sourceFingerprint), - sourceSnapshot: trace.sourceSnapshot - }; -} -function pageQualityReport(page) { - const file = path.join(root, page.path); - const text = fs.readFileSync(file, 'utf8'); - const q = qualityConfig(); const semanticQ = q.semanticMetrics ?? {}; - const words = pageWordCount(text); const headings = headingNames(text); const normalized = headings.map(normalizeHeading); - const modeDefaults=loadConfig().documentationExperience?.modeRequiredSections??{}; - const requiredSections = [...new Set([...(page.requiredSections ?? []), ...(loadConfig().documentationExperience?.enforceModeSections===false?[]:(modeDefaults[page.mode]??[]))])]; - const missingSections = requiredSections.filter((x) => !normalized.some((h) => h.includes(normalizeHeading(x)) || normalizeHeading(x).includes(h))); - const diagramIntents = page.diagramIntents ?? []; const mermaidCount = (text.match(/```mermaid\b/g) ?? []).length; - const minWords = q.minWordsByType?.[page.type] ?? 0; const errors = []; const warnings = []; - if (minWords && words < minWords) warnings.push(`word count ${words} is below advisory target ${minWords} for ${page.type}`); - if ((q.minHeadings ?? 0) && headings.length < q.minHeadings) errors.push(`heading count ${headings.length} is below ${q.minHeadings}`); - if (q.requireDeclaredSections !== false && missingSections.length) errors.push(`missing required sections: ${missingSections.join(', ')}`); - if (q.requirePlannedDiagrams !== false && diagramIntents.length && mermaidCount < 1) errors.push('manifest declares diagram intents but no Mermaid diagram exists'); - const exampleIntents=page.exampleIntents??[]; const extractedExamples=extractExamples(page,text); const exampleHeadingCount=extractedExamples.length; - if(loadConfig().documentationExperience?.requireEvidenceDerivedExamples!==false && exampleIntents.length && exampleHeadingCount<1) errors.push(`manifest declares example intents but no evidence-derived example/scenario section exists`); - if(loadConfig().documentationExperience?.requireEvidenceDerivedExamples!==false && exampleIntents.length && extractedExamples.some((x)=>!x.evidenceDerived)) errors.push(`example/scenario sections lack claim-level evidence or model references: ${extractedExamples.filter((x)=>!x.evidenceDerived).map((x)=>x.title).join(', ')}`); - if(page.status==='deprecated' && !headings.some((x)=>/(deprecat|migration|replacement|sunset)/i.test(x))) errors.push('deprecated page lacks a deprecation/migration/replacement section'); - const semantic = pageSemanticMetrics(page, text); - if (semantic.structuredClaimRatio < Number(semanticQ.minStructuredClaimRatio ?? 0.7)) errors.push(`structured claim ratio ${semantic.structuredClaimRatio.toFixed(2)} is below ${semanticQ.minStructuredClaimRatio ?? 0.7}`); - if (semantic.claimGroundingRatio < Number(semanticQ.minClaimGroundingRatio ?? 0.9)) errors.push(`claim grounding ratio ${semantic.claimGroundingRatio.toFixed(2)} is below ${semanticQ.minClaimGroundingRatio ?? 0.9}`); - if (semantic.evidenceCoverageRatio < Number(semanticQ.minEvidenceCoverageRatio ?? 0.8)) errors.push(`declared evidence coverage ratio ${semantic.evidenceCoverageRatio.toFixed(2)} is below ${semanticQ.minEvidenceCoverageRatio ?? 0.8}`); - if (semantic.modelCoverageRatio < Number(semanticQ.minModelCoverageRatio ?? 0.9)) errors.push(`model-item coverage ratio ${semantic.modelCoverageRatio.toFixed(2)} is below ${semanticQ.minModelCoverageRatio ?? 0.9}`); - if (semantic.catalogCoverageRatio < Number(semanticQ.minCatalogCoverageRatio ?? 1)) errors.push(`catalog coverage ratio ${semantic.catalogCoverageRatio.toFixed(2)} is below ${semanticQ.minCatalogCoverageRatio ?? 1}`); - if (semantic.branchCoverageRatio < Number(semanticQ.minBranchCoverageRatio ?? 0.9)) errors.push(`branch coverage ratio ${semantic.branchCoverageRatio.toFixed(2)} is below ${semanticQ.minBranchCoverageRatio ?? 0.9}`); - if (semantic.unsupportedClaims.length > Number(semanticQ.maxUnsupportedClaims ?? 0)) errors.push(`unsupported claims: ${semantic.unsupportedClaims.join(', ')}`); - if (semantic.stale) errors.push('traceability fingerprint is stale for current page/evidence/model inputs'); - const requiredGroundedClaims = Math.max(1, Math.ceil(words / 1000 * Number(semanticQ.minEvidenceClaimsPer1000Words ?? 1.5))); - semantic.requiredGroundedClaims = requiredGroundedClaims; - if (semantic.groundedClaimCount < requiredGroundedClaims) errors.push(`grounded claim count ${semantic.groundedClaimCount} is below evidence-density requirement ${requiredGroundedClaims}`); - return { pageId: page.id, pagePath: page.path, type: page.type, mode:page.mode, words, exampleIntents, exampleHeadingCount, examples:extractedExamples, headings: headings.length, minWords, requiredSections, missingSections, diagramIntents, mermaidCount, semantic, errors, warnings }; -} -function validatePageQuality(page, hard = true) { - const report = pageQualityReport(page); - if (report.errors.length) { - const message = `${page.path} quality gate failed:\n- ${report.errors.join('\n- ')}`; - if (hard) throw new Error(message); else console.warn(`WARNING: ${message}`); - } - return report; -} -async function doEnrich(id, progressLabel = '', force = false) { - const page = findPage(id); - if (!fs.existsSync(pageFile(page))) fail(`Generate page first: ${page.path}`); - const state = loadPageState().pages?.[id]; - if (!force && state?.enrichedHash && state.enrichedHash === pageCurrentHash(page) && !pageNeedsEnrichment(page)) { - console.log(`[docgen] SKIP enrich:${id} — current page already satisfies quality gates.`); return; - } - await runCommandCode('enrich', renderPrompt('enrich.md', { PAGE_JSON: JSON.stringify(pageRuntimeContract(page), null, 2) }), id, progressLabel); - validatePageFile(page); refreshPageTraceabilityHashes(page); validatePageQuality(page, false); - updatePageState(id, { enrichStatus: 'completed', enrichedAt: now(), enrichedHash: pageCurrentHash(page) }); -} -async function doEnrichBatch(pages, progressLabel = '') { - const pending = pages.filter(pageNeedsEnrichment); - if (!pending.length) return; - await runCommandCode('enrich', renderPrompt('enrich-batch.md', { PAGES_JSON: JSON.stringify(pending.map(pageRuntimeContract), null, 2) }), pending.map((p) => p.id).join(','), progressLabel); - const failures = []; - for (const page of pending) { - try { validatePageFile(page); refreshPageTraceabilityHashes(page); validatePageQuality(page, false); updatePageState(page.id, { enrichStatus: 'completed', enrichedAt: now(), enrichedHash: pageCurrentHash(page) }); } - catch (e) { failures.push(page); } - } - for (const page of failures) await doEnrich(page.id, 'individual fallback after enrich batch', true); -} -async function doEnrichAll() { - const manifest = requireManifestPreflight(); - const pages = manifest.pages.filter(pageNeedsEnrichment); - const size = executionConfig().enrichBatchSize; - for (let i = 0; i < pages.length; i += size) { - const batch = pages.slice(i, i + size); printItemProgress('enrich batch', Math.floor(i / size) + 1, Math.ceil(pages.length / size), batch.map((p) => p.id).join(', ')); - await doEnrichBatch(batch, `batch ${Math.floor(i / size) + 1}/${Math.ceil(pages.length / size)}`); - } -} -function claimSemanticKey(claim) { - if (claim.subject && claim.predicate) return `${normalizeHeading(claim.subject)}|${normalizeHeading(claim.predicate)}`; - return null; -} -function claimValueKey(claim) { return `${normalizeHeading(String(claim.object ?? ''))}|${claim.polarity}`; } -function claimDuplicateKey(claim) { - if (claim.subject && claim.predicate) return `${claimSemanticKey(claim)}|${claimValueKey(claim)}`; - return normalizeHeading(claim.statement); -} -function rebuildTraceabilityIndex() { - if (!fs.existsSync(manifestPath)) return null; - const manifest = loadManifest(); const pages = []; const claims = []; - for (const page of manifest.pages) if (fs.existsSync(pageFile(page))) { - const trace = ensurePageTraceability(page); pages.push({ pageId: page.id, pagePath: page.path, traceabilityPath: traceabilityRelPath(page), pageHash: trace.pageHash, inputHash: trace.inputHash, claimCount: trace.claims.length, sourceSnapshot: trace.sourceSnapshot }); claims.push(...trace.claims); - } - const byDuplicate = new Map(); const bySemantic = new Map(); const claimIds=new Map(); const claimIdCollisions=[]; - for (const claim of claims) { - if(claimIds.has(claim.id)) claimIdCollisions.push({id:claim.id,pages:[claimIds.get(claim.id),claim.pageId]}); else claimIds.set(claim.id,claim.pageId); - const dk = claimDuplicateKey(claim); if (dk) { if (!byDuplicate.has(dk)) byDuplicate.set(dk, []); byDuplicate.get(dk).push(claim); } - const sk = claimSemanticKey(claim); if (sk) { if (!bySemantic.has(sk)) bySemantic.set(sk, []); bySemantic.get(sk).push(claim); } - } - const duplicateGroups = [...byDuplicate.entries()].filter(([, group]) => group.filter((x) => !x.intentionalDuplicate).length > 1).map(([key, group], i) => ({ id: `duplicate-${i + 1}`, key, claims: group.map((x) => ({ id: x.id, pageId: x.pageId, statement: x.statement, intentionalDuplicate: x.intentionalDuplicate })) })); - const contradictions = []; - for (const [key, group] of bySemantic.entries()) { - const factual = group.filter((x) => x.classification !== 'UNKNOWN'); - const byObject = new Map(); for (const claim of factual) { const objectKey=normalizeHeading(String(claim.object ?? '')); if(!byObject.has(objectKey))byObject.set(objectKey,[]); byObject.get(objectKey).push(claim); } - const polarityConflict=[...byObject.values()].some((claims)=>new Set(claims.map((x)=>x.polarity)).size>1); - const exclusiveConflict=factual.some((x)=>x.exclusivePredicate) && new Set(factual.map((x)=>normalizeHeading(String(x.object ?? '')))).size>1; - if (polarityConflict || exclusiveConflict) contradictions.push({ id: `contradiction-${contradictions.length + 1}`, key, severity: factual.some((x) => x.classification === 'FACT') ? 'high' : 'medium', claims: factual.map((x) => ({ id: x.id, pageId: x.pageId, statement: x.statement, classification: x.classification, object: x.object, polarity: x.polarity, exclusivePredicate:x.exclusivePredicate })) }); - } - const currentSource = currentSourceSnapshot(true); - const freshnessPages = pages.map((p) => { const page=findPage(p.pageId); const currentPageHash=pageCurrentHash(page); const currentInputHash=pageInputHash(page); const sourceStale=Boolean(p.sourceSnapshot?.sourceFingerprint && currentSource.sourceFingerprint && p.sourceSnapshot.sourceFingerprint !== currentSource.sourceFingerprint); return { ...p, currentPageHash, currentInputHash, currentSourceSnapshot: currentSource, sourceStale, stale: p.pageHash !== currentPageHash || p.inputHash !== currentInputHash || sourceStale }; }); - const index = { schemaVersion:'1.0', generatedAt:now(), sourceSnapshot:currentSourceSnapshot(), pages, claims, summary:{ pages:pages.length, claims:claims.length, groundedClaims:claims.filter((c)=>(c.evidence??[]).length||(c.sourceModelRefs??[]).length).length, duplicateGroups:duplicateGroups.length, contradictions:contradictions.length, claimIdCollisions:claimIdCollisions.length, stalePages:freshnessPages.filter((x)=>x.stale).length } }; - index.claimIdCollisions=claimIdCollisions; writeJson(traceabilityIndexPath,index); writeJson(duplicatesPath,{schemaVersion:'1.0',generatedAt:now(),groups:duplicateGroups}); writeJson(contradictionsPath,{schemaVersion:'1.0',generatedAt:now(),contradictions}); writeJson(freshnessPath,{schemaVersion:'1.0',generatedAt:now(),sourceSnapshot:index.sourceSnapshot,pages:freshnessPages}); return {index,duplicateGroups,contradictions,claimIdCollisions,freshnessPages}; -} -function writeQualitySummary() { - if (!fs.existsSync(manifestPath)) return null; - const manifest = loadManifest(); const pages = []; - for (const page of manifest.pages) if (fs.existsSync(path.join(root,page.path))) pages.push(pageQualityReport(page)); - const audit = fs.existsSync(auditIndexPath) ? readJson(auditIndexPath) : {summary:{}}; const trace = rebuildTraceabilityIndex(); - const totals = pages.reduce((a,p)=>{ a.claims+=p.semantic.claimCount; a.groundable+=p.semantic.groundableClaimCount; a.grounded+=p.semantic.groundedClaimCount; a.unsupported+=p.semantic.unsupportedClaims.length; a.stale+=p.semantic.stale?1:0; return a; },{claims:0,groundable:0,grounded:0,unsupported:0,stale:0}); - const summary = { schemaVersion:'1.0', generatedAt:now(), qualityProfile:qualityProfile(), sourceSnapshot:currentSourceSnapshot(), pages, localGateFailures:pages.filter((p)=>p.errors.length).length, warningCount:pages.reduce((n,p)=>n+p.warnings.length,0), semanticSummary:{...totals,claimGroundingRatio:ratio(totals.grounded,totals.groundable),contradictions:trace?.contradictions.length??0,duplicateGroups:trace?.duplicateGroups.length??0,claimIdCollisions:trace?.claimIdCollisions.length??0,stalePages:trace?.freshnessPages.filter((x)=>x.stale).length??0}, auditSummary:audit.summary??{} }; - writeJson(path.join(root,'.docgen','audit','quality-summary.json'),summary); return summary; -} -function doTraceability() { - const result = rebuildTraceabilityIndex(); if (!result) fail('Missing manifest. Run plan first.'); - console.log(`Traceability pages: ${result.index.pages.length}`); console.log(`Claims: ${result.index.claims.length}`); console.log(`Contradictions: ${result.contradictions.length}`); console.log(`Duplicate claim groups: ${result.duplicateGroups.length}`); console.log(`Claim ID collisions: ${result.claimIdCollisions.length}`); console.log(`Stale pages: ${result.freshnessPages.filter((x)=>x.stale).length}`); console.log(`Index: ${rel(traceabilityIndexPath)}`); -} -function doQuality() { - const summary = writeQualitySummary(); if (!summary) fail('Missing manifest. Run plan first.'); - for (const p of summary.pages) { const mark=p.errors.length?'FAIL':'PASS'; console.log(`${mark.padEnd(4)} ${p.pageId.padEnd(32)} grounding ${(p.semantic.claimGroundingRatio*100).toFixed(0).padStart(3)}% | evidence ${(p.semantic.evidenceCoverageRatio*100).toFixed(0).padStart(3)}% | model ${(p.semantic.modelCoverageRatio*100).toFixed(0).padStart(3)}% | catalog ${(p.semantic.catalogCoverageRatio*100).toFixed(0).padStart(3)}% | ${p.words} words`); for(const e of p.errors) console.log(` - ERROR: ${e}`); for(const w of p.warnings) console.log(` - WARN: ${w}`); } - const q=qualityConfig(), sq=q.semanticMetrics??{}, a=summary.auditSummary, s=summary.semanticSummary; - const failed=summary.localGateFailures>0||(a.critical??0)>(q.maxCriticalFindings??0)||(a.high??0)>(q.maxHighFindings??0)||s.contradictions>Number(sq.maxContradictions??0)||s.claimIdCollisions>Number(sq.maxClaimIdCollisions??0)||s.stalePages>Number(sq.maxStalePages??0); - console.log(`Quality profile: ${summary.qualityProfile}`); console.log(`Local gate failures: ${summary.localGateFailures}`); console.log(`Semantic summary: ${JSON.stringify(s)}`); console.log(`Audit findings: ${JSON.stringify(a)}`); console.log(`Quality gate: ${failed?'FAIL':'PASS'}`); if(failed) process.exitCode=1; -} - -async function doGenerateAll(force = false) { - const manifest = requireManifestPreflight(); - const cfg = executionConfig(); - const batches = []; - for (let i = 0; i < manifest.pages.length; i += cfg.generateBatchSize) batches.push(manifest.pages.slice(i, i + cfg.generateBatchSize)); - const alreadyValid = manifest.pages.filter(pageIsReusable).length; - console.log(`[docgen] execution plan: ${manifest.pages.length} pages, ${alreadyValid} already valid, up to ${batches.length} generation batch run(s); enrichment only for pages failing local quality gates.`); - updateStage('generate', 'running', { pageCount: manifest.pages.length, batchCount: batches.length, alreadyValid }); - try { - for (let i = 0; i < batches.length; i++) { - const batch = batches[i]; - printItemProgress('generate batch', i + 1, batches.length, batch.map((p) => p.id).join(', ')); - if (force) for (const p of batch) { try { fs.rmSync(pageFile(p)); } catch {} } - await doGenerateBatch(batch, `batch ${i + 1}/${batches.length}`); - updateStage('generate', 'running', { pageCount: manifest.pages.length, batchCount: batches.length, completedBatches: i + 1, currentPages: batch.map((p) => p.id) }); - } - updateStage('generate', 'completed', { pageCount: manifest.pages.length, generated: manifest.pages.filter(pageIsValid).length, batchCount: batches.length }); - } catch (e) { - updateStage('generate', 'failed', { pageCount: manifest.pages.length, generated: manifest.pages.filter(pageIsValid).length, error: e.message }); - throw e; - } -} -function auditIsCurrent(page) { - const audit = path.join(root, '.docgen', 'audit', 'pages', `${page.id}.json`); - if (!fs.existsSync(audit) || !pageIsValid(page)) return false; - try { const report = normalizeJsonFile(audit, (obj) => normalizeAuditReportObject(obj, page), (obj) => assertCanonicalModel(`audit/${page.id}.json`, obj, ['findings'])); return report.pageId === page.id && report.pagePath === page.path && report.pageHash === pageCurrentHash(page) && report.inputHash === pageInputHash(page); } catch { return false; } -} -async function doAudit(id, progressLabel = '', force = false) { - const page = findPage(id); - if (!fs.existsSync(pageFile(page))) fail(`Generate page first: ${page.path}`); - if (!force && auditIsCurrent(page)) { console.log(`[docgen] SKIP audit:${id} — current audit already matches page hash.`); return; } - await runCommandCode('audit', renderPrompt('audit.md', { PAGE_JSON: JSON.stringify(pageRuntimeContract(page), null, 2), PAGE_ID: page.id, PAGE_HASH: pageCurrentHash(page), PAGE_INPUT_HASH: pageInputHash(page) }), id, progressLabel); - const reportPath = path.join(root, '.docgen', 'audit', 'pages', `${id}.json`); - const report = normalizeJsonFile(reportPath, (obj) => normalizeAuditReportObject(obj, page, { defaultHashes: true }), (obj) => { - assertCanonicalModel(`audit/${id}.json`, obj, ['findings']); - if (obj.pageId !== page.id) throw new Error(`Audit report pageId ${obj.pageId} does not match ${page.id}.`); - if (obj.pagePath !== page.path) throw new Error(`Audit report pagePath ${obj.pagePath} does not match ${page.path}.`); - if (obj.pageHash !== pageCurrentHash(page)) throw new Error(`Audit report hash does not match the current page content.`); - if (obj.inputHash !== pageInputHash(page)) throw new Error(`Audit report input hash does not match current evidence/model inputs.`); - }); - updatePageState(id, { auditStatus: 'completed', auditedAt: now(), auditHash: report.pageHash, auditInputHash: report.inputHash }); -} -async function doAuditBatch(pages, progressLabel = '') { - const pending = pages.filter((p) => !auditIsCurrent(p)); - for (const p of pages.filter((p) => !pending.includes(p))) console.log(`[docgen] SKIP audit:${p.id} — current audit exists.`); - if (!pending.length) return; - await runCommandCode('audit', renderPrompt('audit-batch.md', { PAGES_JSON: JSON.stringify(pending.map((p) => ({ ...pageRuntimeContract(p), pageHash: pageCurrentHash(p), inputHash: pageInputHash(p) })), null, 2) }), pending.map((p) => p.id).join(','), progressLabel); - for (const page of pending) { - const reportPath = path.join(root, '.docgen', 'audit', 'pages', `${page.id}.json`); - try { - const report = normalizeJsonFile(reportPath, (obj) => normalizeAuditReportObject(obj, page, { defaultHashes: true }), (obj) => { - assertCanonicalModel(`audit/${page.id}.json`, obj, ['findings']); - if (obj.pageId !== page.id || obj.pagePath !== page.path) throw new Error('Audit report identity mismatch.'); - if (obj.pageHash !== pageCurrentHash(page)) throw new Error('Audit report hash is stale.'); - if (obj.inputHash !== pageInputHash(page)) throw new Error('Audit report input hash is stale.'); - }); - updatePageState(page.id, { auditStatus: 'completed', auditedAt: now(), auditHash: report.pageHash, auditInputHash: report.inputHash }); - } catch { await doAudit(page.id, 'individual fallback after audit batch', true); } - } -} -function rebuildAuditIndex() { - const dir = path.join(root, '.docgen', 'audit', 'pages'); - const pages = []; - const invalidReports = []; - const counts = { critical: 0, high: 0, medium: 0, low: 0 }; - const manifest = fs.existsSync(manifestPath) ? normalizeManifest(false) : { pages: [] }; - const byId = new Map((manifest.pages ?? []).map((p) => [p.id, p])); - if (fs.existsSync(dir)) for (const name of fs.readdirSync(dir)) { - if (!name.endsWith('.json')) continue; - const file = path.join(dir, name); - try { - const raw = readJson(file); - const candidateId = slug(raw.pageId ?? raw.pageID ?? raw.id ?? path.basename(name, '.json')); - const page = byId.get(candidateId); - if (!page) throw new Error(`unknown page id ${candidateId}`); - const report = normalizeJsonFile(file, (obj) => normalizeAuditReportObject(obj, page), (obj) => { - assertCanonicalModel(`audit/${page.id}.json`, obj, ['findings']); - if (obj.pageId !== page.id || obj.pagePath !== page.path) throw new Error('identity mismatch'); - }); - pages.push({ pageId: report.pageId, pagePath: report.pagePath, pageHash: report.pageHash, inputHash: report.inputHash, findingCount: report.findings.length }); - for (const f of report.findings) if (f.severity in counts) counts[f.severity]++; - } catch (e) { invalidReports.push({ file: rel(file), error: e.message }); } - } - const index = { schemaVersion: '1.0', generatedAt: now(), pages, summary: counts, invalidReports }; - writeJson(auditIndexPath, index); - if (invalidReports.length) throw new Error(`Invalid audit report(s):\n- ${invalidReports.map((x) => `${x.file}: ${x.error}`).join('\n- ')}`); - return counts; -} -async function doAuditAll() { - const manifest = requireManifestPreflight(); - const size = executionConfig().auditBatchSize; - const batches = []; for (let i = 0; i < manifest.pages.length; i += size) batches.push(manifest.pages.slice(i, i + size)); - updateStage('audit', 'running', { pageCount: manifest.pages.length, batchCount: batches.length }); - try { - for (let i = 0; i < batches.length; i++) { const batch = batches[i]; printItemProgress('audit batch', i + 1, batches.length, batch.map((p) => p.id).join(', ')); await doAuditBatch(batch, `batch ${i + 1}/${batches.length}`); } - const summary = rebuildAuditIndex(); updateStage('audit', 'completed', { pageCount: manifest.pages.length, findings: summary, batchCount: batches.length }); - } catch (e) { updateStage('audit', 'failed', { pageCount: manifest.pages.length, error: e.message }); throw e; } -} -async function doFix(id, progressLabel = '') { - const page = findPage(id); - const audit = path.join(root, '.docgen', 'audit', 'pages', `${id}.json`); - if (!fs.existsSync(audit)) fail(`Missing audit for ${id}. Run audit first.`); - await runCommandCode('fix', renderPrompt('fix.md', { PAGE_JSON: JSON.stringify(pageRuntimeContract(page), null, 2), PAGE_ID: id }), id, progressLabel); - validatePageFile(page); refreshPageTraceabilityHashes(page); - if (isComprehensive()) validatePageQuality(page, false); -} -async function doFixAll() { - const manifest = loadManifest(); - const fixed = []; - for (const page of manifest.pages) { - const audit = path.join(root, '.docgen', 'audit', 'pages', `${page.id}.json`); - if (!fs.existsSync(audit)) continue; - const report = normalizeJsonFile(audit, (obj) => normalizeAuditReportObject(obj, page), (obj) => assertCanonicalModel(`audit/${page.id}.json`, obj, ['findings'])); - if ((report.findings ?? []).length) { printItemProgress('fix', manifest.pages.indexOf(page) + 1, manifest.pages.length, page.id); await doFix(page.id, `page ${manifest.pages.indexOf(page) + 1}/${manifest.pages.length}`); fixed.push(page.id); } - } - return fixed; -} - -function normalizeRepoPath(value) { - return String(value ?? '').replaceAll('\\', '/').replace(/^\.\//, '').replace(/^\/+/, '').replace(/\/+$/, ''); -} -function regexEscape(value) { return value.replace(/[|\\{}()[\]^$+?.]/g, '\\$&'); } -const DEFAULT_BINARY_EXTENSIONS = new Set([ - '.png','.jpg','.jpeg','.gif','.webp','.bmp','.ico','.tif','.tiff','.avif','.heic','.psd','.ai','.eps', - '.mp3','.wav','.flac','.aac','.ogg','.m4a','.wma','.mid','.midi', - '.mp4','.m4v','.mov','.avi','.mkv','.webm','.wmv','.flv','.mpeg','.mpg','.3gp', - '.pdf','.doc','.docx','.xls','.xlsx','.ppt','.pptx','.odt','.ods','.odp','.rtf', - '.zip','.gz','.tgz','.bz2','.xz','.7z','.rar','.tar','.jar','.war','.ear','.class','.dll','.exe','.so','.dylib','.o','.a','.lib', - '.woff','.woff2','.ttf','.otf','.eot','.bin','.dat','.db','.sqlite','.sqlite3','.pack','.idx','.p12','.pfx','.jks','.keystore', - '.apk','.ipa','.deb','.rpm','.iso','.dmg','.img','.wasm','.pyc','.pyo' -]); -const DEFAULT_TEXT_EXTENSIONS = new Set([ - '.txt','.md','.mdx','.adoc','.rst','.java','.kt','.kts','.groovy','.scala','.clj','.cljs','.go','.rs','.cs','.fs','.fsx', - '.c','.h','.cc','.cpp','.cxx','.hpp','.m','.mm','.swift','.dart','.py','.rb','.php','.pl','.pm','.lua','.r','.jl', - '.js','.mjs','.cjs','.jsx','.ts','.tsx','.vue','.svelte','.html','.htm','.css','.scss','.sass','.less', - '.xml','.xsd','.xsl','.xslt','.svg','.json','.jsonl','.yaml','.yml','.toml','.ini','.cfg','.conf','.properties','.env', - '.sql','.graphql','.gql','.proto','.thrift','.avsc','.bpmn','.dmn','.sh','.bash','.zsh','.fish','.ps1','.bat','.cmd', - '.dockerfile','.gradle','.make','.mk','.tf','.tfvars','.hcl','.rego','.feature','.csv','.tsv','.log','.lock' -]); -function binaryConfig(config = loadConfig()) { - const cfg = config.ignore?.binary ?? {}; - return { - enabled: cfg.enabled !== false, - probeBytes: Math.max(512, Number(cfg.probeBytes ?? 16384)), - maxTextFileBytes: Math.max(1024, Number(cfg.maxTextFileBytes ?? 4 * 1024 * 1024)), - controlCharacterRatio: Math.max(0, Math.min(1, Number(cfg.controlCharacterRatio ?? 0.08))), - allowExtensions: new Set([...(config.sourceExtensions ?? []), ...(cfg.allowExtensions ?? [])].map((x)=>String(x).toLowerCase())), - denyExtensions: new Set([...(cfg.denyExtensions ?? [])].map((x)=>String(x).toLowerCase())) - }; -} -function hasBinaryMagic(buf) { - if (!buf?.length) return false; - const sig=(...bytes)=>bytes.every((b,i)=>buf[i]===b); - return sig(0x89,0x50,0x4e,0x47)||sig(0xff,0xd8,0xff)||sig(0x47,0x49,0x46,0x38)||sig(0x25,0x50,0x44,0x46)||sig(0x50,0x4b,0x03,0x04)||sig(0x1f,0x8b)||sig(0x7f,0x45,0x4c,0x46)||sig(0x4d,0x5a)||sig(0x00,0x61,0x73,0x6d)||sig(0xca,0xfe,0xba,0xbe)||sig(0x52,0x49,0x46,0x46); -} -function classifySourceFile(fullPath, relPath, config = loadConfig()) { - const cfg=binaryConfig(config); - if (!cfg.enabled) return { text:true, reason:null, size:fs.existsSync(fullPath)?fs.statSync(fullPath).size:0 }; - let stat; try { stat=fs.statSync(fullPath); } catch { return { text:false, reason:'unreadable-source-file', size:0 }; } - const ext=path.extname(relPath).toLowerCase(); - if (cfg.denyExtensions.has(ext) || (!cfg.allowExtensions.has(ext) && DEFAULT_BINARY_EXTENSIONS.has(ext))) return { text:false, reason:`binary-extension:${ext || ''}`, size:stat.size }; - if (stat.size > cfg.maxTextFileBytes) return { text:false, reason:`text-file-too-large:${stat.size}`, size:stat.size }; - const declaredText = cfg.allowExtensions.has(ext) || DEFAULT_TEXT_EXTENSIONS.has(ext); - let buf; try { const fd=fs.openSync(fullPath,'r'); buf=Buffer.alloc(Math.min(cfg.probeBytes, stat.size)); const n=fs.readSync(fd,buf,0,buf.length,0); fs.closeSync(fd); buf=buf.subarray(0,n); } catch { return { text:false, reason:'unreadable-source-file', size:stat.size }; } - if (hasBinaryMagic(buf)) return { text:false, reason:'binary-magic-signature', size:stat.size }; - if (buf.includes(0)) return { text:false, reason:'binary-null-byte', size:stat.size }; - let decoded=''; try { decoded=new TextDecoder('utf-8',{fatal:true}).decode(buf); } catch { return { text:false, reason:'non-utf8-content', size:stat.size }; } - let controls=0; for(const ch of decoded){const c=ch.codePointAt(0);if(c<32&&!['\n','\r','\t','\f','\b'].includes(ch))controls++;} - const ratioValue=decoded.length?controls/decoded.length:0; - if(ratioValue>cfg.controlCharacterRatio)return{text:false,reason:`binary-control-ratio:${ratioValue.toFixed(3)}`,size:stat.size}; - return { text:true, reason:null, size:stat.size }; -} -function ignorePatternRegex(pattern, anchored = false) { - let body = ''; let i = 0; - while (i < pattern.length) { - if (pattern[i] === '*') { - if (pattern[i + 1] === '*') { while (pattern[i + 1] === '*') i++; body += '.*'; } - else body += '[^/]*'; - } else if (pattern[i] === '?') body += '[^/]'; - else body += regexEscape(pattern[i]); - i++; - } - return new RegExp(anchored ? `^${body}(?:/.*)?$` : `(?:^|/)${body}(?:/.*)?$`); -} -function loadIgnoreRules(file) { - if (!fs.existsSync(file)) return []; - const stat=fs.statSync(file); const cached=ignoreRulesCache.get(file); if(cached?.mtimeMs===stat.mtimeMs)return cached.rules; - const rules = []; - for (const raw of fs.readFileSync(file, 'utf8').split(/\r?\n/)) { - let line = raw.trim(); - if (!line || line.startsWith('#')) continue; - let negated = false; - if (line.startsWith('!')) { negated = true; line = line.slice(1); } - if (!line) continue; - const directoryOnly = line.endsWith('/'); - if (directoryOnly) line = line.replace(/\/+$/, ''); - const anchored = line.startsWith('/'); - if (anchored) line = line.slice(1); - rules.push({ raw, pattern: line, negated, directoryOnly, anchored, regex: ignorePatternRegex(line, anchored) }); - } - ignoreRulesCache.set(file,{mtimeMs:stat.mtimeMs,rules}); - return rules; -} -function matchIgnoreRules(relPath, isDirectory, rules) { - const normalized = normalizeRepoPath(relPath); - let decision = null; - for (const rule of rules) { - if (rule.directoryOnly && !isDirectory && !normalized.includes(`${rule.pattern}/`) && !normalized.startsWith(`${rule.pattern}/`)) continue; - if (rule.regex.test(normalized)) decision = { ignored: !rule.negated, reason: `.docgenignore:${rule.raw}` }; - } - return decision; -} -function configExcludeDecision(relPath, isDirectory, config) { - const normalized = normalizeRepoPath(relPath); - for (const raw of config.exclude ?? []) { - const pattern = String(raw).replaceAll('\\', '/').replace(/^\.\//, ''); - const directoryOnly = pattern.endsWith('/**') || pattern.endsWith('/'); - const cleaned = pattern.replace(/\/\*\*$/, '').replace(/\/+$/, ''); - const regex = ignorePatternRegex(cleaned, cleaned.startsWith('/')); - if (regex.test(normalized) || (directoryOnly && (normalized === cleaned || normalized.startsWith(`${cleaned}/`)))) return { ignored: true, reason: `config.exclude:${raw}` }; - } - return null; -} -function gitRepositoryAvailable() { - if (gitRepositoryAvailabilityCache !== null) return gitRepositoryAvailabilityCache; - if (!commandExists('git')) return (gitRepositoryAvailabilityCache=false); - const marker = path.join(root, '.git'); - if (fs.existsSync(marker)) return (gitRepositoryAvailabilityCache=true); - const r = spawnSync('git', ['rev-parse', '--is-inside-work-tree'], { cwd: root, encoding: 'utf8', shell: process.platform === 'win32' }); - return (gitRepositoryAvailabilityCache = r.status === 0 && String(r.stdout ?? '').trim() === 'true'); -} -function gitIgnoredBatch(paths) { - if (!paths.length || !gitRepositoryAvailable()) return new Set(); - const input = paths.map(normalizeRepoPath).join('\0') + '\0'; - const r = spawnSync('git', ['check-ignore', '--stdin', '-z', '--no-index'], { cwd: root, input, maxBuffer: 64 * 1024 * 1024, shell: process.platform === 'win32' }); - if (r.status !== 0 && r.status !== 1) return new Set(); - return new Set(Buffer.from(r.stdout ?? []).toString('utf8').split('\0').filter(Boolean).map(normalizeRepoPath)); -} -function gitIgnoredSingle(relPath) { const key=normalizeRepoPath(relPath); if(gitIgnoreSingleCache.has(key))return gitIgnoreSingleCache.get(key); const value=gitIgnoredBatch([key]).has(key); gitIgnoreSingleCache.set(key,value); return value; } -function fallbackGitIgnoreDecision(relPath, isDirectory = false) { - const normalized = normalizeRepoPath(relPath); - const segments = normalized.split('/'); - let decision = null; - for (let depth = 0; depth <= Math.max(0, segments.length - (isDirectory ? 0 : 1)); depth++) { - const base = segments.slice(0, depth).join('/'); - const file = path.join(root, base, '.gitignore'); - if (!fs.existsSync(file)) continue; - const relative = segments.slice(depth).join('/'); - const matched = matchIgnoreRules(relative, isDirectory, loadIgnoreRules(file)); - if (matched) decision = { ignored: matched.ignored, reason: `${base ? `${base}/` : ''}.gitignore:${matched.reason.replace(/^\.docgenignore:/,'')}` }; - } - return decision; -} -function hardIgnoreDecision(relPath, isDirectory, config) { - const normalized = normalizeRepoPath(relPath); - const outputRoot = normalizeRepoPath(config.outputRoot || 'docs'); - const hard = ['.git', '.commandcode', '.docgen', outputRoot, 'node_modules', 'target', 'build', 'dist', 'coverage', 'vendor']; - for (const prefix of hard) if (normalized === prefix || normalized.startsWith(`${prefix}/`)) return { ignored: true, reason: `docgen-hard-exclude:${prefix}/**` }; - return configExcludeDecision(normalized, isDirectory, config); -} -function ignoreDecision(relPath, isDirectory = false, config = loadConfig()) { - const normalized = normalizeRepoPath(relPath); - if (!normalized) return { ignored: false, reason: null }; - const hard = hardIgnoreDecision(normalized, isDirectory, config); - if (hard) return hard; - // The canonical inventory already performed ignore + binary/text classification. Reuse it on hot validation paths. - if (!isDirectory && sourceInventoryCache?.includedSet?.has(normalized)) return { ignored:false, reason:null }; - if (config.ignore?.useGitignore !== false) { - const gitRepo = gitRepositoryAvailable(); - if (gitRepo && gitIgnoredSingle(normalized)) return { ignored: true, reason: '.gitignore' }; - const fallback = fallbackGitIgnoreDecision(normalized, isDirectory); - if (!gitRepo && fallback) return fallback; - } - if (config.ignore?.useDocgenignore !== false) { - const ignoreFile = path.join(root, config.ignore?.docgenignoreFile || '.docgenignore'); - const matched = matchIgnoreRules(normalized, isDirectory, loadIgnoreRules(ignoreFile)); - if (matched) return matched; - } - if (!isDirectory) { - const full = path.join(root, normalized); - if (fs.existsSync(full)) { const classified = classifySourceFile(full, normalized, config); if (!classified.text) return { ignored:true, reason:classified.reason }; } - } - return { ignored: false, reason: null }; -} -function candidateFilesFromGit() { - if (!commandExists('git') || !fs.existsSync(path.join(root, '.git'))) return null; - const r = spawnSync('git', ['ls-files', '-co', '--exclude-standard', '-z'], { cwd: root, maxBuffer: 128 * 1024 * 1024, shell: process.platform === 'win32' }); - if (r.status !== 0) return null; - return Buffer.from(r.stdout ?? []).toString('utf8').split('\0').filter(Boolean).map(normalizeRepoPath); -} -function fallbackWalkCandidates(dir, out = []) { - for (const entry of fs.readdirSync(dir, { withFileTypes: true })) { - const full = path.join(dir, entry.name); const r = rel(full); const hard = hardIgnoreDecision(r, entry.isDirectory(), loadConfig()); - if (hard?.ignored) continue; - if (entry.isDirectory()) fallbackWalkCandidates(full, out); else out.push(normalizeRepoPath(r)); - } - return out; -} -function buildSourceInventory(options = {}) { - const config = loadConfig(); - let candidates = candidateFilesFromGit(); - const usedGit = Boolean(candidates); - if (!candidates) candidates = fallbackWalkCandidates(root); - const gitRepo = gitRepositoryAvailable(); - const gitIgnored = config.ignore?.useGitignore === false ? new Set() : gitIgnoredBatch(candidates); - const docgenRules = config.ignore?.useDocgenignore === false ? [] : loadIgnoreRules(path.join(root, config.ignore?.docgenignoreFile || '.docgenignore')); - const included = []; const ignoredSamples = []; const reasonCounts = {}; let includedBytes=0; let excludedBytes=0; let binaryExcludedCount=0; - for (const item of [...new Set(candidates)].sort()) { - const full = path.join(root, item); - if (!fs.existsSync(full) || !fs.statSync(full).isFile()) continue; - const stat=fs.statSync(full); - let decision = hardIgnoreDecision(item, false, config); - if (!decision && gitIgnored.has(item)) decision = { ignored: true, reason: '.gitignore' }; - if (!decision && !gitRepo && config.ignore?.useGitignore !== false) decision = fallbackGitIgnoreDecision(item, false); - if (!decision && docgenRules.length) decision = matchIgnoreRules(item, false, docgenRules); - if (!decision) { const classified=classifySourceFile(full,item,config); if(!classified.text){decision={ignored:true,reason:classified.reason};binaryExcludedCount++;} } - if (decision?.ignored) { - excludedBytes += stat.size; - reasonCounts[decision.reason] = (reasonCounts[decision.reason] ?? 0) + 1; - if (ignoredSamples.length < 500) ignoredSamples.push({ path: item, reason: decision.reason, size:stat.size }); - } else { included.push(item); includedBytes += stat.size; } - } - const controlFiles = ['.gitignore', config.ignore?.docgenignoreFile || '.docgenignore'].filter((x) => fs.existsSync(path.join(root, x))); - const report = { schemaVersion: '1.1', generatedAt: now(), usedGit, gitAvailable: commandExists('git'), gitRepository: gitRepo, includedCount: included.length, binaryOrNonTextExcludedCount:binaryExcludedCount, includedBytes, excludedBytes, ignoredSampleCount: ignoredSamples.length, reasonCounts, controlFiles, includedFiles: options.includeFiles === false ? undefined : included, ignoredSamples }; - return report; -} -function writeSourceInventory() { - const inventory = buildSourceInventory(); - sourceInventoryCache = { inventory, includedSet:new Set(inventory.includedFiles ?? []) }; gitIgnoreSingleCache.clear(); - writeJson(sourceInventoryPath, inventory); - fs.mkdirSync(path.dirname(sourceFilesPath), { recursive: true }); - fs.writeFileSync(sourceFilesPath, (inventory.includedFiles ?? []).join('\n') + '\n'); - if (loadConfig().ignore?.writeReport !== false) writeJson(ignoreReportPath, { ...inventory, includedFiles: undefined }); - return inventory; -} -function walkFiles(dir, config, out = []) { - const inventory = buildSourceInventory(); - sourceInventoryCache = { inventory, includedSet:new Set(inventory.includedFiles ?? []) }; - for (const relPath of inventory.includedFiles ?? []) out.push(path.join(root, relPath)); - return out; -} -function hashFile(file) { return crypto.createHash('sha256').update(fs.readFileSync(file)).digest('hex'); } -function makeSnapshot() { - const config = readJson(configPath); - const files = walkFiles(root, config); - const entries = {}; - for (const file of files) { - const stat = fs.statSync(file); - if (stat.size > 5 * 1024 * 1024) continue; - entries[rel(file)] = { sha256: hashFile(file), size: stat.size }; - } - const ignoreControls = { configIgnore: config.ignore ?? {}, exclude: config.exclude ?? [], gitignore: fs.existsSync(path.join(root,'.gitignore')) ? fileSha256(path.join(root,'.gitignore')) : null, docgenignore: fs.existsSync(path.join(root, config.ignore?.docgenignoreFile || '.docgenignore')) ? fileSha256(path.join(root, config.ignore?.docgenignoreFile || '.docgenignore')) : null }; - return { schemaVersion: '1.0', generatedAt: now(), ignorePolicyHash: sha256Text(JSON.stringify(ignoreControls)), files: entries }; -} -function doSourceList(pattern = '') { - const inventory = writeSourceInventory(); - const needle = String(pattern ?? '').toLowerCase(); - for (const file of (inventory.includedFiles ?? []).filter((x)=>!needle || x.toLowerCase().includes(needle))) console.log(file); -} -function doSourceGrep(args = []) { - const regexMode = args.includes('--regex'); - const query = args.filter((x)=>x!=='--regex').join(' '); - if (!query) fail('source-grep requires a search string (or --regex ).', 2); - let matcher; - try { matcher = regexMode ? new RegExp(query, 'i') : { test:(x)=>x.toLowerCase().includes(query.toLowerCase()) }; } - catch(e) { fail(`Invalid regex: ${e.message}`,2); } - const inventory = writeSourceInventory(); let matches=0; const maxMatches=500; - for (const relPath of inventory.includedFiles ?? []) { - const file=path.join(root,relPath); let stat; try{stat=fs.statSync(file);}catch{continue;} if(stat.size>2*1024*1024)continue; - let text; try{text=fs.readFileSync(file,'utf8');}catch{continue;} if(text.includes('\0'))continue; - const lines=text.split(/\r?\n/); - for(let i=0;i=maxMatches){console.log(`[docgen] source-grep stopped at ${maxMatches} matches.`);return;} } - } - if(!matches) console.log('[docgen] source-grep found no matches in included source files.'); -} -function doIgnore(target) { - const inventory = writeSourceInventory(); - if (target) { - const normalized = normalizeRepoPath(target); - const full = path.join(root, normalized); - const decision = ignoreDecision(normalized, fs.existsSync(full) && fs.statSync(full).isDirectory()); - console.log(`${normalized}: ${decision.ignored ? 'IGNORED' : 'INCLUDED'}${decision.reason ? ` (${decision.reason})` : ''}`); - return; - } - console.log(`Included source files: ${inventory.includedCount}`); - console.log(`Binary/non-text/oversized files excluded: ${inventory.binaryOrNonTextExcludedCount ?? 0}`); - console.log(`Included source bytes: ${inventory.includedBytes ?? 0}`); - console.log(`Git ignore engine: ${inventory.usedGit ? 'active' : inventory.gitAvailable ? 'fallback inventory' : 'git unavailable'}`); - console.log(`.docgenignore: ${fs.existsSync(path.join(root, loadConfig().ignore?.docgenignoreFile || '.docgenignore')) ? 'present' : 'not present'}`); - console.log(`Source list: ${rel(sourceFilesPath)}`); - console.log(`Ignore report: ${rel(ignoreReportPath)}`); - for (const [reason,count] of Object.entries(inventory.reasonCounts ?? {}).sort()) console.log(`- ${reason}: ${count}`); -} -function doSnapshot() { - const snap = makeSnapshot(); writeJson(fingerprintsPath, snap); console.log(`Snapshot saved: ${Object.keys(snap.files).length} files.`); -} -function changedPaths() { - const current = makeSnapshot(); - if (!fs.existsSync(fingerprintsPath)) return Object.keys(current.files).sort(); - const previous = readJson(fingerprintsPath); - const all = new Set([...Object.keys(previous.files ?? {}), ...Object.keys(current.files)]); - return [...all].filter((p) => previous.files?.[p]?.sha256 !== current.files?.[p]?.sha256).sort(); -} -async function doUpdate(explicitPaths) { - const changed = explicitPaths.length ? explicitPaths : changedPaths(); - if (!changed.length) { console.log('No source changes detected since the last snapshot.'); return; } - const updatePlanPath = path.join(root, '.docgen', 'plan', 'update-plan.json'); - const plan = await runContractStage('update-impact', [updatePlanPath], - (reset) => runCommandCode('update-impact', renderPrompt('update-impact.md', { CHANGED_PATHS_JSON: JSON.stringify(changed, null, 2) }), changed.join(', '), '', { beforeRetry: reset }), - () => normalizeJsonFile(updatePlanPath, (obj) => normalizeUpdatePlanObject(obj, changed), (obj) => assertCanonicalModel('update-plan.json', obj, ['changedPaths', 'affectedEvidenceScopes', 'affectedModels', 'affectedPageIds', 'rationale']))); - const scopes = plan.affectedEvidenceScopes?.length ? plan.affectedEvidenceScopes : changed; - for (const scope of scopes) await doDiscover(scope); - await doAnalyze(`incremental changes: ${changed.join(', ')}`); - await doSemantics(); - if (loadConfig().enterpriseDepth?.enabled !== false) await doEnterprise(); - await doPlan(); - for (const id of plan.affectedPageIds ?? []) { - const currentManifest = loadManifest(); - if (currentManifest.pages.some((p) => p.id === id)) { await doGenerate(id); await doAudit(id); } - } - rebuildAuditIndex(); - doSnapshot(); -} - -function validateStageArtifact(stage) { - if (stage === 'discover') return normalizeEvidenceIndex(); - if (stage === 'analyze') return normalizeJsonFile(systemPath, normalizeSystemObject, assertSystemModel); - if (stage === 'semantics') return [ - normalizeJsonFile(businessPath, normalizeBusinessObject, assertBusinessModel), - normalizeJsonFile(flowsPath, normalizeFlowsObject, assertFlowsModel), - normalizeJsonFile(catalogsPath, normalizeCatalogsObject, assertCatalogsModel) - ]; - if (stage === 'enterprise') return ENTERPRISE_PASSES.flatMap((p)=>p.outputs).map(normalizeEnterpriseFile); - if (stage === 'plan') return requireManifestPreflight(); - return true; -} -function stageCheckpointValid(stage) { - try { validateStageArtifact(stage); return true; } catch (e) { console.warn(`[docgen] checkpoint ${stage} is not reusable: ${e.message}`); return false; } -} -function contractSelfTest() { - const results = []; - const check = (name, fn) => { - try { fn(); results.push({ name, status: 'passed' }); console.log(`PASS contract ${name}`); } - catch (e) { results.push({ name, status: 'failed', error: e.message }); console.error(`FAIL contract ${name}: ${e.message}`); } - }; - check('system aliases', () => assertCanonicalModel('system', normalizeSystemObject({ services: [{}], dependencies: [], processes: [], openQuestions: [] }), ['components','relationships','workflows','unknowns'])); - check('business aliases', () => assertCanonicalModel('business', normalizeBusinessObject({ roles: [], businessCapabilities: [], domainConcepts: [], rules: [{}], decisionPoints: [], conditions: [], stateMachines: [], constraints: [], scenarios: [], gaps: [] }), ['actors','capabilities','concepts','businessRules','decisions','branchConditions','lifecycles','invariants','useCases','unknowns'])); - check('flow aliases', () => assertCanonicalModel('flows', normalizeFlowsObject({ flows: [{ type: 'request' }, { type: 'data' }, { type: 'event' }] }), ['businessFlows','controlFlows','requestFlows','trafficFlows','dataFlows','eventFlows'])); - check('catalog aliases', () => assertCanonicalModel('catalogs', normalizeCatalogsObject({ routes: [{}], handlers: [{}], integrations: [{}], databases: [{}], cronJobs: [{}] }), ['endpoints','messageHandlers','externalDependencies','dataStores','scheduledJobs'])); - check('enterprise aliases', () => { - assertEnterpriseModel('security', normalizeSecurityObject({boundaries:[{}],authnFlows:[],authzRules:[],gaps:[]}), 'security'); - assertEnterpriseModel('operations', normalizeOperationsObject({health:[{}],failures:[],runbooks:[],gaps:[]}), 'operations'); - assertEnterpriseModel('testing', normalizeTestingObject({suites:[{}],testCommands:[],gaps:[]}), 'testing'); - assertEnterpriseModel('data-governance', normalizeDataGovernanceObject({entities:[{}],transactions:[],lineage:[],gaps:[]}), 'dataGovernance'); - assertEnterpriseModel('decisions', normalizeDecisionsObject({adrs:[{}],options:[],gaps:[]}), 'decisions'); - assertEnterpriseModel('configuration', normalizeConfigurationObject({properties:[{}],environmentMatrix:[],gaps:[]}), 'configuration'); - assertEnterpriseModel('change-impact', normalizeChangeImpactObject({changePoints:[{}],blastRadius:[],gaps:[]}), 'changeImpact'); - assertEnterpriseModel('ownership', normalizeOwnershipObject({owners:[{}],raci:[],gaps:[]}), 'ownership'); - }); - check('ignore pattern semantics', () => { const rules=[{raw:'private/**',pattern:'private/**',negated:false,directoryOnly:false,anchored:false,regex:ignorePatternRegex('private/**',false)},{raw:'!private/public.md',pattern:'private/public.md',negated:true,directoryOnly:false,anchored:false,regex:ignorePatternRegex('private/public.md',false)}]; if(!matchIgnoreRules('private/secret.md',false,rules)?.ignored) throw new Error('ignore rule not applied'); if(matchIgnoreRules('private/public.md',false,rules)?.ignored) throw new Error('negation not applied'); }); - check('update-plan aliases', () => assertCanonicalModel('update', normalizeUpdatePlanObject({ changedFiles:['a'], scopes:['.'], models:['system'], pages:['overview'], reasons:['x'] }), ['changedPaths','affectedEvidenceScopes','affectedModels','affectedPageIds','rationale'])); - check('page path variants', () => { for (const x of ['orientation/overview','/orientation/overview.md','docs/orientation/overview','docs/orientation/overview.md']) if (canonicalPagePath(x) !== 'docs/orientation/overview.md') throw new Error(x); }); - check('binary signature detection', () => { if(!hasBinaryMagic(Buffer.from([0x89,0x50,0x4e,0x47])))throw new Error('PNG magic not detected'); if(hasBinaryMagic(Buffer.from('plain text')))throw new Error('text misclassified'); }); - check('documentation mode defaults', () => { const allowed=new Set(loadConfig().documentationExperience?.modes??[]); for(const x of ['tutorial','how-to','explanation','reference','runbook','decision-record','migration-guide','troubleshooting'])if(!allowed.has(x))throw new Error(`missing mode ${x}`); }); - check('audit aliases', () => { const page = { id: 'overview', path: 'docs/orientation/overview.md' }; const x = normalizeAuditReportObject({ id: 'overview', path: 'orientation/overview', hash: 'abc', inputHash: 'def', issues: ['x'] }, page); if (x.pagePath !== page.path || x.findings.length !== 1) throw new Error('audit normalization'); }); - check('normalizer idempotence', () => { - const samples = [ - [normalizeSystemObject, { services:[{id:'a'}], modules:[{id:'b'}], dependencies:[], processes:[], gaps:[] }], - [normalizeBusinessObject, { roles:[], rules:[{id:'r'}], policies:[{id:'p'}] }], - [normalizeFlowsObject, { flows:[{id:'q',type:'request'}], httpFlows:[{id:'q',type:'request'}] }], - [normalizeCatalogsObject, { consumers:[{id:'c'}], producers:[{id:'p'}], listeners:[{id:'l'}] }], - [normalizeUpdatePlanObject, { changedFiles:['a'], pages:['x'] }], - [normalizeSecurityObject, { boundaries:[{id:'b'}], authnFlows:[] }], - [normalizeOperationsObject, { health:[{id:'h'}], deploymentStrategies:[{id:'dep'}], failures:[] }], - [normalizeTestingObject, { suites:[{id:'s'}], testCommands:[] }], - [normalizeDataGovernanceObject, { entities:[{id:'d'}], transactions:[] }], - [normalizeDecisionsObject, { adrs:[{id:'a'}], options:[] }], - [normalizeConfigurationObject, { properties:[{id:'c'}], environmentMatrix:[] }], - [normalizeChangeImpactObject, { changePoints:[{id:'x'}], blastRadius:[] }], - [normalizeOwnershipObject, { owners:[{id:'o'}], raci:[] }] - ]; - for (const [fn, input] of samples) { const once = fn(input); const twice = fn(once); if (JSON.stringify(once) !== JSON.stringify(twice)) throw new Error(`${fn.name} is not idempotent`); } - }); - check('catalog losslessness', () => { const x = normalizeCatalogsObject({ consumers:[{id:'c'}], producers:[{id:'p'}], listeners:[{id:'l'}], kafkaHandlers:[{id:'k'}] }); if (x.messageHandlers.length < 3) throw new Error(`expected at least 3 handlers, got ${x.messageHandlers.length}`); }); - check('evidence path canonicalization', () => { const p = canonicalEvidencePath('repo.json', path.join(root,'.docgen','evidence')); if (p !== '.docgen/evidence/repo.json') throw new Error(p); }); - check('typed semantic items', () => { const x=normalizeBusinessObject({rules:['Only DRAFT may submit']}); const r=x.businessRules[0]; if(r.kind!=='business-rule'||!r.id||!Array.isArray(r.evidence)) throw new Error('typed rule contract'); assertBusinessModel(x); }); - check('evidence line notation', () => { const x=normalizeEvidenceRef('src/A.java#L10-L12'); if(x.path!=='src/A.java'||x.startLine!==10||x.endLine!==12) throw new Error('line notation'); }); - check('duplicate semantic IDs rejected', () => { let failed=false; try{assertBusinessModel(normalizeBusinessObject({rules:[{id:'same',statement:'a'},{id:'same',statement:'b'}]}));}catch{failed=true;} if(!failed) throw new Error('duplicate IDs accepted'); }); - check('FACT requires direct evidence', () => { let failed=false; try{assertBusinessModel(normalizeBusinessObject({rules:[{id:'r',statement:'unsupported',classification:'FACT'}]}));}catch{failed=true;} if(!failed) throw new Error('unsupported FACT was accepted'); }); - check('traceability aliases', () => { const page={id:'overview',path:'docs/orientation/overview.md',evidence:[],models:[]}; const x=normalizeTraceabilityObject({facts:[{claim:'Service uses PostgreSQL',status:'fact',sources:['pom.xml'],subject:'service',predicate:'database',object:'postgresql'}]},page); if(x.claims.length!==1||x.claims[0].kind!=='claim'||x.claims[0].classification!=='FACT') throw new Error('traceability normalization'); }); - check('contradiction detection keys', () => { const a={subject:'quote',predicate:'mutable',object:'yes',polarity:'positive',statement:'a',exclusivePredicate:true}, b={subject:'quote',predicate:'mutable',object:'no',polarity:'positive',statement:'b',exclusivePredicate:true}; if(claimSemanticKey(a)!==claimSemanticKey(b)||claimValueKey(a)===claimValueKey(b)) throw new Error('contradiction keys'); }); - check('word count advisory only', () => { const q=qualityConfig(); if(q.wordCountGate==='hard') throw new Error('word count must not be primary hard gate'); }); - const failures = results.filter((x) => x.status === 'failed'); - const report = { - schemaVersion: '1.0', kitVersion, checkedAt: now(), passed: failures.length === 0, - invariants: ['canonicalization', 'idempotence', 'losslessness', 'path-safety', 'identity-consistency', 'transactional-restore', 'typed-items', 'claim-traceability', 'semantic-quality', 'freshness'], - boundaries: ['discover/evidence-index', 'analyze/system-model', 'semantics/business-model', 'semantics/flow-model', 'semantics/catalog-model', 'plan/manifest', 'generate/markdown-path', 'audit/report', 'update/impact-plan', 'generate/traceability-sidecar', 'quality/cross-page-consistency', 'enterprise/security', 'enterprise/operations', 'enterprise/testing', 'enterprise/data-governance', 'enterprise/decisions', 'enterprise/configuration', 'enterprise/change-impact', 'enterprise/ownership', 'source-ignore/gitignore-docgenignore'], - tests: results - }; - writeJson(path.join(root, '.docgen', 'state', 'contract-report.json'), report); - if (failures.length) throw new Error(`Contract self-test failed:\n- ${failures.map((x) => `${x.name}: ${x.error}`).join('\n- ')}`); - console.log('Contract firewall self-test passed.'); - console.log('Report: .docgen/state/contract-report.json'); - return true; -} -function status() { - const state = loadState(); - console.log(`DocGen Kit ${kitVersion}`); - for (const stage of ['discover', 'analyze', 'semantics', 'enterprise', 'plan', 'generate', 'audit']) console.log(`${stage.padEnd(10)} ${state.stages?.[stage]?.status ?? 'pending'}`); - if (fs.existsSync(manifestPath)) { - const m = normalizeManifest(); const generated = (m.pages ?? []).filter(pageIsValid).length; const reusable = (m.pages ?? []).filter(pageIsReusable).length; - console.log(`pages ${generated}/${m.pages?.length ?? 0} generated | ${reusable} reusable for current inputs`); - } - if (fs.existsSync(auditIndexPath)) { - const a = readJson(auditIndexPath); console.log(`audit ${JSON.stringify(a.summary ?? {})}`); - } -} -function runCaptured(bin, args) { - return spawnSync(bin, args, { cwd: root, encoding: 'utf8', shell: process.platform === 'win32' }); -} -function compatibilityReport() { - const report = { - schemaVersion: '1.0', - kitVersion, - checkedAt: now(), - compatible: true, - authenticated: null, - warnings: [], - checks: {} - }; - - report.checks.staticStructure = { ok: validateStatic() }; - if (!report.checks.staticStructure.ok) report.compatible = false; - - const bin = commandCodeBin(); - report.commandCodeBin = bin; - if (!bin) { - report.compatible = false; - report.checks.executable = { ok: false, detail: 'Command Code executable not found.' }; - writeJson(path.join(root, '.docgen', 'state', 'compatibility.json'), report); - return report; - } - report.checks.executable = { ok: true, detail: bin }; - - const version = runCaptured(bin, ['--version']); - const versionText = `${version.stdout ?? ''}${version.stderr ?? ''}`.trim(); - report.commandCodeVersion = versionText || null; - report.checks.version = { ok: version.status === 0, detail: versionText }; - if (version.status !== 0) report.compatible = false; - - const help = runCaptured(bin, ['--help']); - const helpText = `${help.stdout ?? ''}${help.stderr ?? ''}`; - const requiredFlags = ['--trust', '--print', '--max-turns', '--yolo', '--skip-onboarding', '--verbose']; - const missingFlags = requiredFlags.filter((flag) => !helpText.includes(flag)); - report.checks.requiredFlags = { ok: help.status === 0 && missingFlags.length === 0, requiredFlags, missingFlags }; - if (!report.checks.requiredFlags.ok) report.compatible = false; - - const skills = runCaptured(bin, ['skills', 'list', '--debug']); - const skillOutput = `${skills.stdout ?? ''}${skills.stderr ?? ''}`.trim(); - const expectedSkills = fs.readdirSync(path.join(commandCodeHome, 'skills'), { withFileTypes: true }) - .filter((entry) => entry.isDirectory() && (entry.name.startsWith('doc-') || entry.name.startsWith('tech-') || entry.name.startsWith('domain-'))) - .map((entry) => entry.name) - .sort(); - const missingSkills = expectedSkills.filter((name) => !skillOutput.includes(name)); - const reportsSkippedSkills = /(^|\n)\s*Skipped(?:\s*\(|:)/i.test(skillOutput); - const skillsOk = skills.status === 0 && missingSkills.length === 0 && !reportsSkippedSkills; - report.checks.skills = { - ok: skillsOk, exitCode: skills.status, expectedCount: expectedSkills.length, - missingSkills, reportsSkippedSkills, output: skillOutput.slice(0, 12000) - }; - if (!skillsOk) report.compatible = false; - - const auth = runCaptured(bin, ['status', '--json']); - const authText = `${auth.stdout ?? ''}${auth.stderr ?? ''}`.trim(); - let authenticated = auth.status === 0; - if (auth.status === 0 && authText) { - try { - const parsed = JSON.parse(authText.split(/\r?\n/).find((line) => line.trim().startsWith('{')) ?? authText); - if (typeof parsed.authenticated === 'boolean') authenticated = parsed.authenticated; - else if (typeof parsed.loggedIn === 'boolean') authenticated = parsed.loggedIn; - } catch {} - } - report.authenticated = authenticated; - report.checks.authentication = { ok: authenticated, exitCode: auth.status, detail: authText.slice(0, 4000) }; - if (!authenticated) report.warnings.push('Command Code is not authenticated or status could not confirm authentication. Run `cmd login` before generation.'); - - report.effectiveHeadlessArgs = Object.fromEntries( - ['discover', 'analyze', 'semantics', 'enterprise', 'plan', 'generate', 'enrich', 'audit', 'fix', 'update-impact'].map((stage) => [stage, commandCodeArgs(stage)]) - ); - writeJson(path.join(root, '.docgen', 'state', 'compatibility.json'), report); - return report; -} -function printCompatibility(report) { - console.log(`DocGen Kit: ${kitVersion}`); - console.log(`Command Code executable: ${report.commandCodeBin ?? 'NOT FOUND'}`); - console.log(`Command Code version: ${report.commandCodeVersion ?? 'UNKNOWN'}`); - console.log(`Static structure: ${report.checks.staticStructure?.ok ? 'PASS' : 'FAIL'}`); - console.log(`Required CLI flags: ${report.checks.requiredFlags?.ok ? 'PASS' : 'FAIL'}`); - console.log(`Global DocGen skills load: ${report.checks.skills?.ok ? 'PASS' : 'FAIL'}`); - console.log(`Authentication: ${report.authenticated ? 'PASS' : 'NOT READY'}`); - console.log(`Compatibility: ${report.compatible ? 'PASS' : 'FAIL'}`); - if (report.checks.requiredFlags?.missingFlags?.length) console.log(`Missing flags: ${report.checks.requiredFlags.missingFlags.join(', ')}`); - for (const warning of report.warnings ?? []) console.warn(`WARNING: ${warning}`); - console.log('Report: .docgen/state/compatibility.json'); -} -function doctor() { - console.log(`Node.js: ${process.version}`); - contractSelfTest(); - const report = compatibilityReport(); - printCompatibility(report); - if (!report.compatible) process.exit(1); -} -function copyTreeMissing(src, dest, force = false) { - fs.mkdirSync(dest, { recursive: true }); - for (const entry of fs.readdirSync(src, { withFileTypes: true })) { - const from = path.join(src, entry.name); - const to = path.join(dest, entry.name); - if (entry.isDirectory()) copyTreeMissing(from, to, force); - else if (!fs.existsSync(to) || force) { - fs.mkdirSync(path.dirname(to), { recursive: true }); - fs.copyFileSync(from, to); - } - } -} -function initProject(targetArg = '.', force = false) { - const target = path.resolve(targetArg); - if (!fs.existsSync(target) || !fs.statSync(target).isDirectory()) fail(`Init target is not a directory: ${target}`, 2); - const projectTemplate = path.join(engineHome, 'project-template'); - if (!fs.existsSync(projectTemplate)) fail(`Global project template missing: ${projectTemplate}`); - copyTreeMissing(projectTemplate, path.join(target, '.docgen'), force); - const rootTemplate = path.join(engineHome, 'project-root-template'); - // Root-level files such as .docgenignore are user-owned policy and are never overwritten, even by init --force. - if (fs.existsSync(rootTemplate)) copyTreeMissing(rootTemplate, target, false); - fs.mkdirSync(path.join(target, 'docs'), { recursive: true }); - const marker = { - schemaVersion: '1.0', - kitVersion, - initializedAt: now(), - engineScope: 'global', - engineHome: engineHome.replaceAll('\\\\', '/'), - projectRoot: target.replaceAll('\\\\', '/') - }; - writeJson(path.join(target, '.docgen', 'project.json'), marker); - setRoot(target); - const initStatePath = path.join(target, '.docgen', 'state', 'state.json'); - const initState = fs.existsSync(initStatePath) ? readJson(initStatePath) : { schemaVersion: '1.0', kitVersion, stages: {} }; - initState.kitVersion = kitVersion; - initState.updatedAt = now(); - initState.stages ??= {}; - initState.stages.init = { status: 'completed', updatedAt: now(), engineScope: 'global' }; - writeJson(initStatePath, initState); - console.log(`Initialized DocGen project workspace in ${target}`); - console.log('Next:'); - console.log(' docgen doctor'); - console.log(' docgen all'); -} -function globalDoctor() { - const errors = []; - for (const dir of ['agents', 'skills', 'commands']) if (!fs.existsSync(path.join(commandCodeHome, dir))) errors.push(`Missing ${path.join(commandCodeHome, dir)}`); - for (const dir of ['hooks', 'prompts', 'schemas', 'project-template', 'project-root-template', 'bin']) if (!fs.existsSync(path.join(engineHome, dir))) errors.push(`Missing ${path.join(engineHome, dir)}`); - const bin = commandCodeBin(); - console.log(`DocGen Kit: ${kitVersion}`); - console.log(`Engine home: ${engineHome}`); - console.log(`Command Code home: ${commandCodeHome}`); - console.log(`Command Code executable: ${bin ?? 'NOT FOUND'}`); - console.log(`Global structure: ${errors.length ? 'FAIL' : 'PASS'}`); - for (const e of errors) console.error(`- ${e}`); - if (errors.length || !bin) process.exit(1); -} -function ensureInitialized() { - const found = findProjectRoot(process.cwd()); - if (!found) fail('This repository is not initialized for DocGen. Run `docgen init` from the repository root.', 2); - setRoot(found); - migrateProjectConfig(true); -} -function usage() { - console.log(`Command Code DocGen Kit ${kitVersion} - -Global-first usage: - docgen init [repository] initialize repository-local .docgen state - docgen doctor [--global] check global engine and current project - docgen version print version - docgen where print engine/project locations - -Project commands: - docgen status - docgen migrate add new defaults without overwriting custom config - docgen validate - docgen contract-test run zero-token producer/consumer contract regression tests - docgen discover [scope] - docgen analyze [scope] - docgen semantics extract business/flow/catalog models - docgen enterprise extract P1 security/operations/testing/data/decision/config/impact/ownership models - docgen ignore [path] inspect effective .gitignore + .docgenignore source boundary - docgen source-list [filter] list only included repository source files - docgen source-grep [--regex] search only included repository source files - docgen plan - docgen preflight normalize/validate the entire manifest before any page LLM call - docgen generate generate pages; comprehensive profile auto-enriches - docgen enrich run explicit depth/completeness pass - docgen audit - docgen fix - docgen traceability rebuild claim index, contradiction, duplicate, and freshness reports - docgen publish generate frontmatter, llms.txt, search/navigation/backlinks/redirects/examples metadata - docgen quality run evidence-centric semantic + audit quality gates - docgen snapshot - docgen changed - docgen update [path ...] - docgen resume continue from existing artifacts/checkpoints - docgen all [--fresh] resumable by default; --fresh reruns all stages/pages - -System-of-systems commands: - docgen workspace init [dir] - docgen workspace add - docgen workspace list|validate|collect|analyze|generate|quality|status - docgen workspace impact - docgen workspace changed|snapshot|resume|all - -Project-local overrides are optional under .commandcode/** and .docgen/prompts|schemas/**.`); -} - -const [command, ...args] = process.argv.slice(2); -if (command === 'init') { - const force = args.includes('--force'); - const target = args.find((x) => !x.startsWith('--')) ?? '.'; - initProject(target, force); - process.exit(0); -} -if (command === 'workspace') { const { runWorkspace } = await import('./workspace.mjs'); await runWorkspace(args, { kitVersion, engineHome, commandCodeHome }); process.exit(0); } -if (command === 'version' || command === '--version' || command === '-v') { console.log(kitVersion); process.exit(0); } -if (command === 'where') { - console.log(`engineHome=${engineHome}`); - console.log(`commandCodeHome=${commandCodeHome}`); - console.log(`projectRoot=${findProjectRoot(process.cwd()) ?? 'NOT_INITIALIZED'}`); - process.exit(0); -} -if ((command === 'doctor' || command === 'compat') && args.includes('--global')) { globalDoctor(); process.exit(0); } -if (!command) { usage(); process.exit(0); } -ensureInitialized(); -try { -switch (command) { - case 'doctor': doctor(); break; - case 'compat': doctor(); break; - case 'status': status(); break; - case 'migrate': migrateProjectConfig(false); break; - case 'validate': contractSelfTest(); if (!validateStatic() || !validateGenerated()) process.exit(1); break; - case 'contract-test': contractSelfTest(); break; - case 'discover': await doDiscover(args.join(' ') || '.'); break; - case 'analyze': await doAnalyze(args.join(' ') || 'all current evidence'); break; - case 'semantics': await doSemantics(); break; - case 'enterprise': await doEnterprise(); break; - case 'ignore': doIgnore(args[0]); break; - case 'source-list': doSourceList(args.join(' ')); break; - case 'source-grep': doSourceGrep(args); break; - case 'plan': await doPlan(); break; - case 'preflight': { const m = requireManifestPreflight(); console.log(`Manifest preflight PASS: ${m.pages.length} pages. Report: ${rel(preflightPath)}`); break; } - case 'generate': if (args[0] === '--all') await doGenerateAll(args.includes('--force')); else if (args[0]) await doGenerate(args[0], '', true, args.includes('--force')); else fail('generate requires '); break; - case 'enrich': if (args[0] === '--all') await doEnrichAll(); else if (args[0]) await doEnrich(args[0]); else fail('enrich requires '); break; - case 'audit': if (args[0] === '--all') await doAuditAll(); else if (args[0]) { await doAudit(args[0]); rebuildAuditIndex(); writeQualitySummary(); } else fail('audit requires '); break; - case 'fix': if (args[0] === '--all') await doFixAll(); else if (args[0]) await doFix(args[0]); else fail('fix requires '); break; - case 'traceability': doTraceability(); break; - case 'publish': doPublish(); break; - case 'quality': doQuality(); break; - case 'snapshot': doSnapshot(); break; - case 'changed': console.log(changedPaths().join('\n')); break; - case 'update': await doUpdate(args); break; - case 'resume': - case 'all': { - const fresh = args.includes('--fresh'); - console.log(`DocGen full pipeline | quality profile: ${qualityProfile()} | mode: ${fresh ? 'fresh' : 'resume'}`); - const state = loadState(); - const stageComplete = (name, artifact) => !fresh && state.stages?.[name]?.status === 'completed' && (!artifact || fs.existsSync(artifact)) && stageCheckpointValid(name); - let upstreamReran = false; - if (!upstreamReran && stageComplete('discover', evidenceIndexPath)) console.log('[docgen] SKIP phase 1/8 discovery — completed evidence checkpoint exists.'); - else { printItemProgress('phase', 1, 8, 'evidence discovery'); await doDiscover('.', 'phase 1/8'); upstreamReran = true; } - if (!upstreamReran && stageComplete('analyze', systemPath)) console.log('[docgen] SKIP phase 2/8 analysis — completed system model exists.'); - else { printItemProgress('phase', 2, 8, 'technical architecture analysis'); await doAnalyze('all current evidence', 'phase 2/8'); upstreamReran = true; } - if (!upstreamReran && stageComplete('semantics', catalogsPath) && fs.existsSync(businessPath) && fs.existsSync(flowsPath)) console.log('[docgen] SKIP phase 3/8 semantics — completed semantic models exist.'); - else { printItemProgress('phase', 3, 8, 'business, flow, and catalog semantics'); await doSemantics('phase 3/8'); upstreamReran = true; } - const enterpriseFiles = ENTERPRISE_PASSES.flatMap((p)=>p.outputs); - if (loadConfig().enterpriseDepth?.enabled === false) console.log('[docgen] SKIP phase 4/8 enterprise depth — disabled by configuration.'); - else if (!upstreamReran && stageComplete('enterprise', securityPath) && enterpriseFiles.every((f)=>fs.existsSync(f))) console.log('[docgen] SKIP phase 4/8 enterprise depth — completed P1 models exist.'); - else { printItemProgress('phase', 4, 8, 'P1 enterprise depth'); await doEnterprise('phase 4/8'); upstreamReran = true; } - if (!upstreamReran && stageComplete('plan', manifestPath)) { const m = requireManifestPreflight(); console.log(`[docgen] SKIP phase 5/8 planning — valid preflighted manifest exists (${m.pages.length} pages).`); } - else { printItemProgress('phase', 5, 8, 'multi-page documentation planning'); await doPlan('phase 5/8'); upstreamReran = true; } - const manifest = requireManifestPreflight(); console.log(`Plan contains ${manifest.pages.length} pages across ${manifest.navigation?.length ?? 0} navigation categories.`); - printItemProgress('phase', 6, 8, 'batched page generation + targeted enrichment'); await doGenerateAll(fresh); - printItemProgress('phase', 7, 8, 'batched independent audit'); await doAuditAll(); - if (isComprehensive() && qualityConfig().autoFix !== false) { - console.log('Phase 7b/8 — automatic repair only for pages with audit findings'); - const fixed = await doFixAll(); - if (fixed.length && qualityConfig().reAuditAfterFix !== false) { - console.log(`Re-auditing ${fixed.length} repaired page(s)...`); - for (let i = 0; i < fixed.length; i++) { printItemProgress('re-audit', i + 1, fixed.length, fixed[i]); await doAudit(fixed[i], `re-audit ${i + 1}/${fixed.length}`, true); } - rebuildAuditIndex(); - } - } - printItemProgress('phase', 8, 8, 'quality summary + source snapshot'); - writeQualitySummary(); doPublish(); doSnapshot(); doQuality(); - break; - } - default: usage(); process.exit(2); -} -} catch (err) { - const message = err?.message ?? String(err); - console.error(`ERROR: ${message}`); - if (err?.classification) console.error(`Classification: ${err.classification}`); - process.exitCode = Number(err?.exitCode) || 1; -} diff --git a/global-template/docgen/hooks/docgen-guard-read-paths.mjs b/global-template/docgen/hooks/docgen-guard-read-paths.mjs index c12cb85..14a278d 100644 --- a/global-template/docgen/hooks/docgen-guard-read-paths.mjs +++ b/global-template/docgen/hooks/docgen-guard-read-paths.mjs @@ -1,98 +1,57 @@ import fs from 'node:fs'; import path from 'node:path'; -import { spawnSync } from 'node:child_process'; import { active, readStdinJson, resolveWorkspacePath, isWithin, deny } from './docgen-common.mjs'; const payload = await readStdinJson(); if (!active()) process.exit(0); + const cwd = path.resolve(payload.cwd ?? process.env.COMMANDCODE_PROJECT_DIR ?? process.cwd()); const input = payload.tool_input ?? {}; const toolName = String(payload.tool_name ?? payload.tool ?? '').toLowerCase(); +const stage = String(process.env.DOCGEN_STAGE ?? ''); +const contextOnly = ['1', 'true', 'yes', 'on'].includes(String(process.env.DOCGEN_CONTEXT_ONLY ?? '').toLowerCase()); +const norm = (value) => String(value ?? '').replaceAll('\\', '/').replace(/^\.\//, '').replace(/^\/+/, '').replace(/\/+$/, ''); -function norm(value) { return String(value ?? '').replaceAll('\\', '/').replace(/^\.\//, '').replace(/^\/+/, '').replace(/\/+$/, ''); } -function esc(value) { return value.replace(/[|\\{}()[\]^$+?.]/g, '\\$&'); } -function patternRegex(pattern, anchored = false) { - let body=''; - for(let i=0;i{ - let line=raw.trim(); if(!line||line.startsWith('#'))return null; let negated=false; - if(line.startsWith('!')){negated=true;line=line.slice(1);} const directoryOnly=line.endsWith('/'); - if(directoryOnly)line=line.replace(/\/+$/,''); const anchored=line.startsWith('/'); if(anchored)line=line.slice(1); - return {raw,negated,directoryOnly,pattern:line,regex:patternRegex(line,anchored)}; - }).filter(Boolean); -} -function matchRules(rel,isDir,items){let decision=null;for(const rule of items){if(rule.directoryOnly&&!isDir&&!rel.startsWith(`${rule.pattern}/`)&&!rel.includes(`/${rule.pattern}/`))continue;if(rule.regex.test(rel))decision={ignored:!rule.negated,reason:`.docgenignore:${rule.raw}`};}return decision;} -function config(){try{return JSON.parse(fs.readFileSync(path.join(cwd,'.docgen','config','documentation.json'),'utf8'));}catch{return{};}} -const binaryExt=new Set(['.png','.jpg','.jpeg','.gif','.webp','.bmp','.ico','.tif','.tiff','.avif','.heic','.psd','.mp3','.wav','.flac','.aac','.ogg','.m4a','.mp4','.mov','.avi','.mkv','.webm','.wmv','.pdf','.doc','.docx','.xls','.xlsx','.ppt','.pptx','.zip','.gz','.tgz','.bz2','.xz','.7z','.rar','.tar','.jar','.war','.ear','.class','.dll','.exe','.so','.dylib','.o','.a','.lib','.woff','.woff2','.ttf','.otf','.eot','.bin','.dat','.db','.sqlite','.sqlite3','.p12','.pfx','.jks','.keystore','.apk','.ipa','.iso','.dmg','.img','.wasm','.pyc']); -function binaryDecision(target,rel,cfg){ - if(cfg.ignore?.binary?.enabled===false||!fs.existsSync(target)||!fs.statSync(target).isFile())return null; - const ext=path.extname(rel).toLowerCase();const deny=new Set((cfg.ignore?.binary?.denyExtensions??[]).map((x)=>String(x).toLowerCase())); - const allow=new Set([...(cfg.sourceExtensions??[]),...(cfg.ignore?.binary?.allowExtensions??[])].map((x)=>String(x).toLowerCase())); - if(deny.has(ext)||(!allow.has(ext)&&binaryExt.has(ext)))return `binary-extension:${ext||''}`; - const stat=fs.statSync(target);const max=Number(cfg.ignore?.binary?.maxTextFileBytes??4194304);if(stat.size>max)return `text-file-too-large:${stat.size}`; - let buf;try{const fd=fs.openSync(target,'r');buf=Buffer.alloc(Math.min(Number(cfg.ignore?.binary?.probeBytes??16384),stat.size));const n=fs.readSync(fd,buf,0,buf.length,0);fs.closeSync(fd);buf=buf.subarray(0,n);}catch{return'non-text-unreadable';} - const sig=(...xs)=>xs.every((b,i)=>buf[i]===b);if(sig(0x89,0x50,0x4e,0x47)||sig(0xff,0xd8,0xff)||sig(0x25,0x50,0x44,0x46)||sig(0x50,0x4b,0x03,0x04)||sig(0x1f,0x8b)||sig(0x7f,0x45,0x4c,0x46)||sig(0x4d,0x5a)||sig(0x00,0x61,0x73,0x6d))return'binary-magic-signature'; - if(buf.includes(0))return'binary-null-byte'; - try{new TextDecoder('utf-8',{fatal:true}).decode(buf);}catch{return'non-utf8-content';} - return null; -} -function configExcluded(rel,cfg){for(const raw of cfg.exclude??[]){const p=String(raw).replaceAll('\\','/').replace(/^\.\//,'');const cleaned=p.replace(/\/\*\*$/,'').replace(/\/+$/,'');if(patternRegex(cleaned,cleaned.startsWith('/')).test(rel)||rel===cleaned||rel.startsWith(`${cleaned}/`))return `config.exclude:${raw}`;}return null;} -function gitRepositoryAvailable(){ - if(fs.existsSync(path.join(cwd,'.git')))return true; - const r=spawnSync('git',['rev-parse','--is-inside-work-tree'],{cwd,encoding:'utf8',stdio:['ignore','pipe','ignore'],shell:process.platform==='win32'}); - return r.status===0&&String(r.stdout??'').trim()==='true'; +function inventoryFiles() { + const file = path.join(cwd, '.docgen', 'index', 'inventory.json'); + try { return new Set((JSON.parse(fs.readFileSync(file, 'utf8')).files ?? []).map((item) => norm(item.path))); } + catch { return null; } } -function fallbackGitIgnored(rel){ - const seg=norm(rel).split('/');let ignored=false; - for(let depth=0;depthtoolName.includes(x))) { +if (['grep', 'glob', 'read_multiple_files', 'read_directory'].some((name) => toolName.includes(name))) { const explicit = input.path ?? input.directory ?? input.file_path; const pattern = input.pattern ?? input.glob ?? input.query; - if (!explicit || (typeof pattern === 'string' && /[?*\[]/.test(pattern))) { - deny(`DocGen blocks broad ${toolName || 'read/search'} operations because they may include ignored files. Use .docgen/state/source-files.txt, explicit read_file calls, or \`docgen source-grep \`.`); + if (!explicit || typeof pattern === 'string' && /[?*\[]/.test(pattern)) { + deny(`DocGen blocks broad ${toolName || 'read/search'} operations. Read one explicit bounded context or inventory-approved source path.`); process.exit(0); } } -const rawPaths=[]; -for(const key of ['file_path','path','directory']) if(typeof input[key]==='string') rawPaths.push(input[key]); -for(const key of ['paths','files']) if(Array.isArray(input[key])) rawPaths.push(...input[key].filter((x)=>typeof x==='string')); -for(const key of ['pattern','glob']) if(typeof input[key]==='string') { - const value=input[key]; - if(/[?*\[]/.test(value) && !norm(value).startsWith('.docgen/') && !norm(value).startsWith('docs/')) { - deny(`DocGen blocks wildcard source reads because they may bypass .gitignore/.docgenignore. Read .docgen/state/source-files.txt, then read explicit included paths. Blocked pattern: ${value}`); - process.exit(0); +const rawPaths = []; +for (const key of ['file_path', 'path', 'directory']) if (typeof input[key] === 'string') rawPaths.push(input[key]); +for (const key of ['paths', 'files']) if (Array.isArray(input[key])) rawPaths.push(...input[key].filter((value) => typeof value === 'string')); +for (const key of ['pattern', 'glob']) if (typeof input[key] === 'string') rawPaths.push(input[key]); + +const included = inventoryFiles(); +for (const raw of rawPaths) { + if (/[?*\[]/.test(raw)) { deny(`DocGen blocks wildcard reads: ${raw}`); process.exit(0); } + const target = resolveWorkspacePath(raw, cwd); if (!target) continue; + if (!isWithin(target, cwd)) { deny(`DocGen blocks reads outside the repository: ${raw}`); process.exit(0); } + const rel = norm(path.relative(cwd, target)); + if (contextOnly) { + if (!contextAllowed(rel)) { deny(`Context-only ${stage || 'provider'} run cannot read ${raw}. Allowed inputs are precompiled .docgen/context packs and stage outputs only.`); process.exit(0); } + continue; } - rawPaths.push(value); + if (rel === '.docgen' || rel.startsWith('.docgen/') || rel === 'docs' || rel.startsWith('docs/')) continue; + if (!included) { deny('Source inventory is missing. Run `docgen index` before reading repository source.'); process.exit(0); } + if (!included.has(rel)) { deny(`DocGen will not read a path outside the canonical source inventory: ${raw}. Inventory: .docgen/index/source-files.txt`); process.exit(0); } } -for(const raw of rawPaths){const d=decision(raw);if(d?.ignored){deny(`DocGen will not read ignored path: ${raw} (${d.reason}). Active source inventory: .docgen/state/source-files.txt`);process.exit(0);}} diff --git a/global-template/docgen/lib/context.mjs b/global-template/docgen/lib/context.mjs new file mode 100644 index 0000000..7ccd618 --- /dev/null +++ b/global-template/docgen/lib/context.mjs @@ -0,0 +1,69 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import { estimateTokens, loadConfig, now, projectPaths, sha256, stableHash, writeJson } from './core.mjs'; +import { openDatabase } from './indexer.mjs'; + +const STAGE_QUERIES = { + modelCore: 'repository structure architecture component module package symbol interface contract dependency behavior domain rule state lifecycle control flow data flow event automation runtime build deployment', + modelEnterprise: 'security trust identity permission secret operation observability failure recovery testing configuration ownership decision governance consistency concurrency idempotency compatibility change impact', + plan: 'architecture behavior domain interface dependency data security operation testing configuration ownership decision onboarding reference tutorial runbook change impact', + audit: 'fact evidence inference assumption unknown claim contradiction branch failure security consistency compatibility contract' +}; +const CORE_MODELS = ['system', 'business', 'flows', 'catalogs']; + +function ftsQuery(value) { + const tokens = String(value ?? '').toLowerCase().match(/[a-z0-9_.$/@:-]{2,}/g) ?? []; + return [...new Set(tokens)].slice(0, 40).map((token) => `"${token.replaceAll('"', '""')}"`).join(' OR '); +} + +function rowsForFacts(db, query, limit) { + const match = ftsQuery(query); + if (!match) return db.prepare('SELECT id,kind,name,path,line,statement,snippet,metadata,content_hash FROM facts ORDER BY path,line LIMIT ?').all(limit); + try { return db.prepare(`SELECT f.id,f.kind,f.name,f.path,f.line,f.statement,f.snippet,f.metadata,f.content_hash,bm25(facts_fts) score FROM facts_fts JOIN facts f ON f.id=facts_fts.id WHERE facts_fts MATCH ? ORDER BY score LIMIT ?`).all(match, limit); } + catch { return db.prepare('SELECT id,kind,name,path,line,statement,snippet,metadata,content_hash FROM facts WHERE lower(name||\' \'||statement||\' \'||path) LIKE ? LIMIT ?').all(`%${String(query).toLowerCase().split(/\s+/)[0] ?? ''}%`, limit); } +} + +function rowsForModels(db, query, limit, allowedModels) { + if (allowedModels === false) return []; + const allowed = Array.isArray(allowedModels) ? new Set(allowedModels) : null; const fetchLimit = allowed ? Math.max(limit * 4, 1000) : limit; + const match = ftsQuery(query); let rows = []; + if (!match) rows = db.prepare('SELECT id,model,kind,name,statement,classification,confidence,evidence,payload,content_hash FROM model_items LIMIT ?').all(fetchLimit); + else { + try { rows = db.prepare(`SELECT m.id,m.model,m.kind,m.name,m.statement,m.classification,m.confidence,m.evidence,m.payload,m.content_hash,bm25(model_fts) score FROM model_fts JOIN model_items m ON m.id=model_fts.id WHERE model_fts MATCH ? ORDER BY score LIMIT ?`).all(match, fetchLimit); } + catch { rows = []; } + } + if (allowed) rows = rows.filter((row) => allowed.has(row.model)); + return rows.slice(0, limit); +} + +function compactFact(row) { return { id: row.id, kind: row.kind, name: row.name, path: row.path, line: row.line, statement: row.statement, snippet: row.snippet, metadata: JSON.parse(row.metadata || '{}'), hash: row.content_hash }; } +function compactModel(row) { return { id: row.id, model: row.model, kind: row.kind, name: row.name, statement: row.statement, classification: row.classification, confidence: row.confidence, evidence: JSON.parse(row.evidence || '[]'), payload: JSON.parse(row.payload || '{}'), hash: row.content_hash }; } + +function fitBudget(base, facts, models, maxTokens) { + const selectedFacts = []; const selectedModels = []; let used = estimateTokens(JSON.stringify(base)); + const candidates = [...models.map((item) => ({ type: 'model', item, tokens: estimateTokens(JSON.stringify(item)) })), ...facts.map((item) => ({ type: 'fact', item, tokens: estimateTokens(JSON.stringify(item)) }))]; + for (const candidate of candidates) { + if (used + candidate.tokens > maxTokens) continue; + used += candidate.tokens; + if (candidate.type === 'model') selectedModels.push(candidate.item); else selectedFacts.push(candidate.item); + } + return { selectedFacts, selectedModels, estimatedTokens: used }; +} + +export function compileContext(root, { stage, target = '', query = '', maxTokens, factLimit = 800, modelLimit = 500, allowedModels = undefined, metadata = {} }) { + const paths = projectPaths(root); const config = loadConfig(root); const configured = config.context?.maxTokens?.[stage] ?? config.context?.maxTokens?.default ?? 60000; + const modelScope = allowedModels !== undefined ? allowedModels : stage === 'modelCore' ? false : stage === 'modelEnterprise' ? CORE_MODELS : null; + const budget = Math.max(256, Number(maxTokens ?? configured)); const db = openDatabase(paths.database); + const effectiveQuery = [STAGE_QUERIES[stage] ?? '', query, target].filter(Boolean).join(' '); + const facts = rowsForFacts(db, effectiveQuery, factLimit).map(compactFact); const models = rowsForModels(db, effectiveQuery, modelLimit, modelScope).map(compactModel); + const base = { schemaVersion: '2.0', stage, target: target || null, query: effectiveQuery, modelScope: modelScope === false ? [] : modelScope, metadata }; + const fitted = fitBudget(base, facts, models, budget); + const payload = { ...base, generatedAt: now(), tokenBudget: budget, estimatedTokens: fitted.estimatedTokens, facts: fitted.selectedFacts, modelItems: fitted.selectedModels, omissions: { facts: Math.max(0, facts.length - fitted.selectedFacts.length), modelItems: Math.max(0, models.length - fitted.selectedModels.length) } }; + payload.inputHash = stableHash({ stage, target, query: effectiveQuery, modelScope, facts: payload.facts.map((item) => item.hash), models: payload.modelItems.map((item) => item.hash), metadata }); + const id = sha256(`${stage}\0${target}\0${payload.inputHash}`).slice(0, 24); payload.id = id; + const file = path.join(paths.context, stage, `${target ? target.replace(/[^a-z0-9_.-]+/gi, '-') : 'global'}.json`); writeJson(file, payload); + db.prepare('INSERT INTO contexts(id,stage,target,query,input_hash,estimated_tokens,payload,created_at) VALUES(?,?,?,?,?,?,?,?) ON CONFLICT(id) DO UPDATE SET input_hash=excluded.input_hash,estimated_tokens=excluded.estimated_tokens,payload=excluded.payload,created_at=excluded.created_at').run(id, stage, target || null, effectiveQuery, payload.inputHash, payload.estimatedTokens, JSON.stringify(payload), now()); + db.close(); return { file, payload }; +} + +export function loadContext(file) { return JSON.parse(fs.readFileSync(file, 'utf8')); } diff --git a/global-template/docgen/lib/core.mjs b/global-template/docgen/lib/core.mjs new file mode 100644 index 0000000..c5275ca --- /dev/null +++ b/global-template/docgen/lib/core.mjs @@ -0,0 +1,194 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import crypto from 'node:crypto'; +import { spawn, spawnSync } from 'node:child_process'; +import { fileURLToPath } from 'node:url'; + +export const engineHome = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..'); +export const kitVersion = fs.readFileSync(path.join(engineHome, 'VERSION'), 'utf8').trim(); + +export function findProjectRoot(start = process.cwd()) { + let current = path.resolve(start); + while (true) { + if (fs.existsSync(path.join(current, '.docgen', 'project.json'))) return current; + const parent = path.dirname(current); + if (parent === current) return null; + current = parent; + } +} + +export function requireProjectRoot(start = process.cwd()) { + const root = findProjectRoot(start); + if (!root) throw new Error('No DocGen project found. Run `docgen init` from the repository root.'); + return root; +} + +export function projectPaths(root) { + const base = path.join(root, '.docgen'); + return { + root, + base, + config: path.join(base, 'config', 'documentation.json'), + project: path.join(base, 'project.json'), + state: path.join(base, 'state', 'state.json'), + inventory: path.join(base, 'index', 'inventory.json'), + database: path.join(base, 'index', 'semantic.db'), + context: path.join(base, 'context'), + telemetry: path.join(base, 'telemetry'), + budget: path.join(base, 'budget', 'report.json'), + model: path.join(base, 'model'), + plan: path.join(base, 'plan', 'manifest.json'), + docs: path.join(root, 'docs'), + audit: path.join(base, 'audit'), + publish: path.join(base, 'publish'), + traceability: path.join(base, 'traceability'), + runs: path.join(base, 'runs') + }; +} + +export function ensureDir(dir) { fs.mkdirSync(dir, { recursive: true }); } +export function now() { return new Date().toISOString(); } +export function sleep(ms) { return new Promise((resolve) => setTimeout(resolve, ms)); } +export function posix(value) { return String(value).replaceAll('\\', '/'); } +export function rel(root, file) { return posix(path.relative(root, file)); } +export function readJson(file, fallback = undefined) { + try { return JSON.parse(fs.readFileSync(file, 'utf8')); } + catch (error) { if (fallback !== undefined) return fallback; throw error; } +} +export function writeJson(file, value) { ensureDir(path.dirname(file)); fs.writeFileSync(file, JSON.stringify(value, null, 2) + '\n'); } +export function sha256(value) { return crypto.createHash('sha256').update(Buffer.isBuffer(value) ? value : String(value)).digest('hex'); } +export function fileSha256(file) { return fs.existsSync(file) ? sha256(fs.readFileSync(file)) : null; } +export function stableJson(value) { + if (Array.isArray(value)) return value.map(stableJson); + if (value && typeof value === 'object') return Object.fromEntries(Object.keys(value).sort().map((key) => [key, stableJson(value[key])])); + return value; +} +export function stableHash(value) { return sha256(JSON.stringify(stableJson(value))); } +export function estimateTokens(value) { return Math.ceil(Buffer.byteLength(String(value), 'utf8') / 3.6); } +export function slug(value) { return String(value ?? '').trim().toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/^-|-$/g, '') || 'item'; } + +function directCommand(command) { + if (!command) return null; + const value = String(command); + if (path.isAbsolute(value) || value.includes('/') || value.includes('\\')) { + const full = path.resolve(value); + return fs.existsSync(full) ? full : null; + } + return null; +} + +export function resolveCommand(command) { + const direct = directCommand(command); + if (direct) return direct; + const locator = process.platform === 'win32' + ? path.join(process.env.SystemRoot || 'C:\\Windows', 'System32', 'where.exe') + : 'which'; + const result = spawnSync(locator, [String(command)], { + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'ignore'], + shell: false, + windowsHide: true + }); + if (result.status !== 0) return null; + const candidates = String(result.stdout ?? '').split(/\r?\n/).map((line) => line.trim()).filter(Boolean); + if (!candidates.length) return null; + if (process.platform !== 'win32') return candidates[0]; + return candidates.find((item) => /\.(?:exe|com)$/i.test(item)) + ?? candidates.find((item) => /\.(?:cmd|bat)$/i.test(item)) + ?? candidates.find((item) => fs.existsSync(item)) + ?? null; +} + +export function commandExists(command) { return Boolean(resolveCommand(command)); } + +function powershellHost() { + return resolveCommand('pwsh.exe') ?? resolveCommand('powershell.exe'); +} + +export function spawnCommand(command, args = [], options = {}) { + const resolved = resolveCommand(command) ?? String(command); + const baseOptions = { ...options, shell: false }; + if (process.platform !== 'win32' || !/\.(?:cmd|bat)$/i.test(resolved)) { + return spawn(resolved, args.map(String), baseOptions); + } + const host = powershellHost(); + if (!host) throw new Error(`Cannot safely launch Windows command shim: ${resolved}. PowerShell was not found.`); + const encodedArgs = Buffer.from(JSON.stringify(args.map(String)), 'utf8').toString('base64'); + const env = { + ...(options.env ?? process.env), + DOCGEN_CHILD_EXECUTABLE: resolved, + DOCGEN_CHILD_ARGS_B64: encodedArgs + }; + const script = [ + "$ErrorActionPreference='Stop'", + '$exe=$env:DOCGEN_CHILD_EXECUTABLE', + '$json=[Text.Encoding]::UTF8.GetString([Convert]::FromBase64String($env:DOCGEN_CHILD_ARGS_B64))', + '$childArgs=@(ConvertFrom-Json -InputObject $json)', + '& $exe @childArgs', + 'if ($null -eq $LASTEXITCODE) { exit 0 } else { exit $LASTEXITCODE }' + ].join('; '); + return spawn(host, ['-NoLogo', '-NoProfile', '-NonInteractive', '-ExecutionPolicy', 'Bypass', '-Command', script], { + ...baseOptions, + env + }); +} + +export function terminateProcessTree(child) { + if (!child?.pid) return; + if (process.platform === 'win32') { + const taskkill = resolveCommand('taskkill.exe') ?? 'taskkill.exe'; + spawnSync(taskkill, ['/PID', String(child.pid), '/T', '/F'], { + stdio: 'ignore', + shell: false, + windowsHide: true + }); + return; + } + try { child.kill('SIGTERM'); } catch {} + const timer = setTimeout(() => { try { child.kill('SIGKILL'); } catch {} }, 5_000); + timer.unref?.(); +} + +export function formatDuration(milliseconds) { + const seconds = Math.max(0, Math.floor(Number(milliseconds) / 1000)); + const hours = Math.floor(seconds / 3600); + const minutes = Math.floor((seconds % 3600) / 60); + const remaining = seconds % 60; + return hours ? `${hours}h ${String(minutes).padStart(2, '0')}m ${String(remaining).padStart(2, '0')}s` + : `${minutes}m ${String(remaining).padStart(2, '0')}s`; +} + +export function git(root, args, fallback = null) { + const executable = resolveCommand('git') ?? 'git'; + const result = spawnSync(executable, ['-C', root, ...args], { encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore'], shell: false }); + return result.status === 0 ? result.stdout.trim() : fallback; +} +export function sourceSnapshot(root) { + return { + commit: git(root, ['rev-parse', 'HEAD']), + branch: git(root, ['branch', '--show-current']), + dirty: Boolean(git(root, ['status', '--porcelain'], '')), + capturedAt: now() + }; +} +export function loadConfig(root) { return readJson(projectPaths(root).config, {}); } +export function updateStage(root, stage, status, details = {}) { + const paths = projectPaths(root); + const state = readJson(paths.state, { schemaVersion: '2.0', kitVersion, stages: {} }); + state.schemaVersion = '2.0'; state.kitVersion = kitVersion; state.updatedAt = now(); state.stages ??= {}; + state.stages[stage] = { status, updatedAt: now(), ...details }; + writeJson(paths.state, state); +} +export function appendJsonl(file, value) { ensureDir(path.dirname(file)); fs.appendFileSync(file, JSON.stringify(value) + '\n'); } +export function parseArgs(argv) { + const positional = []; const options = {}; + for (let i = 0; i < argv.length; i++) { + const arg = argv[i]; + if (!arg.startsWith('--')) { positional.push(arg); continue; } + const [rawKey, inline] = arg.slice(2).split('=', 2); const key = rawKey.replace(/-([a-z])/g, (_, c) => c.toUpperCase()); + if (inline !== undefined) options[key] = inline; + else if (argv[i + 1] && !argv[i + 1].startsWith('--')) options[key] = argv[++i]; + else options[key] = true; + } + return { positional, options }; +} diff --git a/global-template/docgen/lib/indexer.mjs b/global-template/docgen/lib/indexer.mjs new file mode 100644 index 0000000..2843fda --- /dev/null +++ b/global-template/docgen/lib/indexer.mjs @@ -0,0 +1,168 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import { DatabaseSync } from 'node:sqlite'; +import { buildInventory } from './inventory.mjs'; +import { ensureDir, estimateTokens, now, projectPaths, readJson, sha256, stableHash } from './core.mjs'; + +function lineNumber(text, offset) { return text.slice(0, offset).split('\n').length; } +function snippet(text, start, radius = 4) { + const lines = text.split(/\r?\n/); const from = Math.max(0, start - radius - 1); const to = Math.min(lines.length, start + radius); + return lines.slice(from, to).map((line, i) => `${from + i + 1}: ${line}`).join('\n'); +} +function addMatch(out, text, rel, kind, regex, nameGroup = 1, metadata = {}) { + for (const match of text.matchAll(regex)) { + const line = lineNumber(text, match.index ?? 0); const name = String(match[nameGroup] ?? match[0]).trim(); + out.push({ id: sha256(`${kind}\0${rel}\0${line}\0${name}`).slice(0, 24), kind, name, path: rel, line, statement: match[0].trim(), snippet: snippet(text, line), metadata }); + } +} +function sourceChunks(rel, text, size = 80, overlap = 15) { + const lines = text.split(/\r?\n/); const chunks = []; const step = Math.max(1, size - overlap); + for (let start = 0; start < lines.length; start += step) { + const end = Math.min(lines.length, start + size); const body = lines.slice(start, end).join('\n').trim(); if (!body) continue; + const first = lines.slice(start, end).find((line) => line.trim())?.trim().slice(0, 160) || rel; + chunks.push({ id: sha256(`source-chunk\0${rel}\0${start + 1}\0${end}`).slice(0, 24), kind: 'source-chunk', name: `${rel}#L${start + 1}-L${end}`, path: rel, line: start + 1, statement: first, snippet: lines.slice(start, end).map((line, i) => `${start + i + 1}: ${line}`).join('\n'), metadata: { startLine: start + 1, endLine: end } }); + if (end === lines.length) break; + } + return chunks; +} +function extractFacts(rel, text) { + const ext = path.extname(rel).toLowerCase(); + const facts = [{ + id: sha256(`file-artifact\0${rel}`).slice(0, 24), + kind: 'file-artifact', + name: rel, + path: rel, + line: 1, + statement: `Indexed source artifact: ${rel}`, + snippet: snippet(text, 1), + metadata: { extension: ext || '', basename: path.basename(rel) } + }, ...sourceChunks(rel, text)]; + + // Cross-language structural signals. These are hints only; model synthesis must + // confirm semantics from bounded source chunks and must never assume a framework. + addMatch(facts, text, rel, 'module-reference', /^\s*(?:import|export\s+.+?\s+from|from|require\s*\(|use|include|include_once|require_once|using|open)\s*[('"{<]*([^'";)>\s}]+)/gmi); + addMatch(facts, text, rel, 'symbol', /\b(?:class|interface|enum|record|object|struct|trait|type|protocol|actor|module|namespace|package)\s+([A-Za-z_$][A-Za-z0-9_.$]*)/g); + addMatch(facts, text, rel, 'function', /\b(?:function|def|func|fn|sub|procedure|proc)\s+([A-Za-z_$][A-Za-z0-9_$]*)\s*\(/g); + addMatch(facts, text, rel, 'configuration-key', /^\s*["']?([A-Za-z0-9_.-]+)["']?\s*[=:]\s*.+$/gm); + addMatch(facts, text, rel, 'url-literal', /["']((?:https?:\/\/|\/)[A-Za-z0-9_~!$&'()*+,;=:@%\/.?#[\]-]+)["']/g); + + // Common interface and runtime conventions are retained as optional evidence, + // but they are not required for repositories using other ecosystems. + addMatch(facts, text, rel, 'url-path', /@Path\s*\(\s*["']([^"']+)["']\s*\)/g); + addMatch(facts, text, rel, 'http-method', /@(GET|POST|PUT|PATCH|DELETE|HEAD|OPTIONS)\b/g); + addMatch(facts, text, rel, 'mapping-annotation', /@(GetMapping|PostMapping|PutMapping|PatchMapping|DeleteMapping|RequestMapping)\s*\(([^)]*)\)/g, 1); + addMatch(facts, text, rel, 'route-declaration', /\b(?:app|router|server|route)\s*\.\s*(?:get|post|put|patch|delete|head|options|route)\s*\(\s*["']([^"']+)["']/gi); + addMatch(facts, text, rel, 'route-declaration', /@(?:app|router)\.(?:get|post|put|patch|delete|route)\s*\(\s*["']([^"']+)["']/gi); + addMatch(facts, text, rel, 'message-channel', /(?:topic|topics|queue|exchange|routingKey|channel|stream|subject)\s*[=:]\s*["']([^"']+)["']/gi); + addMatch(facts, text, rel, 'data-entity', /\b(?:from|join|into|update|table|collection|bucket|index)\s+([A-Za-z_][A-Za-z0-9_.$-]*)/gi); + addMatch(facts, text, rel, 'scheduled-automation', /@(Scheduled|Cron|PeriodicTask)\b/g); + addMatch(facts, text, rel, 'security-boundary', /@(RolesAllowed|PermitAll|DenyAll|PreAuthorize|Secured|Authorize|RequireRole)\b/g); + + // Dependency manifests across major ecosystems. + if (/pom\.xml$/i.test(rel)) addMatch(facts, text, rel, 'dependency', /([^<]+)<\/artifactId>/g); + if (/\.csproj$|packages\.config$/i.test(rel)) addMatch(facts, text, rel, 'dependency', /]+Include=["']([^"']+)["']/gi); + if (/package\.json$/i.test(rel)) addMatch(facts, text, rel, 'dependency', /["']([^"']+)["']\s*:\s*["'][~^*<>=\s]*\d/g); + if (/go\.mod$/i.test(rel)) addMatch(facts, text, rel, 'dependency', /^\s*(?:require\s+)?([A-Za-z0-9_.-]+\.[A-Za-z0-9_./-]+)\s+v\d/gm); + if (/Cargo\.toml$/i.test(rel)) addMatch(facts, text, rel, 'dependency', /^\s*([A-Za-z0-9_-]+)\s*=\s*(?:["']|\{)/gm); + if (/(?:requirements[^/]*\.txt|pyproject\.toml|Pipfile)$/i.test(rel)) addMatch(facts, text, rel, 'dependency', /^\s*["']?([A-Za-z0-9_.-]+)(?:\[[^\]]+\])?\s*(?:[<>=~!]|["'])/gm); + if (/(?:build\.gradle(?:\.kts)?|settings\.gradle(?:\.kts)?)$/i.test(rel)) addMatch(facts, text, rel, 'dependency', /\b(?:implementation|api|compileOnly|runtimeOnly|testImplementation)\s*\(?\s*["']([^"']+)["']/g); + if (/Gemfile$/i.test(rel)) addMatch(facts, text, rel, 'dependency', /^\s*gem\s+["']([^"']+)["']/gm); + if (/composer\.json$/i.test(rel)) addMatch(facts, text, rel, 'dependency', /["']([A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+)["']\s*:/g); + if (/mix\.exs$/i.test(rel)) addMatch(facts, text, rel, 'dependency', /\{:\s*([A-Za-z0-9_]+)\s*,/g); + if (/Dockerfile$/i.test(rel)) addMatch(facts, text, rel, 'runtime-base', /^FROM\s+([^\s]+)/gmi); + if (/\.tf$/i.test(rel)) addMatch(facts, text, rel, 'infrastructure-resource', /^\s*(?:resource|data|module|provider)\s+["']([^"']+)["']/gm); + return facts; +} + +function openDatabase(file) { + ensureDir(path.dirname(file)); const db = new DatabaseSync(file); + db.exec(`PRAGMA journal_mode=WAL; PRAGMA synchronous=NORMAL; + CREATE TABLE IF NOT EXISTS metadata(key TEXT PRIMARY KEY,value TEXT NOT NULL); + CREATE TABLE IF NOT EXISTS files(path TEXT PRIMARY KEY,hash TEXT NOT NULL,size INTEGER NOT NULL,extension TEXT,indexed_at TEXT NOT NULL); + CREATE TABLE IF NOT EXISTS facts(id TEXT PRIMARY KEY,kind TEXT NOT NULL,name TEXT NOT NULL,path TEXT NOT NULL,line INTEGER,statement TEXT,snippet TEXT,metadata TEXT,content_hash TEXT NOT NULL); + CREATE INDEX IF NOT EXISTS idx_facts_kind ON facts(kind); + CREATE INDEX IF NOT EXISTS idx_facts_path ON facts(path); + CREATE VIRTUAL TABLE IF NOT EXISTS facts_fts USING fts5(id UNINDEXED,kind,name,path,statement,snippet,tokenize='unicode61 remove_diacritics 2'); + CREATE TABLE IF NOT EXISTS model_items(id TEXT PRIMARY KEY,semantic_id TEXT NOT NULL,model TEXT NOT NULL,kind TEXT,name TEXT,statement TEXT,classification TEXT,confidence REAL,evidence TEXT,payload TEXT,content_hash TEXT NOT NULL); + CREATE INDEX IF NOT EXISTS idx_model_semantic_id ON model_items(semantic_id); + CREATE VIRTUAL TABLE IF NOT EXISTS model_fts USING fts5(id UNINDEXED,model,kind,name,statement,tokenize='unicode61 remove_diacritics 2'); + CREATE TABLE IF NOT EXISTS contexts(id TEXT PRIMARY KEY,stage TEXT NOT NULL,target TEXT,query TEXT,input_hash TEXT NOT NULL,estimated_tokens INTEGER NOT NULL,payload TEXT NOT NULL,created_at TEXT NOT NULL); + `); + return db; +} + +function replaceFacts(db, rel, facts) { + const ids = db.prepare('SELECT id FROM facts WHERE path=?').all(rel).map((x) => x.id); + const deleteFact = db.prepare('DELETE FROM facts WHERE id=?'); const deleteFts = db.prepare('DELETE FROM facts_fts WHERE id=?'); + for (const id of ids) { deleteFact.run(id); deleteFts.run(id); } + const insert = db.prepare('INSERT INTO facts(id,kind,name,path,line,statement,snippet,metadata,content_hash) VALUES(?,?,?,?,?,?,?,?,?)'); + const insertFts = db.prepare('INSERT INTO facts_fts(id,kind,name,path,statement,snippet) VALUES(?,?,?,?,?,?)'); + for (const fact of facts) { + const hash = stableHash(fact); insert.run(fact.id, fact.kind, fact.name, fact.path, fact.line, fact.statement, fact.snippet, JSON.stringify(fact.metadata ?? {}), hash); + insertFts.run(fact.id, fact.kind, fact.name, fact.path, fact.statement, fact.snippet); + } +} + +export function indexRepository(root, { force = false } = {}) { + const paths = projectPaths(root); const inventory = buildInventory(root); const db = openDatabase(paths.database); + const known = new Map(db.prepare('SELECT path,hash FROM files').all().map((row) => [row.path, row.hash])); const current = new Set(inventory.files.map((x) => x.path)); + const progress = process.env.DOCGEN_PROGRESS !== '0'; const started = Date.now(); let lastReport = started; + let changed = 0; let unchanged = 0; let factCount = 0; + if (progress) console.log(`[docgen] index RUNNING | ${inventory.files.length.toLocaleString()} included files | force=${force}`); + db.exec('BEGIN'); + try { + for (let index = 0; index < inventory.files.length; index++) { + const file = inventory.files[index]; + if (!force && known.get(file.path) === file.hash) unchanged++; + else { + const text = fs.readFileSync(path.join(root, file.path), 'utf8'); const facts = extractFacts(file.path, text); factCount += facts.length; + replaceFacts(db, file.path, facts); + db.prepare('INSERT INTO files(path,hash,size,extension,indexed_at) VALUES(?,?,?,?,?) ON CONFLICT(path) DO UPDATE SET hash=excluded.hash,size=excluded.size,extension=excluded.extension,indexed_at=excluded.indexed_at').run(file.path, file.hash, file.size, file.extension, now()); + changed++; + } + const nowMs = Date.now(); + if (progress && nowMs - lastReport >= 5_000) { lastReport = nowMs; console.log(`[docgen] index RUNNING | ${index + 1}/${inventory.files.length} | changed ${changed} | unchanged ${unchanged} | extracted facts ${factCount.toLocaleString()}`); } + } + for (const rel of known.keys()) if (!current.has(rel)) { replaceFacts(db, rel, []); db.prepare('DELETE FROM files WHERE path=?').run(rel); changed++; } + db.prepare("INSERT INTO metadata(key,value) VALUES('inventory_fingerprint',?) ON CONFLICT(key) DO UPDATE SET value=excluded.value").run(inventory.fingerprint); + db.prepare("INSERT INTO metadata(key,value) VALUES('indexed_at',?) ON CONFLICT(key) DO UPDATE SET value=excluded.value").run(now()); + db.exec('COMMIT'); + } catch (error) { db.exec('ROLLBACK'); db.close(); throw error; } + const totals = { files: db.prepare('SELECT COUNT(*) n FROM files').get().n, facts: db.prepare('SELECT COUNT(*) n FROM facts').get().n }; + db.close(); + if (progress) console.log(`[docgen] index COMPLETED | changed ${changed} | unchanged ${unchanged} | facts ${totals.facts.toLocaleString()} | elapsed ${Math.max(0, Math.round((Date.now() - started) / 1000))}s`); + return { schemaVersion: '2.0', indexedAt: now(), inventoryFingerprint: inventory.fingerprint, changedFiles: changed, unchangedFiles: unchanged, extractedFacts: factCount, totals }; +} + +function walkItems(value, model, out, parent = '') { + if (Array.isArray(value)) { for (const item of value) walkItems(item, model, out, parent); return; } + if (!value || typeof value !== 'object') return; + if (value.id || value.name || value.statement) { + const semanticId = String(value.id ?? sha256(`${model}\0${parent}\0${JSON.stringify(value)}`).slice(0, 24)); const id = `${model}:${semanticId}`; + out.push({ id, semanticId, model, kind: String(value.kind ?? parent ?? 'item'), name: String(value.name ?? value.title ?? semanticId), statement: String(value.statement ?? value.summary ?? value.description ?? ''), classification: String(value.classification ?? 'UNKNOWN'), confidence: Number(value.confidence ?? 0), evidence: value.evidence ?? [], payload: value }); + } + for (const [key, child] of Object.entries(value)) if (!['evidence','sourceModelRefs'].includes(key)) walkItems(child, model, out, key); +} + +export function ingestModels(root) { + const paths = projectPaths(root); const db = openDatabase(paths.database); const files = fs.existsSync(paths.model) ? fs.readdirSync(paths.model).filter((x) => x.endsWith('.json') && !x.endsWith('-bundle.json')) : []; + const insert = db.prepare('INSERT INTO model_items(id,semantic_id,model,kind,name,statement,classification,confidence,evidence,payload,content_hash) VALUES(?,?,?,?,?,?,?,?,?,?,?) ON CONFLICT(id) DO UPDATE SET semantic_id=excluded.semantic_id,model=excluded.model,kind=excluded.kind,name=excluded.name,statement=excluded.statement,classification=excluded.classification,confidence=excluded.confidence,evidence=excluded.evidence,payload=excluded.payload,content_hash=excluded.content_hash'); + const insertFts = db.prepare('INSERT INTO model_fts(id,model,kind,name,statement) VALUES(?,?,?,?,?)'); + db.exec('BEGIN'); let count = 0; + try { + db.exec('DELETE FROM model_items; DELETE FROM model_fts;'); + for (const name of files) { + const model = path.basename(name, '.json'); const items = []; walkItems(readJson(path.join(paths.model, name), {}), model, items); + for (const item of items) { insert.run(item.id, item.semanticId, item.model, item.kind, item.name, item.statement, item.classification, item.confidence, JSON.stringify(item.evidence), JSON.stringify(item.payload), stableHash(item.payload)); insertFts.run(item.id, item.model, item.kind, item.name, item.statement); count++; } + } + db.exec('COMMIT'); + } catch (error) { db.exec('ROLLBACK'); db.close(); throw error; } + db.close(); return { models: files.length, items: count }; +} + +export function databaseStats(root) { + const db = openDatabase(projectPaths(root).database); const stats = { files: db.prepare('SELECT COUNT(*) n FROM files').get().n, facts: db.prepare('SELECT COUNT(*) n FROM facts').get().n, sourceChunks: db.prepare("SELECT COUNT(*) n FROM facts WHERE kind='source-chunk'").get().n, modelItems: db.prepare('SELECT COUNT(*) n FROM model_items').get().n, estimatedIndexedTokens: 0 }; + const samples = db.prepare('SELECT statement,snippet FROM facts').all(); stats.estimatedIndexedTokens = samples.reduce((n, row) => n + estimateTokens(`${row.statement}\n${row.snippet}`), 0); db.close(); return stats; +} + +export { openDatabase }; diff --git a/global-template/docgen/lib/inventory.mjs b/global-template/docgen/lib/inventory.mjs new file mode 100644 index 0000000..030d10b --- /dev/null +++ b/global-template/docgen/lib/inventory.mjs @@ -0,0 +1,113 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import { spawnSync } from 'node:child_process'; +import { ensureDir, git, loadConfig, now, posix, projectPaths, sha256, writeJson } from './core.mjs'; + +const HARD_DIRS = new Set(['.git', '.docgen', '.commandcode', 'docs', 'node_modules', 'target', 'build', 'dist', 'coverage', 'vendor']); +const BINARY_EXTENSIONS = new Set(['.png','.jpg','.jpeg','.gif','.webp','.bmp','.ico','.tif','.tiff','.avif','.heic','.psd','.mp3','.wav','.flac','.aac','.ogg','.mp4','.mov','.avi','.mkv','.webm','.pdf','.doc','.docx','.xls','.xlsx','.ppt','.pptx','.zip','.gz','.tgz','.bz2','.xz','.7z','.rar','.tar','.jar','.war','.ear','.class','.dll','.exe','.so','.dylib','.o','.a','.lib','.woff','.woff2','.ttf','.otf','.eot','.bin','.dat','.db','.sqlite','.sqlite3','.p12','.pfx','.jks','.keystore','.apk','.ipa','.iso','.dmg','.img','.wasm','.pyc','.pyo']); +const ruleCache = new Map(); + +function globRegex(pattern) { + let source = posix(pattern.trim()).replace(/^\.\//, ''); + const anchored = source.startsWith('/'); if (anchored) source = source.slice(1); + const directory = source.endsWith('/'); if (directory) source += '**'; + let out = ''; + for (let i = 0; i < source.length; i++) { + const c = source[i]; + if (c === '*' && source[i + 1] === '*') { out += '.*'; i++; } + else if (c === '*') out += '[^/]*'; + else if (c === '?') out += '[^/]'; + else out += c.replace(/[|\\{}()[\]^$+?.]/g, '\\$&'); + } + return new RegExp(anchored ? `^${out}(?:/.*)?$` : `(?:^|/)${out}(?:/.*)?$`); +} + +function loadRules(file) { + if (ruleCache.has(file)) return ruleCache.get(file); + const result = !fs.existsSync(file) ? [] : fs.readFileSync(file, 'utf8').split(/\r?\n/).map((line) => line.trim()).filter((line) => line && !line.startsWith('#')).map((line) => ({ negate: line.startsWith('!'), regex: globRegex(line.replace(/^!/, '')), raw: line })); + ruleCache.set(file, result); return result; +} + +function ignoredByRules(rel, rules) { + let ignored = false; + for (const rule of rules) if (rule.regex.test(rel)) ignored = !rule.negate; + return ignored; +} + +function ignoredByFallbackGitignore(root, rel) { + const parts = rel.split('/'); let ignored = false; + for (let depth = 0; depth < parts.length; depth++) { + const base = parts.slice(0, depth).join('/'); const local = parts.slice(depth).join('/'); + for (const rule of loadRules(path.join(root, base, '.gitignore'))) if (rule.regex.test(local)) ignored = !rule.negate; + } + return ignored; +} + +function walk(root) { + const out = []; const stack = [root]; + while (stack.length) { + const dir = stack.pop(); + for (const entry of fs.readdirSync(dir, { withFileTypes: true })) { + const full = path.join(dir, entry.name); const rel = posix(path.relative(root, full)); + if (entry.isDirectory()) { if (!HARD_DIRS.has(entry.name)) stack.push(full); } + else out.push(rel); + } + } + return out.sort(); +} + +function candidatePaths(root) { + const insideGit = git(root, ['rev-parse', '--is-inside-work-tree']) === 'true'; + if (!insideGit) return { paths: walk(root), insideGit: false }; + const result = spawnSync('git', ['-C', root, 'ls-files', '-co', '--exclude-standard', '-z'], { encoding: 'buffer', stdio: ['ignore', 'pipe', 'ignore'] }); + if (result.status !== 0) return { paths: walk(root), insideGit: false }; + return { paths: result.stdout.toString('utf8').split('\0').filter(Boolean).map(posix).sort(), insideGit: true }; +} + +function binaryMagic(buffer) { + const sig = (...bytes) => bytes.every((byte, index) => buffer[index] === byte); + return sig(0x89,0x50,0x4e,0x47) || sig(0xff,0xd8,0xff) || sig(0x47,0x49,0x46,0x38) || sig(0x25,0x50,0x44,0x46) || sig(0x50,0x4b,0x03,0x04) || sig(0x1f,0x8b) || sig(0x7f,0x45,0x4c,0x46) || sig(0x4d,0x5a) || sig(0x00,0x61,0x73,0x6d) || sig(0xca,0xfe,0xba,0xbe); +} + +export function classifyFile(root, rel, config = loadConfig(root)) { + const full = path.join(root, rel); let stat; + try { stat = fs.statSync(full); } catch { return { included: false, reason: 'unreadable' }; } + if (!stat.isFile()) return { included: false, reason: 'not-file' }; + const binary = config.ignore?.binary ?? {}; const maxBytes = Number(binary.maxTextFileBytes ?? 4 * 1024 * 1024); + const ext = path.extname(rel).toLowerCase(); + if (BINARY_EXTENSIONS.has(ext) || (binary.denyExtensions ?? []).map((x) => String(x).toLowerCase()).includes(ext)) return { included: false, reason: `binary-extension:${ext}`, size: stat.size }; + if (stat.size > maxBytes) return { included: false, reason: 'oversized-text', size: stat.size }; + const probeBytes = Math.max(512, Number(binary.probeBytes ?? 16384)); const fd = fs.openSync(full, 'r'); const buffer = Buffer.alloc(Math.min(stat.size, probeBytes)); + try { fs.readSync(fd, buffer, 0, buffer.length, 0); } finally { fs.closeSync(fd); } + if (binaryMagic(buffer)) return { included: false, reason: 'binary-magic', size: stat.size }; + if (buffer.includes(0)) return { included: false, reason: 'null-byte', size: stat.size }; + const text = buffer.toString('utf8'); if (text.includes('\uFFFD')) return { included: false, reason: 'invalid-utf8', size: stat.size }; + const controls = [...buffer].filter((b) => b < 32 && ![9,10,13].includes(b)).length; + if (buffer.length && controls / buffer.length > Number(binary.controlCharacterRatio ?? 0.08)) return { included: false, reason: 'control-characters', size: stat.size }; + return { included: true, reason: null, size: stat.size, extension: ext || '' }; +} + +export function buildInventory(root) { + ruleCache.clear(); + const paths = projectPaths(root); const config = loadConfig(root); const candidates = candidatePaths(root); const docgenRules = loadRules(path.join(root, '.docgenignore')); const included = []; const excluded = []; + const progress = config.execution?.progress !== false && process.env.DOCGEN_PROGRESS !== '0'; const started = Date.now(); let lastReport = started; + if (progress) console.log(`[docgen] inventory RUNNING | ${candidates.paths.length.toLocaleString()} candidate files | ${candidates.insideGit ? 'git-aware' : 'filesystem fallback'}`); + for (let index = 0; index < candidates.paths.length; index++) { + const rel = candidates.paths[index]; const top = rel.split('/')[0]; + if (HARD_DIRS.has(top)) excluded.push({ path: rel, reason: 'hard-exclusion' }); + else if (!candidates.insideGit && config.ignore?.useGitignore !== false && ignoredByFallbackGitignore(root, rel)) excluded.push({ path: rel, reason: '.gitignore' }); + else if (config.ignore?.useDocgenignore !== false && ignoredByRules(rel, docgenRules)) excluded.push({ path: rel, reason: '.docgenignore' }); + else { + const result = classifyFile(root, rel, config); + if (!result.included) excluded.push({ path: rel, reason: result.reason, size: result.size ?? 0 }); + else { const content = fs.readFileSync(path.join(root, rel)); included.push({ path: rel, size: result.size, extension: result.extension, hash: sha256(content) }); } + } + const nowMs = Date.now(); + if (progress && nowMs - lastReport >= 5_000) { lastReport = nowMs; console.log(`[docgen] inventory RUNNING | ${index + 1}/${candidates.paths.length} | included ${included.length} | excluded ${excluded.length}`); } + } + const inventory = { schemaVersion: '2.0', generatedAt: now(), files: included, excluded, metrics: { includedFiles: included.length, includedBytes: included.reduce((n, x) => n + x.size, 0), excludedFiles: excluded.length }, fingerprint: sha256(included.map((x) => `${x.path}\0${x.hash}`).join('\n')) }; + ensureDir(path.dirname(paths.inventory)); writeJson(paths.inventory, inventory); + fs.writeFileSync(path.join(path.dirname(paths.inventory), 'source-files.txt'), included.map((x) => x.path).join('\n') + (included.length ? '\n' : '')); + if (progress) console.log(`[docgen] inventory COMPLETED | included ${included.length.toLocaleString()} | excluded ${excluded.length.toLocaleString()} | elapsed ${Math.max(0, Math.round((Date.now() - started) / 1000))}s`); + return inventory; +} diff --git a/global-template/docgen/lib/pipeline.mjs b/global-template/docgen/lib/pipeline.mjs new file mode 100644 index 0000000..296fb8b --- /dev/null +++ b/global-template/docgen/lib/pipeline.mjs @@ -0,0 +1,301 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { compileContext } from './context.mjs'; +import { budgetReport, runProvider } from './provider.mjs'; +import { databaseStats, indexRepository, ingestModels } from './indexer.mjs'; +import { auditRepository } from './quality.mjs'; +import { ensureDir, fileSha256, kitVersion, loadConfig, now, projectPaths, readJson, rel, sha256, slug, sourceSnapshot, stableHash, updateStage, writeJson } from './core.mjs'; + +const moduleDir = path.dirname(fileURLToPath(import.meta.url)); + +function prompt(root, name, vars = {}) { + const override = path.join(root, '.docgen', 'prompts', name); const fallback = path.resolve(moduleDir, '..', 'prompts', name); + let text = fs.readFileSync(fs.existsSync(override) ? override : fallback, 'utf8'); + for (const [key, value] of Object.entries(vars)) text = text.replaceAll(`{{${key}}}`, String(value)); + return text; +} +function state(root) { return readJson(projectPaths(root).state, { schemaVersion: '2.0', kitVersion, stages: {}, pages: {} }); } +function writeState(root, next) { writeJson(projectPaths(root).state, { ...next, schemaVersion: '2.0', kitVersion, updatedAt: now() }); } +function stageCurrent(root, stage, inputHash, outputs = []) { const current = state(root).stages?.[stage]; return current?.status === 'completed' && current.inputHash === inputHash && outputs.every((file) => fs.existsSync(file)); } +function completeStage(root, stage, inputHash, details = {}) { const next = state(root); next.stages ??= {}; next.stages[stage] = { status: 'completed', completedAt: now(), inputHash, ...details }; writeState(root, next); } +function failStage(root, stage, error, details = {}) { const next = state(root); next.stages ??= {}; next.stages[stage] = { ...(next.stages[stage] ?? {}), status: 'failed', failedAt: now(), error: error.message, ...details }; writeState(root, next); } +function pageState(root, id) { return state(root).pages?.[id] ?? {}; } +function updatePage(root, id, patch) { const next = state(root); next.pages ??= {}; next.pages[id] = { ...(next.pages[id] ?? {}), ...patch, updatedAt: now() }; writeState(root, next); } +function modelPath(root, name) { return path.join(projectPaths(root).model, `${name}.json`); } +function tracePath(root, page) { return path.join(projectPaths(root).traceability, 'pages', `${page.id}.json`); } +function validateJson(file) { const value = readJson(file); if (!value || typeof value !== 'object' || Array.isArray(value)) throw new Error(`Invalid JSON object: ${file}`); return value; } +function artifactStamp(file) { + if (!fs.existsSync(file)) return null; + const stat = fs.statSync(file, { bigint: true }); + return { hash: fileSha256(file), mtimeNs: String(stat.mtimeNs) }; +} +function artifactChanged(file, before) { + const after = artifactStamp(file); if (!after) return false; if (!before) return true; + return after.hash !== before.hash || after.mtimeNs !== before.mtimeNs; +} +function phase(name, position, total) { console.log(`[docgen] phase ${position}/${total} ${name.toUpperCase()}`); } + +export function index(root, options = {}) { + updateStage(root, 'index', 'running'); + try { const result = indexRepository(root, options); completeStage(root, 'index', result.inventoryFingerprint, result); return result; } + catch (error) { failStage(root, 'index', error); throw error; } +} + +function splitBundle(root, bundle, names) { + ensureDir(projectPaths(root).model); + for (const name of names) { + const value = bundle[name] ?? bundle[name.replaceAll('-', '')] ?? bundle[name.replace(/-([a-z])/g, (_, c) => c.toUpperCase())]; + if (!value || typeof value !== 'object' || Array.isArray(value)) throw new Error(`Model bundle is missing object: ${name}`); + writeJson(modelPath(root, name), { schemaVersion: '2.0', generatedAt: now(), ...value }); + } +} + +async function synthesizeBundle(root, stage, names, query) { + const paths = projectPaths(root); const context = compileContext(root, { stage, query, target: stage, metadata: { expectedModels: names } }); + const output = path.join(paths.model, `${stage}-bundle.json`); const inputHash = context.payload.inputHash; const outputs = names.map((name) => modelPath(root, name)); + if (stageCurrent(root, stage, inputHash, outputs)) return { skipped: true, inputHash }; + ensureDir(paths.model); updateStage(root, stage, 'running', { inputHash, contextId: context.payload.id }); + const body = prompt(root, stage === 'modelCore' ? 'model-core.md' : 'model-enterprise.md', { CONTEXT_PATH: rel(root, context.file), OUTPUT_PATH: rel(root, output), MODEL_NAMES: JSON.stringify(names) }); + const outputBefore = artifactStamp(output); let bundleWritten = false; const acceptBundle = () => { + try { if (!artifactChanged(output, outputBefore)) return false; const bundle = validateJson(output); splitBundle(root, bundle, names); bundleWritten = true; return outputs.every((file) => fs.existsSync(file)); } + catch { return false; } + }; + try { + const provider = await runProvider(root, { stage, target: stage, prompt: body, acceptArtifacts: acceptBundle }); + if (!bundleWritten) { if (!artifactChanged(output, outputBefore)) throw new Error(`${stage}: provider completed without writing a fresh bundle artifact`); const bundle = validateJson(output); splitBundle(root, bundle, names); } + fs.rmSync(output, { force: true }); completeStage(root, stage, inputHash, { models: names, contextId: context.payload.id, contextTokens: context.payload.estimatedTokens, recovered: provider.recovered === true }); + return { skipped: false, recovered: provider.recovered === true, inputHash }; + } catch (error) { failStage(root, stage, error, { inputHash, contextId: context.payload.id }); throw error; } +} + +export async function model(root, { skipIndex = false } = {}) { + if (!skipIndex) index(root); + await synthesizeBundle(root, 'modelCore', ['system', 'business', 'flows', 'catalogs'], 'repository structure architecture components modules symbols interfaces contracts dependencies behavior domain rules states flows data and automation; detect technologies from evidence and do not assume a language or framework'); + ingestModels(root); + await synthesizeBundle(root, 'modelEnterprise', ['security', 'operations', 'testing', 'data-governance', 'decisions', 'configuration', 'change-impact', 'ownership'], 'security operations testing governance configuration ownership decisions change impact reliability consistency and compatibility across any language framework runtime or deployment model'); + return ingestModels(root); +} + +function canonicalPagePath(page, category, id) { + let value = String(page.path ?? '').replaceAll('\\', '/').replace(/^\/+/, ''); + if (!value) value = `docs/${category}/${slug(page.slug ?? page.title ?? id)}.md`; + if (!value.startsWith('docs/')) value = `docs/${value}`; + if (!value.endsWith('.md')) value += '.md'; + return value; +} +function normalizePage(page, indexValue) { + const id = slug(page.id ?? page.title ?? `page-${indexValue + 1}`); const category = slug(page.category ?? page.type ?? 'overview'); + return { id, title: String(page.title ?? id.replaceAll('-', ' ')), summary: String(page.summary ?? page.purpose ?? ''), category, path: canonicalPagePath(page, category, id), mode: String(page.mode ?? 'explanation'), type: String(page.type ?? 'explanation'), order: Number(page.order ?? indexValue + 1), audience: Array.isArray(page.audience) ? page.audience : ['engineer'], coverageTags: Array.isArray(page.coverageTags) ? page.coverageTags.map(String) : [], query: String(page.query ?? [page.title, page.summary, ...(page.coverageTags ?? [])].join(' ')), requiredSections: Array.isArray(page.requiredSections) ? page.requiredSections.map(String) : [], risk: String(page.risk ?? 'normal'), relatedPages: Array.isArray(page.relatedPages) ? page.relatedPages.map(String) : [] }; +} +function validateManifest(root) { + const paths = projectPaths(root); const manifest = readJson(paths.plan); if (!Array.isArray(manifest.pages) || !manifest.pages.length) throw new Error('Manifest must contain non-empty pages[].'); + manifest.pages = manifest.pages.map(normalizePage); const ids = new Set(); const files = new Set(); const max = Number(loadConfig(root).execution?.maxPlannedPages ?? 30); + if (max > 0 && manifest.pages.length > max) throw new Error(`Manifest contains ${manifest.pages.length} pages, above execution.maxPlannedPages=${max}. Consolidate duplicate user intents or raise the limit explicitly.`); + for (const page of manifest.pages) { + if (ids.has(page.id)) throw new Error(`Duplicate page id: ${page.id}`); if (files.has(page.path)) throw new Error(`Duplicate page path: ${page.path}`); + if (!page.title.trim() || !page.summary.trim()) throw new Error(`Page ${page.id} requires non-empty title and summary.`); + ids.add(page.id); files.add(page.path); + } + manifest.schemaVersion = '2.0'; manifest.generatedAt ??= now(); writeJson(paths.plan, manifest); return manifest; +} + +export async function plan(root) { + const paths = projectPaths(root); ingestModels(root); + const context = compileContext(root, { stage: 'plan', query: 'documentation information architecture onboarding reference explanation tutorial operations decisions behavior interfaces dependencies configuration and change; select only concerns evidenced by this repository regardless of technology stack', target: 'manifest', maxTokens: loadConfig(root).context?.maxTokens?.plan ?? 50000 }); + const inputHash = context.payload.inputHash; if (stageCurrent(root, 'plan', inputHash, [paths.plan])) return validateManifest(root); + updateStage(root, 'plan', 'running', { inputHash, contextId: context.payload.id }); + const planBefore = artifactStamp(paths.plan); let validManifest = null; const acceptManifest = () => { try { if (!artifactChanged(paths.plan, planBefore)) return false; validManifest = validateManifest(root); return true; } catch { return false; } }; + try { + const provider = await runProvider(root, { stage: 'plan', target: 'manifest', prompt: prompt(root, 'plan-indexed.md', { CONTEXT_PATH: rel(root, context.file), OUTPUT_PATH: rel(root, paths.plan) }), acceptArtifacts: acceptManifest }); + if (!validManifest && !artifactChanged(paths.plan, planBefore)) throw new Error('plan: provider completed without writing a fresh manifest artifact'); + const manifest = validManifest ?? validateManifest(root); completeStage(root, 'plan', inputHash, { pages: manifest.pages.length, contextId: context.payload.id, recovered: provider.recovered === true }); return manifest; + } catch (error) { failStage(root, 'plan', error, { inputHash, contextId: context.payload.id }); throw error; } +} + +function frontmatter(page) { return ['---', `title: ${JSON.stringify(page.title)}`, `description: ${JSON.stringify(page.summary)}`, `pageId: ${JSON.stringify(page.id)}`, `category: ${JSON.stringify(page.category)}`, `mode: ${JSON.stringify(page.mode)}`, `type: ${JSON.stringify(page.type)}`, `order: ${page.order}`, '---', ''].join('\n'); } +function markdownTable(headers, rows) { return `| ${headers.join(' | ')} |\n| ${headers.map(() => '---').join(' | ')} |\n${rows.map((row) => `| ${row.map((value) => String(value ?? '').replaceAll('|', '\\|').replace(/\r?\n/g, ' ')).join(' | ')} |`).join('\n')}\n`; } +function deterministicKind(page) { + const tags = new Set(page.coverageTags.map((tag) => tag.toLowerCase())); + const mapping = [ + ['endpoint-catalog', 'endpoints'], ['message-handler-catalog', 'messages'], ['external-dependency-catalog', 'dependencies'], ['dependency-catalog', 'dependencies'], + ['data-store-catalog', 'data-assets'], ['data-asset-catalog', 'data-assets'], ['scheduled-job-catalog', 'automations'], ['automation-catalog', 'automations'], + ['interface-catalog', 'interfaces'], ['component-catalog', 'components'], ['configuration-matrix', 'configuration'], ['ownership-responsibilities', 'ownership'], ['change-impact', 'change-impact'] + ]; + return mapping.find(([tag]) => tags.has(tag))?.[1] ?? null; +} +function evidenceText(item) { return (item.evidence ?? []).map((e) => [e.path, e.startLine ? `L${e.startLine}${e.endLine ? `-L${e.endLine}` : ''}` : ''].filter(Boolean).join('#')).join(', '); } +function arraysFor(document, keys) { const items = []; for (const key of keys) if (Array.isArray(document[key])) items.push(...document[key]); return items; } +function referenceData(root, kind) { + const catalog = readJson(modelPath(root, 'catalogs'), {}); const system = readJson(modelPath(root, 'system'), {}); + let model = 'catalogs'; let modelFiles = [modelPath(root, 'catalogs')]; let items = []; let headers = ['Name', 'Kind', 'Statement', 'Evidence']; let row; + if (kind === 'endpoints') { items = arraysFor(catalog, ['endpoints']); headers = ['Endpoint', 'Method', 'Path', 'Statement', 'Evidence']; row = (item) => [item.name ?? item.id, item.method ?? item.httpMethod ?? '', item.path ?? item.route ?? '', item.statement ?? item.summary ?? '', evidenceText(item)]; } + else if (kind === 'messages') { items = arraysFor(catalog, ['messageHandlers']); headers = ['Handler', 'Direction', 'Channel', 'Statement', 'Evidence']; row = (item) => [item.name ?? item.id, item.direction ?? item.role ?? '', item.topic ?? item.queue ?? item.channel ?? '', item.statement ?? item.summary ?? '', evidenceText(item)]; } + else if (kind === 'interfaces') items = arraysFor(catalog, ['interfaces', 'endpoints', 'messageHandlers', 'contracts']); + else if (kind === 'dependencies') items = arraysFor(catalog, ['dependencies', 'externalDependencies']); + else if (kind === 'data-assets') items = arraysFor(catalog, ['dataAssets', 'dataStores']); + else if (kind === 'automations') items = arraysFor(catalog, ['automations', 'scheduledJobs']); + else if (kind === 'components') { model = 'system'; modelFiles = [modelPath(root, 'system')]; items = arraysFor(system, ['components', 'modules', 'services', 'packages']); } + else { + model = kind; modelFiles = [modelPath(root, model)]; const document = readJson(modelPath(root, model), {}); items = Object.values(document).filter(Array.isArray).flat(); + } + const seen = new Set(); items = items.filter((item) => { const id = String(item?.id ?? stableHash(item)); if (seen.has(id)) return false; seen.add(id); return true; }); + row ??= (item) => [item.name ?? item.title ?? item.id, item.kind ?? '', item.statement ?? item.summary ?? item.description ?? '', evidenceText(item)]; + return { model, modelFiles, items, headers, rows: items.map(row) }; +} +function writeTrace(root, page, claims, inputHash, contextId = null) { + const pageFile = path.join(root, page.path); const trace = { schemaVersion: '2.0', pageId: page.id, pagePath: page.path, pageHash: sha256(fs.readFileSync(pageFile)), inputHash, contextId, generatedAt: now(), claims }; + writeJson(tracePath(root, page), trace); return trace; +} +function renderDeterministic(root, page, kind, inputHash) { + const data = referenceData(root, kind); const text = `${frontmatter(page)}# ${page.title}\n\n${page.summary}\n\n${markdownTable(data.headers, data.rows)}\n`; const file = path.join(root, page.path); ensureDir(path.dirname(file)); fs.writeFileSync(file, text); + const claims = data.items.map((item, indexValue) => ({ id: `${page.id}:${item.id ?? indexValue + 1}`, section: page.title, statement: item.statement ?? item.summary ?? item.description ?? item.name ?? item.id, classification: String(item.classification ?? (item.evidence?.length ? 'FACT' : 'UNKNOWN')).toUpperCase(), confidence: Number(item.confidence ?? (item.evidence?.length ? 1 : 0)), evidence: item.evidence ?? [], sourceModelRefs: [`${data.model}:${item.id ?? indexValue + 1}`] })); + writeTrace(root, page, claims, inputHash); return { items: data.items.length, hash: sha256(text) }; +} +function pageInputHash(page, contextHash) { return stableHash({ page, contextHash }); } +function finalizeProviderTrace(root, page, inputHash, contextId) { + const file = tracePath(root, page); const trace = validateJson(file); if (!Array.isArray(trace.claims)) throw new Error(`${rel(root, file)} must contain claims[].`); + const ids = new Set(); + for (const claim of trace.claims) { + claim.id = String(claim.id ?? `${page.id}:claim-${ids.size + 1}`); if (ids.has(claim.id)) throw new Error(`${page.id}: duplicate claim id ${claim.id}`); ids.add(claim.id); + claim.statement = String(claim.statement ?? '').trim(); claim.classification = String(claim.classification ?? 'UNKNOWN').toUpperCase(); claim.confidence = Number(claim.confidence ?? (claim.classification === 'FACT' ? 1 : 0)); claim.evidence = Array.isArray(claim.evidence) ? claim.evidence : []; claim.sourceModelRefs = Array.isArray(claim.sourceModelRefs) ? claim.sourceModelRefs : []; + } + trace.schemaVersion = '2.0'; trace.pageId = page.id; trace.pagePath = page.path; trace.pageHash = sha256(fs.readFileSync(path.join(root, page.path))); trace.inputHash = inputHash; trace.contextId = contextId; trace.generatedAt = now(); writeJson(file, trace); return trace; +} +function validatePage(root, page, inputHash = null) { + const file = path.join(root, page.path); if (!fs.existsSync(file)) throw new Error(`Missing page: ${page.path}`); const text = fs.readFileSync(file, 'utf8'); + if (!/^---\s*\n[\s\S]*?\n---\s*\n/.test(text)) throw new Error(`${page.path}: missing frontmatter`); if (!/^#\s+\S/m.test(text)) throw new Error(`${page.path}: missing H1`); if (/```(?:plantuml|dot|graphviz|puml)/i.test(text)) throw new Error(`${page.path}: non-Mermaid diagram`); + const traceFile = tracePath(root, page); if (!fs.existsSync(traceFile)) throw new Error(`${page.path}: missing traceability sidecar`); const trace = validateJson(traceFile); if (!Array.isArray(trace.claims)) throw new Error(`${rel(root, traceFile)}: missing claims[]`); if (trace.pageId !== page.id || trace.pagePath !== page.path) throw new Error(`${rel(root, traceFile)}: page identity mismatch`); if (trace.pageHash && trace.pageHash !== sha256(text)) throw new Error(`${rel(root, traceFile)}: stale page hash`); if (inputHash && trace.inputHash !== inputHash) throw new Error(`${rel(root, traceFile)}: stale input hash`); + return { file, text, trace, hash: sha256(text) }; +} + +function checkpointGeneratedItems(root, batch, recovered = false, baseline = new Map()) { + const completed = []; const missing = []; + for (const item of batch) { + try { + const pageFile = path.join(root, item.page.path); const traceFile = tracePath(root, item.page); + if (!fs.existsSync(pageFile) || !fs.existsSync(traceFile)) throw new Error('expected output is missing'); + const before = baseline.get(item.page.id) ?? {}; const existingTrace = readJson(traceFile, {}); + const alreadyCurrent = existingTrace.inputHash === item.inputHash && existingTrace.contextId === item.context.payload.id; + if (!alreadyCurrent && !artifactChanged(pageFile, before.page) && !artifactChanged(traceFile, before.trace)) throw new Error('provider did not write a fresh artifact for the current input'); + finalizeProviderTrace(root, item.page, item.inputHash, item.context.payload.id); const checked = validatePage(root, item.page, item.inputHash); + updatePage(root, item.page.id, { status: 'completed', renderer: 'provider', recovered, inputHash: item.inputHash, pageHash: checked.hash, contextId: item.context.payload.id, completedAt: now(), error: null }); completed.push(item); + } catch (error) { missing.push({ ...item, checkpointError: error.message }); } + } + return { completed, missing }; +} + +async function generateBatch(root, batch, config, recoveryRound = 0) { + if (!batch.length) return { completed: 0, recovered: 0 }; + const maxRecoveryRounds = Math.max(1, Number(config.execution?.generationRecoveryAttempts ?? 3)); const batchId = sha256(batch.map((item) => `${item.page.id}:${item.inputHash}`).join('|')).slice(0, 12); + const baseline = new Map(batch.map((item) => [item.page.id, { page: artifactStamp(path.join(root, item.page.path)), trace: artifactStamp(tracePath(root, item.page)) }])); + for (const item of batch) updatePage(root, item.page.id, { status: 'running', renderer: 'provider', inputHash: item.inputHash, contextId: item.context.payload.id, batchId, recoveryRound, startedAt: now(), error: null }); + const contract = batch.map(({ page, context, inputHash }) => ({ page, contextPath: rel(root, context.file), outputPath: page.path, traceabilityPath: rel(root, tracePath(root, page)), inputHash, contextId: context.payload.id })); + let checkpoint = { completed: [], missing: batch }; let provider; + try { + provider = await runProvider(root, { + stage: 'generate', target: batch.map((item) => item.page.id).join(','), prompt: prompt(root, 'write-pages-indexed.md', { PAGE_CONTRACTS: JSON.stringify(contract, null, 2) }), + acceptArtifacts: () => { checkpoint = checkpointGeneratedItems(root, batch, true, baseline); return checkpoint.completed.length > 0; } + }); + checkpoint = checkpointGeneratedItems(root, batch, provider.recovered === true, baseline); + } catch (error) { + checkpoint = checkpointGeneratedItems(root, batch, true, baseline); + for (const item of checkpoint.missing) updatePage(root, item.page.id, { status: 'failed', failedAt: now(), error: item.checkpointError || error.message }); + if (!checkpoint.completed.length) throw error; + } + if (checkpoint.missing.length) { + if (recoveryRound + 1 >= maxRecoveryRounds) { + const error = new Error(`Generation batch ${batchId} produced ${checkpoint.completed.length}/${batch.length} valid page(s); recovery limit ${maxRecoveryRounds} reached for: ${checkpoint.missing.map((item) => item.page.id).join(', ')}`); + for (const item of checkpoint.missing) updatePage(root, item.page.id, { status: 'failed', failedAt: now(), error: error.message }); throw error; + } + console.warn(`[docgen] generate RECOVERY | checkpointed ${checkpoint.completed.length}/${batch.length}; retrying only ${checkpoint.missing.length} missing/invalid page(s).`); + const next = await generateBatch(root, checkpoint.missing, config, recoveryRound + 1); + return { completed: checkpoint.completed.length + next.completed, recovered: checkpoint.completed.length + next.recovered }; + } + return { completed: checkpoint.completed.length, recovered: provider?.recovered === true ? checkpoint.completed.length : 0 }; +} + +export async function generate(root) { + const manifest = validateManifest(root); const config = loadConfig(root); const pending = []; let deterministicPages = 0; let reusedPages = 0; + updateStage(root, 'generate', 'running', { pages: manifest.pages.length }); + try { + for (const page of manifest.pages) { + const deterministic = deterministicKind(page); + if (deterministic) { + const data = referenceData(root, deterministic); const inputHash = stableHash({ page, models: data.modelFiles.map(fileSha256) }); + try { validatePage(root, page, inputHash); updatePage(root, page.id, { status: 'completed', renderer: 'deterministic', inputHash, pageHash: fileSha256(path.join(root, page.path)), recovered: pageState(root, page.id).status !== 'completed' }); reusedPages++; } + catch { const result = renderDeterministic(root, page, deterministic, inputHash); updatePage(root, page.id, { status: 'completed', renderer: 'deterministic', inputHash, pageHash: result.hash, completedAt: now(), error: null }); } + deterministicPages++; continue; + } + const context = compileContext(root, { stage: 'generate', target: page.id, query: page.query, maxTokens: config.context?.maxTokens?.generate ?? 30000, metadata: { page } }); const inputHash = pageInputHash(page, context.payload.inputHash); + try { + const checked = validatePage(root, page, inputHash); updatePage(root, page.id, { status: 'completed', renderer: 'provider', inputHash, pageHash: checked.hash, contextId: context.payload.id, recovered: pageState(root, page.id).status !== 'completed', error: null }); reusedPages++; continue; + } catch {} + pending.push({ page, context, inputHash }); + } + const size = Math.max(1, Number(config.execution?.generationBatchSize ?? 4)); let recoveredPages = 0; + for (let i = 0; i < pending.length; i += size) { const result = await generateBatch(root, pending.slice(i, i + size), config); recoveredPages += result.recovered; } + const inputHash = stableHash(manifest.pages.map((page) => [page.id, pageState(root, page.id).inputHash])); + completeStage(root, 'generate', inputHash, { pages: manifest.pages.length, providerPages: pending.length, deterministicPages, reusedPages, recoveredPages }); + return { pages: manifest.pages.length, providerPages: pending.length, deterministicPages, reusedPages, recoveredPages }; + } catch (error) { failStage(root, 'generate', error, { pages: manifest.pages.length }); throw error; } +} + +export async function audit(root) { + const paths = projectPaths(root); const manifest = validateManifest(root); ensureDir(paths.audit); updateStage(root, 'audit', 'running'); + try { + const quality = auditRepository(root, manifest); writeJson(path.join(paths.audit, 'deterministic.json'), quality); + const config = loadConfig(root); const threshold = Number(config.audit?.llmRiskThreshold ?? 50); const risky = quality.pages.filter((report) => report.riskScore >= threshold && !report.errors.length); const llmOutput = path.join(paths.audit, 'llm-risk.json'); let highRiskFindings = 0; + if (risky.length && config.audit?.llmEnabled !== false) { + const contexts = risky.map((report) => { const page = manifest.pages.find((item) => item.id === report.pageId); const context = compileContext(root, { stage: 'audit', target: page.id, query: page.query, maxTokens: config.context?.maxTokens?.audit ?? 18000, metadata: { page, deterministicReport: report } }); return { page, contextPath: rel(root, context.file), contextId: context.payload.id, pagePath: page.path, report }; }); + const riskInputHash = stableHash(contexts.map((item) => ({ pageId: item.page.id, pageHash: item.report.pageHash, inputHash: item.report.inputHash, contextId: item.contextId }))); + if (!stageCurrent(root, 'auditRisk', riskInputHash, [llmOutput])) { + const llmBefore = artifactStamp(llmOutput); let valid = null; const accept = () => { try { if (!artifactChanged(llmOutput, llmBefore)) return false; const result = validateJson(llmOutput); if (!Array.isArray(result.pages)) return false; valid = result; return true; } catch { return false; } }; + const provider = await runProvider(root, { stage: 'audit', target: risky.map((report) => report.pageId).join(','), prompt: prompt(root, 'audit-risk-indexed.md', { AUDIT_CONTRACTS: JSON.stringify(contexts, null, 2), OUTPUT_PATH: rel(root, llmOutput) }), acceptArtifacts: accept }); + if (!valid && !artifactChanged(llmOutput, llmBefore)) throw new Error('audit: provider completed without writing a fresh risk report artifact'); + const result = valid ?? validateJson(llmOutput); if (!Array.isArray(result.pages)) throw new Error('Risk audit report must contain pages[].'); completeStage(root, 'auditRisk', riskInputHash, { pages: risky.length, recovered: provider.recovered === true }); + } + const llm = readJson(llmOutput, { pages: [] }); highRiskFindings = (llm.pages ?? []).flatMap((page) => page.findings ?? []).filter((finding) => ['critical', 'high'].includes(String(finding.severity).toLowerCase())).length; + } + const pass = quality.pass && highRiskFindings === 0; + const summary = { schemaVersion: '2.0', generatedAt: now(), auditInputHash: quality.auditInputHash, inventoryFingerprint: quality.inventoryFingerprint, manifestHash: quality.manifestHash, pages: quality.metrics.pages, claims: quality.metrics.claims, evidenceReferences: quality.metrics.evidenceReferences, modelItems: quality.metrics.modelItems, referencedModelItems: quality.metrics.referencedModelItems, modelReferenceCoverage: quality.metrics.modelReferenceCoverage, deterministicFailures: quality.errors.length, deterministicWarnings: quality.warnings.length, llmAuditedPages: risky.length, highRiskFindings, pass }; + writeJson(path.join(paths.audit, 'quality-summary.json'), summary); + if (!pass) throw new Error(`Quality failed: deterministicFailures=${quality.errors.length}, highRiskFindings=${highRiskFindings}. See .docgen/audit/deterministic.json.`); + completeStage(root, 'audit', quality.auditInputHash, summary); return summary; + } catch (error) { failStage(root, 'audit', error); throw error; } +} + +export function publish(root) { + const paths = projectPaths(root); const manifest = validateManifest(root); const summary = readJson(path.join(paths.audit, 'quality-summary.json'), null); const auditState = state(root).stages?.audit; + if (!summary?.pass || auditState?.status !== 'completed' || auditState.inputHash !== summary.auditInputHash) throw new Error('Publish requires a current passing audit. Run `docgen audit` or `docgen resume`.'); + const inventory = readJson(paths.inventory, {}); if (summary.inventoryFingerprint !== inventory.fingerprint || summary.manifestHash !== stableHash(manifest.pages)) throw new Error('Audit is stale relative to the current inventory or manifest. Run `docgen audit`.'); + const currentQuality = auditRepository(root, manifest); + if (!currentQuality.pass || currentQuality.auditInputHash !== summary.auditInputHash) throw new Error('Audit is stale relative to current source, model, page, or traceability artifacts. Run `docgen resume`.'); + ensureDir(paths.publish); const navigation = {}; const search = []; const traces = []; + for (const page of manifest.pages) { + const checked = validatePage(root, page, pageState(root, page.id).inputHash); (navigation[page.category] ??= []).push({ id: page.id, title: page.title, path: page.path, order: page.order, summary: page.summary }); + search.push({ id: page.id, title: page.title, path: page.path, category: page.category, summary: page.summary, keywords: page.coverageTags, excerpt: checked.text.replace(/```[\s\S]*?```/g, ' ').replace(/[#>*_`\[\]()]/g, ' ').replace(/\s+/g, ' ').trim().slice(0, 400) }); + traces.push({ pageId: page.id, pagePath: page.path, pageHash: checked.trace.pageHash, inputHash: checked.trace.inputHash, claims: checked.trace.claims.length }); + } + for (const pages of Object.values(navigation)) pages.sort((a, b) => a.order - b.order); + writeJson(path.join(paths.publish, 'navigation.json'), { schemaVersion: '2.0', generatedAt: now(), navigation }); writeJson(path.join(paths.publish, 'search-index.json'), { schemaVersion: '2.0', generatedAt: now(), pages: search }); writeJson(path.join(paths.traceability, 'index.json'), { schemaVersion: '2.0', generatedAt: now(), pages: traces }); + const lines = [`# ${loadConfig(root).projectName || path.basename(root)}`, '']; for (const [category, pages] of Object.entries(navigation)) { lines.push(`## ${category}`, ''); for (const page of pages) lines.push(`- [${page.title}](${page.path.replace(/^docs\//, '')}) — ${page.summary}`); lines.push(''); } + ensureDir(paths.docs); fs.writeFileSync(path.join(paths.docs, 'llms.txt'), lines.join('\n').trimEnd() + '\n'); const full = manifest.pages.map((page) => fs.readFileSync(path.join(root, page.path), 'utf8')).join('\n\n---\n\n'); fs.writeFileSync(path.join(paths.docs, 'llms-full.txt'), full); + completeStage(root, 'publish', stableHash({ audit: summary.auditInputHash, pages: traces.map((trace) => trace.pageHash) }), { pages: search.length, categories: Object.keys(navigation).length }); + return { pages: search.length, categories: Object.keys(navigation).length, traceabilityPages: traces.length }; +} + +export function status(root) { + const current = state(root); const pages = Object.values(current.pages ?? {}); const stageOrder = ['index', 'modelCore', 'modelEnterprise', 'plan', 'generate', 'audit', 'publish']; const nextStage = stageOrder.find((name) => current.stages?.[name]?.status !== 'completed') ?? null; + return { state: current, summary: { nextStage, completedPages: pages.filter((page) => page.status === 'completed').length, failedPages: pages.filter((page) => page.status === 'failed').length, runningPages: pages.filter((page) => page.status === 'running').length }, budget: budgetReport(root), index: fs.existsSync(projectPaths(root).database) ? databaseStats(root) : null }; +} + +export async function all(root) { + phase('index', 1, 6); index(root); + phase('model', 2, 6); await model(root, { skipIndex: true }); + phase('plan', 3, 6); await plan(root); + phase('generate', 4, 6); await generate(root); + phase('audit', 5, 6); await audit(root); + phase('publish', 6, 6); const publishing = publish(root); + return { publishing, budget: budgetReport(root), snapshot: sourceSnapshot(root) }; +} diff --git a/global-template/docgen/lib/provider.mjs b/global-template/docgen/lib/provider.mjs new file mode 100644 index 0000000..8ed2cd5 --- /dev/null +++ b/global-template/docgen/lib/provider.mjs @@ -0,0 +1,196 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import { + appendJsonl, + commandExists, + ensureDir, + estimateTokens, + formatDuration, + loadConfig, + now, + projectPaths, + resolveCommand, + sha256, + sleep, + spawnCommand, + terminateProcessTree, + writeJson +} from './core.mjs'; + +const MIN_MAX_TURNS = 30; + +function executable(config) { + if (process.env.DOCGEN_COMMAND_CODE_BIN) return process.env.DOCGEN_COMMAND_CODE_BIN; + if (config.commandCode?.executable) return config.commandCode.executable; + for (const candidate of process.platform === 'win32' ? ['cmdc', 'command-code'] : ['cmd', 'cmdc', 'command-code']) { + if (commandExists(candidate)) return resolveCommand(candidate) ?? candidate; + } + throw new Error('Command Code executable not found. Set commandCode.executable or DOCGEN_COMMAND_CODE_BIN.'); +} + +function finiteNumber(value, fallback) { + const parsed = Number(value); + return Number.isFinite(parsed) ? parsed : fallback; +} + +export function providerInvocation(config, stage) { + const cc = config.commandCode ?? {}; const bin = executable(config); const args = ['-p']; + if (cc.trust !== false) args.push('--trust'); + if (cc.skipOnboarding !== false) args.push('--skip-onboarding'); + if (cc.yolo !== false) args.push('--yolo'); + const model = process.env.DOCGEN_MODEL || cc.stageModels?.[stage] || cc.model || null; + if (model) args.push('--model', String(model)); + const configuredTurns = process.env.DOCGEN_MAX_TURNS ?? cc.maxTurns?.[stage] ?? cc.maxTurns?.default ?? MIN_MAX_TURNS; + const maxTurns = Math.max(MIN_MAX_TURNS, Math.floor(finiteNumber(configuredTurns, MIN_MAX_TURNS))); + args.push('--max-turns', String(maxTurns)); + if (cc.verbose === true) args.push('--verbose'); + return { bin, resolvedBin: resolveCommand(bin) ?? String(bin), args, model, maxTurns }; +} + +function budgetConfig(config) { + const budget = config.budget ?? {}; + return { + maxProviderCalls: Number(budget.maxProviderCalls ?? 24), + maxEstimatedInputTokens: Number(budget.maxEstimatedInputTokens ?? 2_500_000), + maxEstimatedOutputTokens: Number(budget.maxEstimatedOutputTokens ?? 500_000), + maxContextTokensPerCall: Number(budget.maxContextTokensPerCall ?? 80_000), + onExceeded: String(budget.onExceeded ?? 'stop-and-report') + }; +} + +function executionConfig(config, stage) { + const execution = config.execution ?? {}; const timeouts = execution.stageTimeoutMinutes ?? {}; + const defaults = { modelCore: 30, modelEnterprise: 30, plan: 20, generate: 25, audit: 20 }; + const timeoutMinutes = finiteNumber(process.env.DOCGEN_STAGE_TIMEOUT_MINUTES ?? (typeof timeouts === 'number' ? timeouts : timeouts[stage] ?? timeouts.default) ?? defaults[stage] ?? 20, 20); + const timeoutMsOverride = finiteNumber(process.env.DOCGEN_STAGE_TIMEOUT_MS ?? 0, 0); + return { + heartbeatSeconds: Math.max(0.1, finiteNumber(execution.heartbeatSeconds ?? 10, 10)), + silenceNoticeSeconds: Math.max(0.1, finiteNumber(execution.silenceNoticeSeconds ?? 45, 45)), + timeoutMinutes: Math.max(1, timeoutMinutes), + timeoutMs: timeoutMsOverride > 0 ? timeoutMsOverride : Math.max(1, timeoutMinutes) * 60_000, + timeoutLabel: timeoutMsOverride > 0 ? formatDuration(timeoutMsOverride) : `${Math.max(1, timeoutMinutes)} minute(s)`, + streamProviderOutput: execution.streamProviderOutput === true || config.commandCode?.verbose === true + }; +} + +function label(stage, target) { return `${stage}${target ? `:${target}` : ''}`; } + +export function telemetry(root) { + const file = path.join(projectPaths(root).telemetry, 'provider-runs.jsonl'); + if (!fs.existsSync(file)) return []; + return fs.readFileSync(file, 'utf8').split(/\r?\n/).filter(Boolean).map((line) => JSON.parse(line)); +} + +function completedRuns(root) { + const terminal = new Map(); + for (const record of telemetry(root)) if (record.status === 'completed' || record.status === 'failed') terminal.set(record.runId, record); + return [...terminal.values()]; +} + +export function budgetReport(root) { + const paths = projectPaths(root); const limits = budgetConfig(loadConfig(root)); const runs = completedRuns(root); + const usage = { + providerCalls: runs.length, + successfulCalls: runs.filter((run) => run.status === 'completed').length, + failedCalls: runs.filter((run) => run.status === 'failed').length, + estimatedInputTokens: runs.reduce((n, x) => n + Number(x.estimatedInputTokens ?? 0), 0), + estimatedOutputTokens: runs.reduce((n, x) => n + Number(x.estimatedOutputTokens ?? 0), 0), + cacheHits: runs.filter((x) => x.cacheHit).length + }; + const remaining = { providerCalls: limits.maxProviderCalls - usage.providerCalls, estimatedInputTokens: limits.maxEstimatedInputTokens - usage.estimatedInputTokens, estimatedOutputTokens: limits.maxEstimatedOutputTokens - usage.estimatedOutputTokens }; + const stages = {}; + for (const run of runs) { + const stage = stages[run.stage] ??= { calls: 0, completed: 0, failed: 0, inputTokens: 0, outputTokens: 0 }; + stage.calls++; stage[run.status]++; stage.inputTokens += Number(run.estimatedInputTokens ?? 0); stage.outputTokens += Number(run.estimatedOutputTokens ?? 0); + } + const report = { schemaVersion: '2.0', generatedAt: now(), limits, usage, remaining, exceeded: Object.values(remaining).some((x) => x < 0), stages }; + writeJson(paths.budget, report); return report; +} + +function assertBudget(root, inputTokens) { + const report = budgetReport(root); const next = { calls: report.usage.providerCalls + 1, input: report.usage.estimatedInputTokens + inputTokens }; + if (next.calls > report.limits.maxProviderCalls) throw new Error(`Provider call budget exceeded (${next.calls}/${report.limits.maxProviderCalls}).`); + if (next.input > report.limits.maxEstimatedInputTokens) throw new Error(`Estimated input-token budget exceeded (${next.input}/${report.limits.maxEstimatedInputTokens}).`); + if (inputTokens > report.limits.maxContextTokensPerCall) throw new Error(`Per-call context budget exceeded (${inputTokens}/${report.limits.maxContextTokensPerCall}).`); +} + +function terminalRecord(root, record, patch) { + const completed = { ...record, finishedAt: now(), ...patch }; + appendJsonl(path.join(projectPaths(root).telemetry, 'provider-runs.jsonl'), completed); budgetReport(root); return completed; +} + +function runOnce(root, stage, target, prompt, attempt, maxAttempts) { + const paths = projectPaths(root); const config = loadConfig(root); const invocation = providerInvocation(config, stage); const execution = executionConfig(config, stage); + const startedAt = now(); const startedMs = Date.now(); const runId = `${startedAt.replace(/[:.]/g, '-')}-${stage}-${sha256(target || 'global').slice(0, 8)}-a${attempt}`; + ensureDir(paths.runs); const stdoutFile = path.join(paths.runs, `${runId}.stdout.log`); const stderrFile = path.join(paths.runs, `${runId}.stderr.log`); + fs.writeFileSync(stdoutFile, ''); fs.writeFileSync(stderrFile, ''); + const inputTokens = estimateTokens(prompt); assertBudget(root, inputTokens); + const record = { schemaVersion: '2.0', runId, stage, target: target || null, attempt, maxAttempts, startedAt, estimatedInputTokens: inputTokens, promptHash: sha256(prompt), executable: invocation.resolvedBin, model: invocation.model, maxTurns: invocation.maxTurns, timeoutMs: execution.timeoutMs, status: 'running' }; + appendJsonl(path.join(paths.telemetry, 'provider-runs.jsonl'), record); + const stageLabel = label(stage, target); + console.log(`[docgen] ${stageLabel} RUNNING | attempt ${attempt}/${maxAttempts} | context ~${inputTokens.toLocaleString()} tokens | maxTurns ${invocation.maxTurns} | timeout ${execution.timeoutLabel}`); + console.log(` provider: ${invocation.resolvedBin}${invocation.model ? ` | model: ${invocation.model}` : ''}`); + console.log(` logs: ${path.relative(root, stdoutFile).replaceAll('\\', '/')} | ${path.relative(root, stderrFile).replaceAll('\\', '/')}`); + + return new Promise((resolve, reject) => { + let settled = false; let timedOut = false; let stdout = ''; let stderr = ''; let lastOutputAt = Date.now(); let child; + try { + child = spawnCommand(invocation.bin, invocation.args, { cwd: root, env: { ...process.env, DOCGEN_MODE: '1', DOCGEN_STAGE: stage, DOCGEN_TARGET: target, DOCGEN_CONTEXT_ONLY: '1' }, stdio: ['pipe', 'pipe', 'pipe'], windowsHide: true }); + } catch (error) { + const completed = terminalRecord(root, record, { exitCode: 1, status: 'failed', estimatedOutputTokens: 0, error: error.message }); + console.error(`[docgen] ${stageLabel} FAILED | ${error.message}`); reject(Object.assign(error, { exitCode: completed.exitCode })); return; + } + const heartbeat = setInterval(() => { + if (settled) return; const elapsed = Date.now() - startedMs; const quiet = Date.now() - lastOutputAt; + const quietText = quiet >= execution.silenceNoticeSeconds * 1000 ? ` | no provider output for ${formatDuration(quiet)}` : ''; + console.log(`[docgen] ${stageLabel} RUNNING | elapsed ${formatDuration(elapsed)} | pid ${child.pid ?? '?'} | maxTurns ${invocation.maxTurns}${quietText}`); + }, execution.heartbeatSeconds * 1000); heartbeat.unref?.(); + const timeout = setTimeout(() => { + if (settled) return; timedOut = true; console.error(`[docgen] ${stageLabel} TIMEOUT | exceeded ${execution.timeoutLabel}; terminating process tree ${child.pid ?? '?'}.`); terminateProcessTree(child); + }, execution.timeoutMs); timeout.unref?.(); + const cleanup = () => { clearInterval(heartbeat); clearTimeout(timeout); }; + child.stdout.on('data', (chunk) => { lastOutputAt = Date.now(); stdout += chunk; fs.appendFileSync(stdoutFile, chunk); if (execution.streamProviderOutput) process.stdout.write(chunk); }); + child.stderr.on('data', (chunk) => { lastOutputAt = Date.now(); stderr += chunk; fs.appendFileSync(stderrFile, chunk); if (execution.streamProviderOutput) process.stderr.write(chunk); }); + child.stdin.on('error', (error) => { stderr += `\nstdin: ${error.message}`; }); + child.on('error', (error) => { + if (settled) return; settled = true; cleanup(); + const completed = terminalRecord(root, record, { exitCode: 1, status: 'failed', estimatedOutputTokens: estimateTokens(stdout + stderr), stdoutFile: path.relative(root, stdoutFile).replaceAll('\\', '/'), stderrFile: path.relative(root, stderrFile).replaceAll('\\', '/'), error: error.message }); + console.error(`[docgen] ${stageLabel} FAILED | elapsed ${formatDuration(Date.now() - startedMs)} | ${error.message}`); reject(Object.assign(error, { exitCode: completed.exitCode, stdout, stderr, run: completed })); + }); + child.on('close', (code) => { + if (settled) return; settled = true; cleanup(); const exitCode = timedOut ? 124 : (code ?? 1); const status = exitCode === 0 ? 'completed' : 'failed'; + const completed = terminalRecord(root, record, { exitCode, status, timedOut, estimatedOutputTokens: estimateTokens(stdout + stderr), stdoutFile: path.relative(root, stdoutFile).replaceAll('\\', '/'), stderrFile: path.relative(root, stderrFile).replaceAll('\\', '/') }); + const elapsed = formatDuration(Date.now() - startedMs); + if (exitCode === 0) { console.log(`[docgen] ${stageLabel} COMPLETED | elapsed ${elapsed} | exit 0`); resolve(completed); } + else { const detail = timedOut ? `timed out after ${execution.timeoutLabel}` : `exit ${exitCode}`; console.error(`[docgen] ${stageLabel} FAILED | elapsed ${elapsed} | ${detail}`); reject(Object.assign(new Error(`${stageLabel} failed: ${detail}. ${stderr.slice(-2000)}`), { exitCode, stdout, stderr, timedOut, run: completed })); } + }); + child.stdin.end(prompt); + }); +} + +export async function runProvider(root, { stage, target = '', prompt, acceptArtifacts = null }) { + const config = loadConfig(root); const retry = config.retry ?? {}; const maxAttempts = Math.max(1, Number(retry.maxAttempts ?? 3)); let lastError; + for (let attempt = 1; attempt <= maxAttempts; attempt++) { + try { return await runOnce(root, stage, target, prompt, attempt, maxAttempts); } + catch (error) { + lastError = error; + if (typeof acceptArtifacts === 'function') { + let accepted = false; + try { accepted = Boolean(await acceptArtifacts({ error, attempt, maxAttempts })); } catch (validationError) { error.artifactValidationError = validationError; } + if (accepted) { + console.warn(`[docgen] ${label(stage, target)} RECOVERED | provider failed but valid expected artifacts were checkpointed; retry suppressed.`); + return { ...(error.run ?? {}), status: 'recovered', recovered: true, recoveredFromExitCode: Number(error.exitCode ?? 1) }; + } + } + const retryable = [5, 6, 7].includes(Number(error.exitCode)) || /429|rate.?limit|timeout|ECONN|5\d\d/i.test(`${error.message}\n${error.stderr ?? ''}`); + if (!retryable || attempt === maxAttempts) throw error; + const base = Number(retry.initialDelaySeconds ?? 15); const max = Number(retry.maxDelaySeconds ?? 120); const delay = Math.min(max, base * (2 ** (attempt - 1))); + console.warn(`[docgen] ${label(stage, target)} RETRY | attempt ${attempt + 1}/${maxAttempts} in ${delay}s`); await sleep(delay * 1000); + } + } + throw lastError; +} + +export function resetTelemetry(root) { + const paths = projectPaths(root); fs.rmSync(paths.telemetry, { recursive: true, force: true }); fs.rmSync(path.dirname(paths.budget), { recursive: true, force: true }); +} diff --git a/global-template/docgen/lib/quality.mjs b/global-template/docgen/lib/quality.mjs new file mode 100644 index 0000000..15ef40d --- /dev/null +++ b/global-template/docgen/lib/quality.mjs @@ -0,0 +1,220 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import { loadConfig, now, posix, projectPaths, readJson, sha256, stableHash } from './core.mjs'; + +const CLASSIFICATIONS = new Set(['FACT', 'INFERENCE', 'ASSUMPTION', 'UNKNOWN']); +const GENERIC_FILLER = [ + /\bthis (?:page|document|section) (?:provides|describes|covers) (?:an )?overview\b/i, + /\bin (?:this|the following) (?:page|document|section),? we (?:will|shall)\b/i, + /\bmore details (?:will|can) be added\b/i +]; + +function tracePath(root, page) { return path.join(projectPaths(root).traceability, 'pages', `${page.id}.json`); } +function modelPath(root, name) { return path.join(projectPaths(root).model, `${name}.json`); } +function normalizeText(value) { return String(value ?? '').toLowerCase().replace(/[`*_>#\[\](){}:;,.!?"']/g, ' ').replace(/\s+/g, ' ').trim(); } +function bodyFingerprint(text) { return sha256(String(text).replace(/^---\s*\n[\s\S]*?\n---\s*\n/, '').replace(/^#\s+.*$/m, '').replace(/\s+/g, ' ').trim().toLowerCase()); } +function safeRelative(value) { + const raw = posix(String(value ?? '').trim()).replace(/^\.\//, ''); + if (!raw || path.posix.isAbsolute(raw) || /^[a-z]:\//i.test(raw)) return null; + const normalized = path.posix.normalize(raw); + if (normalized === '..' || normalized.startsWith('../')) return null; + return normalized; +} +function state(root) { return readJson(projectPaths(root).state, { stages: {}, pages: {} }); } + +function inventoryContext(root) { + const inventory = readJson(projectPaths(root).inventory, { files: [], excluded: [], fingerprint: null }); + return { + inventory, + files: new Map((inventory.files ?? []).map((item) => [item.path, item])), + excluded: new Map((inventory.excluded ?? []).map((item) => [item.path, item])) + }; +} + +function sourceInfo(root, rel, cache) { + if (cache.has(rel)) return cache.get(rel); + const file = path.join(root, rel); + if (!fs.existsSync(file)) { cache.set(rel, null); return null; } + const buffer = fs.readFileSync(file); const text = buffer.toString('utf8'); + const value = { hash: sha256(buffer), lines: text.split(/\r?\n/), text }; + cache.set(rel, value); return value; +} + +function validateEvidence(root, evidence, { inventory, sourceCache, requireLine = false, prefix = 'evidence' }) { + const errors = []; const warnings = []; + if (!evidence || typeof evidence !== 'object' || Array.isArray(evidence)) return { errors: [`${prefix}: must be an object`], warnings }; + const rel = safeRelative(evidence.path); + if (!rel) return { errors: [`${prefix}: invalid repository-relative path`], warnings }; + const item = inventory.files.get(rel); + if (!item) { + const excluded = inventory.excluded.get(rel); + errors.push(`${prefix}: ${rel} is not in the indexed source inventory${excluded ? ` (${excluded.reason})` : ''}`); + return { errors, warnings }; + } + const source = sourceInfo(root, rel, sourceCache); + if (!source) { errors.push(`${prefix}: indexed source no longer exists: ${rel}`); return { errors, warnings }; } + if (item.hash && source.hash !== item.hash) errors.push(`${prefix}: source changed after indexing: ${rel}; run docgen index/resume`); + const hasStart = evidence.startLine !== undefined && evidence.startLine !== null; + if (requireLine && !hasStart) errors.push(`${prefix}: FACT evidence requires startLine: ${rel}`); + if (hasStart) { + const start = Number(evidence.startLine); const end = Number(evidence.endLine ?? start); + if (!Number.isInteger(start) || start < 1) errors.push(`${prefix}: invalid startLine for ${rel}`); + else if (!Number.isInteger(end) || end < start) errors.push(`${prefix}: invalid endLine for ${rel}`); + else if (end > source.lines.length) errors.push(`${prefix}: line range L${start}-L${end} exceeds ${rel} (${source.lines.length} lines)`); + else if (!source.lines.slice(start - 1, end).some((line) => line.trim())) warnings.push(`${prefix}: line range is blank in ${rel}#L${start}-L${end}`); + } + return { errors, warnings }; +} + +function walkModelItems(value, model, out, parent = '') { + if (Array.isArray(value)) { for (const item of value) walkModelItems(item, model, out, parent); return; } + if (!value || typeof value !== 'object') return; + if (value.id || value.name || value.statement) { + const semanticId = String(value.id ?? sha256(`${model}\0${parent}\0${JSON.stringify(value)}`).slice(0, 24)); + out.push({ ref: `${model}:${semanticId}`, semanticId, model, kind: String(value.kind ?? parent ?? 'item'), value }); + } + for (const [key, child] of Object.entries(value)) if (!['evidence', 'sourceModelRefs'].includes(key)) walkModelItems(child, model, out, key); +} + +function auditModels(root, inventory, settings) { + const paths = projectPaths(root); const errors = []; const warnings = []; const sourceCache = new Map(); const refs = new Map(); const byModel = {}; + const files = fs.existsSync(paths.model) ? fs.readdirSync(paths.model).filter((name) => name.endsWith('.json') && !name.endsWith('-bundle.json')).sort() : []; + for (const file of files) { + const model = path.basename(file, '.json'); const items = []; let document; + try { document = readJson(modelPath(root, model)); } catch (error) { errors.push(`${model}: invalid JSON: ${error.message}`); continue; } + walkModelItems(document, model, items); byModel[model] = { items: items.length, facts: 0, errors: 0, warnings: 0 }; + const local = new Set(); + for (const item of items) { + const prefix = `model ${item.ref}`; const value = item.value; + if (local.has(item.semanticId)) { errors.push(`${prefix}: duplicate id within ${model}`); byModel[model].errors++; continue; } + local.add(item.semanticId); + if (refs.has(item.ref)) { errors.push(`${prefix}: duplicate qualified model reference`); byModel[model].errors++; continue; } + refs.set(item.ref, item); + const statement = String(value.statement ?? value.summary ?? value.description ?? '').trim(); + const name = String(value.name ?? value.title ?? '').trim(); + if (!statement && !name) { warnings.push(`${prefix}: missing name and statement`); byModel[model].warnings++; } + const classification = String(value.classification ?? 'UNKNOWN').toUpperCase(); + if (!CLASSIFICATIONS.has(classification)) { errors.push(`${prefix}: invalid classification ${classification}`); byModel[model].errors++; } + const confidence = Number(value.confidence ?? (classification === 'FACT' ? 1 : 0)); + if (!Number.isFinite(confidence) || confidence < 0 || confidence > 1) { errors.push(`${prefix}: confidence must be between 0 and 1`); byModel[model].errors++; } + const evidence = Array.isArray(value.evidence) ? value.evidence : []; + if (classification === 'FACT') { + byModel[model].facts++; + if (!evidence.length) { errors.push(`${prefix}: FACT item has no evidence`); byModel[model].errors++; } + } + for (let index = 0; index < evidence.length; index++) { + const result = validateEvidence(root, evidence[index], { inventory, sourceCache, requireLine: classification === 'FACT' && settings.requireLineEvidenceForFacts, prefix: `${prefix} evidence[${index}]` }); + errors.push(...result.errors); warnings.push(...result.warnings); byModel[model].errors += result.errors.length; byModel[model].warnings += result.warnings.length; + } + } + } + return { files, refs, errors, warnings, byModel, items: refs.size, inputHash: stableHash(files.map((name) => [name, sha256(fs.readFileSync(path.join(paths.model, name))) ])) }; +} + +function contextForPage(root, page, trace) { + if (!trace.contextId) return null; + const file = path.join(projectPaths(root).context, 'generate', `${page.id.replace(/[^a-z0-9_.-]+/gi, '-')}.json`); + if (!fs.existsSync(file)) return { error: `missing declared generation context for ${page.id}` }; + const value = readJson(file, {}); + if (value.id !== trace.contextId) return { error: `context identity mismatch for ${page.id}` }; + const refs = new Set((value.modelItems ?? []).map((item) => item.id)); + const evidence = []; + for (const fact of value.facts ?? []) evidence.push({ path: fact.path, startLine: fact.metadata?.startLine ?? fact.line, endLine: fact.metadata?.endLine ?? fact.line }); + for (const item of value.modelItems ?? []) for (const entry of item.evidence ?? []) evidence.push(entry); + return { value, refs, evidence }; +} + +function evidenceWasSupplied(evidence, supplied) { + const rel = safeRelative(evidence?.path); if (!rel) return false; + const line = Number(evidence.startLine ?? 0); + return supplied.some((entry) => { + if (safeRelative(entry?.path) !== rel) return false; + if (!line) return true; + const start = Number(entry.startLine ?? 0); const end = Number(entry.endLine ?? start); + return !start || (line >= start && line <= end); + }); +} + +function markdownLinks(text) { + const links = []; + for (const match of text.matchAll(/\[[^\]]*\]\(([^)]+)\)/g)) links.push(match[1].trim().replace(/^<|>$/g, '')); + return links; +} + +function validateLink(root, page, target) { + if (!target || /^(?:[a-z]+:|#)/i.test(target)) return null; + const clean = target.split('#')[0].split('?')[0]; if (!clean) return null; + const candidate = clean.startsWith('/') ? path.join(root, clean.replace(/^\/+/, '')) : path.resolve(path.dirname(path.join(root, page.path)), clean); + const relative = posix(path.relative(root, candidate)); + if (relative === '..' || relative.startsWith('../')) return `link escapes repository: ${target}`; + if (!fs.existsSync(candidate)) return `broken local link: ${target}`; + return null; +} + +function auditPage(root, page, inventory, modelRefs, manifestIds, settings) { + const errors = []; const warnings = []; const paths = projectPaths(root); const pageFile = path.join(root, page.path); const traceFile = tracePath(root, page); const pageState = state(root).pages?.[page.id] ?? {}; + if (!fs.existsSync(pageFile)) return { pageId: page.id, path: page.path, errors: [`missing page: ${page.path}`], warnings, riskScore: 100, claims: 0, evidence: 0, modelRefs: 0 }; + const text = fs.readFileSync(pageFile, 'utf8'); const pageHash = sha256(text); + if (!/^---\s*\n[\s\S]*?\n---\s*\n/.test(text)) errors.push('missing YAML frontmatter'); + if (!/^#\s+\S/m.test(text)) errors.push('missing H1'); + if (/```(?:plantuml|dot|graphviz|puml)/i.test(text)) errors.push('contains non-Mermaid diagram'); + if (/\b(?:TODO|TBD|FIXME)\b/i.test(text)) warnings.push('contains unresolved placeholder'); + if (GENERIC_FILLER.some((pattern) => pattern.test(text))) warnings.push('contains generic filler language'); + const headings = new Set([...text.matchAll(/^#{2,6}\s+(.+)$/gm)].map((match) => normalizeText(match[1]))); + for (const section of page.requiredSections ?? []) if (![...headings].some((heading) => heading.includes(normalizeText(section)))) errors.push(`missing required section: ${section}`); + for (const related of page.relatedPages ?? []) if (!manifestIds.has(String(related)) && !manifestIds.has(String(related).replace(/\.md$/i, ''))) warnings.push(`unknown related page: ${related}`); + for (const link of markdownLinks(text)) { const problem = validateLink(root, page, link); if (problem) errors.push(problem); } + if (!fs.existsSync(traceFile)) return { pageId: page.id, path: page.path, pageHash, inputHash: pageState.inputHash, errors: [...errors, 'missing traceability sidecar'], warnings, riskScore: 100, claims: 0, evidence: 0, modelRefs: 0, bodyHash: bodyFingerprint(text) }; + let trace; + try { trace = readJson(traceFile); } catch (error) { return { pageId: page.id, path: page.path, pageHash, inputHash: pageState.inputHash, errors: [...errors, `invalid traceability JSON: ${error.message}`], warnings, riskScore: 100, claims: 0, evidence: 0, modelRefs: 0, bodyHash: bodyFingerprint(text) }; } + if (trace.pageId !== page.id || trace.pagePath !== page.path) errors.push('traceability page identity mismatch'); + if (trace.pageHash !== pageHash) errors.push('stale traceability page hash'); + if (!pageState.inputHash) errors.push('missing page checkpoint inputHash'); + else if (trace.inputHash !== pageState.inputHash) errors.push('stale traceability input hash'); + const context = contextForPage(root, page, trace); if (context?.error) errors.push(context.error); + const claims = Array.isArray(trace.claims) ? trace.claims : []; if (!Array.isArray(trace.claims)) errors.push('traceability claims must be an array'); + const ids = new Set(); const sourceCache = new Map(); let evidenceCount = 0; let modelRefCount = 0; + for (let index = 0; index < claims.length; index++) { + const claim = claims[index] ?? {}; const id = String(claim.id ?? '').trim(); const prefix = `claim ${id || index + 1}`; + if (!id) errors.push(`${prefix}: missing id`); else if (ids.has(id)) errors.push(`${prefix}: duplicate id`); else ids.add(id); + const statement = String(claim.statement ?? '').trim(); if (!statement) errors.push(`${prefix}: missing statement`); + else if (!normalizeText(text).includes(normalizeText(statement))) warnings.push(`${prefix}: statement is not directly present in page prose`); + const classification = String(claim.classification ?? 'UNKNOWN').toUpperCase(); if (!CLASSIFICATIONS.has(classification)) errors.push(`${prefix}: invalid classification ${classification}`); + const confidence = Number(claim.confidence ?? (classification === 'FACT' ? 1 : 0)); if (!Number.isFinite(confidence) || confidence < 0 || confidence > 1) errors.push(`${prefix}: confidence must be between 0 and 1`); + const evidence = Array.isArray(claim.evidence) ? claim.evidence : []; evidenceCount += evidence.length; + if (classification === 'FACT' && !evidence.length) errors.push(`${prefix}: FACT claim has no direct evidence`); + for (let evidenceIndex = 0; evidenceIndex < evidence.length; evidenceIndex++) { + const result = validateEvidence(root, evidence[evidenceIndex], { inventory, sourceCache, requireLine: classification === 'FACT' && settings.requireLineEvidenceForFacts, prefix: `${prefix} evidence[${evidenceIndex}]` }); errors.push(...result.errors); warnings.push(...result.warnings); + if (settings.requireContextBoundEvidence && context && !context.error && !evidenceWasSupplied(evidence[evidenceIndex], context.evidence)) errors.push(`${prefix}: evidence was not supplied in bounded context: ${evidence[evidenceIndex]?.path ?? ''}`); + } + const refs = Array.isArray(claim.sourceModelRefs) ? claim.sourceModelRefs : []; modelRefCount += refs.length; + for (const ref of refs) { + if (!modelRefs.has(ref)) errors.push(`${prefix}: unknown sourceModelRef ${ref}`); + if (settings.requireContextBoundEvidence && context && !context.error && !context.refs.has(ref)) errors.push(`${prefix}: sourceModelRef was not supplied in bounded context: ${ref}`); + } + } + const riskScore = (['business', 'security', 'decision-record', 'runbook', 'migration-guide'].includes(page.type) ? 30 : 0) + (page.risk === 'high' ? 40 : page.risk === 'critical' ? 70 : 0) + warnings.length * 5; + return { pageId: page.id, path: page.path, pageHash, inputHash: pageState.inputHash, bodyHash: bodyFingerprint(text), errors, warnings, riskScore, claims: claims.length, evidence: evidenceCount, modelRefs: modelRefCount, referencedModelRefs: claims.flatMap((claim) => Array.isArray(claim.sourceModelRefs) ? claim.sourceModelRefs : []) }; +} + +export function auditRepository(root, manifest) { + const config = loadConfig(root); const settings = { requireLineEvidenceForFacts: config.audit?.requireLineEvidenceForFacts !== false, requireContextBoundEvidence: config.audit?.requireContextBoundEvidence !== false }; + const inventory = inventoryContext(root); const models = auditModels(root, inventory, settings); const manifestIds = new Set(manifest.pages.flatMap((page) => [page.id, page.path, page.path.replace(/^docs\//, '').replace(/\.md$/i, '')])); + const pages = manifest.pages.map((page) => auditPage(root, page, inventory, models.refs, manifestIds, settings)); const errors = [...models.errors]; const warnings = [...models.warnings]; + for (const page of pages) { errors.push(...page.errors.map((message) => `${page.pageId}: ${message}`)); warnings.push(...page.warnings.map((message) => `${page.pageId}: ${message}`)); } + const bodyOwners = new Map(); const statementOwners = new Map(); const referenced = new Set(); + for (const page of pages) { + if (page.bodyHash) { const previous = bodyOwners.get(page.bodyHash); if (previous) errors.push(`duplicate page body: ${previous} and ${page.pageId}`); else bodyOwners.set(page.bodyHash, page.pageId); } + for (const ref of page.referencedModelRefs ?? []) referenced.add(ref); + const trace = fs.existsSync(tracePath(root, { id: page.pageId })) ? readJson(tracePath(root, { id: page.pageId }), {}) : {}; + for (const claim of trace.claims ?? []) { const key = normalizeText(claim.statement); if (!key) continue; const previous = statementOwners.get(key); if (previous && previous !== page.pageId) warnings.push(`duplicate material claim across pages: ${previous} and ${page.pageId}`); else statementOwners.set(key, page.pageId); } + } + const traceDir = path.join(projectPaths(root).traceability, 'pages'); + if (fs.existsSync(traceDir)) for (const file of fs.readdirSync(traceDir).filter((name) => name.endsWith('.json'))) { const id = path.basename(file, '.json'); if (!manifest.pages.some((page) => page.id === id)) warnings.push(`orphan traceability sidecar: ${file}`); } + const coverageRatio = models.items ? referenced.size / models.items : 1; const minimumCoverage = Math.max(0, Math.min(1, Number(config.audit?.minModelReferenceCoverage ?? 0))); + if (coverageRatio < minimumCoverage) errors.push(`model reference coverage ${(coverageRatio * 100).toFixed(1)}% is below configured ${(minimumCoverage * 100).toFixed(1)}%`); + if (config.audit?.failOnWarnings === true && warnings.length) errors.push(`audit.failOnWarnings=true and ${warnings.length} warning(s) were found`); + const metrics = { pages: pages.length, claims: pages.reduce((sum, page) => sum + page.claims, 0), evidenceReferences: pages.reduce((sum, page) => sum + page.evidence, 0), modelItems: models.items, referencedModelItems: referenced.size, modelReferenceCoverage: Number(coverageRatio.toFixed(4)), errors: errors.length, warnings: warnings.length }; + const inventoryFingerprint = inventory.inventory.fingerprint ?? null; const manifestHash = stableHash(manifest.pages); const auditInputHash = stableHash({ inventoryFingerprint, manifestHash, modelInputHash: models.inputHash, pages: pages.map((page) => [page.pageId, page.pageHash, page.inputHash]) }); + return { schemaVersion: '2.0', generatedAt: now(), inventoryFingerprint, manifestHash, modelInputHash: models.inputHash, auditInputHash, pass: errors.length === 0, metrics, models: { files: models.files, byModel: models.byModel, errors: models.errors, warnings: models.warnings }, pages, errors, warnings }; +} diff --git a/global-template/docgen/package.json b/global-template/docgen/package.json index 71587a1..d52edfc 100644 --- a/global-template/docgen/package.json +++ b/global-template/docgen/package.json @@ -1,9 +1,16 @@ { "name": "commandcode-docgen-kit", - "version": "1.0.0", + "version": "2.0.0", "private": true, "type": "module", + "engines": { + "node": ">=22.5.0" + }, "bin": { - "docgen": "bin/docgen.mjs" + "docgen": "bin/docgen-launcher.mjs" + }, + "scripts": { + "test": "node --test test/*.test.mjs", + "check": "node --check bin/docgen-launcher.mjs && node --check bin/docgen-v2.mjs && node --check bin/workspace.mjs && node --check lib/core.mjs && node --check lib/inventory.mjs && node --check lib/indexer.mjs && node --check lib/context.mjs && node --check lib/provider.mjs && node --check lib/quality.mjs && node --check lib/pipeline.mjs && node --check test/fixtures/fake-provider.mjs && node --check test/semantic-index.test.mjs" } } diff --git a/global-template/docgen/project-template/README.md b/global-template/docgen/project-template/README.md index f49a8dc..32eb6ae 100644 --- a/global-template/docgen/project-template/README.md +++ b/global-template/docgen/project-template/README.md @@ -1,35 +1,47 @@ -# DocGen Project Workspace +# DocGen project workspace -This directory contains repository-specific DocGen configuration, generated evidence, models, plans, audits, run metadata, and state. +This `.docgen` directory holds repository-specific configuration, semantic indexes, bounded contexts, normalized models, execution checkpoints, provider logs, audits, traceability, and publishing metadata. The reusable engine is installed separately under `~/.commandcode/docgen/`. -The reusable DocGen engine is installed globally under `~/.commandcode/docgen/`. +Source code and explicitly supplied repository artifacts remain authoritative. Generated models and documentation are derived, validated artifacts—not a replacement for the source. -Do not treat generated evidence or model artifacts as higher authority than source code. +## Technology-neutral scope +DocGen does not require a particular language, framework, library, protocol, database, messaging system, or deployment architecture. It can document applications, packages, libraries, command-line tools, background jobs, plugins, infrastructure, data pipelines, embedded software, monoliths, services, or mixed repositories. -## Core knowledge models +Technology-specific facts are emitted only when supported by indexed evidence. Generic file artifacts and source chunks provide the fallback for unfamiliar ecosystems. -In addition to technical architecture, DocGen generates repository-local normalized models for business semantics, distinct flow types, and exhaustive interface/dependency catalogs: +## Important paths -- `model/business.json` -- `model/flows.json` -- `model/catalogs.json` - -Published diagrams are Mermaid-only. +| Path | Purpose | +|---|---| +| `config/documentation.json` | project policy, budgets, provider settings, execution, and audit gates | +| `index/inventory.json` | canonical included/excluded source boundary and file hashes | +| `index/source-files.txt` | human-readable included source list | +| `index/semantic.db` | incremental SQLite/FTS semantic index | +| `context//*.json` | bounded provider inputs with omissions and input hashes | +| `model/*.json` | normalized core and enterprise semantic models | +| `plan/manifest.json` | canonical documentation page plan | +| `state/state.json` | stage and page checkpoints used by resume | +| `runs/*.stdout.log` / `*.stderr.log` | provider diagnostics per invocation | +| `telemetry/provider-runs.jsonl` | provider status, model, turns, timeout, tokens, and recovery | +| `budget/report.json` | provider call/token usage and remaining budget | +| `traceability/pages/*.json` | claim-level source and model mappings per page | +| `audit/deterministic.json` | detailed structural, grounding, freshness, link, and coverage findings | +| `audit/quality-summary.json` | current publish gate and aggregate quality metrics | +| `publish/navigation.json` | deterministic documentation navigation | +| `publish/search-index.json` | deterministic search metadata | +| `traceability/index.json` | published page trace summary | -## P0 trustworthiness artifacts +## Normalized model surfaces -- `traceability/pages/*.json`: claim-level source mappings per page. -- `traceability/index.json`: aggregated claims and source snapshot. -- `traceability/contradictions.json`: conflicting subject/predicate claims. -- `traceability/duplicates.json`: unintentional repeated claims. -- `traceability/freshness.json`: page/input/source staleness status. -- `audit/quality-summary.json`: evidence-centric quality metrics. +Core models: +- `model/system.json` +- `model/business.json` +- `model/flows.json` +- `model/catalogs.json` -## P1 enterprise-depth models - -The enterprise stage produces repository-local typed models: +Enterprise-depth models: - `model/security.json` - `model/operations.json` @@ -40,30 +52,46 @@ The enterprise stage produces repository-local typed models: - `model/change-impact.json` - `model/ownership.json` -These models cover trust boundaries, AuthN/AuthZ, ownership/RACI, operational health and recovery, test strategy, data correctness/governance, environment configuration, architectural rationale, and change blast radius. +These are extensible, framework-neutral structures. Optional technology-specific catalogs may appear only when the repository provides evidence for them. + +## Resume and recovery + +`docgen all` and `docgen resume` use the same checkpoint-aware pipeline: + +```text +index -> modelCore -> modelEnterprise -> plan -> generate -> audit -> publish +``` + +A full run indexes once. Completed work is reused only while source, model, context, output, and trace hashes are current. If a generation batch fails after producing some valid pages, those pages are checkpointed immediately and only unresolved pages are retried. A non-zero provider exit is recoverable only when the current invocation wrote fresh artifacts that pass their output contracts. + +Use: + +```text +docgen status +docgen budget +docgen resume +``` -## Ignore-aware source inventory +Provider progress shows the effective executable, model, maximum turns, timeout, context size, elapsed time, and log locations. -DocGen follows repository `.gitignore` and root `.docgenignore`. The effective included source set is written to: +## Correctness and publishing -- `state/source-inventory.json` -- `state/source-files.txt` -- `state/ignore-report.json` +A `FACT` must be grounded in the canonical source inventory. By default, evidence also needs valid line locations, the live source must still match its indexed hash, and generated claims may only reference evidence/model items supplied in their bounded page context. -Ignored files are excluded from discovery, fingerprints, change detection, traceability, and FACT evidence. Use `docgen ignore`, `docgen source-list`, and `docgen source-grep` to inspect or search the effective source boundary. +The deterministic audit checks model identities and evidence, page and trace hashes, required sections, local links, duplicate content, conflicting or duplicate claims, stale/orphan artifacts, and model-reference coverage. Selective provider audit is reserved for semantic risk that deterministic checks cannot prove. -## v0.9 P2 documentation experience +`docgen publish` refuses to run unless the audit is passing and still current. It revalidates the source, models, pages, traces, and audit identity before creating navigation, search, trace indexes, `docs/llms.txt`, and `docs/llms-full.txt`. -Published pages are classified by user intent: `tutorial`, `how-to`, `explanation`, `reference`, `runbook`, `decision-record`, `migration-guide`, or `troubleshooting`. DocGen deterministically produces: +## Source boundary -- Markdown frontmatter; -- `docs/llms.txt` and bounded `docs/llms-full.txt`; -- `publish/navigation.json` and `publish/search-index.json`; -- backlinks, aliases/redirects, orphan-page and examples indexes; -- version, status, deprecation, replacement, and migration metadata. +DocGen follows Git-aware source discovery when available and falls back to filesystem traversal otherwise. Repository `.gitignore`, root `.docgenignore`, binary signatures, invalid UTF-8, NUL-containing files, compiled artifacts, archives, database files, fonts, media, office documents, oversized text, and configured deny extensions are excluded before indexing. -Run `docgen publish` to rebuild these assets without an LLM call. +Inspect the effective boundary with: -## Binary and non-text token boundary +```text +docgen ignore +docgen source-list [substring] +docgen source-grep +``` -Known images, audio/video, PDFs and office documents, archives, compiled artifacts, fonts, database files, keystores, invalid UTF-8, NUL-containing files, and oversized text are excluded from the canonical source inventory. The exclusion applies to reads, grep, fingerprints, change detection, freshness, and evidence validation. Configure the boundary under `ignore.binary` in `config/documentation.json`. +Ignored or stale files cannot be used as FACT evidence. diff --git a/global-template/docgen/project-template/config/documentation.json b/global-template/docgen/project-template/config/documentation.json index 91e83e7..acd0256 100644 --- a/global-template/docgen/project-template/config/documentation.json +++ b/global-template/docgen/project-template/config/documentation.json @@ -1,344 +1,98 @@ { - "schemaVersion": "1.6", + "schemaVersion": "2.0", "projectName": "", "outputRoot": "docs", - "discoveryScopes": [ - "." - ], - "exclude": [ - ".git/**", - ".idea/**", - ".vscode/**", - ".commandcode/**", - ".docgen/**", - "docs/**", - "node_modules/**", - "target/**", - "build/**", - "dist/**", - "coverage/**", - "vendor/**", - "**/*.min.js", - "**/*.map", - "**/*.class", - "**/*.jar", - "**/*.war", - "**/*.zip" - ], - "sourceExtensions": [ - ".java", - ".kt", - ".kts", - ".xml", - ".sql", - ".json", - ".yaml", - ".yml", - ".properties", - ".md", - ".go", - ".rs", - ".cs", - ".js", - ".mjs", - ".cjs", - ".ts", - ".tsx", - ".vue", - ".proto", - ".bpmn", - ".dmn", - ".sh", - ".ps1", - ".dockerfile" - ], - "audiences": [ - "engineer", - "architect", - "operator", - "product", - "business-analyst" - ], - "pageTypes": [ - "overview", - "architecture", - "business", - "concept", - "flow", - "guide", - "reference", - "data", - "integration", - "operations", - "troubleshooting", - "security", - "testing", - "configuration", - "decision", - "ownership", - "maintenance", - "tutorial", - "runbook", - "migration" - ], - "requireEvidenceReferences": true, - "diagramFormat": "mermaid", - "maxDiscoveryScopeHint": "Prefer one module or bounded subsystem per discovery run for large repositories.", "commandCode": { "executable": "", "trust": true, "skipOnboarding": true, "yolo": true, + "verbose": false, "model": "", - "stageModels": {}, - "maxTurns": { - "default": 20, - "discover": 30, - "analyze": 30, - "plan": 20, - "generate": 20, - "audit": 20, - "fix": 20, - "update-impact": 20, - "enrich": 20, - "semantics": 30, - "enterprise": 30 - } - }, - "quality": { - "profile": "comprehensive", - "autoEnrich": true, - "autoFix": true, - "reAuditAfterFix": true, - "minWordsByType": { - "overview": 900, - "architecture": 1200, - "concept": 900, - "guide": 1000, - "reference": 700, - "operations": 1000, - "business": 1000, - "flow": 1000, - "data": 1000, - "integration": 900, - "troubleshooting": 1000 + "stageModels": { + "modelCore": "", + "modelEnterprise": "", + "plan": "", + "generate": "", + "audit": "" }, - "minHeadings": 4, - "requireDeclaredSections": true, - "requirePlannedDiagrams": true, - "maxCriticalFindings": 0, - "maxHighFindings": 0, - "requireMermaidOnly": true, - "requireCoverageTags": true, - "requiredCoverageTagsWhenEvidenceExists": [ - "business-rules", - "branch-conditions", - "request-flow", - "traffic-flow", - "data-flow", - "endpoint-catalog", - "message-handler-catalog", - "external-dependency-catalog", - "security-trust-boundaries", - "authorization-model", - "data-governance", - "consistency-transactions", - "operations-observability", - "failure-recovery", - "testing-strategy", - "configuration-matrix", - "architecture-decisions", - "change-impact", - "ownership-responsibilities" - ], - "enrichmentMode": "on-failure", - "wordCountGate": "advisory", - "semanticMetrics": { - "minClaimGroundingRatio": 0.9, - "minEvidenceCoverageRatio": 0.8, - "minCatalogCoverageRatio": 1.0, - "minBranchCoverageRatio": 0.9, - "minEvidenceClaimsPer1000Words": 1.5, - "maxUnsupportedClaims": 0, - "maxContradictions": 0, - "maxStalePages": 0, - "duplicateClaimsAsWarning": true, - "minModelCoverageRatio": 0.9, - "evidenceClaimDensityGate": "hard", - "minStructuredClaimRatio": 0.7, - "maxClaimIdCollisions": 0 + "maxTurns": { + "default": 30, + "modelCore": 30, + "modelEnterprise": 30, + "plan": 30, + "generate": 30, + "audit": 30 } }, - "progress": { - "heartbeatSeconds": 10, - "noOutputWarningSeconds": 45, - "showCommandOutput": true, - "verboseCommandCode": true + "budget": { + "maxProviderCalls": 24, + "maxEstimatedInputTokens": 2500000, + "maxEstimatedOutputTokens": 500000, + "maxContextTokensPerCall": 80000, + "onExceeded": "stop-and-report" }, - "knowledgeBase": { - "target": "deep-multi-page-system-knowledge-base", - "navigation": "category-and-page hierarchy", - "preferFocusedPagesOverCatchAll": true, - "exhaustiveCatalogs": true, - "flowTypes": [ - "business", - "control", - "request", - "traffic", - "data", - "event" - ], - "diagramFormat": "mermaid-only" + "context": { + "maxTokens": { + "default": 60000, + "modelCore": 80000, + "modelEnterprise": 80000, + "plan": 50000, + "generate": 30000, + "audit": 18000 + } }, "execution": { - "resumeByDefault": true, - "skipValidPages": true, - "generateBatchSize": 4, - "enrichBatchSize": 4, - "auditBatchSize": 6, - "failFastPreflight": true, + "generationBatchSize": 4, + "maxPlannedPages": 30, + "resumeByContentHash": true, + "heartbeatSeconds": 10, + "silenceNoticeSeconds": 45, + "streamProviderOutput": false, "stageTimeoutMinutes": { "default": 20, - "discover": 35, - "analyze": 35, - "semantics": 35, - "plan": 25, - "generate": 20, - "enrich": 15, - "audit": 15, - "fix": 15, - "update-impact": 15, - "enterprise": 30 + "modelCore": 30, + "modelEnterprise": 30, + "plan": 20, + "generate": 25, + "audit": 20 }, - "adoptLegacyValidPages": true + "generationRecoveryAttempts": 3, + "recoverValidArtifacts": true + }, + "audit": { + "llmEnabled": true, + "llmRiskThreshold": 50, + "failOnCritical": true, + "failOnHigh": true, + "failOnWarnings": false, + "requireLineEvidenceForFacts": true, + "requireContextBoundEvidence": true, + "minModelReferenceCoverage": 0 }, "retry": { - "enabled": true, - "maxAttempts": 4, - "retryableExitCodes": [ - 5, - 6, - 7 - ], + "maxAttempts": 3, "initialDelaySeconds": 15, - "rateLimitDelaySeconds": 30, - "maxDelaySeconds": 120, - "multiplier": 2, - "jitterRatio": 0.2, - "countdownSeconds": 10, - "interRequestDelaySeconds": 3 - }, - "enterpriseDepth": { - "enabled": true, - "passes": [ - "governance", - "operability", - "data-and-configuration", - "evolution" - ], - "models": [ - ".docgen/model/security.json", - ".docgen/model/operations.json", - ".docgen/model/testing.json", - ".docgen/model/data-governance.json", - ".docgen/model/decisions.json", - ".docgen/model/configuration.json", - ".docgen/model/change-impact.json", - ".docgen/model/ownership.json" - ], - "requiredCoverageTagsWhenEvidenceExists": [ - "security-trust-boundaries", - "authorization-model", - "data-governance", - "consistency-transactions", - "operations-observability", - "failure-recovery", - "testing-strategy", - "configuration-matrix", - "architecture-decisions", - "change-impact", - "ownership-responsibilities" - ] + "maxDelaySeconds": 120 }, "ignore": { "useGitignore": true, "useDocgenignore": true, - "docgenignoreFile": ".docgenignore", - "writeReport": true, "rejectIgnoredEvidence": true, "binary": { "enabled": true, "probeBytes": 16384, "maxTextFileBytes": 4194304, "controlCharacterRatio": 0.08, - "allowExtensions": [], "denyExtensions": [] } }, - "documentationExperience": { - "enabled": true, - "modes": [ - "tutorial", - "how-to", - "explanation", - "reference", - "runbook", - "decision-record", - "migration-guide", - "troubleshooting" - ], - "requireMode": true, - "generateFrontmatter": true, - "generateLlmsTxt": true, - "generateLlmsFull": true, - "llmsFullMaxBytes": 5242880, - "generateSearchIndex": true, - "generateBacklinks": true, - "generateRedirects": true, - "generateExamplesIndex": true, - "requireEvidenceDerivedExamples": true, - "versioning": true, - "deprecationMetadata": true, - "adoptPreP2Pages": true, - "enforceModeSections": true, - "modeRequiredSections": { - "tutorial": [ - "Prerequisites", - "Walkthrough", - "Next Steps" - ], - "how-to": [ - "Prerequisites", - "Steps", - "Verification" - ], - "explanation": [], - "reference": [], - "runbook": [ - "Symptoms", - "Diagnosis", - "Procedure", - "Verification", - "Escalation" - ], - "decision-record": [ - "Context", - "Decision", - "Alternatives", - "Consequences" - ], - "migration-guide": [ - "Prerequisites", - "Migration Steps", - "Verification", - "Rollback" - ], - "troubleshooting": [ - "Symptoms", - "Diagnosis", - "Resolution", - "Verification" - ] - } + "publishing": { + "frontmatter": true, + "llmsTxt": true, + "llmsFullTxt": true, + "searchIndex": true, + "navigationIndex": true, + "mermaidOnly": true } } diff --git a/global-template/docgen/project-template/state/state.json b/global-template/docgen/project-template/state/state.json index 407f2f3..7e795f6 100644 --- a/global-template/docgen/project-template/state/state.json +++ b/global-template/docgen/project-template/state/state.json @@ -1,28 +1,15 @@ { - "schemaVersion": "1.0", - "kitVersion": "1.0.0", + "schemaVersion": "2.0", + "kitVersion": "2.0.0", "updatedAt": null, "stages": { - "discover": { - "status": "pending" - }, - "analyze": { - "status": "pending" - }, - "plan": { - "status": "pending" - }, - "generate": { - "status": "pending" - }, - "audit": { - "status": "pending" - }, - "semantics": { - "status": "pending" - }, - "enterprise": { - "status": "pending" - } - } + "index": { "status": "pending" }, + "modelCore": { "status": "pending" }, + "modelEnterprise": { "status": "pending" }, + "plan": { "status": "pending" }, + "generate": { "status": "pending" }, + "auditRisk": { "status": "pending" }, + "audit": { "status": "pending" } + }, + "pages": {} } diff --git a/global-template/docgen/prompts/analyze.md b/global-template/docgen/prompts/analyze.md deleted file mode 100644 index 8249575..0000000 --- a/global-template/docgen/prompts/analyze.md +++ /dev/null @@ -1,12 +0,0 @@ -You are running a bounded DocGen architecture analysis stage. - -Scope: {{SCOPE}} - -Delegate synthesis to the `doc-architect` custom agent. Use existing `.docgen/evidence/**` as the primary input and inspect source only for targeted verification. Reconcile `.docgen/model/**` and ensure `.docgen/model/system.json` exists and is valid JSON. - -Do not write published documentation. Preserve FACT/INFERENCE/UNKNOWN classifications and evidence references. - -Typed semantic contract: -- Every component, relationship, workflow, and unknown must be an object with stable `id`, `kind`, `classification`, `confidence`, `evidence[]`, `sourceModelRefs[]`, and `unknowns[]`. -- `FACT` items require direct source evidence. Use `INFERENCE` or `UNKNOWN` when evidence is incomplete. -- Evidence references should include repository-relative path and symbol/line range when available. diff --git a/global-template/docgen/prompts/audit-batch.md b/global-template/docgen/prompts/audit-batch.md deleted file mode 100644 index 407b51e..0000000 --- a/global-template/docgen/prompts/audit-batch.md +++ /dev/null @@ -1,10 +0,0 @@ -Mandatory source boundary: never read or cite repository source absent from `.docgen/state/source-files.txt`. Existing `.docgen/**` and `docs/**` artifacts are allowed. - -You are running a bounded DocGen batched independent audit stage. - -Pages to audit, including their current content hashes: -{{PAGES_JSON}} - -Delegate to `doc-auditor`. For every page, verify claims against declared evidence/models and write one report to `.docgen/audit/pages/.json`. Each report must contain `schemaVersion`, `pageId`, `pagePath`, `pageHash` and `inputHash` copied from the batch input, and `findings`. Do not skip a page. - -For each page, audit its traceability sidecar as well as Markdown. Report unsupported/misclassified claims, incomplete evidence/catalog/branch coverage, and stale traceability hashes. Include `claimIds` in findings. diff --git a/global-template/docgen/prompts/audit-risk-indexed.md b/global-template/docgen/prompts/audit-risk-indexed.md new file mode 100644 index 0000000..9893d4c --- /dev/null +++ b/global-template/docgen/prompts/audit-risk-indexed.md @@ -0,0 +1,36 @@ +You are the DocGen selective high-risk auditor. + +Audit contracts: +{{AUDIT_CONTRACTS}} + +Read only: +- each declared `pagePath`; +- each declared bounded `contextPath`. + +Do not read repository source, the semantic database, unrelated pages, broad model directories, agent trees, or external resources. +Do not assume any language, framework, library, protocol, database, messaging system, deployment model, or application architecture. Judge only what the supplied page and bounded context establish. + +Audit only material semantic risk that deterministic checks cannot prove: +- unsupported, overconfident, or materially ambiguous business, security, operational, data, or architectural claims; +- incorrect `FACT`, `INFERENCE`, `ASSUMPTION`, or `UNKNOWN` classification; +- conclusions that exceed the supplied evidence or omit a necessary uncertainty qualifier; +- missing failure branches, exceptions, constraints, preconditions, or recovery implications that materially change meaning; +- contradictions between page prose and supplied context; +- unsafe migration, recovery, security, operational, or data-correctness instructions. + +Write exactly one JSON report: `{{OUTPUT_PATH}}`. +Shape: +{ + "schemaVersion": "2.0", + "generatedAt": "ISO timestamp", + "pages": [ + { + "pageId": "...", + "findings": [ + {"severity":"critical|high|medium|low","statement":"...","evidenceIds":[],"recommendation":"..."} + ] + } + ] +} + +Use an empty `findings` array when no material issue is established. Do not rewrite pages and do not delegate. diff --git a/global-template/docgen/prompts/audit.md b/global-template/docgen/prompts/audit.md deleted file mode 100644 index d55b1e0..0000000 --- a/global-template/docgen/prompts/audit.md +++ /dev/null @@ -1,33 +0,0 @@ -Mandatory source boundary: never read or cite repository source absent from `.docgen/state/source-files.txt`. Existing `.docgen/**` and `docs/**` artifacts are allowed. - -You are running an independent DocGen page audit. - -Page manifest entry: -{{PAGE_JSON}} - -Delegate to the `doc-auditor` custom agent. Audit the generated page against its declared evidence/models and normalized business/flows/catalog models. - -Check: -- unsupported or incorrect claims; -- FACT/INFERENCE confusion; -- missing required sections or coverage tags; -- omitted evidenced business rules, decisions or branch conditions; -- omitted lifecycle/state transitions; -- incomplete business/control/request/traffic/data/event flow steps; -- incomplete endpoint catalog coverage; -- incomplete message producer/consumer/listener/handler coverage; -- incomplete external/cloud/internal dependency coverage; -- missing failure/retry/DLQ/idempotency/order implications when evidenced; -- contradictions with source-grounded models; -- broken cross-links; -- non-Mermaid diagrams or missing planned Mermaid diagrams; -- shallow generic prose that avoids repository-specific detail. - -Write/update only the audit artifact for this page. Never modify application source. - -Current page hash (copy to report as `pageHash`): {{PAGE_HASH}} -Current evidence/model contract hash (copy to report as `inputHash`): {{PAGE_INPUT_HASH}} - -Traceability audit: -- Read the page traceability sidecar. Identify claims with no evidence/model grounding, claims misclassified as FACT, stale hashes, missing catalog/branch item coverage, and claim statements contradicting source-backed models. -- Audit findings should include relevant `claimIds` and evidence references. diff --git a/global-template/docgen/prompts/discover.md b/global-template/docgen/prompts/discover.md deleted file mode 100644 index d129857..0000000 --- a/global-template/docgen/prompts/discover.md +++ /dev/null @@ -1,29 +0,0 @@ -You are running a bounded DocGen discovery stage. - -Scope: {{SCOPE}} - -Delegate evidence extraction to the `doc-discoverer` custom agent. Require the canonical evidence contract and all relevant technology/domain skills. Inspect the requested scope and reconcile `.docgen/evidence/**` plus `.docgen/evidence/index.json`. - -Discovery coverage must be broad enough to support a system knowledge base. When evidenced, extract: - -- repository/module/build/runtime structure and entry points; -- domain nouns, actors, states, validations, guards, decisions and branch conditions; -- HTTP/RPC endpoints and their handlers/contracts; -- Kafka/RabbitMQ/queue/stream producers, consumers, listeners, processors, retry and DLQ handlers; -- persistence entities, tables, migrations, repositories/mappers, transactions and caches; -- external/internal services, cloud services, databases, brokers, object stores, identity providers and other integrations; -- scheduled/background jobs; -- configuration and security boundaries; -- deployment/runtime/network clues relevant to traffic flow; -- tests and examples that reveal supported behavior. - -The evidence index MUST use the canonical top-level `artifacts` array. Each important fact must include source paths. Keep business semantics and implementation facts distinguishable. Preserve unknowns explicitly. - -Completion contract: -- no application source changes; -- no published docs generation; -- evidence index exists and is valid JSON; -- important facts include source paths; -- unknowns remain unknown rather than invented. - -Keep this run bounded to the stated scope. diff --git a/global-template/docgen/prompts/enrich-batch.md b/global-template/docgen/prompts/enrich-batch.md deleted file mode 100644 index 8dab380..0000000 --- a/global-template/docgen/prompts/enrich-batch.md +++ /dev/null @@ -1,23 +0,0 @@ -Mandatory source boundary: never read or cite repository source absent from `.docgen/state/source-files.txt`. Existing `.docgen/**` and `docs/**` artifacts are allowed. - -You are running a targeted DocGen batched enrichment stage. - -Only these pages failed deterministic local quality gates: -{{PAGES_JSON}} - -Delegate to `doc-writer`. Improve each existing target in place. Add only evidence-supported depth needed to satisfy required sections, catalog completeness, flow steps/branches, Mermaid diagram intents, examples, invariants and failure behavior. Do not rewrite unrelated pages. Verify all targets before finishing. - -Claim-level traceability contract: -- In the SAME run, write the companion JSON declared by `traceabilityPath` in the page contract. -- The sidecar must contain `claims[]`; each material repository-specific claim needs: `id`, `section`, `statement`, `classification`, confidence, optional `subject`/`predicate`/`object`/`polarity`, `evidence[]`, and `sourceModelRefs[]`. -- `FACT` claims require direct evidence. Do not manufacture source paths or symbols. -- Populate `coverage.evidenceRefsUsed` with declared evidence/model inputs actually used, and populate model/catalog/branch item refs with their stable IDs. -- Omit pageHash/inputHash if unknown; the orchestrator fills them after the page is written. -- Unknown or disputed behavior belongs in `unknowns[]`, not as a confident claim. - -Traceability contradiction precision: -- Set `exclusivePredicate: true` only when the subject/predicate is single-valued and different objects would be mutually exclusive. -- Leave it false for multi-valued relations such as “has component”, “uses service”, or “emits event”. - - -P2 page experience: follow the page `mode` precisely. Generate evidence-derived examples for declared `exampleIntents`; label placeholders. Include deprecation/migration/rollback information when declared. The orchestrator owns canonical frontmatter and publishing indexes; do not invent renderer-specific metadata. diff --git a/global-template/docgen/prompts/enrich.md b/global-template/docgen/prompts/enrich.md deleted file mode 100644 index 5e2bd66..0000000 --- a/global-template/docgen/prompts/enrich.md +++ /dev/null @@ -1,41 +0,0 @@ -Mandatory source boundary: never read or cite repository source absent from `.docgen/state/source-files.txt`. Existing `.docgen/**` and `docs/**` artifacts are allowed. - -You are running a bounded DocGen depth-and-completeness enrichment pass. - -Page manifest entry: -{{PAGE_JSON}} - -Delegate exactly this page to the `doc-writer` custom agent. Read the existing page, all declared evidence/model inputs, normalized business/flows/catalog models, style guide, glossary, and quality configuration. - -Improve the existing page rather than replacing it with generic prose. Close omissions and shallow areas. Aim for documentation that remains useful across three reading depths: orientation, working understanding, and deep technical/reference use. - -Check for missing supported detail such as: -- business intent, actors, capabilities and outcomes; -- business rules, validations, decisions and explicit branch conditions; -- lifecycle/state transitions and invariants; -- control/execution sequence; -- request path from entry point to response; -- traffic/network/trust boundaries; -- data origin, transformation, ownership, persistence and propagation; -- event/message producer-channel-consumer behavior; -- complete endpoint/message/integration catalog coverage where applicable; -- failure, retry, recovery, idempotency and operational implications; -- concrete examples, decision tables and troubleshooting cues; -- Mermaid diagrams and navigation links. - -All diagrams must be Mermaid. Do not invent unsupported behavior. Preserve useful existing material. Modify only the requested page. - -Claim-level traceability contract: -- In the SAME run, write the companion JSON declared by `traceabilityPath` in the page contract. -- The sidecar must contain `claims[]`; each material repository-specific claim needs: `id`, `section`, `statement`, `classification`, confidence, optional `subject`/`predicate`/`object`/`polarity`, `evidence[]`, and `sourceModelRefs[]`. -- `FACT` claims require direct evidence. Do not manufacture source paths or symbols. -- Populate `coverage.evidenceRefsUsed` with declared evidence/model inputs actually used, and populate model/catalog/branch item refs with their stable IDs. -- Omit pageHash/inputHash if unknown; the orchestrator fills them after the page is written. -- Unknown or disputed behavior belongs in `unknowns[]`, not as a confident claim. - -Traceability contradiction precision: -- Set `exclusivePredicate: true` only when the subject/predicate is single-valued and different objects would be mutually exclusive. -- Leave it false for multi-valued relations such as “has component”, “uses service”, or “emits event”. - - -P2 page experience: follow the page `mode` precisely. Generate evidence-derived examples for declared `exampleIntents`; label placeholders. Include deprecation/migration/rollback information when declared. The orchestrator owns canonical frontmatter and publishing indexes; do not invent renderer-specific metadata. diff --git a/global-template/docgen/prompts/enterprise.md b/global-template/docgen/prompts/enterprise.md deleted file mode 100644 index 53fba56..0000000 --- a/global-template/docgen/prompts/enterprise.md +++ /dev/null @@ -1,28 +0,0 @@ -You are running the DocGen P1 enterprise-depth analysis pass: `{{ENTERPRISE_PASS}}`. - -Delegate to the `doc-enterprise-analyst` custom agent. - -Mandatory source boundary: -- Read `.docgen/state/source-files.txt` before any source inspection. -- Do not read or cite paths excluded by `.gitignore`, `.docgenignore`, or DocGen config. -- Use `.docgen/evidence/**` and `.docgen/model/*.json` as primary inputs. - -Produce exactly the output files listed below and no published Markdown: - -{{OUTPUT_PATHS_JSON}} - -Pass contracts: - -## governance -Produce `security.json` and `ownership.json` covering trust boundaries, principals, authentication, authorization, permissions, service identities, secrets, sensitive data, threats, controls, team/component/data/operations ownership, RACI, approval authority, and escalation paths. - -## operability -Produce `operations.json` and `testing.json` covering runtime components, health/readiness/liveness, logs/metrics/traces, SLI/SLO, alerts, capacity limits, scaling, failure modes, recovery, backup/restore, deployment/rollback, runbooks, test suites/types, fixtures/data, test environments/commands, contract tests, failure injection, quality gates, and coverage gaps. - -## data-and-configuration -Produce `data-governance.json` and `configuration.json` covering data entities, source of truth, ownership, classification, retention/deletion, transaction boundaries, consistency, concurrency/locking, idempotency, reconciliation, lineage, migrations, auditability, settings, environment matrix, flags, secrets, validation, reload/restart behavior, tuning, and deprecations. - -## evolution -Produce `decisions.json` and `change-impact.json` covering recorded ADR decisions, inferred decisions clearly labeled as inference, alternatives, trade-offs, constraints, consequences, superseded decisions, change surfaces, blast-radius edges, compatibility boundaries, safe extension points, migration risks, affected tests/operations/contracts. - -Use empty arrays where evidence is absent. Never invent a security control, SLO, owner, decision rationale, test guarantee, data policy, or configuration default. diff --git a/global-template/docgen/prompts/fix.md b/global-template/docgen/prompts/fix.md deleted file mode 100644 index 249e93f..0000000 --- a/global-template/docgen/prompts/fix.md +++ /dev/null @@ -1,18 +0,0 @@ -Mandatory source boundary: never read or cite repository source absent from `.docgen/state/source-files.txt`. Existing `.docgen/**` and `docs/**` artifacts are allowed. - -You are running a bounded DocGen repair stage. - -Page manifest entry: -{{PAGE_JSON}} - -Audit file: `.docgen/audit/pages/{{PAGE_ID}}.json` - -Delegate exactly this page to the `doc-writer` custom agent. Repair supported audit findings using evidence/models. Do not broaden scope or modify unrelated pages. The repaired page must remain source-grounded and structurally valid. - -Claim-level traceability contract: -- In the SAME run, write the companion JSON declared by `traceabilityPath` in the page contract. -- The sidecar must contain `claims[]`; each material repository-specific claim needs: `id`, `section`, `statement`, `classification`, confidence, optional `subject`/`predicate`/`object`/`polarity`, `evidence[]`, and `sourceModelRefs[]`. -- `FACT` claims require direct evidence. Do not manufacture source paths or symbols. -- Populate `coverage.evidenceRefsUsed` with declared evidence/model inputs actually used, and populate model/catalog/branch item refs with their stable IDs. -- Omit pageHash/inputHash if unknown; the orchestrator fills them after the page is written. -- Unknown or disputed behavior belongs in `unknowns[]`, not as a confident claim. diff --git a/global-template/docgen/prompts/generate-batch.md b/global-template/docgen/prompts/generate-batch.md deleted file mode 100644 index 99d92af..0000000 --- a/global-template/docgen/prompts/generate-batch.md +++ /dev/null @@ -1,30 +0,0 @@ -Mandatory source boundary: never read or cite repository source absent from `.docgen/state/source-files.txt`. Existing `.docgen/**` and `docs/**` artifacts are allowed. - -You are running a bounded DocGen batched page generation stage. - -Page manifest entries: -{{PAGES_JSON}} - -Delegate the batch to the `doc-writer` custom agent. Generate every listed page at its exact canonical `docs/**/*.md` target. Treat each manifest entry as an independent content contract. - -Rules: -- write all listed pages and no unrelated page; -- use exact evidence/model paths from each entry; never substitute invented filenames; -- preserve deep, evidence-grounded coverage, required sections, catalogs, flows, rules, branches, failure behavior, cross-links and Mermaid-only diagrams; -- do not stop after the first page; verify every target exists before finishing; -- if one page cannot be completed, still finish the others and clearly report the missing target. - -Claim-level traceability contract: -- In the SAME run, write the companion JSON declared by `traceabilityPath` in the page contract. -- The sidecar must contain `claims[]`; each material repository-specific claim needs: `id`, `section`, `statement`, `classification`, confidence, optional `subject`/`predicate`/`object`/`polarity`, `evidence[]`, and `sourceModelRefs[]`. -- `FACT` claims require direct evidence. Do not manufacture source paths or symbols. -- Populate `coverage.evidenceRefsUsed` with declared evidence/model inputs actually used, and populate model/catalog/branch item refs with their stable IDs. -- Omit pageHash/inputHash if unknown; the orchestrator fills them after the page is written. -- Unknown or disputed behavior belongs in `unknowns[]`, not as a confident claim. - -Traceability contradiction precision: -- Set `exclusivePredicate: true` only when the subject/predicate is single-valued and different objects would be mutually exclusive. -- Leave it false for multi-valued relations such as “has component”, “uses service”, or “emits event”. - - -P2 page experience: follow the page `mode` precisely. Generate evidence-derived examples for declared `exampleIntents`; label placeholders. Include deprecation/migration/rollback information when declared. The orchestrator owns canonical frontmatter and publishing indexes; do not invent renderer-specific metadata. diff --git a/global-template/docgen/prompts/generate.md b/global-template/docgen/prompts/generate.md deleted file mode 100644 index a4d2590..0000000 --- a/global-template/docgen/prompts/generate.md +++ /dev/null @@ -1,44 +0,0 @@ -Mandatory source boundary: never read or cite repository source absent from `.docgen/state/source-files.txt`. Existing `.docgen/**` and `docs/**` artifacts are allowed. - -You are running a bounded DocGen page generation stage. - -Page manifest entry: -{{PAGE_JSON}} - -Delegate exactly this page to the `doc-writer` custom agent. - -This page is part of a deep multi-page system knowledge base. Treat the manifest as a content contract, not a suggestion. - -Quality contract: -- cover every required section and every coverage tag relevant to this page; -- use all relevant declared evidence/model inputs, including business/flows/catalog models; -- begin with orientation and purpose, then progressively deepen into implementation and operational detail; -- explain boundaries, actors, inputs/outputs, ownership, lifecycle, invariants, rules, decisions, branch conditions and failure behavior when supported; -- for flow pages, enumerate steps and branches before or alongside the diagram; -- for reference pages, be exhaustive over the corresponding normalized catalog rather than sampling a few entries; -- endpoint pages must list all catalogued endpoints in scope and explain handler/security/validation/downstream effects when known; -- messaging pages must list all catalogued producers/consumers/listeners/processors/handlers in scope and explain channel, payload, delivery, retry/DLQ/idempotency/order behavior when known; -- integration pages must list all catalogued external/cloud/internal dependencies in scope and explain direction, protocol, data, auth and failure behavior when known; -- use tables for dense catalogs and decision matrices; -- include practical examples and implementation orientation when supported; -- include cross-links to related pages; -- include source-grounding notes where they materially help maintainers verify behavior; -- all diagrams MUST use fenced `mermaid`; never use PlantUML, Graphviz, ASCII-art diagrams, or image-only diagrams; -- never pad with generic textbook prose and never invent unsupported behavior. - -Write exactly the manifest target path. Do not modify unrelated pages or application source. Validate Markdown structure before finishing. - -Claim-level traceability contract: -- In the SAME run, write the companion JSON declared by `traceabilityPath` in the page contract. -- The sidecar must contain `claims[]`; each material repository-specific claim needs: `id`, `section`, `statement`, `classification`, confidence, optional `subject`/`predicate`/`object`/`polarity`, `evidence[]`, and `sourceModelRefs[]`. -- `FACT` claims require direct evidence. Do not manufacture source paths or symbols. -- Populate `coverage.evidenceRefsUsed` with declared evidence/model inputs actually used, and populate model/catalog/branch item refs with their stable IDs. -- Omit pageHash/inputHash if unknown; the orchestrator fills them after the page is written. -- Unknown or disputed behavior belongs in `unknowns[]`, not as a confident claim. - -Traceability contradiction precision: -- Set `exclusivePredicate: true` only when the subject/predicate is single-valued and different objects would be mutually exclusive. -- Leave it false for multi-valued relations such as “has component”, “uses service”, or “emits event”. - - -P2 page experience: follow the page `mode` precisely. Generate evidence-derived examples for declared `exampleIntents`; label placeholders. Include deprecation/migration/rollback information when declared. The orchestrator owns canonical frontmatter and publishing indexes; do not invent renderer-specific metadata. diff --git a/global-template/docgen/prompts/model-core.md b/global-template/docgen/prompts/model-core.md new file mode 100644 index 0000000..0fceb91 --- /dev/null +++ b/global-template/docgen/prompts/model-core.md @@ -0,0 +1,30 @@ +You are the DocGen core model synthesizer. + +Read exactly one input artifact: `{{CONTEXT_PATH}}`. +Do not read repository source, `.docgen/index/semantic.db`, arbitrary model files, or any path not named in this prompt. +The context pack is already selected, deduplicated, and bounded by the orchestrator. + +Write exactly one JSON file: `{{OUTPUT_PATH}}`. +It must contain these top-level model objects: {{MODEL_NAMES}}. + +Repository-neutral rules: +- detect languages, frameworks, libraries, runtimes, protocols, storage, messaging, build systems, and deployment models only when the supplied evidence supports them; +- never assume HTTP, SQL, a message broker, a database, microservices, a particular programming language, or a named framework; +- represent whatever the repository actually contains: applications, libraries, CLIs, jobs, infrastructure, data pipelines, plugins, embedded systems, monoliths, services, packages, or mixed workspaces; +- preserve repository-relative evidence paths and exact line ranges from the context pack; +- classify every semantic item as FACT, INFERENCE, ASSUMPTION, or UNKNOWN; +- FACT requires direct evidence present in the context pack; +- use stable IDs and explicit kinds; +- keep empty arrays when a concern is not evidenced; +- never invent interfaces, dependencies, behavior, rules, states, flows, data assets, automations, or infrastructure; +- do not write Markdown or modify application source. + +Core shape: +- system: components, modules, packages, relationships, workflows, runtimes, deploymentUnits, unknowns; +- business: actors, capabilities, concepts, businessRules, decisions, branchConditions, lifecycles, invariants, useCases, unknowns; +- flows: businessFlows, controlFlows, requestFlows, trafficFlows, dataFlows, eventFlows, executionFlows; +- catalogs: interfaces, contracts, endpoints, messageHandlers, dependencies, externalDependencies, dataAssets, dataStores, automations, scheduledJobs, buildArtifacts, configurationSurfaces. + +The named arrays are a broad vocabulary, not a checklist. Populate only evidenced concerns and use generic arrays such as `interfaces`, `dependencies`, `dataAssets`, and `automations` when technology-specific categories do not fit. + +Before completion, parse the JSON you wrote and verify every requested top-level object exists. diff --git a/global-template/docgen/prompts/model-enterprise.md b/global-template/docgen/prompts/model-enterprise.md new file mode 100644 index 0000000..fe37ef5 --- /dev/null +++ b/global-template/docgen/prompts/model-enterprise.md @@ -0,0 +1,28 @@ +You are the DocGen enterprise model synthesizer. + +Read exactly one input artifact: `{{CONTEXT_PATH}}`. +Do not read repository source, the SQLite database, or arbitrary files. The context pack is the complete allowed context for this run. + +Write exactly one JSON file: `{{OUTPUT_PATH}}`. +It must contain these top-level model objects: {{MODEL_NAMES}}. + +Repository-neutral rules: +- infer the technology stack only from supplied evidence and do not assume a language, framework, database, broker, protocol, cloud, or deployment style; +- use only evidence and model items present in the context pack; +- FACT items require direct repository-relative evidence with line ranges; +- use INFERENCE, ASSUMPTION, or UNKNOWN when information is incomplete; +- use stable IDs and typed objects; +- preserve explicit unknowns instead of guessing; +- do not write Markdown or application code. + +Expected concerns, only when evidenced: +- security: trust boundaries, principals, authentication, authorization, permissions, identities, secrets, sensitive data, threats, controls; +- operations: runtime, health, observability, service indicators, alerts, capacity, scaling, failures, recovery, backup, deployment, runbooks; +- testing: suites, types, fixtures, environments, commands, contract tests, failure injection, quality gates, gaps; +- data-governance: ownership, source of truth, classification, retention, consistency, concurrency, idempotency, reconciliation, lineage, migrations; +- decisions: recorded and inferred decisions, alternatives, trade-offs, constraints, consequences, supersession; +- configuration: settings, environment matrix, flags, secrets, validation, reload/restart, tuning, deprecation; +- change-impact: change surfaces, direct/transitive effects, compatibility, migration risks, tests and operations affected; +- ownership: team, component, data and operational ownership, RACI, approval, escalation. + +Before completion, parse the JSON and verify every requested top-level object exists. diff --git a/global-template/docgen/prompts/plan-indexed.md b/global-template/docgen/prompts/plan-indexed.md new file mode 100644 index 0000000..b5fde08 --- /dev/null +++ b/global-template/docgen/prompts/plan-indexed.md @@ -0,0 +1,38 @@ +You are the DocGen documentation planner. + +Read exactly one bounded input artifact: `{{CONTEXT_PATH}}`. +Do not read repository source or other files. + +Write exactly one JSON file: `{{OUTPUT_PATH}}`. +The JSON must contain: +- schemaVersion; +- metadata with project description; +- pages[]. + +Each page requires: +- id, title, summary, category, mode, type, order; +- audience[]; +- coverageTags[]; +- query: concise retrieval terms used by the context compiler; +- requiredSections[]; +- risk: low, normal, high, or critical; +- relatedPages[]. + +Plan for the repository that is actually evidenced. Do not assume it is an HTTP service, database application, message-driven system, microservice, Java project, or any other specific stack. +Choose documentation intents from the detected artifacts and behavior: architecture, modules, packages, interfaces, contracts, data, workflows, configuration, operations, testing, security, decisions, onboarding, usage, extension points, or deployment. + +Plan a useful knowledge base, not an arbitrary maximum page count. +Prefer one page per distinct user intent or independently maintainable contract. +Avoid duplicate pages and catch-all pages. +Use reference pages for exhaustive catalogs and narrative pages for explanation, behavior, architecture, decisions, runbooks, and migrations. + +Token-efficiency rules: +- do not create a dedicated page when a section in an existing page answers the same user intent; +- group homogeneous low-risk references into catalogs; +- split only when evidence volume, audience, lifecycle, or change ownership differs; +- target no more than 30 pages unless the context clearly justifies more; +- assign high/critical risk only to material business, security, financial, migration, recovery, safety, or architectural-decision content. + +Useful deterministic coverage tags include generic tags such as `component-catalog`, `interface-catalog`, `dependency-catalog`, `data-asset-catalog`, `automation-catalog`, `configuration-matrix`, `ownership-responsibilities`, and `change-impact`. Technology-specific tags may be used only when evidence supports them. + +Before completion, parse the JSON and verify page IDs and intended paths are unique. diff --git a/global-template/docgen/prompts/plan.md b/global-template/docgen/prompts/plan.md deleted file mode 100644 index 72f4879..0000000 --- a/global-template/docgen/prompts/plan.md +++ /dev/null @@ -1,70 +0,0 @@ -You are running the DocGen documentation planning stage. - -Delegate to the `doc-planner` custom agent. Read: - -- `.docgen/evidence/index.json` and relevant evidence artifacts; -- `.docgen/model/system.json`; -- `.docgen/model/business.json` when present; -- `.docgen/model/flows.json` when present; -- `.docgen/model/catalogs.json` when present; -- `.docgen/model/security.json`, `operations.json`, `testing.json`, `data-governance.json`, `decisions.json`, `configuration.json`, `change-impact.json`, and `ownership.json` when present; -- documentation config, style guide and glossary; -- the existing manifest when present. - -Produce or reconcile `.docgen/plan/manifest.json` conforming to the manifest schema. - -The target is a deep multi-page system knowledge base with the breadth and navigation density of a curated Mintlify-style documentation site. Do not optimize for a small page count. Split major concepts into focused pages when that improves discoverability or depth. Do not create one giant catch-all page. - -Build a navigation taxonomy with categories and pages appropriate to the repository. Cover every evidence-backed surface that matters, including when present: - -- orientation, quickstart, repository map and architecture at a glance; -- business/domain overview, actors, capabilities, glossary and conceptual model; -- business logic, rules, validations, decisions and branch conditions; -- lifecycle/state-machine documentation; -- business flows and use cases; -- control/execution flows; -- inbound request flows; -- traffic/network/trust-boundary flows; -- data models, ownership, transformations, persistence and data flows; -- event/message flows; -- complete endpoint catalog and deeper API behavior pages; -- complete Kafka/RabbitMQ/queue/stream handler catalog; -- external services, cloud services, internal service dependencies and integrations; -- module/component deep dives; -- security overview, trust boundaries, authentication, authorization/permissions, secrets, sensitive-data protection, threats and controls; -- data governance: ownership, source of truth, classification, retention, transactions, consistency, concurrency, idempotency, reconciliation, lineage, migrations and auditability; -- operations: health/readiness, logs/metrics/traces, SLI/SLO when evidenced, alerts, capacity, scaling, failure modes, recovery, backup/restore, deployment/rollback and runbooks; -- testing strategy, suites, fixtures/data, environments, commands, contract tests, failure injection, gates and coverage gaps; -- configuration overview, environment matrix, flags, secrets, validation, reload/restart behavior, tuning and deprecation; -- architecture decisions, alternatives, trade-offs, constraints, consequences and supersession; -- ownership/RACI/approval/escalation and change-impact/blast-radius/compatibility/extension-point documentation; -- persistence, configuration, security and observability; -- local development and common engineering tasks; -- deployment/runtime architecture; -- operations, failure modes, recovery and troubleshooting; -- reference pages where exhaustive lists are useful. - -Every page must define category, purpose, summary, audience, evidence/models, required sections, Mermaid diagram intents, coverageTags, related pages, document mode, search keywords, aliases where relevant, lifecycle/version/deprecation metadata, evidence-derived example intents, and optional required tables/quality hints. - -Required coverage tags are conditional on evidence. Examples: -`system-overview`, `architecture`, `security-trust-boundaries`, `authorization-model`, `data-governance`, `consistency-transactions`, `operations-observability`, `failure-recovery`, `testing-strategy`, `configuration-matrix`, `architecture-decisions`, `change-impact`, `ownership-responsibilities`, `business-domain`, `business-rules`, `branch-conditions`, `state-lifecycle`, `business-flow`, `control-flow`, `request-flow`, `traffic-flow`, `data-model`, `data-flow`, `event-flow`, `endpoint-catalog`, `message-handler-catalog`, `external-dependency-catalog`, `persistence`, `security`, `configuration`, `operations`, `troubleshooting`. - -{{MISSING_COVERAGE}} - -Avoid duplicate ownership of the same concept. Preserve stable page ids/paths where reasonable. All planned diagrams must be Mermaid. - - -Hard manifest rules: -- every page path must be canonical: `docs//.md`; -- `evidence[]` and `models[]` must contain exact existing repository-relative paths or exact evidence artifact IDs from `.docgen/evidence/index.json`; -- never invent shorthand filenames such as `system.json` when the actual path is `.docgen/model/system.json`; -- before finishing, verify all page ids, paths, navigation references, evidence references and model references. - - -P2 documentation experience rules: -- choose exactly one primary mode: tutorial, how-to, explanation, reference, runbook, decision-record, migration-guide, or troubleshooting; -- plan user journeys from orientation to task completion to deep reference; -- create migration/deprecation pages when evidence exists; -- use stable aliases for renamed/moved pages; -- declare exampleIntents only when examples can be grounded in evidence/tests/contracts; -- include searchKeywords and clear relatedPages/backlinks. diff --git a/global-template/docgen/prompts/semantics.md b/global-template/docgen/prompts/semantics.md deleted file mode 100644 index 91eede4..0000000 --- a/global-template/docgen/prompts/semantics.md +++ /dev/null @@ -1,38 +0,0 @@ -You are running the DocGen business-and-system semantics stage. - -Respect `.docgen/state/source-files.txt`; ignored repository files must not be read or cited. Existing `.docgen/evidence/**` and `.docgen/model/**` are allowed. - -Delegate to the `doc-domain-analyst` custom agent. Read `.docgen/evidence/**` and `.docgen/model/system.json` as primary inputs; inspect source only for targeted verification. - -Produce exactly these normalized model files: - -1. `.docgen/model/business.json` - Required arrays: actors, capabilities, concepts, businessRules, decisions, branchConditions, lifecycles, invariants, useCases, unknowns. - -2. `.docgen/model/flows.json` - Required arrays: businessFlows, controlFlows, requestFlows, trafficFlows, dataFlows, eventFlows. - -3. `.docgen/model/catalogs.json` - Required arrays: endpoints, messageHandlers, externalDependencies, dataStores, scheduledJobs. - -Coverage requirements: -- capture business logic, rules, validation, eligibility, state/lifecycle logic and explicit branch conditions; -- distinguish business flow from execution/control flow; -- model inbound request flow end-to-end; -- model traffic/network hops and trust boundaries when evidenced; -- model data origin, validation, transformation, ownership, persistence and propagation; -- model event/message flow including producer, channel, consumer, retry/DLQ/idempotency/order semantics when evidenced; -- inventory every evidenced endpoint; -- inventory every evidenced Kafka/RabbitMQ/queue/stream handler and producer; -- inventory external services, cloud services, internal services, databases, caches, brokers and other dependencies. - -Use FACT / INFERENCE / UNKNOWN and evidence references. Empty arrays are valid when no evidence exists. Never invent a business rule or infrastructure hop. - -Do not write published documentation. - -P0 typed-output contract: -- Do not emit scalar strings in semantic arrays. Emit typed objects with stable IDs. -- Every item requires `kind`, `classification` (`FACT`, `INFERENCE`, or `UNKNOWN`), confidence 0..1, and `evidence[]`. -- Business rules must identify statement, trigger/conditions, outcome, exceptions/failure outcome when evidenced. -- Decisions and flows must enumerate branches; catalog entries must expose their identifying contract fields. -- Facts without direct evidence are invalid; downgrade them to INFERENCE or UNKNOWN rather than inventing evidence. diff --git a/global-template/docgen/prompts/update-impact.md b/global-template/docgen/prompts/update-impact.md deleted file mode 100644 index 54fd8d3..0000000 --- a/global-template/docgen/prompts/update-impact.md +++ /dev/null @@ -1,6 +0,0 @@ -You are running DocGen incremental impact analysis. - -Changed paths: -{{CHANGED_PATHS_JSON}} - -Use the `doc-architect` and `doc-planner` capabilities as needed to determine impact. Produce `.docgen/plan/update-plan.json` conforming to its schema with affected evidence scopes, model artifacts, stable page ids, and concise rationale. Do not regenerate pages in this run. diff --git a/global-template/docgen/prompts/workspace-synthesis.md b/global-template/docgen/prompts/workspace-synthesis.md deleted file mode 100644 index edacd57..0000000 --- a/global-template/docgen/prompts/workspace-synthesis.md +++ /dev/null @@ -1,13 +0,0 @@ -You are the P3 system-of-systems synthesis stage. - -Inputs are validated `.docgen-workspace/model/**`, repository registries, shared contracts, and traceability. Do not rescan arbitrary source repositories. - -Improve only evidence-supported explanations for: -- repository/system boundaries; -- cross-repository dependencies; -- shared contracts and ownership; -- end-to-end business journeys; -- request, event, control, and data flows; -- cross-repository blast radius. - -Preserve unresolved edges and UNKNOWN classifications. All diagrams must be Mermaid. Never invent a service, contract, transition, owner, or data movement. diff --git a/global-template/docgen/prompts/write-pages-indexed.md b/global-template/docgen/prompts/write-pages-indexed.md new file mode 100644 index 0000000..7137d50 --- /dev/null +++ b/global-template/docgen/prompts/write-pages-indexed.md @@ -0,0 +1,45 @@ +You are the DocGen bounded documentation writer. + +Page contracts: +{{PAGE_CONTRACTS}} + +For every contract: +1. Read only its declared `contextPath`. +2. Write exactly its declared `outputPath`. +3. Write exactly its declared `traceabilityPath` in the same run. +4. Do not read repository source, SQLite, broad `.docgen/model/**`, existing unrelated pages, agents, or skills. +5. Treat the context pack as the complete allowed factual context. + +Repository-neutral writing rules: +- describe only languages, frameworks, libraries, protocols, storage, messaging, build systems, infrastructure, or deployment models directly supported by context evidence; +- never add generic framework background or assume a conventional architecture; +- preserve FACT / INFERENCE / ASSUMPTION / UNKNOWN distinctions; +- cite repository-relative evidence paths and exact line ranges inline where useful; +- never invent behavior absent from the context pack; +- include every required section and honor the page mode; +- use Mermaid for diagrams; never PlantUML, Graphviz, or external image generation; +- explain branches, failures, unknowns, compatibility, and operational consequences when evidenced; +- keep cross-links limited to related page IDs/paths in the contract; +- include YAML frontmatter with title, description, pageId, category, mode, type, and order; +- verify every output exists and contains one H1 before finishing. + +Traceability sidecar shape: +{ + "schemaVersion": "2.0", + "pageId": "contract page id", + "pagePath": "contract output path", + "claims": [ + { + "id": "stable page-scoped claim id", + "section": "heading containing the claim", + "statement": "material repository-specific claim copied or faithfully represented in the page", + "classification": "FACT|INFERENCE|ASSUMPTION|UNKNOWN", + "confidence": 0.0, + "evidence": [{"path":"repository-relative path","startLine":1,"endLine":1}], + "sourceModelRefs": ["qualified model item id from context"] + } + ] +} + +A FACT claim requires direct evidence that was supplied in the bounded context. Every sourceModelRef must also be present in that context. The orchestrator will fill pageHash, inputHash, and contextId. +Do not delegate to another agent. Complete the bounded write directly. diff --git a/global-template/docgen/schemas/quality-summary.schema.json b/global-template/docgen/schemas/quality-summary.schema.json index d8cc83e..aa26f96 100644 --- a/global-template/docgen/schemas/quality-summary.schema.json +++ b/global-template/docgen/schemas/quality-summary.schema.json @@ -1,57 +1,30 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "docgen:quality-summary:1.0", + "$id": "docgen:quality-summary:2.0", "type": "object", "required": [ - "schemaVersion", - "generatedAt", - "qualityProfile", - "sourceSnapshot", - "pages", - "localGateFailures", - "warningCount", - "semanticSummary", - "auditSummary" + "schemaVersion", "generatedAt", "auditInputHash", "inventoryFingerprint", + "manifestHash", "pages", "claims", "evidenceReferences", "modelItems", + "referencedModelItems", "modelReferenceCoverage", "deterministicFailures", + "deterministicWarnings", "llmAuditedPages", "highRiskFindings", "pass" ], "properties": { - "schemaVersion": { - "const": "1.0" - }, - "generatedAt": { - "type": "string" - }, - "qualityProfile": { - "type": "string" - }, - "sourceSnapshot": { - "type": "object" - }, - "pages": { - "type": "array" - }, - "localGateFailures": { - "type": "integer", - "minimum": 0 - }, - "warningCount": { - "type": "integer", - "minimum": 0 - }, - "semanticSummary": { - "type": "object", - "required": [ - "claims", - "grounded", - "unsupported", - "stale", - "claimGroundingRatio", - "contradictions", - "duplicateGroups", - "stalePages" - ] - }, - "auditSummary": { - "type": "object" - } - } + "schemaVersion": { "const": "2.0" }, + "generatedAt": { "type": "string" }, + "auditInputHash": { "type": "string" }, + "inventoryFingerprint": { "type": ["string", "null"] }, + "manifestHash": { "type": "string" }, + "pages": { "type": "integer", "minimum": 0 }, + "claims": { "type": "integer", "minimum": 0 }, + "evidenceReferences": { "type": "integer", "minimum": 0 }, + "modelItems": { "type": "integer", "minimum": 0 }, + "referencedModelItems": { "type": "integer", "minimum": 0 }, + "modelReferenceCoverage": { "type": "number", "minimum": 0, "maximum": 1 }, + "deterministicFailures": { "type": "integer", "minimum": 0 }, + "deterministicWarnings": { "type": "integer", "minimum": 0 }, + "llmAuditedPages": { "type": "integer", "minimum": 0 }, + "highRiskFindings": { "type": "integer", "minimum": 0 }, + "pass": { "type": "boolean" } + }, + "additionalProperties": true } diff --git a/global-template/docgen/schemas/semantic-item.schema.json b/global-template/docgen/schemas/semantic-item.schema.json index 6d4f10f..87806d1 100644 --- a/global-template/docgen/schemas/semantic-item.schema.json +++ b/global-template/docgen/schemas/semantic-item.schema.json @@ -88,6 +88,7 @@ "enum": [ "FACT", "INFERENCE", + "ASSUMPTION", "UNKNOWN" ] }, diff --git a/global-template/docgen/schemas/traceability.schema.json b/global-template/docgen/schemas/traceability.schema.json index f60ab95..9c40b43 100644 --- a/global-template/docgen/schemas/traceability.schema.json +++ b/global-template/docgen/schemas/traceability.schema.json @@ -1,134 +1,45 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "docgen:traceability:1.0", + "$id": "docgen:traceability:2.0", "type": "object", - "required": [ - "schemaVersion", - "generatedAt", - "pageId", - "pagePath", - "pageHash", - "inputHash", - "sourceSnapshot", - "claims", - "coverage", - "unknowns", - "legacyUnmapped" - ], + "required": ["schemaVersion", "pageId", "pagePath", "pageHash", "inputHash", "claims"], "properties": { - "schemaVersion": { - "const": "1.0" - }, - "generatedAt": { - "type": "string" - }, - "pageId": { - "type": "string" - }, - "pagePath": { - "type": "string", - "pattern": "^docs/.+\\.md$" - }, - "pageHash": { - "type": [ - "string", - "null" - ] - }, - "inputHash": { - "type": [ - "string", - "null" - ] - }, - "sourceSnapshot": { - "type": "object", - "required": [ - "capturedAt", - "commit", - "branch", - "dirty", - "sourceFingerprint" - ], - "properties": { - "capturedAt": { - "type": "string" - }, - "commit": { - "type": [ - "string", - "null" - ] - }, - "branch": { - "type": [ - "string", - "null" - ] - }, - "dirty": { - "type": [ - "boolean", - "null" - ] - }, - "sourceFingerprint": { - "type": [ - "string", - "null" - ] - } - } - }, + "schemaVersion": { "const": "2.0" }, + "generatedAt": { "type": "string" }, + "pageId": { "type": "string", "minLength": 1 }, + "pagePath": { "type": "string", "pattern": "^docs/.+\\.md$" }, + "pageHash": { "type": "string", "minLength": 1 }, + "inputHash": { "type": "string", "minLength": 1 }, + "contextId": { "type": ["string", "null"] }, "claims": { "type": "array", "items": { - "$ref": "semantic-item.schema.json#/$defs/claim" - } - }, - "coverage": { - "type": "object", - "required": [ - "evidenceRefsUsed", - "modelItemRefs", - "catalogItemRefs", - "branchItemRefs" - ], - "properties": { - "evidenceRefsUsed": { - "type": "array", - "items": { - "type": "string" - } + "type": "object", + "required": ["id", "section", "statement", "classification", "confidence", "evidence", "sourceModelRefs"], + "properties": { + "id": { "type": "string", "minLength": 1 }, + "section": { "type": "string" }, + "statement": { "type": "string", "minLength": 1 }, + "classification": { "enum": ["FACT", "INFERENCE", "ASSUMPTION", "UNKNOWN"] }, + "confidence": { "type": "number", "minimum": 0, "maximum": 1 }, + "evidence": { + "type": "array", + "items": { + "type": "object", + "required": ["path"], + "properties": { + "path": { "type": "string", "minLength": 1 }, + "startLine": { "type": "integer", "minimum": 1 }, + "endLine": { "type": "integer", "minimum": 1 } + }, + "additionalProperties": true + } + }, + "sourceModelRefs": { "type": "array", "items": { "type": "string" } } }, - "modelItemRefs": { - "type": "array", - "items": { - "type": "string" - } - }, - "catalogItemRefs": { - "type": "array", - "items": { - "type": "string" - } - }, - "branchItemRefs": { - "type": "array", - "items": { - "type": "string" - } - } - } - }, - "unknowns": { - "type": "array", - "items": { - "type": "string" + "additionalProperties": true } - }, - "legacyUnmapped": { - "type": "boolean" } - } + }, + "additionalProperties": true } diff --git a/global-template/docgen/test/fixtures/fake-provider.mjs b/global-template/docgen/test/fixtures/fake-provider.mjs new file mode 100755 index 0000000..4b8318a --- /dev/null +++ b/global-template/docgen/test/fixtures/fake-provider.mjs @@ -0,0 +1,114 @@ +#!/usr/bin/env node +import fs from 'node:fs'; +import path from 'node:path'; + +const maxTurnsIndex = process.argv.indexOf('--max-turns'); +const maxTurns = maxTurnsIndex >= 0 ? Number(process.argv[maxTurnsIndex + 1]) : 0; +if (!Number.isFinite(maxTurns) || maxTurns < 30) { + console.error(`Warning: Reached maximum conversation turns (${maxTurns || 'missing'}). Retry with --max-turns 30.`); + process.exit(8); +} + +const prompt = fs.readFileSync(0, 'utf8').replace(/\r\n?/g, '\n'); +const stage = process.env.DOCGEN_STAGE; +const cwd = process.cwd(); +const tick = String.fromCharCode(96); +const between = (text, start, end) => { + const tail = text.split(start)[1]; + return tail ? tail.split(end)[0] : null; +}; +const target = between(prompt, `Write exactly one JSON file: ${tick}`, tick); +const write = (rel, value) => { + if (!rel) throw new Error(`missing output path for ${stage}`); + const file = path.join(cwd, rel); + fs.mkdirSync(path.dirname(file), { recursive: true }); + fs.writeFileSync(file, JSON.stringify(value, null, 2) + '\n'); +}; + +if (process.env.DOCGEN_TEST_FAIL_BEFORE_WRITE_STAGE === stage) { + console.error(`simulated provider failure before writing ${stage} artifacts`); + process.exit(8); +} + +if (stage === 'modelCore') { + write(target, { + system: { + components: [{ id: 'resource', kind: 'component', name: 'Resource', statement: 'Source component', classification: 'FACT', confidence: 1, evidence: [{ path: 'src/Resource.java', startLine: 1 }] }], + relationships: [], workflows: [], unknowns: [] + }, + business: { actors: [], capabilities: [], concepts: [], businessRules: [], decisions: [], branchConditions: [], lifecycles: [], invariants: [], useCases: [], unknowns: [] }, + flows: { businessFlows: [], controlFlows: [], requestFlows: [], trafficFlows: [], dataFlows: [], eventFlows: [] }, + catalogs: { interfaces: [], contracts: [], endpoints: [], messageHandlers: [], dependencies: [], externalDependencies: [], dataAssets: [], dataStores: [], automations: [], scheduledJobs: [], buildArtifacts: [], configurationSurfaces: [] } + }); +} else if (stage === 'modelEnterprise') { + write(target, { + security: { unknowns: [] }, operations: { unknowns: [] }, testing: { unknowns: [] }, + 'data-governance': { unknowns: [] }, decisions: { unknowns: [] }, configuration: { unknowns: [] }, + 'change-impact': { unknowns: [] }, ownership: { unknowns: [] } + }); +} else if (stage === 'plan') { + const pageCount = Math.max(1, Number(process.env.DOCGEN_TEST_PAGE_COUNT || 1)); + write(target, { + schemaVersion: '2.0', metadata: { description: 'Fixture docs' }, + pages: Array.from({ length: pageCount }, (_, index) => ({ + id: index === 0 ? 'overview' : `detail-${index + 1}`, + title: index === 0 ? 'System Overview' : `System Detail ${index + 1}`, + summary: index === 0 ? 'Fixture overview.' : `Fixture detail ${index + 1}.`, + category: 'orientation', mode: 'explanation', type: 'overview', order: index + 1, + audience: ['engineer'], coverageTags: ['architecture'], query: 'resource architecture', + requiredSections: [], risk: 'low', relatedPages: [] + })) + }); +} else if (stage === 'generate') { + const json = between(prompt, 'Page contracts:\n', '\n\nFor every contract:'); + const contracts = JSON.parse(json); + const partialMarker = path.join(cwd, '.docgen', 'test-partial-generate.marker'); + const partialFirstAttempt = process.env.DOCGEN_TEST_PARTIAL_GENERATE === '1' && !fs.existsSync(partialMarker); + const selected = partialFirstAttempt ? contracts.slice(0, 1) : contracts; + for (const contract of selected) { + const page = contract.page; + const md = `--- +title: ${JSON.stringify(page.title)} +description: ${JSON.stringify(page.summary)} +pageId: ${JSON.stringify(page.id)} +category: ${JSON.stringify(page.category)} +mode: ${JSON.stringify(page.mode)} +type: ${JSON.stringify(page.type)} +order: ${page.order} +--- +# ${page.title} + +${page.summary} + +The repository contains a source component. +`; + const output = path.join(cwd, contract.outputPath); + fs.mkdirSync(path.dirname(output), { recursive: true }); + fs.writeFileSync(output, md); + write(contract.traceabilityPath, { + schemaVersion: '2.0', pageId: page.id, pagePath: contract.outputPath, + claims: [{ + id: `${page.id}:resource`, section: page.title, + statement: 'The repository contains a source component.', classification: 'FACT', confidence: 1, + evidence: [{ path: 'src/Resource.java', startLine: 1 }], sourceModelRefs: ['system:resource'] + }] + }); + } + if (partialFirstAttempt) { + fs.writeFileSync(partialMarker, 'partial\n'); + console.error('simulated provider exit after partial valid generation'); + process.exit(8); + } +} else if (stage === 'audit') { + const output = between(prompt, `report: ${tick}`, tick) || '.docgen/audit/llm-risk.json'; + write(output, { schemaVersion: '2.0', pages: [] }); +} else { + console.error(`unexpected stage ${stage}`); + process.exitCode = 2; +} + + +if (process.env.DOCGEN_TEST_EXIT_AFTER_WRITE_STAGE === stage) { + console.error(`simulated provider exit after valid ${stage} artifacts`); + process.exit(8); +} diff --git a/global-template/docgen/test/git-inventory-regression.test.mjs b/global-template/docgen/test/git-inventory-regression.test.mjs new file mode 100644 index 0000000..680fe7b --- /dev/null +++ b/global-template/docgen/test/git-inventory-regression.test.mjs @@ -0,0 +1,92 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { spawnSync } from 'node:child_process'; +import { fileURLToPath } from 'node:url'; +import { buildInventory } from '../lib/inventory.mjs'; +import { projectPaths, writeJson } from '../lib/core.mjs'; + +const testDir = path.dirname(fileURLToPath(import.meta.url)); +const engineRoot = path.resolve(testDir, '..'); +const repositoryRoot = path.resolve(engineRoot, '..', '..'); +const misspelledBoolean = ['Bole', 'an'].join(''); + +function fixture() { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'docgen-git-inventory-')); + const paths = projectPaths(root); + fs.mkdirSync(path.join(root, 'src'), { recursive: true }); + fs.mkdirSync(path.dirname(paths.config), { recursive: true }); + writeJson(paths.project, { schemaVersion: '2.0', kitVersion: '2.0.0' }); + writeJson(paths.config, { + schemaVersion: '2.0', + ignore: { + useGitignore: true, + useDocgenignore: true, + binary: { enabled: true, maxTextFileBytes: 1024 * 1024 } + }, + execution: { progress: false } + }); + return root; +} + +function initializeGitFixture() { + const root = fixture(); + const init = spawnSync('git', ['init'], { cwd: root, encoding: 'utf8' }); + assert.equal(init.status, 0, init.stderr || init.stdout); + fs.writeFileSync(path.join(root, 'src', 'Tracked.java'), 'class Tracked {}\n'); + fs.writeFileSync(path.join(root, 'ignored.log'), 'ignore me\n'); + fs.writeFileSync(path.join(root, '.gitignore'), '*.log\n'); + return root; +} + +test('git-aware inventory executes native git path enumeration', () => { + const root = initializeGitFixture(); + const inventory = buildInventory(root); + assert(inventory.files.some((item) => item.path === 'src/Tracked.java')); + assert(!inventory.files.some((item) => item.path === 'ignored.log')); +}); + +test('installed launcher indexes a real Git repository', () => { + const commandCodeHome = fs.mkdtempSync(path.join(os.tmpdir(), 'docgen-installed-home-')); + const install = spawnSync(process.execPath, [ + path.join(repositoryRoot, 'install.mjs'), + '--force', + '--no-link-cli', + '--no-hooks', + '--commandcode-home', + commandCodeHome + ], { cwd: repositoryRoot, encoding: 'utf8' }); + assert.equal(install.status, 0, `INSTALL STDERR:\n${install.stderr}\nINSTALL STDOUT:\n${install.stdout}`); + + const root = initializeGitFixture(); + const launcher = path.join(commandCodeHome, 'docgen', 'bin', 'docgen-launcher.mjs'); + const run = spawnSync(process.execPath, [launcher, 'index'], { + cwd: root, + encoding: 'utf8', + env: { ...process.env, DOCGEN_PROGRESS: '0' } + }); + const combinedOutput = `${run.stdout}\n${run.stderr}`; + assert.equal(run.status, 0, `INDEX STDERR:\n${run.stderr}\nINDEX STDOUT:\n${run.stdout}`); + assert(!combinedOutput.includes(`${misspelledBoolean} is not defined`)); + assert(fs.existsSync(path.join(root, '.docgen', 'index', 'semantic.db'))); +}); + +test('shipped JavaScript contains no misspelled Boolean global', () => { + const pattern = new RegExp(`\\b${misspelledBoolean}\\b`); + const stack = [engineRoot]; + const offenders = []; + while (stack.length) { + const dir = stack.pop(); + for (const entry of fs.readdirSync(dir, { withFileTypes: true })) { + if (['node_modules', '.git'].includes(entry.name)) continue; + const file = path.join(dir, entry.name); + if (entry.isDirectory()) stack.push(file); + else if (/\.(?:mjs|js|cjs)$/.test(entry.name) && pattern.test(fs.readFileSync(file, 'utf8'))) { + offenders.push(path.relative(engineRoot, file)); + } + } + } + assert.deepEqual(offenders, []); +}); diff --git a/global-template/docgen/test/semantic-index.test.mjs b/global-template/docgen/test/semantic-index.test.mjs new file mode 100644 index 0000000..c428923 --- /dev/null +++ b/global-template/docgen/test/semantic-index.test.mjs @@ -0,0 +1,216 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { spawnSync } from 'node:child_process'; +import { fileURLToPath } from 'node:url'; +import { buildInventory } from '../lib/inventory.mjs'; +import { compileContext } from '../lib/context.mjs'; +import { databaseStats, indexRepository, openDatabase } from '../lib/indexer.mjs'; +import { audit, generate, publish } from '../lib/pipeline.mjs'; +import { projectPaths, readJson, writeJson } from '../lib/core.mjs'; + +const testDir = path.dirname(fileURLToPath(import.meta.url)); +const cli = path.resolve(testDir, '..', 'bin', 'docgen-v2.mjs'); + +function fixture() { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'docgen-v2-')); const p = projectPaths(root); + fs.mkdirSync(path.join(root, 'src'), { recursive: true }); + fs.mkdirSync(path.dirname(p.config), { recursive: true }); + writeJson(p.project, { schemaVersion: '2.0', kitVersion: '2.0.0' }); + writeJson(p.config, { schemaVersion: '2.0', projectName: 'Fixture', ignore: { useGitignore: true, useDocgenignore: true, binary: { enabled: true, maxTextFileBytes: 1024 * 1024 } }, context: { maxTokens: { default: 4000, modelCore: 4000, modelEnterprise: 4000, plan: 4000, generate: 4000, audit: 2000 } }, budget: { maxProviderCalls: 10, maxEstimatedInputTokens: 200000, maxEstimatedOutputTokens: 50000, maxContextTokensPerCall: 10000 }, execution: { generationBatchSize: 2, maxPlannedPages: 30 }, audit: { llmEnabled: false }, retry: { maxAttempts: 1 } }); + writeJson(p.state, { schemaVersion: '2.0', kitVersion: '2.0.0', stages: {}, pages: {} }); + return root; +} + +function catalogFixture(root) { + const p = projectPaths(root); + fs.writeFileSync(path.join(root, 'src', 'Resource.java'), '@Path("/quotes")\nclass Resource { @POST void create() {} }\n'); + indexRepository(root, { force: true }); + fs.mkdirSync(p.model, { recursive: true }); + writeJson(path.join(p.model, 'catalogs.json'), { schemaVersion: '2.0', endpoints: [{ id: 'create-quote', name: 'Create quote', statement: 'Creates a quote.', classification: 'FACT', confidence: 1, method: 'POST', path: '/quotes', evidence: [{ path: 'src/Resource.java', startLine: 1 }] }], messageHandlers: [], externalDependencies: [], dataStores: [], scheduledJobs: [] }); + writeJson(p.plan, { schemaVersion: '2.0', pages: [{ id: 'endpoint-catalog', title: 'Endpoint Catalog', summary: 'HTTP API reference.', category: 'api', mode: 'reference', type: 'reference', order: 1, audience: ['engineer'], coverageTags: ['endpoint-catalog'], query: 'endpoints', requiredSections: [], relatedPages: [] }] }); + return p; +} + +function installFakeProvider(root) { + const p = projectPaths(root); const dir = path.join(p.base, 'test-bin'); fs.mkdirSync(dir, { recursive: true }); + const source = path.join(testDir, 'fixtures', 'fake-provider.mjs'); + const file = path.join(dir, 'fake-provider.mjs'); + fs.copyFileSync(source, file); fs.chmodSync(file, 0o755); + const check = spawnSync(process.execPath, ['--check', file], { encoding: 'utf8' }); + assert.equal(check.status, 0, check.stderr || check.stdout); + if (process.platform !== 'win32') return file; + const shim = path.join(dir, 'fake-provider.cmd'); + fs.writeFileSync(shim, `@echo off\r\n"${process.execPath}" "${file}" %*\r\n`); + return shim; +} + +test('inventory excludes binary and docgenignore paths', () => { + const root = fixture(); + fs.writeFileSync(path.join(root, 'src', 'Resource.java'), '@Path("/quotes")\nclass Resource { @GET void get() {} }\n'); + fs.writeFileSync(path.join(root, 'secret.txt'), 'ignore me'); + fs.writeFileSync(path.join(root, 'image.png'), Buffer.from([0x89,0x50,0x4e,0x47,0,1])); + fs.writeFileSync(path.join(root, '.docgenignore'), 'secret.txt\n'); + const inv = buildInventory(root); + assert(inv.files.some((item) => item.path === 'src/Resource.java')); + assert(!inv.files.some((item) => item.path === 'secret.txt')); + assert(inv.excluded.some((item) => item.path === 'image.png')); +}); + +test('non-git inventory respects nested gitignore files', () => { + const root = fixture(); + fs.mkdirSync(path.join(root, 'src', 'generated'), { recursive: true }); + fs.writeFileSync(path.join(root, 'src', 'generated', '.gitignore'), '*.txt\n'); + fs.writeFileSync(path.join(root, 'src', 'generated', 'secret.txt'), 'ignored'); + fs.writeFileSync(path.join(root, 'src', 'generated', 'public.java'), 'class Public {}'); + const inv = buildInventory(root); + assert(!inv.files.some((item) => item.path === 'src/generated/secret.txt')); + assert(inv.files.some((item) => item.path === 'src/generated/public.java')); +}); + +test('index is incremental and extracts source chunks and facts', () => { + const root = fixture(); + fs.writeFileSync(path.join(root, 'src', 'Resource.java'), '@Path("/quotes")\nclass Resource { @POST void create() {} }\n'); + const first = indexRepository(root, { force: true }); + const second = indexRepository(root); + const stats = databaseStats(root); + assert.equal(first.changedFiles > 0, true); + assert.equal(second.changedFiles, 0); + assert.equal(second.unchangedFiles > 0, true); + assert.equal(stats.facts > 0, true); + assert.equal(stats.sourceChunks > 0, true); +}); + +test('context compiler stays within configured budget', () => { + const root = fixture(); + for (let i = 0; i < 60; i++) fs.writeFileSync(path.join(root, 'src', `Resource${i}.java`), `@Path("/items/${i}")\nclass Resource${i} { @GET void get${i}() {} }\n`); + indexRepository(root, { force: true }); + const { payload } = compileContext(root, { stage: 'generate', target: 'api', query: 'endpoint path resource', maxTokens: 1200 }); + assert(payload.estimatedTokens <= 1200); + assert(payload.facts.length > 0); + assert(payload.omissions.facts > 0); +}); + +test('reference catalog page is deterministic, traced, auditable, and reusable', async () => { + const root = fixture(); const p = catalogFixture(root); + const first = await generate(root); const second = await generate(root); + assert.equal(first.providerPages, 0); assert.equal(second.providerPages, 0); + const output = fs.readFileSync(path.join(root, 'docs', 'api', 'endpoint-catalog.md'), 'utf8'); + assert.match(output, /POST/); assert.match(output, /\/quotes/); + const trace = readJson(path.join(p.traceability, 'pages', 'endpoint-catalog.json')); + assert.equal(trace.claims.length, 1); assert.equal(trace.claims[0].classification, 'FACT'); assert.equal(trace.claims[0].evidence[0].path, 'src/Resource.java'); + const quality = await audit(root); assert.equal(quality.pass, true); + assert.equal(fs.existsSync(path.join(p.telemetry, 'provider-runs.jsonl')), false); +}); + +test('deterministic audit rejects FACT evidence outside inventory', async () => { + const root = fixture(); const p = catalogFixture(root); await generate(root); + const file = path.join(p.traceability, 'pages', 'endpoint-catalog.json'); const trace = readJson(file); trace.claims[0].evidence = [{ path: 'ignored/secret.java' }]; writeJson(file, trace); + await assert.rejects(() => audit(root), /Quality failed/); +}); + +test('full indexed pipeline uses four provider calls, one index pass, minimum 30 turns, then zero calls on resume', () => { + const root = fixture(); const p = projectPaths(root); const provider = installFakeProvider(root); + fs.writeFileSync(path.join(root, 'src', 'Resource.java'), '@Path("/quotes")\nclass Resource { @GET void get() {} }\n'); + const config = readJson(p.config); config.commandCode = { executable: provider, trust: false, skipOnboarding: false, yolo: false, verbose: false, maxTurns: { default: 4, generate: 12 } }; writeJson(p.config, config); + const env = { ...process.env, DOCGEN_PROGRESS: '1', DOCGEN_MAX_TURNS: '12' }; + const first = spawnSync(process.execPath, [cli, 'all'], { cwd: root, encoding: 'utf8', env }); assert.equal(first.status, 0, `FIRST STDERR:\n${first.stderr}\nFIRST STDOUT:\n${first.stdout}`); + assert.equal((first.stdout.match(/\[docgen\] index RUNNING/g) ?? []).length, 1, first.stdout); + assert.match(first.stdout, /maxTurns 30/); + const healed = readJson(p.config).commandCode.maxTurns; for (const value of Object.values(healed)) assert.equal(value, 30); + const firstBudget = readJson(p.budget); assert.equal(firstBudget.usage.providerCalls, 4); assert.equal(firstBudget.usage.failedCalls, 0); + const second = spawnSync(process.execPath, [cli, 'all'], { cwd: root, encoding: 'utf8', env }); assert.equal(second.status, 0, `SECOND STDERR:\n${second.stderr}\nSECOND STDOUT:\n${second.stdout}`); + const secondBudget = readJson(p.budget); assert.equal(secondBudget.usage.providerCalls, 4); + const summary = readJson(path.join(p.audit, 'quality-summary.json')); assert.equal(summary.pass, true); assert.equal(summary.claims, 1); assert.equal(summary.evidenceReferences, 1); +}); + + +test('generic semantic index extracts cross-language artifacts without requiring a specific stack', () => { + const root = fixture(); + fs.writeFileSync(path.join(root, 'src', 'worker.py'), 'import asyncio\nclass Worker:\n def run(self):\n return True\n'); + fs.writeFileSync(path.join(root, 'src', 'lib.rs'), 'use std::collections::HashMap;\npub struct Cache {}\npub fn build() {}\n'); + fs.writeFileSync(path.join(root, 'go.mod'), 'module example.test/tool\n\nrequire github.com/acme/lib v1.2.3\n'); + fs.writeFileSync(path.join(root, 'main.tf'), 'resource "example_service" "main" {}\n'); + indexRepository(root, { force: true }); + const db = openDatabase(projectPaths(root).database); + const facts = db.prepare('SELECT kind,path,name FROM facts').all(); db.close(); + assert(facts.some((fact) => fact.kind === 'file-artifact' && fact.path === 'src/worker.py')); + assert(facts.some((fact) => fact.kind === 'module-reference' && fact.path === 'src/worker.py')); + assert(facts.some((fact) => fact.kind === 'symbol' && fact.name === 'Worker')); + assert(facts.some((fact) => fact.kind === 'function' && fact.name === 'build')); + assert(facts.some((fact) => fact.kind === 'dependency' && fact.path === 'go.mod')); + assert(facts.some((fact) => fact.kind === 'infrastructure-resource' && fact.path === 'main.tf')); +}); + +test('audit rejects out-of-range line evidence and source changes after indexing', async () => { + const root = fixture(); const p = catalogFixture(root); await generate(root); await audit(root); + const traceFile = path.join(p.traceability, 'pages', 'endpoint-catalog.json'); const trace = readJson(traceFile); + trace.claims[0].evidence = [{ path: 'src/Resource.java', startLine: 999, endLine: 999 }]; writeJson(traceFile, trace); + await assert.rejects(() => audit(root), /Quality failed/); + let report = readJson(path.join(p.audit, 'deterministic.json')); assert(report.errors.some((error) => /line range/.test(error))); + trace.claims[0].evidence = [{ path: 'src/Resource.java', startLine: 1 }]; writeJson(traceFile, trace); + fs.appendFileSync(path.join(root, 'src', 'Resource.java'), '// changed after index\n'); + await assert.rejects(() => audit(root), /Quality failed/); + report = readJson(path.join(p.audit, 'deterministic.json')); assert(report.errors.some((error) => /source changed after indexing/.test(error))); +}); + +test('provider exit after valid artifacts is recovered without repeating completed work', () => { + const root = fixture(); const p = projectPaths(root); const provider = installFakeProvider(root); + fs.writeFileSync(path.join(root, 'src', 'Resource.java'), 'class Resource {}\n'); + const config = readJson(p.config); config.commandCode = { executable: provider, trust: false, skipOnboarding: false, yolo: false, maxTurns: { default: 12 } }; writeJson(p.config, config); + const run = spawnSync(process.execPath, [cli, 'all'], { cwd: root, encoding: 'utf8', env: { ...process.env, DOCGEN_PROGRESS: '0', DOCGEN_TEST_EXIT_AFTER_WRITE_STAGE: 'generate' } }); + assert.equal(run.status, 0, `STDERR:\n${run.stderr}\nSTDOUT:\n${run.stdout}`); assert.match(run.stderr, /RECOVERED/); + const state = readJson(p.state); assert.equal(state.pages.overview.status, 'completed'); assert.equal(state.pages.overview.recovered, true); + const budget = readJson(p.budget); assert.equal(budget.usage.providerCalls, 4); assert.equal(budget.usage.failedCalls, 1); +}); + +test('partial batch generation checkpoints valid pages and retries only missing pages', () => { + const root = fixture(); const p = projectPaths(root); const provider = installFakeProvider(root); + fs.writeFileSync(path.join(root, 'src', 'Resource.java'), 'class Resource {}\n'); + const config = readJson(p.config); config.commandCode = { executable: provider, trust: false, skipOnboarding: false, yolo: false, maxTurns: { default: 30 } }; config.execution.generationBatchSize = 2; config.execution.generationRecoveryAttempts = 3; writeJson(p.config, config); + const run = spawnSync(process.execPath, [cli, 'all'], { cwd: root, encoding: 'utf8', env: { ...process.env, DOCGEN_PROGRESS: '0', DOCGEN_TEST_PAGE_COUNT: '2', DOCGEN_TEST_PARTIAL_GENERATE: '1' } }); + assert.equal(run.status, 0, `STDERR:\n${run.stderr}\nSTDOUT:\n${run.stdout}`); assert.match(run.stderr, /RECOVERY/); + const state = readJson(p.state); assert.equal(state.pages.overview.status, 'completed'); assert.equal(state.pages['detail-2'].status, 'completed'); + const budget = readJson(p.budget); assert.equal(budget.usage.providerCalls, 5); assert.equal(budget.usage.failedCalls, 1); +}); + + +test('recovery never accepts stale pre-existing plan artifacts when provider writes nothing', () => { + const root = fixture(); const p = projectPaths(root); const provider = installFakeProvider(root); + fs.writeFileSync(path.join(root, 'src', 'Resource.java'), 'class Resource {}\n'); + const config = readJson(p.config); config.commandCode = { executable: provider, trust: false, skipOnboarding: false, yolo: false, maxTurns: { default: 30 } }; writeJson(p.config, config); + const first = spawnSync(process.execPath, [cli, 'all'], { cwd: root, encoding: 'utf8', env: { ...process.env, DOCGEN_PROGRESS: '0' } }); assert.equal(first.status, 0, first.stderr || first.stdout); + const currentState = readJson(p.state); currentState.stages.plan.status = 'failed'; writeJson(p.state, currentState); + const staleManifestHash = fs.readFileSync(p.plan, 'utf8'); + const second = spawnSync(process.execPath, [cli, 'plan'], { cwd: root, encoding: 'utf8', env: { ...process.env, DOCGEN_PROGRESS: '0', DOCGEN_TEST_FAIL_BEFORE_WRITE_STAGE: 'plan' } }); + assert.notEqual(second.status, 0, second.stdout); assert.match(second.stderr, /failed: exit 8/i); assert.equal(fs.readFileSync(p.plan, 'utf8'), staleManifestHash); + assert.equal(readJson(p.state).stages.plan.status, 'failed'); +}); + +test('audit rejects unknown model references and publish rejects stale source artifacts', async () => { + const root = fixture(); const p = projectPaths(root); const provider = installFakeProvider(root); + fs.writeFileSync(path.join(root, 'src', 'Resource.java'), 'class Resource {}\n'); + const config = readJson(p.config); config.commandCode = { executable: provider, trust: false, skipOnboarding: false, yolo: false, maxTurns: { default: 30 } }; writeJson(p.config, config); + const run = spawnSync(process.execPath, [cli, 'all'], { cwd: root, encoding: 'utf8', env: { ...process.env, DOCGEN_PROGRESS: '0' } }); assert.equal(run.status, 0, run.stderr || run.stdout); + const traceFile = path.join(p.traceability, 'pages', 'overview.json'); const trace = readJson(traceFile); trace.claims[0].sourceModelRefs = ['system:does-not-exist']; writeJson(traceFile, trace); + await assert.rejects(() => audit(root), /Quality failed/); const report = readJson(path.join(p.audit, 'deterministic.json')); assert(report.errors.some((error) => /unknown sourceModelRef/.test(error))); + trace.claims[0].sourceModelRefs = ['system:resource']; writeJson(traceFile, trace); await audit(root); + fs.appendFileSync(path.join(root, 'src', 'Resource.java'), '// stale\n'); + assert.throws(() => publish(root), /stale relative to current source/); +}); + +test('v1 migration preserves docs and ignore policy while archiving workflow state', () => { + const root = fixture(); const p = projectPaths(root); + writeJson(p.config, { schemaVersion: '1.6', projectName: 'Migrated', commandCode: { executable: 'custom-cmdc', model: 'cheap-model' }, ignore: { useGitignore: true, binary: { maxTextFileBytes: 123456 } } }); + writeJson(p.project, { schemaVersion: '1.0', kitVersion: '1.0.0' }); + fs.mkdirSync(path.join(root, 'docs'), { recursive: true }); fs.writeFileSync(path.join(root, 'docs', 'keep.md'), '# Keep\n'); fs.writeFileSync(path.join(root, '.docgenignore'), 'private/**\n'); + fs.mkdirSync(path.join(p.base, 'evidence'), { recursive: true }); fs.writeFileSync(path.join(p.base, 'evidence', 'legacy.json'), '{}'); + const run = spawnSync(process.execPath, [cli, 'migrate'], { cwd: root, encoding: 'utf8' }); + assert.equal(run.status, 0, run.stderr); + assert.equal(fs.readFileSync(path.join(root, 'docs', 'keep.md'), 'utf8'), '# Keep\n'); + assert.equal(fs.readFileSync(path.join(root, '.docgenignore'), 'utf8'), 'private/**\n'); + const next = readJson(p.config); assert.equal(next.schemaVersion, '2.0'); assert.equal(next.projectName, 'Migrated'); assert.equal(next.commandCode.executable, 'custom-cmdc'); assert.equal(next.ignore.binary.maxTextFileBytes, 123456); + const marker = readJson(p.project); assert.match(marker.migrationBackup, /^\.docgen\/migration-backup\//); assert(fs.existsSync(path.join(root, marker.migrationBackup, 'evidence', 'legacy.json'))); +}); diff --git a/global-template/docgen/test/windows-runtime.test.mjs b/global-template/docgen/test/windows-runtime.test.mjs new file mode 100644 index 0000000..a9a40f8 --- /dev/null +++ b/global-template/docgen/test/windows-runtime.test.mjs @@ -0,0 +1,81 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { once } from 'node:events'; +import { spawnSync } from 'node:child_process'; +import { fileURLToPath } from 'node:url'; +import { projectPaths, spawnCommand, writeJson } from '../lib/core.mjs'; +import { runProvider } from '../lib/provider.mjs'; + +const testDir = path.dirname(fileURLToPath(import.meta.url)); +const launcher = path.resolve(testDir, '..', 'bin', 'docgen-launcher.mjs'); + +function fixture() { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'docgen-runtime-')); + const paths = projectPaths(root); + fs.mkdirSync(path.dirname(paths.config), { recursive: true }); + writeJson(paths.project, { schemaVersion: '2.0', kitVersion: '2.0.0' }); + writeJson(paths.state, { schemaVersion: '2.0', kitVersion: '2.0.0', stages: {}, pages: {} }); + return { root, paths }; +} + +function providerShim(root) { + const script = path.join(root, 'silent-provider.mjs'); + fs.writeFileSync(script, '#!/usr/bin/env node\nawait new Promise((resolve) => setTimeout(resolve, Number(process.env.DOCGEN_TEST_DELAY_MS || 0)));\n'); + fs.chmodSync(script, 0o755); + if (process.platform !== 'win32') return script; + const shim = path.join(root, 'silent-provider.cmd'); + fs.writeFileSync(shim, `@echo off\r\n"${process.execPath}" "${script}" %*\r\n`); + return shim; +} + +test('launcher suppresses node:sqlite ExperimentalWarning for user CLI', () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'docgen-launcher-')); + const run = spawnSync(process.execPath, [launcher, 'init', root], { encoding: 'utf8' }); + assert.equal(run.status, 0, run.stderr || run.stdout); + assert.doesNotMatch(run.stderr, /ExperimentalWarning|SQLite is an experimental feature/); +}); + +test('safe command launcher preserves Windows shim execution without shell:true', async () => { + if (process.platform !== 'win32') return; + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'docgen-cmd-')); + const script = path.join(root, 'args.mjs'); + fs.writeFileSync(script, 'console.log(JSON.stringify(process.argv.slice(2)));\n'); + const shim = path.join(root, 'args.cmd'); + fs.writeFileSync(shim, `@echo off\r\n"${process.execPath}" "${script}" %*\r\n`); + const child = spawnCommand(shim, ['alpha', 'space value'], { stdio: ['ignore', 'pipe', 'pipe'], windowsHide: true }); + let stdout = ''; child.stdout.on('data', (chunk) => { stdout += chunk; }); + const [code] = await once(child, 'close'); + assert.equal(code, 0); + assert.deepEqual(JSON.parse(stdout.trim()), ['alpha', 'space value']); +}); + +test('provider timeout terminates silent process and records exit 124', async () => { + const { root, paths } = fixture(); + const executable = providerShim(root); + writeJson(paths.config, { + schemaVersion: '2.0', + commandCode: { executable, trust: false, skipOnboarding: false, yolo: false, verbose: false, maxTurns: { default: 1 } }, + budget: { maxProviderCalls: 5, maxEstimatedInputTokens: 10000, maxEstimatedOutputTokens: 10000, maxContextTokensPerCall: 5000 }, + execution: { heartbeatSeconds: 0.05, silenceNoticeSeconds: 0.05, streamProviderOutput: false }, + retry: { maxAttempts: 1 } + }); + const previousDelay = process.env.DOCGEN_TEST_DELAY_MS; + const previousTimeout = process.env.DOCGEN_STAGE_TIMEOUT_MS; + process.env.DOCGEN_TEST_DELAY_MS = '5000'; + process.env.DOCGEN_STAGE_TIMEOUT_MS = '250'; + try { + await assert.rejects( + () => runProvider(root, { stage: 'plan', target: 'timeout-fixture', prompt: 'timeout test' }), + (error) => error.exitCode === 124 && error.timedOut === true + ); + } finally { + if (previousDelay === undefined) delete process.env.DOCGEN_TEST_DELAY_MS; else process.env.DOCGEN_TEST_DELAY_MS = previousDelay; + if (previousTimeout === undefined) delete process.env.DOCGEN_STAGE_TIMEOUT_MS; else process.env.DOCGEN_STAGE_TIMEOUT_MS = previousTimeout; + } + const records = fs.readFileSync(path.join(paths.telemetry, 'provider-runs.jsonl'), 'utf8').trim().split(/\r?\n/).map(JSON.parse); + assert.equal(records.at(-1).timedOut, true); + assert.equal(records.at(-1).exitCode, 124); +}); diff --git a/install.mjs b/install.mjs index cab7860..1a7c07e 100644 --- a/install.mjs +++ b/install.mjs @@ -15,90 +15,96 @@ const noHooks = argv.includes('--no-hooks'); const noLinkCli = argv.includes('--no-link-cli'); const localIndex = argv.indexOf('--project-local'); const homeIndex = argv.indexOf('--commandcode-home'); -const commandCodeHome = path.resolve(homeIndex >= 0 && argv[homeIndex+1] ? argv[homeIndex+1] : path.join(os.homedir(), '.commandcode')); +const commandCodeHome = path.resolve(homeIndex >= 0 && argv[homeIndex + 1] ? argv[homeIndex + 1] : path.join(os.homedir(), '.commandcode')); const timestamp = new Date().toISOString().replace(/[:.]/g, '-'); -function sha256(data){ return crypto.createHash('sha256').update(data).digest('hex'); } -function walk(dir){ const out=[]; for(const e of fs.readdirSync(dir,{withFileTypes:true})){ const f=path.join(dir,e.name); e.isDirectory()?out.push(...walk(f)):out.push(f);} return out; } -function ensureDir(dir){ if(!dryRun) fs.mkdirSync(dir,{recursive:true}); } -function copyWithPolicy(src,dest,backupRoot,installed,skipped){ - const data=fs.readFileSync(src); const rel=path.relative(commandCodeHome,dest).replaceAll('\\','/'); - if(fs.existsSync(dest)){ - const existing=fs.readFileSync(dest); if(sha256(existing)===sha256(data)){installed.push({path:rel,action:'unchanged'}); return;} - if(!force){skipped.push({path:rel,reason:'conflict; use --force to overwrite'}); return;} - if(!dryRun){const b=path.join(backupRoot,rel); fs.mkdirSync(path.dirname(b),{recursive:true}); fs.copyFileSync(dest,b);} +const LEGACY_ENGINE = [ + 'docgen/bin/docgen.mjs', + ...['discover','analyze','semantics','enterprise','plan','generate','generate-batch','enrich','enrich-batch','audit','audit-batch','fix','update-impact','workspace-synthesis'].map((name) => `docgen/prompts/${name}.md`) +]; +const LEGACY_AGENTS = ['doc-discoverer','doc-architect','doc-domain-analyst','doc-enterprise-analyst','doc-planner','doc-writer','doc-auditor','doc-system-analyst'].map((name) => `agents/${name}.md`); +const LEGACY_COMMANDS = ['docgen-discover','docgen-analyze','docgen-fix','docgen-update','docgen-enrich','docgen-quality','docgen-semantics','docgen-preflight','docgen-contract-test','docgen-traceability','docgen-enterprise'].map((name) => `commands/${name}.md`); +const LEGACY_MANAGED = [...LEGACY_ENGINE, ...LEGACY_AGENTS, ...LEGACY_COMMANDS]; + +function sha256(data) { return crypto.createHash('sha256').update(data).digest('hex'); } +function walk(dir) { if (!fs.existsSync(dir)) return []; const out = []; for (const entry of fs.readdirSync(dir, { withFileTypes: true })) { const file = path.join(dir, entry.name); entry.isDirectory() ? out.push(...walk(file)) : out.push(file); } return out; } +function ensureDir(dir) { if (!dryRun) fs.mkdirSync(dir, { recursive: true }); } +function copyWithPolicy(src, dest, backupRoot, installed, skipped) { + const data = fs.readFileSync(src); const rel = path.relative(commandCodeHome, dest).replaceAll('\\', '/'); + if (fs.existsSync(dest)) { + const existing = fs.readFileSync(dest); if (sha256(existing) === sha256(data)) { installed.push({ path: rel, action: 'unchanged' }); return; } + if (!force) { skipped.push({ path: rel, reason: 'conflict; use --force to overwrite' }); return; } + if (!dryRun) { const backup = path.join(backupRoot, rel); fs.mkdirSync(path.dirname(backup), { recursive: true }); fs.copyFileSync(dest, backup); } + } + console.log(`${dryRun ? '[dry-run] ' : ''}copy ${dest}`); + if (!dryRun) { fs.mkdirSync(path.dirname(dest), { recursive: true }); fs.writeFileSync(dest, data); try { fs.chmodSync(dest, fs.statSync(src).mode); } catch {} } + installed.push({ path: rel, action: fs.existsSync(dest) ? 'updated' : 'created' }); +} +function removeLegacy(root, relPaths, backupRoot, installed) { + for (const rel of relPaths) { + const file = path.join(root, rel); if (!fs.existsSync(file) || !fs.statSync(file).isFile()) continue; + console.log(`${dryRun ? '[dry-run] ' : ''}remove legacy ${file}`); + if (!dryRun) { const backup = path.join(backupRoot, 'removed-legacy', rel); fs.mkdirSync(path.dirname(backup), { recursive: true }); fs.copyFileSync(file, backup); fs.rmSync(file, { force: true }); } + installed.push({ path: rel.replaceAll('\\', '/'), action: 'removed-legacy' }); } - console.log(`${dryRun?'[dry-run] ':''}copy ${dest}`); - if(!dryRun){fs.mkdirSync(path.dirname(dest),{recursive:true}); fs.writeFileSync(dest,data); try{fs.chmodSync(dest,fs.statSync(src).mode)}catch{}} - installed.push({path:rel,action:fs.existsSync(dest)?'updated':'created'}); } -function hookCommand(file){ return `node ${JSON.stringify(path.join(commandCodeHome,'docgen','hooks',file))}`; } -function mergeGlobalSettings(backupRoot,installed){ - if(noHooks) return; - const dest=path.join(commandCodeHome,'settings.json'); let current={}; - if(fs.existsSync(dest)){try{current=JSON.parse(fs.readFileSync(dest,'utf8'))}catch(e){console.error(`Invalid JSON: ${dest}: ${e.message}`);process.exit(2)}; if(!dryRun){const b=path.join(backupRoot,'settings.json');fs.mkdirSync(path.dirname(b),{recursive:true});fs.copyFileSync(dest,b)}} +function hookCommand(file) { return `node ${JSON.stringify(path.join(commandCodeHome, 'docgen', 'hooks', file))}`; } +function mergeGlobalSettings(backupRoot, installed) { + if (noHooks) return; + const dest = path.join(commandCodeHome, 'settings.json'); let current = {}; + if (fs.existsSync(dest)) { try { current = JSON.parse(fs.readFileSync(dest, 'utf8')); } catch (error) { console.error(`Invalid JSON: ${dest}: ${error.message}`); process.exit(2); } if (!dryRun) { const backup = path.join(backupRoot, 'settings.json'); fs.mkdirSync(path.dirname(backup), { recursive: true }); fs.copyFileSync(dest, backup); } } current.hooks ??= {}; - const defs={ - SessionStart:[{hooks:[{type:'command',command:hookCommand('docgen-session-context.mjs'),timeout:5}]}], - PreToolUse:[ - {matcher:'write|edit',hooks:[{type:'command',command:hookCommand('docgen-guard-write-paths.mjs'),timeout:5}]}, - {matcher:'read',hooks:[{type:'command',command:hookCommand('docgen-guard-read-paths.mjs'),timeout:5}]}, - {matcher:'shell',hooks:[{type:'command',command:hookCommand('docgen-guard-shell.mjs'),timeout:5}]} + const defs = { + SessionStart: [{ hooks: [{ type: 'command', command: hookCommand('docgen-session-context.mjs'), timeout: 5 }] }], + PreToolUse: [ + { matcher: 'write|edit', hooks: [{ type: 'command', command: hookCommand('docgen-guard-write-paths.mjs'), timeout: 5 }] }, + { matcher: 'read', hooks: [{ type: 'command', command: hookCommand('docgen-guard-read-paths.mjs'), timeout: 5 }] }, + { matcher: 'shell', hooks: [{ type: 'command', command: hookCommand('docgen-guard-shell.mjs'), timeout: 5 }] } ], - PostToolUse:[{matcher:'write|edit',hooks:[{type:'command',command:hookCommand('docgen-validate-written-artifact.mjs'),timeout:10}]}] + PostToolUse: [{ matcher: 'write|edit', hooks: [{ type: 'command', command: hookCommand('docgen-validate-written-artifact.mjs'), timeout: 10 }] }] }; - for(const [event,arr] of Object.entries(defs)){ - current.hooks[event] ??= []; - const sig=new Set(current.hooks[event].flatMap(d=>(d.hooks??[]).map(h=>`${d.matcher??''}|${h.command??''}`))); - for(const d of arr){const fresh=(d.hooks??[]).filter(h=>!sig.has(`${d.matcher??''}|${h.command??''}`));if(fresh.length) current.hooks[event].push({...d,hooks:fresh});} + for (const [event, definitions] of Object.entries(defs)) { + current.hooks[event] ??= []; const signatures = new Set(current.hooks[event].flatMap((definition) => (definition.hooks ?? []).map((hook) => `${definition.matcher ?? ''}|${hook.command ?? ''}`))); + for (const definition of definitions) { const fresh = (definition.hooks ?? []).filter((hook) => !signatures.has(`${definition.matcher ?? ''}|${hook.command ?? ''}`)); if (fresh.length) current.hooks[event].push({ ...definition, hooks: fresh }); } } - console.log(`${dryRun?'[dry-run] ':''}merge ${dest}`); - if(!dryRun){fs.mkdirSync(path.dirname(dest),{recursive:true});fs.writeFileSync(dest,JSON.stringify(current,null,2)+'\n')} - installed.push({path:'settings.json',action:'merged-docgen-hooks'}); + console.log(`${dryRun ? '[dry-run] ' : ''}merge ${dest}`); + if (!dryRun) { fs.mkdirSync(path.dirname(dest), { recursive: true }); fs.writeFileSync(dest, JSON.stringify(current, null, 2) + '\n'); } + installed.push({ path: 'settings.json', action: 'merged-docgen-hooks' }); } -function installGlobal(){ - const template=path.join(here,'global-template'); const backupRoot=path.join(commandCodeHome,'docgen-backup',timestamp); const installed=[]; const skipped=[]; +function installGlobal() { + const template = path.join(here, 'global-template'); const backupRoot = path.join(commandCodeHome, 'docgen-backup', timestamp); const installed = []; const skipped = []; ensureDir(commandCodeHome); - for(const area of ['agents','skills','commands']) for(const src of walk(path.join(template,area))) copyWithPolicy(src,path.join(commandCodeHome,area,path.relative(path.join(template,area),src)),backupRoot,installed,skipped); - for(const src of walk(path.join(template,'docgen'))) copyWithPolicy(src,path.join(commandCodeHome,'docgen',path.relative(path.join(template,'docgen'),src)),backupRoot,installed,skipped); - mergeGlobalSettings(backupRoot,installed); - if(!dryRun){ - fs.writeFileSync(path.join(commandCodeHome,'docgen','installation.json'),JSON.stringify({schemaVersion:'1.0',kitVersion:version,scope:'global',installedAt:new Date().toISOString(),commandCodeHome,files:installed,skipped},null,2)+'\n'); - if(!noLinkCli){ - const npm=process.platform==='win32'?'npm.cmd':'npm'; - const link=spawnSync(npm,['link'],{cwd:path.join(commandCodeHome,'docgen'),stdio:'inherit',shell:process.platform==='win32'}); - if(link.status!==0) console.warn('WARNING: npm link failed. Use `node ~/.commandcode/docgen/bin/docgen.mjs` or rerun without --no-link-cli after fixing npm.'); - } + for (const area of ['agents', 'skills', 'commands']) { const areaRoot = path.join(template, area); for (const src of walk(areaRoot)) copyWithPolicy(src, path.join(commandCodeHome, area, path.relative(areaRoot, src)), backupRoot, installed, skipped); } + const engineRoot = path.join(template, 'docgen'); for (const src of walk(engineRoot)) copyWithPolicy(src, path.join(commandCodeHome, 'docgen', path.relative(engineRoot, src)), backupRoot, installed, skipped); + removeLegacy(commandCodeHome, LEGACY_MANAGED, backupRoot, installed); + mergeGlobalSettings(backupRoot, installed); + if (!dryRun) { + fs.writeFileSync(path.join(commandCodeHome, 'docgen', 'installation.json'), JSON.stringify({ schemaVersion: '2.0', kitVersion: version, scope: 'global', installedAt: new Date().toISOString(), commandCodeHome, files: installed, skipped }, null, 2) + '\n'); + if (!noLinkCli) { const npm = process.platform === 'win32' ? 'npm.cmd' : 'npm'; const link = spawnSync(npm, ['link'], { cwd: path.join(commandCodeHome, 'docgen'), stdio: 'inherit', shell: process.platform === 'win32' }); if (link.status !== 0) console.warn('WARNING: npm link failed. Use `node ~/.commandcode/docgen/bin/docgen-v2.mjs` or rerun after fixing npm.'); } } console.log(`\nInstalled Command Code DocGen Kit ${version} globally into ${commandCodeHome}`); - if(skipped.length){console.log('\nSkipped conflicts:');for(const x of skipped)console.log(`- ${x.path}: ${x.reason}`)} - console.log('\nNext:'); console.log(' cd '); console.log(' docgen init'); console.log(' docgen doctor'); console.log(' docgen all'); + if (skipped.length) { console.log('\nSkipped conflicts:'); for (const item of skipped) console.log(`- ${item.path}: ${item.reason}`); } + console.log('\nNext:'); console.log(' cd '); console.log(' docgen init # new repository'); console.log(' docgen migrate # existing v1 repository'); console.log(' docgen doctor'); console.log(' docgen all'); } -function installProjectLocal(target){ - const template=path.join(here,'project-local-template'); const abs=path.resolve(target); if(!fs.existsSync(abs)||!fs.statSync(abs).isDirectory()){console.error(`Target is not a directory: ${abs}`);process.exit(2)} - const backupRoot=path.join(abs,'.docgen','install-backup',timestamp); const installed=[]; const skipped=[]; - function localCopy(src,dest){const data=fs.readFileSync(src);const rel=path.relative(abs,dest).replaceAll('\\','/');if(fs.existsSync(dest)){if(sha256(fs.readFileSync(dest))===sha256(data)){installed.push({path:rel,action:'unchanged'});return}if(rel==='.docgenignore'){skipped.push({path:rel,reason:'preserved user-owned ignore policy'});return}if(!force){skipped.push({path:rel,reason:'conflict; use --force'});return}if(!dryRun){const b=path.join(backupRoot,rel);fs.mkdirSync(path.dirname(b),{recursive:true});fs.copyFileSync(dest,b)}}console.log(`${dryRun?'[dry-run] ':''}copy ${rel}`);if(!dryRun){fs.mkdirSync(path.dirname(dest),{recursive:true});fs.writeFileSync(dest,data)}installed.push({path:rel,action:'copied'})} - for(const src of walk(template)){const rel=path.relative(template,src);if(rel==='AGENTS.md'||rel==='.commandcode/settings.json')continue;localCopy(src,path.join(abs,rel))} - // Install the same current engine used by global mode under the project's .commandcode scope. - const localEngineTemplate=path.join(here,'global-template','docgen'); - for(const src of walk(localEngineTemplate)){const rel=path.relative(localEngineTemplate,src);localCopy(src,path.join(abs,'.commandcode','docgen',rel))} - if(!dryRun){ - const marker={schemaVersion:'1.0',kitVersion:version,initializedAt:new Date().toISOString(),engineScope:'project-local',engineHome:path.join(abs,'.commandcode','docgen').replaceAll('\\','/'),projectRoot:abs.replaceAll('\\','/')}; - fs.mkdirSync(path.join(abs,'.docgen'),{recursive:true});fs.writeFileSync(path.join(abs,'.docgen','project.json'),JSON.stringify(marker,null,2)+'\n'); +function installProjectLocal(target) { + const template = path.join(here, 'project-local-template'); const abs = path.resolve(target); if (!fs.existsSync(abs) || !fs.statSync(abs).isDirectory()) { console.error(`Target is not a directory: ${abs}`); process.exit(2); } + const backupRoot = path.join(abs, '.docgen', 'install-backup', timestamp); const installed = []; const skipped = []; + function localCopy(src, dest) { const data = fs.readFileSync(src); const rel = path.relative(abs, dest).replaceAll('\\', '/'); if (fs.existsSync(dest)) { if (sha256(fs.readFileSync(dest)) === sha256(data)) { installed.push({ path: rel, action: 'unchanged' }); return; } if (rel === '.docgenignore') { skipped.push({ path: rel, reason: 'preserved user-owned ignore policy' }); return; } if (!force) { skipped.push({ path: rel, reason: 'conflict; use --force' }); return; } if (!dryRun) { const backup = path.join(backupRoot, rel); fs.mkdirSync(path.dirname(backup), { recursive: true }); fs.copyFileSync(dest, backup); } } console.log(`${dryRun ? '[dry-run] ' : ''}copy ${rel}`); if (!dryRun) { fs.mkdirSync(path.dirname(dest), { recursive: true }); fs.writeFileSync(dest, data); } installed.push({ path: rel, action: 'copied' }); } + for (const src of walk(template)) { const rel = path.relative(template, src); if (rel === 'AGENTS.md' || rel === '.commandcode/settings.json') continue; localCopy(src, path.join(abs, rel)); } + const localEngineTemplate = path.join(here, 'global-template', 'docgen'); + for (const src of walk(localEngineTemplate)) { const rel = path.relative(localEngineTemplate, src); localCopy(src, path.join(abs, '.commandcode', 'docgen', rel)); } + removeLegacy(path.join(abs, '.commandcode'), LEGACY_MANAGED, backupRoot, installed); + if (!dryRun) { const marker = { schemaVersion: '2.0', kitVersion: version, initializedAt: new Date().toISOString(), engineScope: 'project-local', engineHome: path.join(abs, '.commandcode', 'docgen').replaceAll('\\', '/'), projectRoot: abs.replaceAll('\\', '/') }; fs.mkdirSync(path.join(abs, '.docgen'), { recursive: true }); fs.writeFileSync(path.join(abs, '.docgen', 'project.json'), JSON.stringify(marker, null, 2) + '\n'); } + const markerStart = '', markerEnd = ''; + const memorySrc = fs.readFileSync(path.join(template, 'AGENTS.md'), 'utf8'); const memoryDest = path.join(abs, 'AGENTS.md'); const existingMemory = fs.existsSync(memoryDest) ? fs.readFileSync(memoryDest, 'utf8') : ''; + if (!existingMemory.includes(markerStart) || !existingMemory.includes(markerEnd)) { console.log(`${dryRun ? '[dry-run] ' : ''}${existingMemory ? 'append' : 'create'} ${memoryDest}`); if (!dryRun) fs.writeFileSync(memoryDest, existingMemory.trimEnd() + (existingMemory ? '\n\n' : '') + memorySrc.trim() + '\n'); } + if (!noHooks) { + const settingsDest = path.join(abs, '.commandcode', 'settings.json'); let current = {}; const source = JSON.parse(fs.readFileSync(path.join(template, '.commandcode', 'settings.json'), 'utf8')); + if (fs.existsSync(settingsDest)) { try { current = JSON.parse(fs.readFileSync(settingsDest, 'utf8')); } catch (error) { console.error(`Invalid JSON: ${settingsDest}: ${error.message}`); process.exit(2); } } + current.hooks ??= {}; for (const [event, definitions] of Object.entries(source.hooks ?? {})) { current.hooks[event] ??= []; const signatures = new Set(current.hooks[event].flatMap((definition) => (definition.hooks ?? []).map((hook) => `${definition.matcher ?? ''}|${hook.command ?? ''}`))); for (const definition of definitions) { const fresh = (definition.hooks ?? []).filter((hook) => !signatures.has(`${definition.matcher ?? ''}|${hook.command ?? ''}`)); if (fresh.length) current.hooks[event].push({ ...definition, hooks: fresh }); } } + console.log(`${dryRun ? '[dry-run] ' : ''}merge ${settingsDest}`); if (!dryRun) { fs.mkdirSync(path.dirname(settingsDest), { recursive: true }); fs.writeFileSync(settingsDest, JSON.stringify(current, null, 2) + '\n'); } } - const markerStart='', markerEnd=''; - const memorySrc=fs.readFileSync(path.join(template,'AGENTS.md'),'utf8'); const memoryDest=path.join(abs,'AGENTS.md'); - const existingMemory=fs.existsSync(memoryDest)?fs.readFileSync(memoryDest,'utf8'):''; - if(!existingMemory.includes(markerStart)||!existingMemory.includes(markerEnd)){ - console.log(`${dryRun?'[dry-run] ':''}${existingMemory?'append':'create'} ${memoryDest}`); - if(!dryRun){fs.writeFileSync(memoryDest,existingMemory.trimEnd()+(existingMemory?'\n\n':'')+memorySrc.trim()+'\n')} - } - if(!noHooks){ - const settingsDest=path.join(abs,'.commandcode','settings.json'); let current={}; const source=JSON.parse(fs.readFileSync(path.join(template,'.commandcode','settings.json'),'utf8')); - if(fs.existsSync(settingsDest)){try{current=JSON.parse(fs.readFileSync(settingsDest,'utf8'))}catch(e){console.error(`Invalid JSON: ${settingsDest}: ${e.message}`);process.exit(2)}} - current.hooks ??= {}; for(const [event,defs] of Object.entries(source.hooks??{})){current.hooks[event]??=[];const sig=new Set(current.hooks[event].flatMap(d=>(d.hooks??[]).map(h=>`${d.matcher??''}|${h.command??''}`)));for(const d of defs){const fresh=(d.hooks??[]).filter(h=>!sig.has(`${d.matcher??''}|${h.command??''}`));if(fresh.length)current.hooks[event].push({...d,hooks:fresh})}} - console.log(`${dryRun?'[dry-run] ':''}merge ${settingsDest}`); if(!dryRun){fs.mkdirSync(path.dirname(settingsDest),{recursive:true});fs.writeFileSync(settingsDest,JSON.stringify(current,null,2)+'\n')} - } - console.log(`\nInstalled self-contained project-local DocGen ${version} into ${abs}`); if(skipped.length){console.log('\nSkipped or preserved files:');for(const x of skipped)console.log(`- ${x.path}: ${x.reason}`)} + console.log(`\nInstalled self-contained project-local DocGen ${version} into ${abs}`); if (skipped.length) { console.log('\nSkipped or preserved files:'); for (const item of skipped) console.log(`- ${item.path}: ${item.reason}`); } } -if(localIndex>=0){const target=argv[localIndex+1];if(!target){console.error('Usage: node install.mjs --project-local [--force]');process.exit(2)}installProjectLocal(target)}else installGlobal(); + +if (localIndex >= 0) { const target = argv[localIndex + 1]; if (!target) { console.error('Usage: node install.mjs --project-local [--force]'); process.exit(2); } installProjectLocal(target); } else installGlobal(); diff --git a/project-local-template/.commandcode/agents/doc-architect.md b/project-local-template/.commandcode/agents/doc-architect.md deleted file mode 100644 index 22ba8b1..0000000 --- a/project-local-template/.commandcode/agents/doc-architect.md +++ /dev/null @@ -1,25 +0,0 @@ ---- -name: "doc-architect" -description: "Use to synthesize normalized components, relationships, workflows, state transitions, ownership, and failure boundaries from existing evidence." -tools: "read_file, read_multiple_files, read_directory, glob, grep, write_file, edit_file, todo_write" ---- -You are the architecture synthesis worker. - -Before working, apply these installed Command Code skills by capability name: - -- `doc-evidence-contract` -- `doc-architecture-analysis` -- `doc-workflow-analysis` -- relevant `tech-*` and `domain-*` skills discovered for this repository - -Evidence is authoritative over prior prose. Produce normalized JSON under `.docgen/model/**`. Every non-obvious conclusion must carry evidence references and an epistemic classification. Verify uncertain claims against source only when needed; do not broaden scope unnecessarily. - -Do not write published documentation and never modify application source. - -## P0 Trustworthiness - -Apply `doc-traceability` and `doc-semantic-quality`. Produce typed semantic objects, claim-level evidence mappings, and explicit UNKNOWNs. Never promote unsupported prose to FACT. -## Ignore boundary - -Before reading repository source, read `.docgen/state/source-files.txt`. Do not read, search, cite, or derive facts from repository paths absent from that inventory. Existing `.docgen/**` and `docs/**` workflow artifacts remain available. - diff --git a/project-local-template/.commandcode/agents/doc-auditor.md b/project-local-template/.commandcode/agents/doc-auditor.md deleted file mode 100644 index 733964d..0000000 --- a/project-local-template/.commandcode/agents/doc-auditor.md +++ /dev/null @@ -1,20 +0,0 @@ ---- -name: "doc-auditor" -description: "Use to independently audit one generated page for factual grounding, coverage completeness, catalog completeness, flow depth, and structural quality." -tools: "read_file, read_multiple_files, read_directory, glob, grep, write_file, edit_file, todo_write" ---- -You are the independent documentation auditor. - -Apply `doc-claim-verification` and relevant analysis/catalog skills. Compare the page to its manifest, evidence, and normalized technical/business/flow/catalog models. - -Find unsupported claims, omissions, shallow treatment, missing branches, missing catalog entries, missing failure semantics, contradictions, broken links, and any non-Mermaid diagram. Audit independently from the writer. - -Write only `.docgen/audit/**`. Never modify published docs or application source. - -## P0 Trustworthiness - -Apply `doc-traceability` and `doc-semantic-quality`. Produce typed semantic objects, claim-level evidence mappings, and explicit UNKNOWNs. Never promote unsupported prose to FACT. -## Ignore boundary - -Before reading repository source, read `.docgen/state/source-files.txt`. Do not read, search, cite, or derive facts from repository paths absent from that inventory. Existing `.docgen/**` and `docs/**` workflow artifacts remain available. - diff --git a/project-local-template/.commandcode/agents/doc-discoverer.md b/project-local-template/.commandcode/agents/doc-discoverer.md deleted file mode 100644 index fb3a0fd..0000000 --- a/project-local-template/.commandcode/agents/doc-discoverer.md +++ /dev/null @@ -1,20 +0,0 @@ ---- -name: "doc-discoverer" -description: "Use to inspect repository source and produce factual evidence artifacts without writing published documentation." -tools: "read_file, read_multiple_files, read_directory, glob, grep, shell_command, write_file, edit_file, todo_write" ---- -You are the repository evidence extraction worker. - -Apply these installed skills by capability name: - -- `doc-evidence-contract` -- `doc-repository-discovery` -- relevant `tech-*` and `domain-*` skills discovered for this repository - -Discover not only structure but also evidence required for deep system documentation: entry points, endpoints, handlers, business/domain clues, guards and branch conditions, states, persistence, messages, integrations, jobs, configuration, security, runtime/deployment and tests/examples. - -Write only under `.docgen/evidence/**`. Maintain canonical `.docgen/evidence/index.json` with top-level `artifacts[]`. Each important fact must be source-grounded. Preserve unknowns. Do not write published documentation and never modify application source. -## Ignore boundary - -Before reading repository source, read `.docgen/state/source-files.txt`. Do not read, search, cite, or derive facts from repository paths absent from that inventory. Existing `.docgen/**` and `docs/**` workflow artifacts remain available. - diff --git a/project-local-template/.commandcode/agents/doc-domain-analyst.md b/project-local-template/.commandcode/agents/doc-domain-analyst.md deleted file mode 100644 index 4154c47..0000000 --- a/project-local-template/.commandcode/agents/doc-domain-analyst.md +++ /dev/null @@ -1,45 +0,0 @@ ---- -name: "doc-domain-analyst" -description: "Use to extract business semantics, rules, decisions, branch conditions, lifecycles, data semantics, flow models, API/message catalogs, and external dependency catalogs from evidence." -tools: "read_file, read_multiple_files, read_directory, glob, grep, write_file, edit_file, todo_write" ---- -You are the business-and-system semantics synthesis worker. - -Apply these installed Command Code skills by capability name: - -- `doc-evidence-contract` -- `doc-business-analysis` -- `doc-flow-analysis` -- `doc-data-model-analysis` -- `doc-api-catalog` -- `doc-messaging-catalog` -- `doc-integration-catalog` -- relevant `tech-*` and `domain-*` skills discovered for this repository - -Primary input is `.docgen/evidence/**` plus `.docgen/model/system.json`. Inspect source only for targeted verification. - -Produce and reconcile: - -- `.docgen/model/business.json` -- `.docgen/model/flows.json` -- `.docgen/model/catalogs.json` - -Rules: - -- distinguish business semantics from implementation mechanics; -- extract explicit rules, validations, guards, eligibility checks, decisions, branch conditions, state transitions, invariants, actors, outcomes, and data semantics; -- model business flow, control flow, request flow, traffic flow, data flow, and event flow separately; -- inventory all evidenced HTTP endpoints and message handlers/consumers/producers/listeners; -- inventory external systems, cloud services, databases, brokers, caches, identity providers, downstream/upstream services, scheduled jobs, and other runtime dependencies; -- preserve FACT / INFERENCE / UNKNOWN classification and source evidence; -- use empty arrays when a category has no evidence; never invent missing behavior. - -Do not write published documentation and never modify application source. - -## P0 Trustworthiness - -Apply `doc-traceability` and `doc-semantic-quality`. Produce typed semantic objects, claim-level evidence mappings, and explicit UNKNOWNs. Never promote unsupported prose to FACT. -## Ignore boundary - -Before reading repository source, read `.docgen/state/source-files.txt`. Do not read, search, cite, or derive facts from repository paths absent from that inventory. Existing `.docgen/**` and `docs/**` workflow artifacts remain available. - diff --git a/project-local-template/.commandcode/agents/doc-enterprise-analyst.md b/project-local-template/.commandcode/agents/doc-enterprise-analyst.md deleted file mode 100644 index 6c4cda4..0000000 --- a/project-local-template/.commandcode/agents/doc-enterprise-analyst.md +++ /dev/null @@ -1,28 +0,0 @@ ---- -name: "doc-enterprise-analyst" -description: "Use to extract enterprise-depth security, operations, testing, data governance, architecture decisions, configuration, ownership, and change-impact models from grounded repository evidence." -tools: "read_file, read_multiple_files, read_directory, glob, grep, write_file, edit_file, todo_write" ---- -You are the enterprise-depth documentation analysis worker. - -Apply these installed skills by capability name as relevant: - -- `doc-security-analysis` -- `doc-operations-analysis` -- `doc-testing-analysis` -- `doc-data-governance-analysis` -- `doc-decision-analysis` -- `doc-configuration-analysis` -- `doc-change-impact-analysis` -- `doc-ownership-analysis` -- `doc-evidence-contract` -- `doc-traceability` -- `doc-semantic-quality` - -Read `.docgen/state/source-files.txt` first. Never read a repository source path that is absent from that inventory. `.gitignore`, `.docgenignore`, and configured exclusions are mandatory boundaries. - -Primary inputs are existing evidence and normalized technical/business models. Inspect source only for targeted verification. - -Produce only the output files named by the invoking pass. Every semantic array item must be a typed object with stable ID, FACT/INFERENCE/UNKNOWN classification, confidence, resolvable evidence, model references, and explicit unknowns. FACT without direct non-ignored evidence is invalid. - -Do not write published documentation and do not modify application source. diff --git a/project-local-template/.commandcode/agents/doc-planner.md b/project-local-template/.commandcode/agents/doc-planner.md deleted file mode 100644 index 718785e..0000000 --- a/project-local-template/.commandcode/agents/doc-planner.md +++ /dev/null @@ -1,21 +0,0 @@ ---- -name: "doc-planner" -description: "Use to design a deep multi-page documentation information architecture and coverage-driven manifest from evidence and normalized models." -tools: "read_file, read_multiple_files, read_directory, glob, grep, write_file, edit_file, todo_write" ---- -You are the documentation information-architecture planner. - -Apply the installed `doc-page-planning` skill by capability name. - -Read project documentation config/style/glossary plus all normalized model surfaces. Plan a navigable, category-rich system knowledge base. Do not mechanically create one page per class/file, but also do not collapse a complex repository into a handful of oversized pages. - -The plan must make exhaustive catalogs discoverable and give important concepts/flows their own deep-dive pages. Preserve stable page ids and paths when reconciling unless evidence supports restructuring. - -Produce `.docgen/plan/manifest.json` conforming to its schema. Do not write published pages and never modify application source. -## Ignore boundary - -Before reading repository source, read `.docgen/state/source-files.txt`. Do not read, search, cite, or derive facts from repository paths absent from that inventory. Existing `.docgen/**` and `docs/**` workflow artifacts remain available. - - - -Apply `doc-documentation-experience` and `doc-migration-versioning`. Every page contract must choose a primary document mode, search keywords, aliases when relevant, lifecycle/version metadata, and evidence-derived example intents. diff --git a/project-local-template/.commandcode/agents/doc-system-analyst.md b/project-local-template/.commandcode/agents/doc-system-analyst.md deleted file mode 100644 index 0cd6f5d..0000000 --- a/project-local-template/.commandcode/agents/doc-system-analyst.md +++ /dev/null @@ -1,25 +0,0 @@ ---- -name: "doc-system-analyst" -description: "Use for system-of-systems analysis across multiple DocGen repositories: dependency graphs, shared contracts, business journeys, cross-service flows, data lineage, ownership, and blast radius." -tools: "read_file, read_multiple_files, read_directory, glob, grep, write_file, edit_file, todo_write" ---- -You are the P3 system-of-systems documentation analyst. - -Consume only validated repository-level DocGen artifacts and workspace registries. Do not rescan arbitrary repository source unless a workspace command explicitly asks for targeted verification. - -Apply these installed skills by capability name: - -- `doc-workspace-aggregation` -- `doc-cross-repo-dependency` -- `doc-contract-registry` -- `doc-business-journey` -- `doc-cross-repo-flow` -- `doc-data-lineage-workspace` -- `doc-workspace-change-impact` -- `doc-workspace-publishing` -- `doc-traceability` -- `doc-semantic-quality` - -Preserve repository identity, contract ownership, evidence provenance, FACT/INFERENCE/UNKNOWN classification, and unresolved edges. Never invent a cross-service connection merely because two names look related. Ambiguous matches remain unresolved. - -All published diagrams must use Mermaid. diff --git a/project-local-template/.commandcode/agents/doc-writer.md b/project-local-template/.commandcode/agents/doc-writer.md deleted file mode 100644 index a189390..0000000 --- a/project-local-template/.commandcode/agents/doc-writer.md +++ /dev/null @@ -1,25 +0,0 @@ ---- -name: "doc-writer" -description: "Use to write or enrich exactly one evidence-grounded Markdown page with deep repository-specific explanation and Mermaid diagrams." -tools: "read_file, read_multiple_files, read_directory, glob, grep, write_file, edit_file, todo_write" ---- -You are the bounded documentation page writer. - -Apply the writing skill matching the page type plus `doc-mermaid` when diagrams are planned. Use the page manifest as a strict content contract. - -Write for multiple reading depths: orient a newcomer, give a maintainer a working model, and preserve deep technical/reference details for expert use. Prefer repository-specific facts, tables, decision matrices, step-by-step flows and explicit caveats over generic prose. - -For catalog pages, be exhaustive over the normalized catalog in scope. For flow pages, preserve branches and alternate/failure paths. For business pages, distinguish rules and inferred semantics. All diagrams must be Mermaid. - -Modify exactly one target page under `docs/**`. Never modify application source. - -## P0 Trustworthiness - -Apply `doc-traceability` and `doc-semantic-quality`. Produce typed semantic objects, claim-level evidence mappings, and explicit UNKNOWNs. Never promote unsupported prose to FACT. -## Ignore boundary - -Before reading repository source, read `.docgen/state/source-files.txt`. Do not read, search, cite, or derive facts from repository paths absent from that inventory. Existing `.docgen/**` and `docs/**` workflow artifacts remain available. - - - -Apply `doc-documentation-experience`, `doc-example-scenario-writing`, `doc-migration-versioning`, and `doc-publishing-metadata`. Respect the page mode: tutorials teach progressively, how-to pages are goal-oriented, references are exhaustive, runbooks are executable under pressure, decision records explain rationale/trade-offs, and migration guides include verification/rollback. diff --git a/project-local-template/.commandcode/commands/docgen-analyze.md b/project-local-template/.commandcode/commands/docgen-analyze.md deleted file mode 100644 index 6933475..0000000 --- a/project-local-template/.commandcode/commands/docgen-analyze.md +++ /dev/null @@ -1 +0,0 @@ -Run the globally installed DocGen orchestrator command `docgen analyze` for scope `$ARGUMENTS` (use all current evidence when empty). Report completion or the exact failure. diff --git a/project-local-template/.commandcode/commands/docgen-audit.md b/project-local-template/.commandcode/commands/docgen-audit.md index b7c3d90..f818a04 100644 --- a/project-local-template/.commandcode/commands/docgen-audit.md +++ b/project-local-template/.commandcode/commands/docgen-audit.md @@ -1 +1 @@ -Run the globally installed DocGen orchestrator command `docgen audit $1`. Report the audit result for exactly that page. +Run `docgen audit` in the current repository. Execute deterministic structural/traceability checks for every page and invoke the LLM auditor only for pages above the configured risk threshold whose audit input hash changed. Report deterministic failures, high-risk findings, audited-page count, and budget usage. diff --git a/project-local-template/.commandcode/commands/docgen-budget.md b/project-local-template/.commandcode/commands/docgen-budget.md new file mode 100644 index 0000000..826ab59 --- /dev/null +++ b/project-local-template/.commandcode/commands/docgen-budget.md @@ -0,0 +1 @@ +Run `docgen budget $ARGUMENTS`. With no argument, report provider calls, estimated input/output tokens, stage totals, remaining limits, and whether the hard budget is exceeded. Use `reset` only when the user explicitly wants telemetry cleared. diff --git a/project-local-template/.commandcode/commands/docgen-context.md b/project-local-template/.commandcode/commands/docgen-context.md new file mode 100644 index 0000000..7a9fc5b --- /dev/null +++ b/project-local-template/.commandcode/commands/docgen-context.md @@ -0,0 +1 @@ +Run `docgen context $ARGUMENTS`. Compile one bounded context pack from SQLite/FTS5 for the requested stage/query, then report the output path, estimated tokens, and omitted fact/model-item counts. Do not invoke a provider. diff --git a/project-local-template/.commandcode/commands/docgen-contract-test.md b/project-local-template/.commandcode/commands/docgen-contract-test.md deleted file mode 100644 index d63624a..0000000 --- a/project-local-template/.commandcode/commands/docgen-contract-test.md +++ /dev/null @@ -1,5 +0,0 @@ -Run the global DocGen contract firewall regression suite without calling an LLM provider: - -`docgen contract-test` - -Report the command output and stop if any producer/consumer contract test fails. diff --git a/project-local-template/.commandcode/commands/docgen-discover.md b/project-local-template/.commandcode/commands/docgen-discover.md deleted file mode 100644 index 2a460e0..0000000 --- a/project-local-template/.commandcode/commands/docgen-discover.md +++ /dev/null @@ -1 +0,0 @@ -Run the globally installed DocGen orchestrator command `docgen discover` for scope `$ARGUMENTS` (use `.` when empty). The orchestrator owns bounded headless execution, safety mode, state updates, and validation. Report completion or the exact failure. diff --git a/project-local-template/.commandcode/commands/docgen-enrich.md b/project-local-template/.commandcode/commands/docgen-enrich.md deleted file mode 100644 index 5b67adb..0000000 --- a/project-local-template/.commandcode/commands/docgen-enrich.md +++ /dev/null @@ -1 +0,0 @@ -Run the globally installed DocGen orchestrator command `docgen enrich $1`. Perform the bounded depth-and-completeness pass for exactly the requested manifest page. diff --git a/project-local-template/.commandcode/commands/docgen-enterprise.md b/project-local-template/.commandcode/commands/docgen-enterprise.md deleted file mode 100644 index 16a7572..0000000 --- a/project-local-template/.commandcode/commands/docgen-enterprise.md +++ /dev/null @@ -1,11 +0,0 @@ -Run the global DocGen enterprise-depth stage from the current repository: - -```bash -node ~/.commandcode/docgen/bin/docgen.mjs enterprise -``` - -On native Windows use: - -```powershell -node "$env:USERPROFILE\.commandcode\docgen\bin\docgen.mjs" enterprise -``` diff --git a/project-local-template/.commandcode/commands/docgen-fix.md b/project-local-template/.commandcode/commands/docgen-fix.md deleted file mode 100644 index 9409f98..0000000 --- a/project-local-template/.commandcode/commands/docgen-fix.md +++ /dev/null @@ -1 +0,0 @@ -Run the globally installed DocGen orchestrator command `docgen fix $1`, then report the repaired page. Re-audit separately when requested. diff --git a/project-local-template/.commandcode/commands/docgen-generate.md b/project-local-template/.commandcode/commands/docgen-generate.md index a142773..a154869 100644 --- a/project-local-template/.commandcode/commands/docgen-generate.md +++ b/project-local-template/.commandcode/commands/docgen-generate.md @@ -1 +1 @@ -Run the globally installed DocGen orchestrator command `docgen generate $1`. Generate exactly the requested manifest page through the bounded orchestrated workflow. +Run `docgen generate` in the current repository. Render low-risk reference/catalog pages deterministically, compile bounded item-level context packs for narrative pages, batch only pages with compatible context, generate traceability sidecars in the same invocation, and skip pages whose input hash is unchanged. diff --git a/project-local-template/.commandcode/commands/docgen-index.md b/project-local-template/.commandcode/commands/docgen-index.md new file mode 100644 index 0000000..81f9d2e --- /dev/null +++ b/project-local-template/.commandcode/commands/docgen-index.md @@ -0,0 +1 @@ +Run `docgen index $ARGUMENTS` in the current repository. Build or incrementally refresh the deterministic SQLite/FTS5 semantic index from the canonical text-source inventory, then report changed files, extracted facts, source chunks, and totals. diff --git a/project-local-template/.commandcode/commands/docgen-migrate.md b/project-local-template/.commandcode/commands/docgen-migrate.md new file mode 100644 index 0000000..0602ed8 --- /dev/null +++ b/project-local-template/.commandcode/commands/docgen-migrate.md @@ -0,0 +1 @@ +Run `docgen migrate` in the current initialized repository. This is a breaking v1-to-v2 migration: preserve `docs/**` and `.docgenignore`, archive legacy `.docgen` workflow artifacts, install the v2 config/state contract, and report the backup path. diff --git a/project-local-template/.commandcode/commands/docgen-model.md b/project-local-template/.commandcode/commands/docgen-model.md new file mode 100644 index 0000000..00c49f1 --- /dev/null +++ b/project-local-template/.commandcode/commands/docgen-model.md @@ -0,0 +1 @@ +Run `docgen model` in the current repository. Synthesize the core and enterprise model bundles from bounded semantic-index context packs, ingest the resulting typed model items back into SQLite, and report model/item counts and budget usage. diff --git a/project-local-template/.commandcode/commands/docgen-preflight.md b/project-local-template/.commandcode/commands/docgen-preflight.md deleted file mode 100644 index 62e4643..0000000 --- a/project-local-template/.commandcode/commands/docgen-preflight.md +++ /dev/null @@ -1 +0,0 @@ -Run `docgen preflight` from the current repository using the shell. Report whether manifest paths and all evidence/model references are valid before generation. diff --git a/project-local-template/.commandcode/commands/docgen-quality.md b/project-local-template/.commandcode/commands/docgen-quality.md deleted file mode 100644 index cf81ef5..0000000 --- a/project-local-template/.commandcode/commands/docgen-quality.md +++ /dev/null @@ -1 +0,0 @@ -Run `docgen quality` for the current initialized repository. Report local page-quality metrics, audit severity totals, and whether the configured quality gate passes. diff --git a/project-local-template/.commandcode/commands/docgen-semantics.md b/project-local-template/.commandcode/commands/docgen-semantics.md deleted file mode 100644 index cd58c99..0000000 --- a/project-local-template/.commandcode/commands/docgen-semantics.md +++ /dev/null @@ -1,7 +0,0 @@ -Run the global DocGen orchestrator semantics stage in the current initialized repository: - -```bash -docgen semantics -``` - -This extracts business rules/decisions/branches, six flow types, endpoints, message handlers, external dependencies, data stores and scheduled jobs into normalized model files. diff --git a/project-local-template/.commandcode/commands/docgen-traceability.md b/project-local-template/.commandcode/commands/docgen-traceability.md deleted file mode 100644 index e9c261b..0000000 --- a/project-local-template/.commandcode/commands/docgen-traceability.md +++ /dev/null @@ -1,9 +0,0 @@ -Run the global DocGen traceability and consistency index for the current repository. - -Use the shell tool: - -```bash -docgen traceability -``` - -This is deterministic and does not call an LLM provider. diff --git a/project-local-template/.commandcode/commands/docgen-update.md b/project-local-template/.commandcode/commands/docgen-update.md deleted file mode 100644 index 16ffd4a..0000000 --- a/project-local-template/.commandcode/commands/docgen-update.md +++ /dev/null @@ -1 +0,0 @@ -Run the globally installed DocGen orchestrator command `docgen update $ARGUMENTS`. When no paths are supplied, let DocGen use its fingerprint snapshot to detect changes. diff --git a/project-local-template/.docgen/README.md b/project-local-template/.docgen/README.md index 6e1d087..cce0e17 100644 --- a/project-local-template/.docgen/README.md +++ b/project-local-template/.docgen/README.md @@ -1,71 +1,67 @@ -# `.docgen` workspace +# `.docgen` workspace — v2 -This directory is the explicit machine-readable state of Command Code DocGen Kit. +This directory stores the machine-readable state of the token-efficient DocGen pipeline. ## Flow ```text -source - -> evidence/ - -> model/ +included text source + -> index/inventory.json + -> index/semantic.db + -> context/** bounded packs + -> model/** typed knowledge -> plan/manifest.json - -> ../docs/**/*.md - -> audit/ + -> ../docs/**/*.md + traceability/pages/*.json + -> audit/** + -> publish/** ``` ## Directories -- `config/` — project documentation policy, glossary, style, and Command Code runtime configuration. -- `schemas/` — JSON contracts for generated intermediate artifacts. -- `prompts/` — bounded stage prompts used by the orchestrator. -- `evidence/` — source-grounded facts and their index. -- `model/` — components, relationships, workflows, unknowns, and system model. -- `plan/` — documentation manifest and incremental update plan. -- `audit/` — per-page findings and aggregate audit index. -- `runs/` — metadata for each orchestrated Command Code headless invocation. -- `state/` — pipeline state, fingerprints, and compatibility report. - -## Most important files +- `config/` — runtime, budget, context, ignore, audit, and publishing policy. +- `index/` — canonical inventory plus SQLite/FTS5 files, facts, source chunks, model items, and context metadata. +- `context/` — content-addressed bounded inputs supplied to provider runs. +- `model/` — core and enterprise typed semantic models. +- `plan/` — the bounded page manifest. +- `traceability/` — claim-level page sidecars and aggregate index. +- `audit/` — deterministic checks, selective high-risk LLM audit, and quality summary. +- `telemetry/` — provider-run JSONL telemetry. +- `budget/` — current call/token budget report. +- `runs/` — stdout/stderr logs for provider calls. +- `publish/` — deterministic navigation and search metadata. +- `state/` — content-hash stage and page checkpoints. +- `migration-backup/` — archived v1 artifacts created by `docgen migrate`. + +## Important files ```text config/documentation.json -config/style-guide.md -config/glossary.md -evidence/index.json -model/system.json -plan/manifest.json -audit/index.json +index/inventory.json +index/source-files.txt +index/semantic.db state/state.json -state/fingerprints.json -state/compatibility.json +plan/manifest.json +traceability/index.json +audit/quality-summary.json +budget/report.json +publish/navigation.json +publish/search-index.json ``` -Do not treat generated documentation as more authoritative than source evidence. The pipeline intentionally preserves FACT / INFERENCE / UNKNOWN boundaries. +## Provider boundary +Provider sessions may read only their declared `.docgen/context/**` pack and stage output paths. They may not scan repository source, query SQLite, load broad model directories, inspect unrelated pages, or delegate to installed agents. This boundary is enforced by hooks as well as prompts. -## Core knowledge models +## Models -In addition to technical architecture, DocGen generates repository-local normalized models for business semantics, distinct flow types, and exhaustive interface/dependency catalogs: +Core: +- `model/system.json` - `model/business.json` - `model/flows.json` - `model/catalogs.json` -Published diagrams are Mermaid-only. - -## P0 trustworthiness artifacts - -- `traceability/pages/*.json`: claim-level source mappings per page. -- `traceability/index.json`: aggregated claims and source snapshot. -- `traceability/contradictions.json`: conflicting subject/predicate claims. -- `traceability/duplicates.json`: unintentional repeated claims. -- `traceability/freshness.json`: page/input/source staleness status. -- `audit/quality-summary.json`: evidence-centric quality metrics. - - -## P1 enterprise-depth models - -The enterprise stage produces repository-local typed models: +Enterprise: - `model/security.json` - `model/operations.json` @@ -76,40 +72,35 @@ The enterprise stage produces repository-local typed models: - `model/change-impact.json` - `model/ownership.json` -These models cover trust boundaries, AuthN/AuthZ, ownership/RACI, operational health and recovery, test strategy, data correctness/governance, environment configuration, architectural rationale, and change blast radius. +All material items and page claims preserve `FACT`, `INFERENCE`, and `UNKNOWN`. FACT requires evidence from `index/inventory.json`. -## Ignore-aware source inventory +## Commands -DocGen follows repository `.gitignore` and root `.docgenignore`. The effective included source set is written to: - -- `state/source-inventory.json` -- `state/source-files.txt` -- `state/ignore-report.json` - -Ignored files are excluded from discovery, fingerprints, change detection, traceability, and FACT evidence. Use `docgen ignore`, `docgen source-list`, and `docgen source-grep` to inspect or search the effective source boundary. - -## v0.9 P2 documentation experience - -Published pages are classified by user intent: `tutorial`, `how-to`, `explanation`, `reference`, `runbook`, `decision-record`, `migration-guide`, or `troubleshooting`. DocGen deterministically produces: - -- Markdown frontmatter; -- `docs/llms.txt` and bounded `docs/llms-full.txt`; -- `publish/navigation.json` and `publish/search-index.json`; -- backlinks, aliases/redirects, orphan-page and examples indexes; -- version, status, deprecation, replacement, and migration metadata. +```bash +docgen migrate # v1 repositories only +docgen index # deterministic and incremental +docgen model # two bounded synthesis calls +docgen plan +docgen generate # deterministic references + bounded narratives +docgen audit # deterministic + selective risk audit +docgen publish # no provider call +docgen budget +docgen status +``` -Run `docgen publish` to rebuild these assets without an LLM call. +`docgen all` and `docgen resume` run the full content-hash-resumable flow. -## Binary and non-text token boundary +## Ignore and binary boundary -Known images, audio/video, PDFs and office documents, archives, compiled artifacts, fonts, database files, keystores, invalid UTF-8, NUL-containing files, and oversized text are excluded from the canonical source inventory. The exclusion applies to reads, grep, fingerprints, change detection, freshness, and evidence validation. Configure the boundary under `ignore.binary` in `config/documentation.json`. +`.gitignore`, `.docgenignore`, hard exclusions, binary signatures, invalid UTF-8, NUL bytes, and oversized text are applied before indexing. Use `docgen ignore`, `docgen source-list`, and `docgen source-grep` to inspect the effective source boundary. -## Multi-repository P3 +## Multi-repository workspace -This repository can be registered into a parent system workspace with: +Register this repository into a parent workspace only after its models are current: ```bash docgen workspace add /path/to/this-repository +docgen workspace all ``` -P3 consumes this repository's validated `.docgen/model/**`, catalogs, ownership, traceability, source fingerprint, and commit metadata. It does not blindly rescan ignored or binary source files. +Workspace aggregation consumes validated repository models and hashes; it does not rescan source. diff --git a/project-local-template/.docgen/config/documentation.json b/project-local-template/.docgen/config/documentation.json index 91e83e7..40c577b 100644 --- a/project-local-template/.docgen/config/documentation.json +++ b/project-local-template/.docgen/config/documentation.json @@ -1,344 +1,92 @@ { - "schemaVersion": "1.6", + "schemaVersion": "2.0", "projectName": "", "outputRoot": "docs", - "discoveryScopes": [ - "." - ], - "exclude": [ - ".git/**", - ".idea/**", - ".vscode/**", - ".commandcode/**", - ".docgen/**", - "docs/**", - "node_modules/**", - "target/**", - "build/**", - "dist/**", - "coverage/**", - "vendor/**", - "**/*.min.js", - "**/*.map", - "**/*.class", - "**/*.jar", - "**/*.war", - "**/*.zip" - ], - "sourceExtensions": [ - ".java", - ".kt", - ".kts", - ".xml", - ".sql", - ".json", - ".yaml", - ".yml", - ".properties", - ".md", - ".go", - ".rs", - ".cs", - ".js", - ".mjs", - ".cjs", - ".ts", - ".tsx", - ".vue", - ".proto", - ".bpmn", - ".dmn", - ".sh", - ".ps1", - ".dockerfile" - ], - "audiences": [ - "engineer", - "architect", - "operator", - "product", - "business-analyst" - ], - "pageTypes": [ - "overview", - "architecture", - "business", - "concept", - "flow", - "guide", - "reference", - "data", - "integration", - "operations", - "troubleshooting", - "security", - "testing", - "configuration", - "decision", - "ownership", - "maintenance", - "tutorial", - "runbook", - "migration" - ], - "requireEvidenceReferences": true, - "diagramFormat": "mermaid", - "maxDiscoveryScopeHint": "Prefer one module or bounded subsystem per discovery run for large repositories.", "commandCode": { "executable": "", "trust": true, "skipOnboarding": true, "yolo": true, + "verbose": false, "model": "", - "stageModels": {}, - "maxTurns": { - "default": 20, - "discover": 30, - "analyze": 30, - "plan": 20, - "generate": 20, - "audit": 20, - "fix": 20, - "update-impact": 20, - "enrich": 20, - "semantics": 30, - "enterprise": 30 - } - }, - "quality": { - "profile": "comprehensive", - "autoEnrich": true, - "autoFix": true, - "reAuditAfterFix": true, - "minWordsByType": { - "overview": 900, - "architecture": 1200, - "concept": 900, - "guide": 1000, - "reference": 700, - "operations": 1000, - "business": 1000, - "flow": 1000, - "data": 1000, - "integration": 900, - "troubleshooting": 1000 + "stageModels": { + "modelCore": "", + "modelEnterprise": "", + "plan": "", + "generate": "", + "audit": "" }, - "minHeadings": 4, - "requireDeclaredSections": true, - "requirePlannedDiagrams": true, - "maxCriticalFindings": 0, - "maxHighFindings": 0, - "requireMermaidOnly": true, - "requireCoverageTags": true, - "requiredCoverageTagsWhenEvidenceExists": [ - "business-rules", - "branch-conditions", - "request-flow", - "traffic-flow", - "data-flow", - "endpoint-catalog", - "message-handler-catalog", - "external-dependency-catalog", - "security-trust-boundaries", - "authorization-model", - "data-governance", - "consistency-transactions", - "operations-observability", - "failure-recovery", - "testing-strategy", - "configuration-matrix", - "architecture-decisions", - "change-impact", - "ownership-responsibilities" - ], - "enrichmentMode": "on-failure", - "wordCountGate": "advisory", - "semanticMetrics": { - "minClaimGroundingRatio": 0.9, - "minEvidenceCoverageRatio": 0.8, - "minCatalogCoverageRatio": 1.0, - "minBranchCoverageRatio": 0.9, - "minEvidenceClaimsPer1000Words": 1.5, - "maxUnsupportedClaims": 0, - "maxContradictions": 0, - "maxStalePages": 0, - "duplicateClaimsAsWarning": true, - "minModelCoverageRatio": 0.9, - "evidenceClaimDensityGate": "hard", - "minStructuredClaimRatio": 0.7, - "maxClaimIdCollisions": 0 + "maxTurns": { + "default": 12, + "modelCore": 16, + "modelEnterprise": 16, + "plan": 12, + "generate": 12, + "audit": 10 } }, - "progress": { - "heartbeatSeconds": 10, - "noOutputWarningSeconds": 45, - "showCommandOutput": true, - "verboseCommandCode": true + "budget": { + "maxProviderCalls": 24, + "maxEstimatedInputTokens": 2500000, + "maxEstimatedOutputTokens": 500000, + "maxContextTokensPerCall": 80000, + "onExceeded": "stop-and-report" }, - "knowledgeBase": { - "target": "deep-multi-page-system-knowledge-base", - "navigation": "category-and-page hierarchy", - "preferFocusedPagesOverCatchAll": true, - "exhaustiveCatalogs": true, - "flowTypes": [ - "business", - "control", - "request", - "traffic", - "data", - "event" - ], - "diagramFormat": "mermaid-only" + "context": { + "maxTokens": { + "default": 60000, + "modelCore": 80000, + "modelEnterprise": 80000, + "plan": 50000, + "generate": 30000, + "audit": 18000 + } }, "execution": { - "resumeByDefault": true, - "skipValidPages": true, - "generateBatchSize": 4, - "enrichBatchSize": 4, - "auditBatchSize": 6, - "failFastPreflight": true, + "generationBatchSize": 4, + "maxPlannedPages": 30, + "resumeByContentHash": true, + "heartbeatSeconds": 10, + "silenceNoticeSeconds": 45, + "streamProviderOutput": false, "stageTimeoutMinutes": { "default": 20, - "discover": 35, - "analyze": 35, - "semantics": 35, - "plan": 25, - "generate": 20, - "enrich": 15, - "audit": 15, - "fix": 15, - "update-impact": 15, - "enterprise": 30 - }, - "adoptLegacyValidPages": true + "modelCore": 30, + "modelEnterprise": 30, + "plan": 20, + "generate": 25, + "audit": 20 + } + }, + "audit": { + "llmEnabled": true, + "llmRiskThreshold": 50, + "failOnCritical": true, + "failOnHigh": true }, "retry": { - "enabled": true, - "maxAttempts": 4, - "retryableExitCodes": [ - 5, - 6, - 7 - ], + "maxAttempts": 3, "initialDelaySeconds": 15, - "rateLimitDelaySeconds": 30, - "maxDelaySeconds": 120, - "multiplier": 2, - "jitterRatio": 0.2, - "countdownSeconds": 10, - "interRequestDelaySeconds": 3 - }, - "enterpriseDepth": { - "enabled": true, - "passes": [ - "governance", - "operability", - "data-and-configuration", - "evolution" - ], - "models": [ - ".docgen/model/security.json", - ".docgen/model/operations.json", - ".docgen/model/testing.json", - ".docgen/model/data-governance.json", - ".docgen/model/decisions.json", - ".docgen/model/configuration.json", - ".docgen/model/change-impact.json", - ".docgen/model/ownership.json" - ], - "requiredCoverageTagsWhenEvidenceExists": [ - "security-trust-boundaries", - "authorization-model", - "data-governance", - "consistency-transactions", - "operations-observability", - "failure-recovery", - "testing-strategy", - "configuration-matrix", - "architecture-decisions", - "change-impact", - "ownership-responsibilities" - ] + "maxDelaySeconds": 120 }, "ignore": { "useGitignore": true, "useDocgenignore": true, - "docgenignoreFile": ".docgenignore", - "writeReport": true, "rejectIgnoredEvidence": true, "binary": { "enabled": true, "probeBytes": 16384, "maxTextFileBytes": 4194304, "controlCharacterRatio": 0.08, - "allowExtensions": [], "denyExtensions": [] } }, - "documentationExperience": { - "enabled": true, - "modes": [ - "tutorial", - "how-to", - "explanation", - "reference", - "runbook", - "decision-record", - "migration-guide", - "troubleshooting" - ], - "requireMode": true, - "generateFrontmatter": true, - "generateLlmsTxt": true, - "generateLlmsFull": true, - "llmsFullMaxBytes": 5242880, - "generateSearchIndex": true, - "generateBacklinks": true, - "generateRedirects": true, - "generateExamplesIndex": true, - "requireEvidenceDerivedExamples": true, - "versioning": true, - "deprecationMetadata": true, - "adoptPreP2Pages": true, - "enforceModeSections": true, - "modeRequiredSections": { - "tutorial": [ - "Prerequisites", - "Walkthrough", - "Next Steps" - ], - "how-to": [ - "Prerequisites", - "Steps", - "Verification" - ], - "explanation": [], - "reference": [], - "runbook": [ - "Symptoms", - "Diagnosis", - "Procedure", - "Verification", - "Escalation" - ], - "decision-record": [ - "Context", - "Decision", - "Alternatives", - "Consequences" - ], - "migration-guide": [ - "Prerequisites", - "Migration Steps", - "Verification", - "Rollback" - ], - "troubleshooting": [ - "Symptoms", - "Diagnosis", - "Resolution", - "Verification" - ] - } + "publishing": { + "frontmatter": true, + "llmsTxt": true, + "llmsFullTxt": true, + "searchIndex": true, + "navigationIndex": true, + "mermaidOnly": true } } diff --git a/project-local-template/.docgen/state/state.json b/project-local-template/.docgen/state/state.json index 407f2f3..7e795f6 100644 --- a/project-local-template/.docgen/state/state.json +++ b/project-local-template/.docgen/state/state.json @@ -1,28 +1,15 @@ { - "schemaVersion": "1.0", - "kitVersion": "1.0.0", + "schemaVersion": "2.0", + "kitVersion": "2.0.0", "updatedAt": null, "stages": { - "discover": { - "status": "pending" - }, - "analyze": { - "status": "pending" - }, - "plan": { - "status": "pending" - }, - "generate": { - "status": "pending" - }, - "audit": { - "status": "pending" - }, - "semantics": { - "status": "pending" - }, - "enterprise": { - "status": "pending" - } - } + "index": { "status": "pending" }, + "modelCore": { "status": "pending" }, + "modelEnterprise": { "status": "pending" }, + "plan": { "status": "pending" }, + "generate": { "status": "pending" }, + "auditRisk": { "status": "pending" }, + "audit": { "status": "pending" } + }, + "pages": {} } diff --git a/project-local-template/AGENTS.md b/project-local-template/AGENTS.md index f618ab5..dcc4784 100644 --- a/project-local-template/AGENTS.md +++ b/project-local-template/AGENTS.md @@ -1,66 +1,57 @@ -# Documentation Engineering System +# DocGen 2 Documentation Engineering -This repository contains an evidence-grounded documentation workflow under `.docgen/**` with published Markdown under `docs/**`. +This repository uses a token-efficient, evidence-grounded documentation pipeline. Repository source is indexed once into `.docgen/index/semantic.db`; provider runs receive bounded `.docgen/context/**` packs rather than broad source access. -## Authority Order +## Authority order -When documentation claims conflict, use this authority order: +1. application source and executable behavior; +2. configuration, API/message contracts, schemas, migrations, and deployment manifests; +3. deterministic facts/source chunks in the semantic index; +4. typed `.docgen/model/**` items; +5. generated documentation. -1. application source code and executable behavior -2. configuration, contracts, schemas, migrations, and deployment manifests -3. generated evidence artifacts in `.docgen/evidence/**` -4. normalized models in `.docgen/model/**` -5. existing documentation +Existing prose is not authoritative over contradictory source evidence. -Existing prose is never authoritative over contradictory source evidence. +## Epistemic rules -## Epistemic Rules +Material statements are `FACT`, `INFERENCE`, or `UNKNOWN`. -Important technical statements must be treated as one of: +- `FACT` requires direct repository-relative evidence present in the canonical source inventory. +- `INFERENCE` must identify supporting facts/model items. +- `UNKNOWN` preserves missing or disputed information without guessing. -- `FACT`: directly supported by source evidence -- `INFERENCE`: derived from multiple facts; the supporting evidence must be recorded -- `UNKNOWN`: insufficient evidence; do not invent a conclusion +Never invent endpoints, rules, states, integrations, ownership, security behavior, retry behavior, failure semantics, or operational guarantees. -Never invent endpoints, business rules, state transitions, integrations, data ownership, security behavior, retry behavior, failure semantics, or operational guarantees. - -## Documentation Workflow - -Use this order: +## Workflow ```text -discover -> analyze -> plan -> generate -> audit -> fix as needed +index -> model -> plan -> generate -> audit -> publish ``` -Do not skip directly from source code to broad user-facing documentation for non-trivial systems. - -## Artifact Boundaries +Use `docgen all` or `docgen resume` for the complete content-hash-resumable pipeline. For a v1 project, run `docgen migrate` first. -- evidence: `.docgen/evidence/**` -- architecture/workflow model: `.docgen/model/**` -- documentation plan: `.docgen/plan/**` -- audit findings: `.docgen/audit/**` -- generated documentation: `docs/**` +## Token boundary -DocGen workflows must not modify application source, build files, migrations, infrastructure, or tests. +- source inventory: `.docgen/index/inventory.json` and `.docgen/index/source-files.txt`; +- semantic index: `.docgen/index/semantic.db`; +- bounded provider inputs: `.docgen/context/**`; +- provider telemetry/budget: `.docgen/telemetry/**` and `.docgen/budget/report.json`; +- typed models: `.docgen/model/**`; +- page plan: `.docgen/plan/manifest.json`; +- traceability: `.docgen/traceability/**`; +- audit results: `.docgen/audit/**`; +- published documentation: `docs/**`. -## Writing Standard +Provider runs are context-only: they must not read repository source, the SQLite database, broad model directories, unrelated pages, agents, or skills. Deterministic index/render/audit code owns those boundaries. -Write for engineers who do not yet know the codebase. Prefer purpose, mental model, responsibilities, boundaries, interactions, workflows, state transitions, failure behavior, and actionable guides. Avoid file-by-file or class-by-class narration. - -Use standard Markdown and Mermaid. Keep claims precise and traceable to evidence. Mark genuine uncertainty instead of smoothing it over. - +## Writing standard +Write for engineers who do not yet know the codebase. Prefer purpose, mental models, boundaries, interactions, lifecycles, branches, failure behavior, and actionable guidance over file-by-file narration. Use standard Markdown and Mermaid only. Keep claims precise and traceable. -## Source Inventory and Ignore Boundary +## Enterprise and workspace depth -During DocGen workflows, repository source access must follow `.docgen/state/source-files.txt`. Do not read, search, cite, fingerprint, or use as FACT evidence any file excluded by `.gitignore`, `.docgenignore`, DocGen hard exclusions, or project `config.exclude`. Use explicit included paths or `docgen source-grep` instead of broad wildcard reads. +When evidence supports it, model security, operations, testing, data governance, decisions, configuration, change impact, and ownership. Multi-repository workspace analysis consumes validated repository models and must preserve unresolved edges rather than inventing cross-service links. -## P1 Enterprise Depth - -When supported by evidence, build typed models for security, operations, testing, data governance, decisions, configuration, change impact, and ownership. Keep policy, business semantics, implementation behavior, operational guarantees, and inferred rationale epistemically distinct. Do not invent SLOs, permissions, owners, retention periods, recovery guarantees, or architectural rationale. - -## P3 system-of-systems workspace - -For multi-repository documentation, use `docgen workspace ...` from a parent workspace. Workspace analysis must consume validated `.docgen/model/**` artifacts from member repositories rather than bypassing repository ignore, binary, traceability, and contract boundaries. Cross-repository relationships require explicit contract/dependency evidence. All system diagrams use Mermaid. +DocGen workflows may write only under `.docgen/**` and `docs/**`; they must not modify application source, build files, migrations, infrastructure, or tests. +