ci: stage exhaustive model bundle hardening validation

Author: fajarnugraha37

.github/model-bundle-hardening/pipeline.part-00

@@ -0,0 +1,104 @@
+import fs from 'node:fs';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { compileContext } from './context.mjs';
+import { budgetReport, runProvider } from './provider.mjs';
+import { databaseStats, indexRepository, ingestModels } from './indexer.mjs';
+import { auditRepository } from './quality.mjs';
+import { ensureDir, fileSha256, kitVersion, loadConfig, now, projectPaths, readJson, rel, sha256, slug, sourceSnapshot, stableHash, updateStage, writeJson } from './core.mjs';
+import { evidenceFromAliases, normalizeSemanticDocument, normalizeSourceModelRefs, semanticMetadata } from './semantic.mjs';
+import { extractModelObjects, mergeModelObjects, safeModelPlaceholder } from './model-bundle.mjs';
+
+const moduleDir = path.dirname(fileURLToPath(import.meta.url));
+
+function prompt(root, name, vars = {}) {
+  const override = path.join(root, '.docgen', 'prompts', name); const fallback = path.resolve(moduleDir, '..', 'prompts', name);
+  let text = fs.readFileSync(fs.existsSync(override) ? override : fallback, 'utf8');
+  for (const [key, value] of Object.entries(vars)) text = text.replaceAll(`{{${key}}}`, String(value));
+  return text;
+}
+function state(root) { return readJson(projectPaths(root).state, { schemaVersion: '2.0', kitVersion, stages: {}, pages: {} }); }
+function writeState(root, next) { writeJson(projectPaths(root).state, { ...next, schemaVersion: '2.0', kitVersion, updatedAt: now() }); }
+function stageCurrent(root, stage, inputHash, outputs = []) { const current = state(root).stages?.[stage]; return current?.status === 'completed' && current.inputHash === inputHash && outputs.every((file) => fs.existsSync(file)); }
+function completeStage(root, stage, inputHash, details = {}) { const next = state(root); next.stages ??= {}; next.stages[stage] = { status: 'completed', completedAt: now(), inputHash, ...details }; writeState(root, next); }
+function failStage(root, stage, error, details = {}) { const next = state(root); next.stages ??= {}; next.stages[stage] = { ...(next.stages[stage] ?? {}), status: 'failed', failedAt: now(), error: error.message, ...details }; writeState(root, next); }
+function pageState(root, id) { return state(root).pages?.[id] ?? {}; }
+function updatePage(root, id, patch) { const next = state(root); next.pages ??= {}; next.pages[id] = { ...(next.pages[id] ?? {}), ...patch, updatedAt: now() }; writeState(root, next); }
+function modelPath(root, name) { return path.join(projectPaths(root).model, `${name}.json`); }
+function tracePath(root, page) { return path.join(projectPaths(root).traceability, 'pages', `${page.id}.json`); }
+function validateJson(file) { const value = readJson(file); if (!value || typeof value !== 'object' || Array.isArray(value)) throw new Error(`Invalid JSON object: ${file}`); return value; }
+function artifactStamp(file) {
+  if (!fs.existsSync(file)) return null;
+  const stat = fs.statSync(file, { bigint: true });
+  return { hash: fileSha256(file), mtimeNs: String(stat.mtimeNs) };
+}
+function artifactChanged(file, before) {
+  const after = artifactStamp(file); if (!after) return false; if (!before) return true;
+  return after.hash !== before.hash || after.mtimeNs !== before.mtimeNs;
+}
+function phase(name, position, total) { console.log(`[docgen] phase ${position}/${total} ${name.toUpperCase()}`); }
+
+export function index(root, options = {}) {
+  updateStage(root, 'index', 'running');
+  try { const result = indexRepository(root, options); completeStage(root, 'index', result.inventoryFingerprint, result); return result; }
+  catch (error) { failStage(root, 'index', error); throw error; }
+}
+
+function validateBundleJson(file) {
+  const value = readJson(file);
+  if (!value || typeof value !== 'object') throw new Error(`Invalid model bundle JSON: ${file}`);
+  return value;
+}
+
+function commitModelObjects(root, stage, names, objects) {
+  const modelDir = projectPaths(root).model;
+  const transaction = `${stage}-${process.pid}-${Date.now()}`;
+  const stagingDir = path.join(modelDir, `.staging-${transaction}`);
+  const backupDir = path.join(modelDir, `.backup-${transaction}`);
+  ensureDir(stagingDir); ensureDir(backupDir);
+  const existed = new Set();
+  try {
+    for (const name of names) {
+      if (!Object.hasOwn(objects, name)) throw new Error(`${stage}: unresolved model object before commit: ${name}`);
+      const value = structuredClone(objects[name]); normalizeSemanticDocument(value);
+      const staged = path.join(stagingDir, `${name}.json`);
+      writeJson(staged, { schemaVersion: '2.0', generatedAt: now(), ...value }); validateJson(staged);
+    }
+    for (const name of names) {
+      const final = modelPath(root, name); const backup = path.join(backupDir, `${name}.json`);
+      if (fs.existsSync(final)) { fs.copyFileSync(final, backup); existed.add(name); }
+    }
+    for (const name of names) fs.copyFileSync(path.join(stagingDir, `${name}.json`), modelPath(root, name));
+  } catch (error) {
+    for (const name of names) {
+      const final = modelPath(root, name); const backup = path.join(backupDir, `${name}.json`);
+      if (existed.has(name) && fs.existsSync(backup)) fs.copyFileSync(backup, final);
+      else fs.rmSync(final, { force: true });
+    }
+    throw error;
+  } finally {
+    fs.rmSync(stagingDir, { recursive: true, force: true }); fs.rmSync(backupDir, { recursive: true, force: true });
+  }
+}
+
+async function synthesizeBundle(root, stage, names, query) {
+  const paths = projectPaths(root); const config = loadConfig(root);
+  const context = compileContext(root, { stage, query, target: stage, metadata: { expectedModels: names } });
+  const output = path.join(paths.model, `${stage}-bundle.json`); const inputHash = context.payload.inputHash; const outputs = names.map((name) => modelPath(root, name));
+  if (stageCurrent(root, stage, inputHash, outputs)) return { skipped: true, inputHash };
+  ensureDir(paths.model); updateStage(root, stage, 'running', { inputHash, contextId: context.payload.id });
+  const temporaryArtifacts = new Set([output]);
+  const renderPrompt = (expected, target, repair = false) => {
+    const base = prompt(root, stage === 'modelCore' ? 'model-core.md' : 'model-enterprise.md', { CONTEXT_PATH: rel(root, context.file), OUTPUT_PATH: rel(root, target), MODEL_NAMES: JSON.stringify(expected) });
+    if (!repair) return base;
+    return `${base}\n\nRecovery request: write only the requested model object(s). For a single requested name, a direct object or a named wrapper is accepted. Do not repeat unrelated models.`;
+  };
+  async function requestBundle(expected, target, repair = false) {
+    temporaryArtifacts.add(target); const before = artifactStamp(target); let extraction = null;
+    const inspect = () => {
+      if (!artifactChanged(target, before)) return null;
+      extraction = extractModelObjects(validateBundleJson(target), expected);
+      return extraction;
+    };
+    const accept = () => Boolean(Object.keys(inspect()?.objects ?? {}).length);
+    const provider = await runProvider(

.github/model-bundle-hardening/pipeline.part-01

@@ -0,0 +1,85 @@
+root, { stage, target: repair ? `${stage}:repair:${expected.join(',')}` : stage, prompt: renderPrompt(expected, target, repair), acceptArtifacts: accept });
+    if (!extraction) {
+      if (!artifactChanged(target, before)) throw new Error(`${stage}: provider completed without writing a fresh bundle artifact`);
+      extraction = extractModelObjects(validateBundleJson(target), expected);
+    }
+    return { provider, ...extraction };
+  }
+
+  const resolved = {}; const diagnostics = []; const recoveryErrors = []; let recovered = false; let providerCalls = 0;
+  const merge = (result) => {
+    providerCalls++; recovered ||= result.provider?.recovered === true; mergeModelObjects(resolved, result); diagnostics.push(...(result.diagnostics ?? []));
+  };
+  try {
+    const first = await requestBundle(names, output); merge(first);
+    if (!Object.keys(resolved).length) throw new Error(`${stage}: provider bundle contained no recognizable requested model objects`);
+
+    let missing = names.filter((name) => !Object.hasOwn(resolved, name));
+    if (missing.length) {
+      console.warn(`[docgen] ${stage} REPAIR | bundle omitted ${missing.join(', ')}; requesting only unresolved model object(s).`);
+      const repairOutput = path.join(paths.model, `${stage}-repair-${sha256(missing.join('|')).slice(0, 10)}-bundle.json`);
+      try { merge(await requestBundle(missing, repairOutput, true)); }
+      catch (error) { recoveryErrors.push(`batch repair: ${error.message}`); console.warn(`[docgen] ${stage} REPAIR WARNING | ${error.message}`); }
+    }
+
+    missing = names.filter((name) => !Object.hasOwn(resolved, name));
+    for (const name of missing) {
+      const singleOutput = path.join(paths.model, `${stage}-repair-${sha256(name).slice(0, 10)}-${name}.json`);
+      console.warn(`[docgen] ${stage} OBJECT REPAIR | requesting ${name} independently.`);
+      try { merge(await requestBundle([name], singleOutput, true)); }
+      catch (error) { recoveryErrors.push(`${name}: ${error.message}`); console.warn(`[docgen] ${stage} OBJECT REPAIR WARNING | ${name}: ${error.message}`); }
+    }
+
+    missing = names.filter((name) => !Object.hasOwn(resolved, name)); const degradedModels = [];
+    const missingPolicy = String(config.execution?.missingModelPolicy ?? 'placeholder').trim().toLowerCase();
+    if (missing.length && missingPolicy === 'fail') throw new Error(`Model recovery exhausted for: ${missing.join(', ')}`);
+    for (const name of missing) {
+      resolved[name] = safeModelPlaceholder(name, `${stage} provider output omitted ${name} after a batch repair and an independent object repair.`);
+      degradedModels.push(name);
+      console.warn(`[docgen] ${stage} DEGRADED | ${name} was replaced with an explicit UNKNOWN placeholder; no repository fact was invented.`);
+    }
+
+    commitModelObjects(root, stage, names, resolved);
+    completeStage(root, stage, inputHash, { models: names, degradedModels, recoveryErrors, providerCalls, contextId: context.payload.id, contextTokens: context.payload.estimatedTokens, recovered });
+    return { skipped: false, recovered, degradedModels, providerCalls, inputHash };
+  } catch (error) {
+    failStage(root, stage, error, { inputHash, contextId: context.payload.id, recoveryErrors }); throw error;
+  } finally {
+    for (const file of temporaryArtifacts) fs.rmSync(file, { force: true });
+  }
+}
+
+export async function model(root, { skipIndex = false } = {}) {
+  if (!skipIndex) index(root);
+  await synthesizeBundle(root, 'modelCore', ['system', 'business', 'flows', 'catalogs'], 'repository structure architecture components modules symbols interfaces contracts dependencies behavior domain rules states flows data and automation; detect technologies from evidence and do not assume a language or framework');
+  ingestModels(root);
+  await synthesizeBundle(root, 'modelEnterprise', ['security', 'operations', 'testing', 'data-governance', 'decisions', 'configuration', 'change-impact', 'ownership'], 'security operations testing governance configuration ownership decisions change impact reliability consistency and compatibility across any language framework runtime or deployment model');
+  return ingestModels(root);
+}
+
+function canonicalPagePath(page, category, id) {
+  let value = String(page.path ?? '').replaceAll('\\', '/').replace(/^\/+/, '');
+  if (!value) value = `docs/${category}/${slug(page.slug ?? page.title ?? id)}.md`;
+  if (!value.startsWith('docs/')) value = `docs/${value}`;
+  if (!value.endsWith('.md')) value += '.md';
+  return value;
+}
+function normalizePage(page, indexValue) {
+  const id = slug(page.id ?? page.title ?? `page-${indexValue + 1}`); const category = slug(page.category ?? page.type ?? 'overview');
+  return { id, title: String(page.title ?? id.replaceAll('-', ' ')), summary: String(page.summary ?? page.purpose ?? ''), category, path: canonicalPagePath(page, category, id), mode: String(page.mode ?? 'explanation'), type: String(page.type ?? 'explanation'), order: Number(page.order ?? indexValue + 1), audience: Array.isArray(page.audience) ? page.audience : ['engineer'], coverageTags: Array.isArray(page.coverageTags) ? page.coverageTags.map(String) : [], query: String(page.query ?? [page.title, page.summary, ...(page.coverageTags ?? [])].join(' ')), requiredSections: Array.isArray(page.requiredSections) ? page.requiredSections.map(String) : [], risk: String(page.risk ?? 'normal'), relatedPages: Array.isArray(page.relatedPages) ? page.relatedPages.map(String) : [] };
+}
+function validateManifest(root) {
+  const paths = projectPaths(root); const manifest = readJson(paths.plan); if (!Array.isArray(manifest.pages) || !manifest.pages.length) throw new Error('Manifest must contain non-empty pages[].');
+  manifest.pages = manifest.pages.map(normalizePage); const ids = new Set(); const files = new Set(); const max = Number(loadConfig(root).execution?.maxPlannedPages ?? 30);
+  if (max > 0 && manifest.pages.length > max) throw new Error(`Manifest contains ${manifest.pages.length} pages, above execution.maxPlannedPages=${max}. Consolidate duplicate user intents or raise the limit explicitly.`);
+  for (const page of manifest.pages) {
+    if (ids.has(page.id)) throw new Error(`Duplicate page id: ${page.id}`); if (files.has(page.path)) throw new Error(`Duplicate page path: ${page.path}`);
+    if (!page.title.trim() || !page.summary.trim()) throw new Error(`Page ${page.id} requires non-empty title and summary.`);
+    ids.add(page.id); files.add(page.path);
+  }
+  manifest.schemaVersion = '2.0'; manifest.generatedAt ??= now(); writeJson(paths.plan, manifest); return manifest;
+}
+
+export async function plan(root) {
+  const paths = projectPaths(root); ingestModels(root);
+  const context = compileContext(root, { stage: 'plan', query: 'documentation information architecture onboarding reference explanation tutorial operations decisions behavior interfaces dependencies configuration and change; select only concerns evidenced by this repository regardless of technology stack', targe