fix: eliminate repeated missing model object failures

Author: fajarnugraha37

global-template/docgen/CONTRACTS.md

@@ -74,6 +74,19 @@ index -> modelCore -> modelEnterprise -> plan -> generate -> audit -> publish
 5. bounded recovery retries only the unresolved subset;
 6. a page becomes `completed` only after Markdown and traceability validation succeeds.
 
+## Model bundle recovery contract
+
+Model synthesis treats provider output as an untrusted transport shape. The orchestrator:
+
+1. extracts requested models from exact keys, normalized key variants, nested wrappers, descriptor arrays, JSON-string payloads, and direct singleton repair objects;
+2. salvages every recognized object from a partial bundle without writing partial model state;
+3. performs at most one batch repair for unresolved names, then at most one independent request per unresolved model;
+4. commits the reconciled model set only after every requested name is resolved;
+5. defaults unresolved names to an explicit `UNKNOWN` placeholder with no evidence and records them in `state.stages.<stage>.degradedModels`;
+6. supports `execution.missingModelPolicy = "fail"` for environments that prefer a hard gate after bounded recovery.
+
+A completed degraded stage is reusable on `docgen resume`, preventing an unbounded provider retry loop. `docgen status` exposes degraded model names in `summary.degradedModels`.
+
 ## Correctness gate
 
 The deterministic audit validates, at minimum:

global-template/docgen/lib/model-bundle.mjs

@@ -0,0 +1,197 @@
+const WRAPPER_KEYS = new Set([
+  'artifacts', 'bundle', 'data', 'documents', 'items', 'model', 'models', 'modules',
+  'objects', 'output', 'outputs', 'payload', 'result', 'results', 'response'
+]);
+const STRONG_SINGLETON_WRAPPERS = new Set(['artifacts', 'bundle', 'documents', 'model', 'models', 'modules', 'output', 'outputs', 'payload', 'response', 'result', 'results']);
+const DESCRIPTOR_KEYS = ['modelName', 'model', 'filename', 'fileName', 'path', 'name', 'key', 'type', 'kind', 'id'];
+const PAYLOAD_KEYS = ['value', 'content', 'payload', 'data', 'document', 'object', 'result', 'body', 'model'];
+const GENERIC_NAME_TOKENS = new Set(['artifact', 'document', 'model', 'object', 'output', 'result']);
+const NON_MODEL_SINGLETON_KEYS = new Set(['code', 'error', 'errors', 'message', 'messages', 'note', 'status', 'success', 'warning', 'warnings']);
+const MODEL_SIGNAL_KEYS = new Set(['id', 'name', 'title', 'kind', 'statement', 'summary', 'description', 'classification', 'confidence', 'evidence', 'unknowns', 'items']);
+
+function isPlainObject(value) {
+  return Boolean(value) && typeof value === 'object' && !Array.isArray(value);
+}
+
+function parseJsonValue(value) {
+  if (typeof value !== 'string') return value;
+  const text = value.trim();
+  if (!text || !['{', '['].includes(text[0])) return value;
+  try { return JSON.parse(text); } catch { return value; }
+}
+
+function words(value) {
+  return String(value ?? '')
+    .replace(/\.json$/i, '')
+    .replace(/([a-z0-9])([A-Z])/g, '$1-$2')
+    .toLowerCase()
+    .split(/[^a-z0-9]+/)
+    .filter(Boolean);
+}
+
+function singular(word) {
+  if (word.endsWith('ies') && word.length > 4) return `${word.slice(0, -3)}y`;
+  if (word.endsWith('s') && word.length > 3 && !word.endsWith('ss')) return word.slice(0, -1);
+  return word;
+}
+
+export function canonicalModelName(value) {
+  const parts = words(value);
+  while (parts.length && GENERIC_NAME_TOKENS.has(parts[0])) parts.shift();
+  while (parts.length && GENERIC_NAME_TOKENS.has(parts.at(-1))) parts.pop();
+  if (parts.length) parts[parts.length - 1] = singular(parts.at(-1));
+  return parts.join('');
+}
+
+function wrapperKey(value) {
+  return words(value).join('');
+}
+
+function matchesExpected(value, expected) {
+  const candidate = canonicalModelName(value);
+  return Boolean(candidate) && candidate === canonicalModelName(expected);
+}
+
+function asModelObject(value) {
+  const parsed = parseJsonValue(value);
+  if (isPlainObject(parsed)) return parsed;
+  if (Array.isArray(parsed)) return { items: parsed };
+  return null;
+}
+
+function descriptorIdentity(value) {
+  for (const key of DESCRIPTOR_KEYS) {
+    const candidate = value?.[key];
+    if (typeof candidate === 'string' || typeof candidate === 'number') return String(candidate);
+  }
+  return null;
+}
+
+function descriptorPayload(value) {
+  for (const key of PAYLOAD_KEYS) {
+    if (!(key in value)) continue;
+    const candidate = asModelObject(value[key]);
+    if (candidate) return candidate;
+  }
+  const ignored = new Set([...DESCRIPTOR_KEYS, ...PAYLOAD_KEYS]);
+  const remainder = Object.fromEntries(Object.entries(value).filter(([key]) => !ignored.has(key)));
+  return Object.keys(remainder).length ? remainder : null;
+}
+
+function directSingletonCandidate(value, expectedNames) {
+  if (expectedNames.length !== 1) return null;
+  const root = asModelObject(value);
+  if (!root) return null;
+  const keys = Object.keys(root);
+  if (!keys.length) return root;
+  if (keys.length === 1 && keys.some((key) => STRONG_SINGLETON_WRAPPERS.has(wrapperKey(key)))) return null;
+  if (keys.every((key) => NON_MODEL_SINGLETON_KEYS.has(wrapperKey(key)))) return null;
+  if (keys.some((key) => expectedNames.some((name) => matchesExpected(key, name)))) return null;
+  const structured = Object.values(root).some((entry) => entry && typeof entry === 'object');
+  const semantic = keys.some((key) => MODEL_SIGNAL_KEYS.has(wrapperKey(key)));
+  return structured || semantic ? root : null;
+}
+
+export function extractModelObjects(bundle, expectedNames, { maxDepth = 8, maxNodes = 10000 } = {}) {
+  const expected = [...new Set(expectedNames.map(String))];
+  const candidates = new Map();
+  const diagnostics = [];
+  const seen = new WeakSet();
+  let nodes = 0;
+
+  function offer(name, rawValue, score, origin) {
+    const value = asModelObject(rawValue);
+    if (!value) return;
+    const current = candidates.get(name);
+    if (!current || score > current.score) {
+      candidates.set(name, { value, score, origin });
+      diagnostics.push({ type: 'accepted', model: name, origin, score });
+    }
+  }
+
+  function visit(rawValue, depth = 0, origin = '$') {
+    if (depth > maxDepth || nodes >= maxNodes) return;
+    const value = parseJsonValue(rawValue);
+    if (Array.isArray(value)) {
+      nodes++;
+      for (let index = 0; index < value.length; index++) visit(value[index], depth + 1, `${origin}[${index}]`);
+      return;
+    }
+    if (!isPlainObject(value) || seen.has(value)) return;
+    seen.add(value); nodes++;
+
+    const matchedModelKeys = new Set();
+    for (const [key, child] of Object.entries(value)) {
+      for (const name of expected) {
+        if (matchesExpected(key, name)) { offer(name, child, 1000 - (depth * 10), `${origin}.${key}`); matchedModelKeys.add(key); }
+      }
+    }
+
+    const identity = descriptorIdentity(value);
+    if (identity) {
+      for (const name of expected) {
+        if (matchesExpected(identity, name)) offer(name, descriptorPayload(value), 800 - (depth * 10), `${origin}<${identity}>`);
+      }
+    }
+
+    const entries = Object.entries(value).sort(([left], [right]) => {
+      const leftWrapper = WRAPPER_KEYS.has(wrapperKey(left)) ? 0 : 1;
+      const rightWrapper = WRAPPER_KEYS.has(wrapperKey(right)) ? 0 : 1;
+      return leftWrapper - rightWrapper;
+    });
+    for (const [key, child] of entries) if (!matchedModelKeys.has(key)) visit(child, depth + 1, `${origin}.${key}`);
+  }
+
+  visit(bundle);
+  if (!candidates.size) {
+    const singleton = directSingletonCandidate(bundle, expected);
+    if (singleton) offer(expected[0], singleton, 100, '$<direct-singleton>');
+  }
+
+  const objects = Object.fromEntries(expected.filter((name) => candidates.has(name)).map((name) => [name, candidates.get(name).value]));
+  const missing = expected.filter((name) => !Object.hasOwn(objects, name));
+  if (nodes >= maxNodes) diagnostics.push({ type: 'limit', reason: 'maxNodes', maxNodes });
+  return { objects, missing, diagnostics, visitedNodes: nodes };
+}
+
+export function mergeModelObjects(target, extraction) {
+  for (const [name, value] of Object.entries(extraction.objects ?? {})) if (!Object.hasOwn(target, name)) target[name] = value;
+  return target;
+}
+
+export function safeModelPlaceholder(name, reason = 'Provider omitted the requested model object after bounded recovery.') {
+  return {
+    status: 'degraded',
+    providerOutputStatus: 'missing',
+    classification: 'UNKNOWN',
+    confidence: 0,
+    evidence: [],
+    unknowns: [{
+      id: `${canonicalModelName(name) || 'model'}-provider-output-missing`,
+      kind: 'provider-output-gap',
+      name: `${name} model unavailable`,
+      statement: reason,
+      classification: 'UNKNOWN',
+      confidence: 0,
+      evidence: []
+    }],
+    normalizationNotes: [`A deterministic UNKNOWN placeholder was created for ${name}; no repository fact was invented.`]
+  };
+}
+
+export function resolveModelObjects(expectedNames, bundles, { missingPolicy = 'placeholder', placeholderReason } = {}) {
+  const expected = [...new Set(expectedNames.map(String))];
+  const objects = {};
+  const diagnostics = [];
+  for (const [index, bundle] of bundles.entries()) {
+    const extraction = extractModelObjects(bundle, expected.filter((name) => !Object.hasOwn(objects, name)));
+    mergeModelObjects(objects, extraction);
+    diagnostics.push(...extraction.diagnostics.map((entry) => ({ ...entry, attempt: index + 1 })));
+  }
+  const unresolved = expected.filter((name) => !Object.hasOwn(objects, name));
+  const degraded = [];
+  if (unresolved.length && missingPolicy !== 'fail') {
+    for (const name of unresolved) { objects[name] = safeModelPlaceholder(name, placeholderReason); degraded.push(name); }
+  }
+  return { objects, missing: expected.filter((name) => !Object.hasOwn(objects, name)), degraded, diagnostics };
+}

global-template/docgen/lib/model-io.mjs

@@ -0,0 +1,55 @@
+import fs from 'node:fs';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { ensureDir, fileSha256, now, projectPaths, readJson, writeJson } from './core.mjs';
+import { normalizeSemanticDocument } from './semantic.mjs';
+
+const moduleDir = path.dirname(fileURLToPath(import.meta.url));
+export const modelPath = (root, name) => path.join(projectPaths(root).model, `${name}.json`);
+export const stamp = (file) => fs.existsSync(file) ? { hash: fileSha256(file), mtimeMs: fs.statSync(file).mtimeMs } : null;
+export const changed = (file, before) => { const after = stamp(file); return Boolean(after && (!before || after.hash !== before.hash || after.mtimeMs !== before.mtimeMs)); };
+
+export function renderModelPrompt(root, name, vars, repair = false) {
+  const override = path.join(root, '.docgen', 'prompts', name);
+  let text = fs.readFileSync(fs.existsSync(override) ? override : path.resolve(moduleDir, '..', 'prompts', name), 'utf8');
+  for (const [key, value] of Object.entries(vars)) text = text.replaceAll(`{{${key}}}`, String(value));
+  return repair ? `${text}\n\nRecovery request: write only the requested model object(s). A direct object is accepted for a single requested name.` : text;
+}
+export function stageCurrent(root, stage, inputHash, outputs) {
+  const current = readJson(projectPaths(root).state, {}).stages?.[stage];
+  return current?.status === 'completed' && current.inputHash === inputHash && outputs.every(fs.existsSync);
+}
+export function readBundle(file) {
+  const value = readJson(file);
+  if (!value || typeof value !== 'object') throw new Error(`Invalid model bundle JSON: ${file}`);
+  return value;
+}
+export function commitModels(root, stage, names, objects) {
+  const dir = projectPaths(root).model;
+  const token = `${stage}-${process.pid}-${Date.now()}`;
+  const staging = path.join(dir, `.staging-${token}`);
+  const backup = path.join(dir, `.backup-${token}`);
+  ensureDir(staging); ensureDir(backup);
+  const existed = new Set();
+  try {
+    for (const name of names) {
+      const value = structuredClone(objects[name]);
+      normalizeSemanticDocument(value);
+      writeJson(path.join(staging, `${name}.json`), { schemaVersion: '2.0', generatedAt: now(), ...value });
+    }
+    for (const name of names) {
+      const final = modelPath(root, name);
+      if (fs.existsSync(final)) { fs.copyFileSync(final, path.join(backup, `${name}.json`)); existed.add(name); }
+    }
+    for (const name of names) fs.copyFileSync(path.join(staging, `${name}.json`), modelPath(root, name));
+  } catch (error) {
+    for (const name of names) {
+      const final = modelPath(root, name); const saved = path.join(backup, `${name}.json`);
+      if (existed.has(name) && fs.existsSync(saved)) fs.copyFileSync(saved, final); else fs.rmSync(final, { force: true });
+    }
+    throw error;
+  } finally {
+    fs.rmSync(staging, { recursive: true, force: true });
+    fs.rmSync(backup, { recursive: true, force: true });
+  }
+}

global-template/docgen/lib/model-synthesis.mjs

@@ -0,0 +1,75 @@
+import fs from 'node:fs';
+import path from 'node:path';
+import { compileContext } from './context.mjs';
+import { runProvider } from './provider.mjs';
+import { ensureDir, loadConfig, now, projectPaths, rel, sha256, updateStage } from './core.mjs';
+import { extractModelObjects, mergeModelObjects, safeModelPlaceholder } from './model-bundle.mjs';
+import { changed, commitModels, modelPath, readBundle, renderModelPrompt, stageCurrent, stamp } from './model-io.mjs';
+
+export async function synthesizeModels(root, stage, names, query) {
+  const paths = projectPaths(root); const config = loadConfig(root);
+  const context = compileContext(root, { stage, query, target: stage, metadata: { expectedModels: names } });
+  const inputHash = context.payload.inputHash; const outputs = names.map((name) => modelPath(root, name));
+  if (stageCurrent(root, stage, inputHash, outputs)) return { skipped: true, inputHash };
+  ensureDir(paths.model); updateStage(root, stage, 'running', { inputHash, contextId: context.payload.id });
+  const temporary = new Set(); const resolved = {}; const recoveryErrors = [];
+  let providerCalls = 0; let recovered = false;
+  async function request(expected, target, repair = false) {
+    temporary.add(target); const before = stamp(target); let extraction = null;
+    const inspect = () => {
+      if (!changed(target, before)) return null;
+      extraction = extractModelObjects(readBundle(target), expected);
+      return extraction;
+    };
+    providerCalls++;
+    const provider = await runProvider(root, {
+      stage,
+      target: repair ? `${stage}:repair:${expected.join(',')}` : stage,
+      prompt: renderModelPrompt(root, stage === 'modelCore' ? 'model-core.md' : 'model-enterprise.md', {
+        CONTEXT_PATH: rel(root, context.file), OUTPUT_PATH: rel(root, target), MODEL_NAMES: JSON.stringify(expected)
+      }, repair),
+      acceptArtifacts: () => Boolean(Object.keys(inspect()?.objects ?? {}).length)
+    });
+    recovered ||= provider.recovered === true;
+    extraction ??= inspect();
+    if (!extraction || !Object.keys(extraction.objects ?? {}).length) throw new Error(`${stage}: provider completed without a fresh recognizable model artifact`);
+    return extraction;
+  }
+  const merge = (result) => mergeModelObjects(resolved, result);
+  try {
+    try { merge(await request(names, path.join(paths.model, `${stage}-bundle.json`))); }
+    catch (error) { recoveryErrors.push(`initial: ${error.message}`); }
+    let missing = names.filter((name) => !Object.hasOwn(resolved, name));
+    if (missing.length) {
+      console.warn(`[docgen] ${stage} REPAIR | unresolved: ${missing.join(', ')}`);
+      try { merge(await request(missing, path.join(paths.model, `${stage}-repair-${sha256(missing.join('|')).slice(0, 10)}-bundle.json`), true)); }
+      catch (error) { recoveryErrors.push(`batch: ${error.message}`); }
+    }
+    missing = names.filter((name) => !Object.hasOwn(resolved, name));
+    for (const name of missing) {
+      console.warn(`[docgen] ${stage} OBJECT REPAIR | ${name}`);
+      try { merge(await request([name], path.join(paths.model, `${stage}-repair-${sha256(name).slice(0, 10)}-${name}.json`), true)); }
+      catch (error) { recoveryErrors.push(`${name}: ${error.message}`); }
+    }
+    missing = names.filter((name) => !Object.hasOwn(resolved, name));
+    const policy = String(config.execution?.missingModelPolicy ?? 'placeholder').toLowerCase();
+    if (missing.length && policy === 'fail') throw new Error(`Model recovery exhausted for: ${missing.join(', ')}`);
+    const degradedModels = [];
+    for (const name of missing) {
+      resolved[name] = safeModelPlaceholder(name, `${stage} provider output omitted ${name} after bounded recovery.`);
+      degradedModels.push(name);
+      console.warn(`[docgen] ${stage} DEGRADED | ${name} -> explicit UNKNOWN placeholder`);
+    }
+    commitModels(root, stage, names, resolved);
+    updateStage(root, stage, 'completed', {
+      inputHash, completedAt: now(), models: names, degradedModels, recoveryErrors, providerCalls,
+      contextId: context.payload.id, contextTokens: context.payload.estimatedTokens, recovered
+    });
+    return { skipped: false, inputHash, degradedModels, providerCalls, recovered };
+  } catch (error) {
+    updateStage(root, stage, 'failed', { inputHash, failedAt: now(), error: error.message, recoveryErrors, contextId: context.payload.id });
+    throw error;
+  } finally {
+    for (const file of temporary) fs.rmSync(file, { force: true });
+  }
+}