fix(seed-loader): report deferred reference back-fill failures instead of swallowing them (#2805)

A circular reference between two seed records is resolved in two passes: the
parent is inserted first without the back-reference (deferred), then pass 2
issues an engine.update to fill it in. If that pass-2 update threw,
resolveDeferredUpdates only logged a warning — the reference stayed NULL yet
the load still returned success: true, errors: [], totalErrored: 0. An
incomplete relationship was reported as a clean load.

Route both pass-2 failure modes (target still missing, or the back-fill write
threw) through a single recordDeferredError helper that pushes a
ReferenceResolutionError into the object's per-result errors, bumps its
errored count so summary.totalErrored is truthful, and joins allErrors so the
load reports success: false — matching how pass 1's update path already
reports write failures.

Note: a lone transient blip on the back-fill still recovers via
withTransientRetry (covered by a test); only a failure that outlasts the
retry budget — or a non-transient error — is reported.


Claude-Session: https://claude.ai/code/session_01MAg2Tzkv69j9zx6ZyyrbHC

Co-authored-by: Claude <noreply@anthropic.com>

Author: os-zhuang

packages/metadata-protocol/src/seed-loader-deferred-failure.test.ts

@@ -0,0 +1,190 @@
+// Copyright (c) 2025 ObjectStack. Licensed under the Apache-2.0 license.
+
+import { describe, it, expect, vi } from 'vitest';
+import { SeedLoaderService } from './seed-loader';
+import type { IDataEngine, IMetadataService } from '@objectstack/spec/contracts';
+
+/**
+ * framework#2805: a pass-2 (deferred) reference back-fill that FAILS must be
+ * reported — not silently swallowed.
+ *
+ * Two records reference each other (a circular dependency). The parent is
+ * inserted first without the back-reference (deferred to pass 2); pass 2 then
+ * issues an `engine.update` to fill the reference in. Before this fix, if that
+ * update threw, `resolveDeferredUpdates` only logged a warning: the reference
+ * stayed NULL yet the loader still returned `success: true`, `errors: []`,
+ * `totalErrored: 0` — an incomplete relationship reported as a clean load.
+ */
+
+function createLogger() {
+  return { info: vi.fn(), warn: vi.fn(), error: vi.fn(), debug: vi.fn() };
+}
+
+function createFaithfulEngine(): { engine: IDataEngine; store: Record<string, any[]> } {
+  const store: Record<string, any[]> = {};
+  let idCounter = 0;
+
+  const engine = {
+    find: vi.fn(async (objectName: string, query?: any) => {
+      let records = store[objectName] || [];
+      if (query?.where) {
+        records = records.filter((r) =>
+          Object.entries(query.where).every(([k, v]) => r[k] === v),
+        );
+      }
+      if (typeof query?.limit === 'number') records = records.slice(0, query.limit);
+      return records;
+    }),
+    findOne: vi.fn(async (objectName: string, query?: any) => {
+      const rows = await (engine.find as any)(objectName, { ...query, limit: 1 });
+      return rows[0] ?? null;
+    }),
+    insert: vi.fn(async (objectName: string, data: any) => {
+      if (!store[objectName]) store[objectName] = [];
+      if (Array.isArray(data)) {
+        const records = data.map((d) => ({ id: `gen-${++idCounter}`, ...d }));
+        store[objectName].push(...records);
+        return records;
+      }
+      const record = { id: `gen-${++idCounter}`, ...data };
+      store[objectName].push(record);
+      return record;
+    }),
+    update: vi.fn(async (objectName: string, data: any) => {
+      const records = store[objectName] || [];
+      const idx = records.findIndex((r) => r.id === data.id);
+      if (idx >= 0) { records[idx] = { ...records[idx], ...data }; return records[idx]; }
+      return data;
+    }),
+    delete: vi.fn(async () => ({ deleted: 1 })),
+    count: vi.fn(async (objectName: string) => (store[objectName] || []).length),
+    aggregate: vi.fn(async () => []),
+  } as unknown as IDataEngine;
+
+  return { engine, store };
+}
+
+// Two objects that reference each other → a circular dependency that forces
+// the multi-pass deferred back-fill (audit_department.head_id is filled in
+// during pass 2, once audit_worker "Alice" exists).
+function createMetadata(): IMetadataService {
+  const objects: Record<string, any> = {
+    audit_department: {
+      name: 'audit_department',
+      fields: {
+        name: { type: 'text' },
+        head_id: { type: 'lookup', reference: 'audit_worker' },
+      },
+    },
+    audit_worker: {
+      name: 'audit_worker',
+      fields: {
+        name: { type: 'text' },
+        department_id: { type: 'lookup', reference: 'audit_department' },
+      },
+    },
+  };
+  return {
+    getObject: vi.fn(async (name: string) => objects[name]),
+    listObjects: vi.fn(async () => Object.values(objects)),
+    register: vi.fn(async () => {}),
+    get: vi.fn(async (_t: string, name: string) => objects[name]),
+    list: vi.fn(async () => []),
+    unregister: vi.fn(async () => {}),
+    exists: vi.fn(async () => false),
+    listNames: vi.fn(async () => []),
+  } as unknown as IMetadataService;
+}
+
+const CONFIG = {
+  dryRun: false,
+  haltOnError: false,
+  multiPass: true,
+  defaultMode: 'insert',
+  batchSize: 1000,
+  transaction: false,
+} as any;
+
+const SEEDS = [
+  {
+    object: 'audit_department',
+    externalId: 'name',
+    mode: 'insert',
+    env: ['prod', 'dev', 'test'],
+    records: [{ name: 'Engineering', head_id: 'Alice' }],
+  },
+  {
+    object: 'audit_worker',
+    externalId: 'name',
+    mode: 'insert',
+    env: ['prod', 'dev', 'test'],
+    records: [{ name: 'Alice', department_id: 'Engineering' }],
+  },
+] as any[];
+
+describe('seed deferred back-fill failure is reported, not swallowed (framework#2805)', () => {
+  it('a failing pass-2 reference update flips success=false and counts an error', async () => {
+    const { engine, store } = createFaithfulEngine();
+    const metadata = createMetadata();
+
+    // The ONLY update in this load is pass-2's back-fill of
+    // audit_department.head_id. Make every attempt fail (a persistent
+    // "fetch failed" outlasts the transient-retry budget) so the deferred
+    // reference genuinely never lands.
+    const realUpdate = (engine.update as any).getMockImplementation();
+    let deptUpdateAttempts = 0;
+    (engine.update as any).mockImplementation(async (obj: string, data: any, opts: any) => {
+      if (obj === 'audit_department') {
+        deptUpdateAttempts++;
+        throw new Error('fetch failed');
+      }
+      return realUpdate(obj, data, opts);
+    });
+
+    const result = await new SeedLoaderService(engine, metadata, createLogger()).load({
+      seeds: SEEDS,
+      config: CONFIG,
+    });
+
+    // The back-fill was attempted (and exhausted its retries).
+    expect(deptUpdateAttempts).toBeGreaterThan(0);
+
+    // The relationship is genuinely incomplete: Engineering.head_id is still null.
+    const engineeringRow = store.audit_department.find((r) => r.name === 'Engineering')!;
+    expect(engineeringRow.head_id == null).toBe(true);
+
+    // ...so the load must NOT report clean success.
+    expect(result.success).toBe(false);
+    expect(result.summary.totalErrored).toBeGreaterThan(0);
+    expect(result.errors.length).toBeGreaterThan(0);
+    expect(result.errors.some((e: { field: string }) => e.field === 'head_id')).toBe(true);
+  });
+
+  it('a transient blip that recovers on retry still reports clean success', async () => {
+    const { engine, store } = createFaithfulEngine();
+    const metadata = createMetadata();
+
+    // First back-fill attempt blips, the retry succeeds — the reference lands,
+    // so this is NOT an error.
+    const realUpdate = (engine.update as any).getMockImplementation();
+    let deptUpdateAttempts = 0;
+    (engine.update as any).mockImplementation(async (obj: string, data: any, opts: any) => {
+      if (obj === 'audit_department') {
+        deptUpdateAttempts++;
+        if (deptUpdateAttempts === 1) throw new Error('fetch failed');
+      }
+      return realUpdate(obj, data, opts);
+    });
+
+    const result = await new SeedLoaderService(engine, metadata, createLogger()).load({
+      seeds: SEEDS,
+      config: CONFIG,
+    });
+
+    expect(deptUpdateAttempts).toBe(2); // blipped once, then succeeded
+    const aliceId = store.audit_worker.find((r) => r.name === 'Alice')!.id;
+    expect(store.audit_department.find((r) => r.name === 'Engineering')!.head_id).toBe(aliceId);
+    expect(result.success).toBe(true);
+    expect(result.summary.totalErrored).toBe(0);
+  });
+});

packages/metadata-protocol/src/seed-loader.ts

@@ -770,34 +770,61 @@ export class SeedLoaderService implements ISeedLoaderService {
               resultEntry.referencesDeferred--;
             }
           } catch (err: any) {
-            this.logger.warn('[SeedLoader] Failed to resolve deferred reference', {
+            // LOUD FAILURE (framework#2805): the target resolved but the
+            // back-fill WRITE failed (a transient error that outlasted the
+            // retry budget, a validation veto, …). The reference stays NULL —
+            // the very corruption pass 2 exists to prevent — so this must be a
+            // reported, counted error, never a silent warning. Swallowing it
+            // returned `success: true` / `totalErrored: 0` over a load that
+            // left a circular relationship half-written.
+            this.logger.warn('[SeedLoader] Failed to write deferred reference', {
               object: deferred.objectName,
               field: deferred.field,
-              error: err.message,
+              error: err?.message,
             });
+            this.recordDeferredError(deferred, allResults, allErrors,
+              `Failed to write deferred reference: ${deferred.objectName}.${deferred.field} = '${deferred.attemptedValue}' → ${deferred.targetObject}.${deferred.targetField}: ${err?.message ?? String(err)}`);
           }
         }
       } else {
-        // Still unresolved after pass 2
-        const error: ReferenceResolutionError = {
-          sourceObject: deferred.objectName,
-          field: deferred.field,
-          targetObject: deferred.targetObject,
-          targetField: deferred.targetField,
-          attemptedValue: deferred.attemptedValue,
-          recordIndex: deferred.recordIndex,
-          message: `Deferred reference unresolved after pass 2: ${deferred.objectName}.${deferred.field} = '${deferred.attemptedValue}' → ${deferred.targetObject}.${deferred.targetField} not found`,
-        };
-
-        const resultEntry = allResults.find(r => r.object === deferred.objectName);
-        if (resultEntry) {
-          resultEntry.errors.push(error);
-        }
-        allErrors.push(error);
+        // Still unresolved after pass 2 — the target never materialized.
+        this.recordDeferredError(deferred, allResults, allErrors,
+          `Deferred reference unresolved after pass 2: ${deferred.objectName}.${deferred.field} = '${deferred.attemptedValue}' → ${deferred.targetObject}.${deferred.targetField} not found`);
       }
     }
   }
 
+  /**
+   * Record a pass-2 (deferred) reference failure as a first-class error: it
+   * lands in the object's per-result `errors`, bumps its `errored` count (so
+   * `summary.totalErrored` is truthful), and joins `allErrors` (so the load
+   * reports `success: false`). Both pass-2 failure modes — target still
+   * missing, or the back-fill write threw — route through here so neither can
+   * leave an incomplete relationship reported as a clean load (framework#2805).
+   */
+  private recordDeferredError(
+    deferred: DeferredUpdate,
+    allResults: SeedLoadResult[],
+    allErrors: ReferenceResolutionError[],
+    message: string,
+  ): void {
+    const error: ReferenceResolutionError = {
+      sourceObject: deferred.objectName,
+      field: deferred.field,
+      targetObject: deferred.targetObject,
+      targetField: deferred.targetField,
+      attemptedValue: deferred.attemptedValue,
+      recordIndex: deferred.recordIndex,
+      message,
+    };
+    const resultEntry = allResults.find(r => r.object === deferred.objectName);
+    if (resultEntry) {
+      resultEntry.errors.push(error);
+      resultEntry.errored++;
+    }
+    allErrors.push(error);
+  }
+
   // ==========================================================================
   // Internal: Write Operations
   // ==========================================================================