Implement metadata history schemas and database tracking

Agent-Logs-Url: https://github.com/objectstack-ai/framework/sessions/f60fe20f-aa2a-44ef-9eea-fb72d6cc454f

Co-authored-by: hotlong <50353452+hotlong@users.noreply.github.com>

Author: Claude

packages/metadata/src/loaders/database-loader.ts

@@ -17,10 +17,13 @@ import type {
   MetadataSaveOptions,
   MetadataSaveResult,
   MetadataRecord,
+  MetadataHistoryRecord,
 } from '@objectstack/spec/system';
 import { SysMetadataObject } from '../objects/sys-metadata.object.js';
+import { SysMetadataHistoryObject } from '../objects/sys-metadata-history.object.js';
 import type { IDataDriver } from '@objectstack/spec/contracts';
 import type { MetadataLoader } from './loader-interface.js';
+import { calculateChecksum } from '../utils/metadata-history-utils.js';
 
 /**
  * Configuration for the DatabaseLoader.
@@ -32,8 +35,14 @@ export interface DatabaseLoaderOptions {
   /** The table name to store metadata records (default: 'sys_metadata') */
   tableName?: string;
 
+  /** The table name to store history records (default: 'sys_metadata_history') */
+  historyTableName?: string;
+
   /** Tenant ID for multi-tenant isolation */
   tenantId?: string;
+
+  /** Enable history tracking (default: true) */
+  trackHistory?: boolean;
 }
 
 /**
@@ -57,13 +66,18 @@ export class DatabaseLoader implements MetadataLoader {
 
   private driver: IDataDriver;
   private tableName: string;
+  private historyTableName: string;
   private tenantId?: string;
+  private trackHistory: boolean;
   private schemaReady = false;
+  private historySchemaReady = false;
 
   constructor(options: DatabaseLoaderOptions) {
     this.driver = options.driver;
     this.tableName = options.tableName ?? 'sys_metadata';
+    this.historyTableName = options.historyTableName ?? 'sys_metadata_history';
     this.tenantId = options.tenantId;
+    this.trackHistory = options.trackHistory !== false; // Default to true
   }
 
   /**
@@ -86,6 +100,25 @@ export class DatabaseLoader implements MetadataLoader {
     }
   }
 
+  /**
+   * Ensure the history table exists.
+   * Uses IDataDriver.syncSchema with the SysMetadataHistoryObject definition.
+   */
+  private async ensureHistorySchema(): Promise<void> {
+    if (!this.trackHistory || this.historySchemaReady) return;
+
+    try {
+      await this.driver.syncSchema(this.historyTableName, {
+        ...SysMetadataHistoryObject,
+        name: this.historyTableName,
+      });
+      this.historySchemaReady = true;
+    } catch {
+      // If syncSchema fails (e.g. table already exists), mark ready and continue
+      this.historySchemaReady = true;
+    }
+  }
+
   /**
    * Build base filter conditions for queries.
    * Always includes tenantId when configured.
@@ -101,6 +134,83 @@ export class DatabaseLoader implements MetadataLoader {
     return filter;
   }
 
+  /**
+   * Create a history record for a metadata change.
+   *
+   * @param metadataId - The metadata record ID
+   * @param type - Metadata type
+   * @param name - Metadata name
+   * @param version - Version number
+   * @param metadata - The metadata payload
+   * @param operationType - Type of operation
+   * @param previousChecksum - Checksum of previous version (if any)
+   * @param changeNote - Optional change description
+   * @param recordedBy - Optional user who made the change
+   */
+  private async createHistoryRecord(
+    metadataId: string,
+    type: string,
+    name: string,
+    version: number,
+    metadata: unknown,
+    operationType: 'create' | 'update' | 'publish' | 'revert' | 'delete',
+    previousChecksum?: string,
+    changeNote?: string,
+    recordedBy?: string
+  ): Promise<void> {
+    if (!this.trackHistory) return;
+
+    await this.ensureHistorySchema();
+
+    const now = new Date().toISOString();
+    const checksum = await calculateChecksum(metadata);
+
+    // Skip if checksum matches previous version (no actual change)
+    if (previousChecksum && checksum === previousChecksum && operationType === 'update') {
+      return;
+    }
+
+    const historyId = generateId();
+    const metadataJson = JSON.stringify(metadata);
+
+    const historyRecord: Partial<MetadataHistoryRecord> = {
+      id: historyId,
+      metadataId,
+      name,
+      type,
+      version,
+      operationType,
+      metadata: metadataJson as any,
+      checksum,
+      previousChecksum,
+      changeNote,
+      recordedBy,
+      recordedAt: now,
+      ...(this.tenantId ? { tenantId: this.tenantId } : {}),
+    };
+
+    try {
+      await this.driver.create(this.historyTableName, {
+        id: historyRecord.id,
+        metadata_id: historyRecord.metadataId,
+        name: historyRecord.name,
+        type: historyRecord.type,
+        version: historyRecord.version,
+        operation_type: historyRecord.operationType,
+        metadata: historyRecord.metadata,
+        checksum: historyRecord.checksum,
+        previous_checksum: historyRecord.previousChecksum,
+        change_note: historyRecord.changeNote,
+        recorded_by: historyRecord.recordedBy,
+        recorded_at: historyRecord.recordedAt,
+        ...(this.tenantId ? { tenant_id: this.tenantId } : {}),
+      });
+    } catch (error) {
+      // Log error but don't fail the main operation
+      console.error(`Failed to create history record for ${type}/${name}:`, error);
+    }
+  }
+
   /**
    * Convert a database row to a metadata payload.
    * Parses the JSON `metadata` column back into an object.
@@ -280,6 +390,7 @@ export class DatabaseLoader implements MetadataLoader {
 
     const now = new Date().toISOString();
     const metadataJson = JSON.stringify(data);
+    const newChecksum = await calculateChecksum(data);
 
     try {
       const existing = await this.driver.findOne(this.tableName, {
@@ -290,13 +401,27 @@ export class DatabaseLoader implements MetadataLoader {
       if (existing) {
         // Update existing record
         const version = ((existing.version as number) ?? 0) + 1;
+        const previousChecksum = existing.checksum as string | undefined;
+
         await this.driver.update(this.tableName, existing.id as string, {
           metadata: metadataJson,
           version,
+          checksum: newChecksum,
           updated_at: now,
           state: 'active',
         });
 
+        // Create history record for update
+        await this.createHistoryRecord(
+          existing.id as string,
+          type,
+          name,
+          version,
+          data,
+          'update',
+          previousChecksum
+        );
+
         return {
           success: true,
           path: `datasource://${this.tableName}/${type}/${name}`,
@@ -313,6 +438,7 @@ export class DatabaseLoader implements MetadataLoader {
           namespace: 'default',
           scope: (data as any)?.scope ?? 'platform',
           metadata: metadataJson,
+          checksum: newChecksum,
           strategy: 'merge',
           state: 'active',
           version: 1,
@@ -322,6 +448,16 @@ export class DatabaseLoader implements MetadataLoader {
           updated_at: now,
         });
 
+        // Create history record for creation
+        await this.createHistoryRecord(
+          id,
+          type,
+          name,
+          1,
+          data,
+          'create'
+        );
+
         return {
           success: true,
           path: `datasource://${this.tableName}/${type}/${name}`,

packages/metadata/src/objects/sys-metadata-history.object.ts

@@ -0,0 +1,148 @@
+// Copyright (c) 2025 ObjectStack. Licensed under the Apache-2.0 license.
+
+import { ObjectSchema, Field } from '@objectstack/spec/data';
+
+/**
+ * sys_metadata_history — Metadata Version History Object
+ *
+ * Stores historical snapshots of metadata changes for version tracking,
+ * audit trail, and rollback capabilities.
+ *
+ * This is a system object (isSystem: true) — protected from deletion and
+ * automatically provisioned when metadata history is enabled.
+ *
+ * Each record represents a single version snapshot of a metadata item,
+ * created whenever the metadata is modified, published, or reverted.
+ *
+ * @see MetadataHistoryRecordSchema in metadata-persistence.zod.ts
+ */
+export const SysMetadataHistoryObject = ObjectSchema.create({
+  namespace: 'sys',
+  name: 'metadata_history',
+  label: 'Metadata History',
+  pluralLabel: 'Metadata History',
+  icon: 'history',
+  isSystem: true,
+  description: 'Version history and audit trail for metadata changes',
+
+  fields: {
+    /** Primary Key (UUID) */
+    id: Field.text({
+      label: 'ID',
+      required: true,
+      readonly: true,
+    }),
+
+    /** Foreign key to sys_metadata.id */
+    metadata_id: Field.text({
+      label: 'Metadata ID',
+      required: true,
+      readonly: true,
+      maxLength: 255,
+    }),
+
+    /** Machine name (denormalized for easier querying) */
+    name: Field.text({
+      label: 'Name',
+      required: true,
+      searchable: true,
+      readonly: true,
+      maxLength: 255,
+    }),
+
+    /** Metadata type (denormalized for easier querying) */
+    type: Field.text({
+      label: 'Metadata Type',
+      required: true,
+      searchable: true,
+      readonly: true,
+      maxLength: 100,
+    }),
+
+    /** Version number at this snapshot */
+    version: Field.number({
+      label: 'Version',
+      required: true,
+      readonly: true,
+    }),
+
+    /** Type of operation that created this history entry */
+    operation_type: Field.select(['create', 'update', 'publish', 'revert', 'delete'], {
+      label: 'Operation Type',
+      required: true,
+      readonly: true,
+    }),
+
+    /** Historical metadata snapshot (JSON payload) */
+    metadata: Field.textarea({
+      label: 'Metadata',
+      required: true,
+      readonly: true,
+      description: 'JSON-serialized metadata snapshot at this version',
+    }),
+
+    /** SHA-256 checksum of metadata content */
+    checksum: Field.text({
+      label: 'Checksum',
+      required: true,
+      readonly: true,
+      maxLength: 64,
+    }),
+
+    /** Checksum of the previous version */
+    previous_checksum: Field.text({
+      label: 'Previous Checksum',
+      required: false,
+      readonly: true,
+      maxLength: 64,
+    }),
+
+    /** Human-readable description of changes */
+    change_note: Field.textarea({
+      label: 'Change Note',
+      required: false,
+      readonly: true,
+      description: 'Description of what changed in this version',
+    }),
+
+    /** Tenant ID for multi-tenant isolation */
+    tenant_id: Field.text({
+      label: 'Tenant ID',
+      required: false,
+      readonly: true,
+      maxLength: 255,
+    }),
+
+    /** User who made this change */
+    recorded_by: Field.text({
+      label: 'Recorded By',
+      required: false,
+      readonly: true,
+      maxLength: 255,
+    }),
+
+    /** When was this version recorded */
+    recorded_at: Field.datetime({
+      label: 'Recorded At',
+      required: true,
+      readonly: true,
+    }),
+  },
+
+  indexes: [
+    { fields: ['metadata_id', 'version'], unique: true },
+    { fields: ['metadata_id', 'recorded_at'] },
+    { fields: ['type', 'name'] },
+    { fields: ['recorded_at'] },
+    { fields: ['operation_type'] },
+    { fields: ['tenant_id'] },
+  ],
+
+  enable: {
+    trackHistory: false, // Don't track history of history records
+    searchable: false,
+    apiEnabled: true,
+    apiMethods: ['get', 'list'], // Read-only via API
+    trash: false,
+  },
+});