Merge branch 'main' into op-sqlite-tests

Author: Chriztiaan

.changeset/cuddly-news-carry.md

@@ -0,0 +1,5 @@
+---
+'@powersync/diagnostics-app': patch
+---
+
+Fix handling of partial checkpoints

.changeset/giant-ladybugs-dress.md

@@ -0,0 +1,8 @@
+---
+'@powersync/common': minor
+---
+
+- Add `trackPreviousValues` option on `Table` which sets `CrudEntry.previousValues` to previous values on updates.
+- Add `trackMetadata` option on `Table` which adds a `_metadata` column that can be used for updates.
+  The configured metadata is available through `CrudEntry.metadata`.
+- Add `ignoreEmptyUpdates` option which skips creating CRUD entries for updates that don't change any values.

.changeset/polite-news-sneeze.md

@@ -0,0 +1,5 @@
+---
+'@powersync/common': patch
+---
+
+Fix applying bucket state around partial syncs.

packages/common/src/client/sync/bucket/CrudEntry.ts

@@ -25,9 +25,11 @@ export type CrudEntryJSON = {
 
 type CrudEntryDataJSON = {
   data: Record<string, any>;
+  old?: Record<string, any>;
   op: UpdateType;
   type: string;
   id: string;
+  metadata?: string;
 };
 
 /**
@@ -62,6 +64,13 @@ export class CrudEntry {
    * Data associated with the change.
    */
   opData?: Record<string, any>;
+
+  /**
+   * For tables where the `trackPreviousValues` option has been enabled, this tracks previous values for
+   * `UPDATE` and `DELETE` statements.
+   */
+  previousValues?: Record<string, any>;
+
   /**
    * Table that contained the change.
    */
@@ -71,9 +80,26 @@ export class CrudEntry {
    */
   transactionId?: number;
 
+  /**
+   * Client-side metadata attached with this write.
+   *
+   * This field is only available when the `trackMetadata` option was set to `true` when creating a table
+   * and the insert or update statement set the `_metadata` column.
+   */
+  metadata?: string;
+
   static fromRow(dbRow: CrudEntryJSON) {
     const data: CrudEntryDataJSON = JSON.parse(dbRow.data);
-    return new CrudEntry(parseInt(dbRow.id), data.op, data.type, data.id, dbRow.tx_id, data.data);
+    return new CrudEntry(
+      parseInt(dbRow.id),
+      data.op,
+      data.type,
+      data.id,
+      dbRow.tx_id,
+      data.data,
+      data.old,
+      data.metadata
+    );
   }
 
   constructor(
@@ -82,14 +108,18 @@ export class CrudEntry {
     table: string,
     id: string,
     transactionId?: number,
-    opData?: Record<string, any>
+    opData?: Record<string, any>,
+    previousValues?: Record<string, any>,
+    metadata?: string
   ) {
     this.clientId = clientId;
     this.id = id;
     this.op = op;
     this.opData = opData;
     this.table = table;
     this.transactionId = transactionId;
+    this.previousValues = previousValues;
+    this.metadata = metadata;
   }
 
   /**

packages/common/src/client/sync/bucket/SqliteBucketStorage.ts

@@ -154,9 +154,9 @@ export class SqliteBucketStorage extends BaseObserver<BucketStorageListener> imp
       return { ready: false, checkpointValid: false, checkpointFailures: r.checkpointFailures };
     }
 
-    const buckets = checkpoint.buckets;
+    let buckets = checkpoint.buckets;
     if (priority !== undefined) {
-      buckets.filter((b) => hasMatchingPriority(priority, b));
+      buckets = buckets.filter((b) => hasMatchingPriority(priority, b));
     }
     const bucketNames = buckets.map((b) => b.bucket);
     await this.writeTransaction(async (tx) => {

packages/common/src/db/crud/SyncProgress.ts

@@ -54,7 +54,7 @@ export interface ProgressWithOperations {
  * Additionally, the {@link SyncProgress.untilPriority} method can be used to otbain progress towards
  * a specific priority (instead of the progress for the entire download).
  *
- * The reported progress always reflects the status towards th end of a sync iteration (after
+ * The reported progress always reflects the status towards the end of a sync iteration (after
  * which a consistent snapshot of all buckets is available locally).
  *
  * In rare cases (in particular, when a [compacting](https://docs.powersync.com/usage/lifecycle-maintenance/compacting-buckets)

packages/common/src/db/schema/Schema.ts

@@ -53,15 +53,7 @@ export class Schema<S extends SchemaType = SchemaType> {
 
   private convertToClassicTables(props: S) {
     return Object.entries(props).map(([name, table]) => {
-      const convertedTable = new Table({
-        name,
-        columns: table.columns,
-        indexes: table.indexes,
-        localOnly: table.localOnly,
-        insertOnly: table.insertOnly,
-        viewName: table.viewNameOverride || name
-      });
-      return convertedTable;
+      return table.copyWithName(name);
     });
   }
 }

packages/common/src/db/schema/Table.ts

@@ -10,16 +10,34 @@ import { Index } from './Index.js';
 import { IndexedColumn } from './IndexedColumn.js';
 import { TableV2 } from './TableV2.js';
 
-export interface TableOptions {
+interface SharedTableOptions {
+  localOnly?: boolean;
+  insertOnly?: boolean;
+  viewName?: string;
+  trackPrevious?: boolean | TrackPreviousOptions;
+  trackMetadata?: boolean;
+  ignoreEmptyUpdates?: boolean;
+}
+
+/** Whether to include previous column values when PowerSync tracks local changes.
+ *
+ * Including old values may be helpful for some backend connector implementations, which is
+ * why it can be enabled on per-table or per-columm basis.
+ */
+export interface TrackPreviousOptions {
+  /** When defined, a list of column names for which old values should be tracked. */
+  columns?: string[];
+  /** When enabled, only include values that have actually been changed by an update. */
+  onlyWhenChanged?: boolean;
+}
+
+export interface TableOptions extends SharedTableOptions {
   /**
    * The synced table name, matching sync rules
    */
   name: string;
   columns: Column[];
   indexes?: Index[];
-  localOnly?: boolean;
-  insertOnly?: boolean;
-  viewName?: string;
 }
 
 export type RowType<T extends TableV2<any>> = {
@@ -30,17 +48,17 @@ export type RowType<T extends TableV2<any>> = {
 
 export type IndexShorthand = Record<string, string[]>;
 
-export interface TableV2Options {
+export interface TableV2Options extends SharedTableOptions {
   indexes?: IndexShorthand;
-  localOnly?: boolean;
-  insertOnly?: boolean;
-  viewName?: string;
 }
 
 export const DEFAULT_TABLE_OPTIONS = {
   indexes: [],
   insertOnly: false,
-  localOnly: false
+  localOnly: false,
+  trackPrevious: false,
+  trackMetadata: false,
+  ignoreEmptyUpdates: false
 };
 
 export const InvalidSQLCharacters = /["'%,.#\s[\]]/;
@@ -137,17 +155,23 @@ export class Table<Columns extends ColumnsType = ColumnsType> {
     }
   }
 
+  copyWithName(name: string): Table {
+    return new Table({
+      ...this.options,
+      name
+    });
+  }
+
   private isTableV1(arg: TableOptions | Columns): arg is TableOptions {
     return 'columns' in arg && Array.isArray(arg.columns);
   }
 
   private initTableV1(options: TableOptions) {
     this.options = {
       ...options,
-      indexes: options.indexes || [],
-      insertOnly: options.insertOnly ?? DEFAULT_TABLE_OPTIONS.insertOnly,
-      localOnly: options.localOnly ?? DEFAULT_TABLE_OPTIONS.localOnly
+      indexes: options.indexes || []
     };
+    this.applyDefaultOptions();
   }
 
   private initTableV2(columns: Columns, options?: TableV2Options) {
@@ -173,14 +197,26 @@ export class Table<Columns extends ColumnsType = ColumnsType> {
       name: '',
       columns: convertedColumns,
       indexes: convertedIndexes,
-      insertOnly: options?.insertOnly ?? DEFAULT_TABLE_OPTIONS.insertOnly,
-      localOnly: options?.localOnly ?? DEFAULT_TABLE_OPTIONS.localOnly,
-      viewName: options?.viewName
+      viewName: options?.viewName,
+      insertOnly: options?.insertOnly,
+      localOnly: options?.localOnly,
+      trackPrevious: options?.trackPrevious,
+      trackMetadata: options?.trackMetadata,
+      ignoreEmptyUpdates: options?.ignoreEmptyUpdates
     };
+    this.applyDefaultOptions();
 
     this._mappedColumns = columns;
   }
 
+  private applyDefaultOptions() {
+    this.options.insertOnly ??= DEFAULT_TABLE_OPTIONS.insertOnly;
+    this.options.localOnly ??= DEFAULT_TABLE_OPTIONS.localOnly;
+    this.options.trackPrevious ??= DEFAULT_TABLE_OPTIONS.trackPrevious;
+    this.options.trackMetadata ??= DEFAULT_TABLE_OPTIONS.trackMetadata;
+    this.options.ignoreEmptyUpdates ??= DEFAULT_TABLE_OPTIONS.ignoreEmptyUpdates;
+  }
+
   get name() {
     return this.options.name;
   }
@@ -212,11 +248,23 @@ export class Table<Columns extends ColumnsType = ColumnsType> {
   }
 
   get localOnly() {
-    return this.options.localOnly ?? false;
+    return this.options.localOnly!;
   }
 
   get insertOnly() {
-    return this.options.insertOnly ?? false;
+    return this.options.insertOnly!;
+  }
+
+  get trackPrevious() {
+    return this.options.trackPrevious!;
+  }
+
+  get trackMetadata() {
+    return this.options.trackMetadata!;
+  }
+
+  get ignoreEmptyUpdates() {
+    return this.options.ignoreEmptyUpdates!;
   }
 
   get internalName() {
@@ -250,6 +298,13 @@ export class Table<Columns extends ColumnsType = ColumnsType> {
       throw new Error(`Table has too many columns. The maximum number of columns is ${MAX_AMOUNT_OF_COLUMNS}.`);
     }
 
+    if (this.trackMetadata && this.localOnly) {
+      throw new Error(`Can't include metadata for local-only tables.`);
+    }
+    if (this.trackPrevious != false && this.localOnly) {
+      throw new Error(`Can't include old values for local-only tables.`);
+    }
+
     const columnNames = new Set<string>();
     columnNames.add('id');
     for (const column of this.columns) {
@@ -286,11 +341,17 @@ export class Table<Columns extends ColumnsType = ColumnsType> {
   }
 
   toJSON() {
+    const trackPrevious = this.trackPrevious;
+
     return {
       name: this.name,
       view_name: this.viewName,
       local_only: this.localOnly,
       insert_only: this.insertOnly,
+      include_old: trackPrevious && ((trackPrevious as any).columns ?? true),
+      include_old_only_when_changed: typeof trackPrevious == 'object' && trackPrevious.onlyWhenChanged == true,
+      include_metadata: this.trackMetadata,
+      ignore_empty_update: this.ignoreEmptyUpdates,
       columns: this.columns.map((c) => c.toJSON()),
       indexes: this.indexes.map((e) => e.toJSON(this))
     };

packages/common/tests/db/schema/Schema.test.ts

@@ -90,6 +90,10 @@ describe('Schema', () => {
           view_name: 'users',
           local_only: false,
           insert_only: false,
+          ignore_empty_update: false,
+          include_metadata: false,
+          include_old: false,
+          include_old_only_when_changed: false,
           columns: [
             { name: 'name', type: 'TEXT' },
             { name: 'age', type: 'INTEGER' }
@@ -101,6 +105,10 @@ describe('Schema', () => {
           view_name: 'posts',
           local_only: false,
           insert_only: false,
+          ignore_empty_update: false,
+          include_metadata: false,
+          include_old: false,
+          include_old_only_when_changed: false,
           columns: [
             { name: 'title', type: 'TEXT' },
             { name: 'content', type: 'TEXT' }