Raw table improvements (#855)

Author: simolus3

.changeset/rare-doors-repeat.md

@@ -0,0 +1,9 @@
+---
+'@powersync/common': minor
+'@powersync/node': minor
+'@powersync/capacitor': minor
+'@powersync/react-native': minor
+'@powersync/web': minor
+---
+
+Improve raw tables by making `put` and `delete` statements optional if a local name is given.

packages/common/src/db/schema/RawTable.ts

@@ -1,8 +1,23 @@
+import { TableOrRawTableOptions } from './Table.js';
+
 /**
- * A pending variant of a {@link RawTable} that doesn't have a name (because it would be inferred when creating the
- * schema).
+ * Instructs PowerSync to sync data into a "raw" table.
+ *
+ * Since raw tables are not backed by JSON, running complex queries on them may be more efficient. Further, they allow
+ * using client-side table and column constraints.
+ *
+ * To collect local writes to raw tables with PowerSync, custom triggers are required. See
+ * {@link https://docs.powersync.com/usage/use-case-examples/raw-tables the documentation} for details and an example on
+ * using raw tables.
+ *
+ * Note that raw tables are only supported when using the new `SyncClientImplementation.rust` sync client.
+ *
+ * @experimental Please note that this feature is experimental at the moment, and not covered by PowerSync semver or
+ * stability guarantees.
  */
-export type RawTableType = {
+export type RawTableType = RawTableTypeWithStatements | InferredRawTableType;
+
+interface RawTableTypeWithStatements {
   /**
    * The statement to run when PowerSync detects that a row needs to be inserted or updated.
    */
@@ -11,7 +26,44 @@ export type RawTableType = {
    * The statement to run when PowerSync detects that a row needs to be deleted.
    */
   delete: PendingStatement;
-};
+
+  /**
+   * An optional statement to run when `disconnectAndClear()` is called on a PowerSync database.
+   */
+  clear?: string;
+}
+
+/**
+ * The schema of a {@link RawTableType} in the local database.
+ *
+ * This information is optional when declaring raw tables. However, providing it allows the sync client to infer `put`
+ * and `delete` statements automatically.
+ */
+interface RawTableSchema extends TableOrRawTableOptions {
+  /**
+   * The actual name of the raw table in the local schema.
+   *
+   * Unlike {@link RawTable.name}, which describes the name of synced tables to match, this reflects  the SQLite table
+   * name. This is used to infer {@link RawTableType.put} and {@link RawTableType.delete} statements for the sync
+   * client. It can also be used to auto-generate triggers forwarding writes on raw tables into the CRUD upload queue
+   * (using the `powersync_create_raw_table_crud_trigger` SQL function).
+   *
+   * When absent, defaults to {@link RawTable.name}.
+   */
+  tableName?: string;
+
+  /**
+   * An optional filter of columns that should be synced.
+   *
+   * By default, all columns in a raw table are considered for sync. If a filter is specified, PowerSync treats
+   * unmatched columns as local-only and will not attempt to sync them.
+   */
+  syncedColumns?: string[];
+}
+
+interface InferredRawTableType extends Partial<RawTableTypeWithStatements> {
+  schema: RawTableSchema;
+}
 
 /**
  * A parameter to use as part of {@link PendingStatement}.
@@ -21,8 +73,10 @@ export type RawTableType = {
  *
  * For insert and replace operations, the values of columns in the table are available as parameters through
  * `{Column: 'name'}`.
+ * The `"Rest"` parameter gets resolved to a JSON object covering all values from the synced row that haven't been
+ * covered by a `Column` parameter.
  */
-export type PendingStatementParameter = 'Id' | { Column: string };
+export type PendingStatementParameter = 'Id' | { Column: string } | 'Rest';
 
 /**
  * A statement that the PowerSync client should use to insert or delete data into a table managed by the user.
@@ -33,35 +87,16 @@ export type PendingStatement = {
 };
 
 /**
- * Instructs PowerSync to sync data into a "raw" table.
- *
- * Since raw tables are not backed by JSON, running complex queries on them may be more efficient. Further, they allow
- * using client-side table and column constraints.
- *
- * To collect local writes to raw tables with PowerSync, custom triggers are required. See
- * {@link https://docs.powersync.com/usage/use-case-examples/raw-tables the documentation} for details and an example on
- * using raw tables.
- *
- * Note that raw tables are only supported when using the new `SyncClientImplementation.rust` sync client.
- *
- * @experimental Please note that this feature is experimental at the moment, and not covered by PowerSync semver or
- * stability guarantees.
+ * @internal
  */
-export class RawTable implements RawTableType {
+export type RawTable<T extends RawTableType = RawTableType> = T & {
   /**
    * The name of the table.
    *
-   * This does not have to match the actual table name in the schema - {@link put} and {@link delete} are free to use
-   * another table. Instead, this name is used by the sync client to recognize that operations on this table (as it
-   * appears in the source / backend database) are to be handled specially.
+   * This does not have to match the actual table name in the schema - {@link RawTableType.put} and
+   * {@link RawTableType.delete} are free to use another table. Instead, this name is used by the sync client to
+   * recognize that operations on this table (as it appears in the source / backend database) are to be handled
+   * specially.
    */
   name: string;
-  put: PendingStatement;
-  delete: PendingStatement;
-
-  constructor(name: string, type: RawTableType) {
-    this.name = name;
-    this.put = type.put;
-    this.delete = type.delete;
-  }
-}
+};

packages/common/src/db/schema/Schema.ts

@@ -1,3 +1,4 @@
+import { encodeTableOptions } from './internal.js';
 import { RawTable, RawTableType } from './RawTable.js';
 import { RowType, Table } from './Table.js';
 
@@ -57,7 +58,7 @@ export class Schema<S extends SchemaType = SchemaType> {
    */
   withRawTables(tables: Record<string, RawTableType>) {
     for (const [name, rawTableDefinition] of Object.entries(tables)) {
-      this.rawTables.push(new RawTable(name, rawTableDefinition));
+      this.rawTables.push({ name, ...rawTableDefinition });
     }
   }
 
@@ -70,7 +71,31 @@ export class Schema<S extends SchemaType = SchemaType> {
   toJSON() {
     return {
       tables: this.tables.map((t) => t.toJSON()),
-      raw_tables: this.rawTables
+      raw_tables: this.rawTables.map(Schema.rawTableToJson)
     };
   }
+
+  /**
+   * Returns a representation of the raw table that is understood by the PowerSync SQLite core extension.
+   *
+   * The output of this can be passed through `JSON.serialize` and then used in `powersync_create_raw_table_crud_trigger`
+   * to define triggers for this table.
+   */
+  static rawTableToJson(table: RawTable): unknown {
+    const serialized: any = {
+      name: table.name,
+      put: table.put,
+      delete: table.delete,
+      clear: table.clear
+    };
+    if ('schema' in table) {
+      // We have schema options, those are flattened into the outer JSON object for the core extension.
+      const schema = table.schema;
+      serialized.table_name = schema.tableName ?? table.name;
+      serialized.synced_columns = schema.syncedColumns;
+      Object.assign(serialized, encodeTableOptions(table.schema));
+    }
+
+    return serialized;
+  }
 }

packages/common/src/db/schema/Table.ts

@@ -8,17 +8,24 @@ import {
 } from './Column.js';
 import { Index } from './Index.js';
 import { IndexedColumn } from './IndexedColumn.js';
+import { encodeTableOptions } from './internal.js';
 import { TableV2 } from './TableV2.js';
 
-interface SharedTableOptions {
+/**
+ * Options that apply both to JSON-based tables and raw tables.
+ */
+export interface TableOrRawTableOptions {
   localOnly?: boolean;
   insertOnly?: boolean;
-  viewName?: string;
   trackPrevious?: boolean | TrackPreviousOptions;
   trackMetadata?: boolean;
   ignoreEmptyUpdates?: boolean;
 }
 
+interface SharedTableOptions extends TableOrRawTableOptions {
+  viewName?: string;
+}
+
 /** Whether to include previous column values when PowerSync tracks local changes.
  *
  * Including old values may be helpful for some backend connector implementations, which is
@@ -341,19 +348,12 @@ export class Table<Columns extends ColumnsType = ColumnsType> {
   }
 
   toJSON() {
-    const trackPrevious = this.trackPrevious;
-
     return {
       name: this.name,
       view_name: this.viewName,
-      local_only: this.localOnly,
-      insert_only: this.insertOnly,
-      include_old: trackPrevious && ((trackPrevious as any).columns ?? true),
-      include_old_only_when_changed: typeof trackPrevious == 'object' && trackPrevious.onlyWhenChanged == true,
-      include_metadata: this.trackMetadata,
-      ignore_empty_update: this.ignoreEmptyUpdates,
       columns: this.columns.map((c) => c.toJSON()),
-      indexes: this.indexes.map((e) => e.toJSON(this))
+      indexes: this.indexes.map((e) => e.toJSON(this)),
+      ...encodeTableOptions(this)
     };
   }
 }

packages/common/src/db/schema/internal.ts

@@ -0,0 +1,17 @@
+import { TableOrRawTableOptions } from './Table.js';
+
+/**
+ * @internal Not exported from `index.ts`.
+ */
+export function encodeTableOptions(options: TableOrRawTableOptions) {
+  const trackPrevious = options.trackPrevious;
+
+  return {
+    local_only: options.localOnly,
+    insert_only: options.insertOnly,
+    include_old: trackPrevious && ((trackPrevious as any).columns ?? true),
+    include_old_only_when_changed: typeof trackPrevious == 'object' && trackPrevious.onlyWhenChanged == true,
+    include_metadata: options.trackMetadata,
+    ignore_empty_update: options.ignoreEmptyUpdates
+  };
+}

packages/common/src/index.ts

@@ -39,7 +39,7 @@ export * from './db/DBAdapter.js';
 export * from './db/schema/Column.js';
 export * from './db/schema/Index.js';
 export * from './db/schema/IndexedColumn.js';
-export * from './db/schema/RawTable.js';
+export { RawTableType, PendingStatementParameter, PendingStatement } from './db/schema/RawTable.js';
 export * from './db/schema/Schema.js';
 export * from './db/schema/Table.js';
 export * from './db/schema/TableV2.js';

packages/node/tests/PowerSyncDatabase.test.ts

@@ -1,7 +1,7 @@
 import * as path from 'node:path';
 import { Worker } from 'node:worker_threads';
 
-import { LockContext } from '@powersync/common';
+import { LockContext, Schema } from '@powersync/common';
 import { randomUUID } from 'node:crypto';
 import { expect, test, vi } from 'vitest';
 import { CrudEntry, CrudTransaction, PowerSyncDatabase } from '../lib';
@@ -227,3 +227,21 @@ tempDirectoryTest.skipIf(process.versions.node < '22.5.0')(
     }
   }
 );
+
+databaseTest('clear raw tables', async ({ database }) => {
+  await database.init();
+  const schema = new Schema({});
+  schema.withRawTables({
+    users: {
+      table_name: 'lists',
+      clear: 'DELETE FROM lists'
+    }
+  });
+  await database.updateSchema(schema);
+  await database.execute('CREATE TABLE lists (id TEXT NOT NULL PRIMARY KEY, name TEXT)');
+  await database.execute('INSERT INTO lists (id, name) VALUES (uuid(), ?)', ['list']);
+
+  expect(await database.getAll('SELECT * FROM lists')).toHaveLength(1);
+  await database.disconnectAndClear();
+  expect(await database.getAll('SELECT * FROM lists')).toHaveLength(0);
+});

packages/node/tests/crud.test.ts

@@ -1,6 +1,7 @@
-import { expect } from 'vitest';
-import { column, Schema, Table } from '@powersync/common';
+import { expect, describe } from 'vitest';
+import { column, Schema, Table, RawTable } from '@powersync/common';
 import { databaseTest } from './utils';
+import { PowerSyncDatabase } from '../src';
 
 databaseTest('include metadata', async ({ database }) => {
   await database.init();
@@ -103,3 +104,60 @@ databaseTest('ignore empty update', async ({ database }) => {
   const batch = await database.getNextCrudTransaction();
   expect(batch).toBeNull();
 });
+
+describe('raw table', () => {
+  async function createTrigger(db: PowerSyncDatabase, table: RawTable, write: string) {
+    await db.execute('SELECT powersync_create_raw_table_crud_trigger(?, ?, ?)', [
+      JSON.stringify(Schema.rawTableToJson(table)),
+      `users_${write}`,
+      write
+    ]);
+  }
+
+  databaseTest('inferred crud trigger', async ({ database }) => {
+    const table: RawTable = { name: 'users', schema: {} };
+    await database.execute('CREATE TABLE users (id TEXT, name TEXT);');
+    await createTrigger(database, table, 'INSERT');
+
+    await database.execute('INSERT INTO users (id, name) VALUES (?, ?);', ['id', 'user']);
+    const tx = await database.getNextCrudTransaction()!!;
+    expect(tx.crud).toHaveLength(1);
+    const write = tx.crud[0];
+    expect(write.op).toStrictEqual('PUT');
+    expect(write.table).toStrictEqual('users');
+    expect(write.id).toStrictEqual('id');
+    expect(write.opData).toStrictEqual({
+      name: 'user'
+    });
+  });
+
+  databaseTest('with options', async ({ database }) => {
+    const table: RawTable = {
+      name: 'custom_sync_name',
+      schema: {
+        tableName: 'users',
+        syncedColumns: ['name'],
+        ignoreEmptyUpdates: true,
+        trackPrevious: true
+      }
+    };
+    await database.execute('CREATE TABLE users (id TEXT, name TEXT, local TEXT);');
+    await database.execute('INSERT INTO users (id, name, local) VALUES (?, ?, ?);', ['id', 'name', 'local']);
+    await createTrigger(database, table, 'UPDATE');
+
+    await database.execute('UPDATE users SET name = ?, local = ?;', ['updated_name', 'updated_local']);
+    // This should not generate a CRUD entry because the only synced column is not affected.
+    await database.execute('UPDATE users SET name = ?, local = ?;', ['name', 'updated_local_2']);
+
+    const tx = await database.getNextCrudTransaction()!!;
+    expect(tx.crud).toHaveLength(1);
+    const write = tx.crud[0];
+    expect(write.op).toStrictEqual('PATCH');
+    expect(write.table).toStrictEqual('custom_sync_name');
+    expect(write.id).toStrictEqual('id');
+    expect(write.opData).toStrictEqual({ name: 'updated_name' });
+    expect(write.previousValues).toStrictEqual({
+      name: 'name'
+    });
+  });
+});