feat(attachments): streaming attachment transport + saveFileFromUri

.changeset/attachment-streaming-transport.md

@@ -0,0 +1,7 @@
+---
+'@powersync/common': minor
+'@powersync/attachments-storage-react-native': minor
+'@powersync/node': minor
+---
+
+Add a streaming attachment transport (`AttachmentTransportAdapter` with a default `BufferedAttachmentTransport`, plus native `ExpoFileSystemTransportAdapter` and `ReactNativeFSTransportAdapter` implementations) and `AttachmentQueue.saveFileFromUri`, so large files can be uploaded and downloaded without being buffered in JS memory.

packages/attachments-storage-react-native/README.md

@@ -5,9 +5,12 @@
 >
 > Do not rely on this package for production use.
 
-React Native file system storage adapters for [PowerSync](https://powersync.com) attachments.
+React Native storage and transport adapters for [PowerSync](https://powersync.com) attachments.
 
-This package provides `LocalStorageAdapter` implementations for React Native environments, allowing you to store and retrieve attachment files on device.
+This package provides:
+
+- **Local storage adapters** (`LocalStorageAdapter`) — persist attachment files on device.
+- **Streaming transport adapters** (`AttachmentTransportAdapter`) — move attachment bytes directly between the local file and remote storage using native APIs, without buffering the whole file in JS memory. Recommended for large files (recordings, videos) on lower-end devices.
 
 ## Installation
 
@@ -19,7 +22,7 @@ pnpm add @powersync/attachments-storage-react-native
 yarn add @powersync/attachments-storage-react-native
 ```
 
-You'll also need to install one of the supported file system libraries:
+You'll also need to install one of the supported file system libraries. The same library powers both the local storage adapter and the native transport adapter for that platform.
 
 ### For Expo projects
 
@@ -36,20 +39,23 @@ npx expo install expo-file-system
 npm install @dr.pogodin/react-native-fs
 ```
 
-## Usage
+## Local storage adapters
+
+The local storage adapter handles file persistence on the device. Pass it to the queue as `localStorage`.
 
 ### With Expo File System
 
 ```typescript
 import { ExpoFileSystemStorageAdapter } from '@powersync/attachments-storage-react-native';
 import { AttachmentQueue } from '@powersync/react-native';
 
-const storageAdapter = new ExpoFileSystemStorageAdapter();
+const localStorage = new ExpoFileSystemStorageAdapter();
 
 const attachmentQueue = new AttachmentQueue({
-  powersync: db,
-  storage: cloudStorage,
-  storageAdapter
+  db,
+  localStorage,
+  remoteStorage, // your RemoteStorageAdapter (buffered upload/download/delete)
+  watchAttachments
 });
 ```
 
@@ -59,43 +65,137 @@ const attachmentQueue = new AttachmentQueue({
 import { ReactNativeFileSystemStorageAdapter } from '@powersync/attachments-storage-react-native';
 import { AttachmentQueue } from '@powersync/react-native';
 
-const storageAdapter = new ReactNativeFileSystemStorageAdapter();
+const localStorage = new ReactNativeFileSystemStorageAdapter();
+
+const attachmentQueue = new AttachmentQueue({
+  db,
+  localStorage,
+  remoteStorage,
+  watchAttachments
+});
+```
+
+### Custom storage directory
+
+Both local adapters accept an optional `storageDirectory` parameter:
+
+```typescript
+const localStorage = new ExpoFileSystemStorageAdapter('/custom/path/to/attachments/');
+```
+
+## Streaming transport adapters
+
+A transport adapter owns **all** remote operations (`upload` / `download` / `delete`) and transfers bytes natively — the file never enters the JS heap. Implement remote delete via the `deleteFile` callback.
+
+The queue takes **exactly one** remote mechanism: either `remoteStorage` or `transportAdapter`. A `transportAdapter` replaces `remoteStorage` entirely, so none is needed alongside it.
+
+Both transports are backend-agnostic: you supply resolver callbacks that map an attachment to a request (typically a presigned URL from your backend).
+
+### With Expo File System (`uploadAsync` / `downloadAsync`)
+
+```typescript
+import {
+  ExpoFileSystemStorageAdapter,
+  ExpoFileSystemTransportAdapter
+} from '@powersync/attachments-storage-react-native';
+import { AttachmentQueue } from '@powersync/react-native';
+
+const localStorage = new ExpoFileSystemStorageAdapter();
+
+const transportAdapter = new ExpoFileSystemTransportAdapter({
+  resolveUpload: async (attachment) => ({
+    url: await getSignedUploadUrl(attachment.filename), // from your backend
+    httpMethod: 'PUT',
+    mimeType: attachment.mediaType ?? 'application/octet-stream'
+  }),
+  resolveDownload: async (attachment) => ({
+    url: await getSignedDownloadUrl(attachment.filename)
+  }),
+  deleteFile: async (attachment) => {
+    await deleteFromRemoteStorage(attachment.filename); // your SDK / DELETE call
+  }
+});
 
 const attachmentQueue = new AttachmentQueue({
-  powersync: db,
-  storage: cloudStorage,
-  storageAdapter
+  db,
+  localStorage,
+  transportAdapter, // owns upload/download/delete — no remoteStorage needed
+  watchAttachments
+});
+```
+
+### With React Native FS (`uploadFiles` / `downloadFile`)
+
+Identical options shape. The upload is sent as a raw binary `PUT` (`binaryStreamOnly`), suitable for presigned S3/Supabase URLs.
+
+```typescript
+import {
+  ReactNativeFileSystemStorageAdapter,
+  ReactNativeFSTransportAdapter
+} from '@powersync/attachments-storage-react-native';
+
+const localStorage = new ReactNativeFileSystemStorageAdapter();
+
+const transportAdapter = new ReactNativeFSTransportAdapter({
+  resolveUpload: async (attachment) => ({
+    url: await getSignedUploadUrl(attachment.filename),
+    httpMethod: 'PUT',
+    mimeType: attachment.mediaType ?? 'application/octet-stream'
+  }),
+  resolveDownload: async (attachment) => ({
+    url: await getSignedDownloadUrl(attachment.filename)
+  }),
+  deleteFile: async (attachment) => {
+    await deleteFromRemoteStorage(attachment.filename);
+  }
 });
 ```
 
-### Custom Storage Directory
+## Registering an on-disk file (buffer-free)
 
-Both adapters accept an optional `storageDirectory` parameter:
+For files already written to disk (recordings, camera/picker output), use `AttachmentQueue.saveFileFromUri` to register them without reading the bytes into memory — the local adapter's `moveFile` relocates the file into managed storage.
 
 ```typescript
-const storageAdapter = new ExpoFileSystemStorageAdapter('/custom/path/to/attachments/');
+await attachmentQueue.saveFileFromUri({
+  localUri, // path to the existing file
+  fileExtension: 'm4a',
+  mediaType: 'audio/m4a'
+});
 ```
 
 ## API
 
-Both adapters implement the `LocalStorageAdapter` interface from `@powersync/common`:
+### Local storage adapters
+
+Implement the `LocalStorageAdapter` interface from `@powersync/common`:
 
 - `initialize()` - Create the storage directory if it doesn't exist
 - `clear()` - Remove all files from the storage directory
 - `getLocalUri(filename)` - Get the full path for a filename
 - `saveFile(filePath, data, options?)` - Save data to a file
 - `readFile(filePath, options?)` - Read a file as ArrayBuffer
+- `moveFile(sourceUri, targetUri)` - Move a file into managed storage without buffering (enables `saveFileFromUri`)
 - `deleteFile(filePath)` - Delete a file
 - `fileExists(filePath)` - Check if a file exists
 - `makeDir(path)` - Create a directory
 - `rmDir(path)` - Remove a directory
 
+### Transport adapters
+
+Implement the `AttachmentTransportAdapter` interface from `@powersync/common`:
+
+- `upload(attachment)` - Transfer the local file to remote storage
+- `download(attachment)` - Transfer the remote file into `attachment.localUri`
+- `delete(attachment)` - Delete the file from remote storage
+
 ## Supported Versions
 
-| Adapter                               | Library                       | Supported Versions |
-| ------------------------------------- | ----------------------------- | ------------------ |
-| `ExpoFileSystemStorageAdapter`        | `expo-file-system`            | >=19.0.0 (Expo 54+)|
-| `ReactNativeFileSystemStorageAdapter` | `@dr.pogodin/react-native-fs` | ^2.25.0            |
+| Adapter                               | Library                       | Supported Versions  |
+| ------------------------------------- | ----------------------------- | ------------------- |
+| `ExpoFileSystemStorageAdapter`        | `expo-file-system`            | >=19.0.0 (Expo 54+) |
+| `ExpoFileSystemTransportAdapter`      | `expo-file-system`            | >=19.0.0 (Expo 54+) |
+| `ReactNativeFileSystemStorageAdapter` | `@dr.pogodin/react-native-fs` | ^2.25.0             |
+| `ReactNativeFSTransportAdapter`       | `@dr.pogodin/react-native-fs` | ^2.25.0             |
 
 ## License
 

packages/attachments-storage-react-native/src/ExpoFileSystemStorageAdapter.ts

@@ -1,5 +1,5 @@
 import { decode as decodeBase64 } from 'base64-arraybuffer';
-import type { AttachmentData, LocalStorageAdapter } from '@powersync/common';
+import type { AttachmentData, StreamingLocalStorageAdapter } from '@powersync/common';
 import type { File, Directory } from 'expo-file-system';
 
 /**
@@ -9,7 +9,7 @@ import type { File, Directory } from 'expo-file-system';
  * @experimental
  * @alpha This is currently experimental and may change without a major version bump.
  */
-export class ExpoFileSystemStorageAdapter implements LocalStorageAdapter {
+export class ExpoFileSystemStorageAdapter implements StreamingLocalStorageAdapter {
   private File: typeof File;
   private Directory: typeof Directory;
   private storageDir: Directory;
@@ -81,6 +81,17 @@ To use the Expo File System attachment adapter please install expo-file-system (
     return buffer;
   }
 
+  async moveFile(sourceUri: string, targetUri: string): Promise<number> {
+    if (sourceUri !== targetUri) {
+      const target = new this.File(targetUri);
+      if (target.exists) {
+        target.delete();
+      }
+      new this.File(sourceUri).move(target);
+    }
+    return new this.File(targetUri).size ?? 0;
+  }
+
   async deleteFile(filePath: string): Promise<void> {
     try {
       const file = new this.File(filePath);

packages/attachments-storage-react-native/src/ExpoFileSystemTransportAdapter.ts

@@ -0,0 +1,150 @@
+import type { AttachmentRecord, AttachmentTransportAdapter, LocatedAttachmentRecord } from '@powersync/common';
+
+/**
+ * Describes the HTTP request used to upload a file's bytes to remote storage.
+ * Typically points at a presigned URL.
+ */
+export interface ExpoUploadRequest {
+  /** Destination URL (e.g. a presigned upload URL). */
+  url: string;
+  /** HTTP method. Defaults to `PUT`. */
+  httpMethod?: 'POST' | 'PUT' | 'PATCH';
+  /** Additional request headers. */
+  headers?: Record<string, string>;
+  /**
+   * Upload encoding, matching Expo's `FileSystemUploadType`
+   * (`0` = binary content, `1` = multipart). Defaults to binary content.
+   */
+  uploadType?: number;
+  /** Form field name, used only for multipart uploads. */
+  fieldName?: string;
+  /** MIME type of the file. Defaults to the attachment's `mediaType`. */
+  mimeType?: string;
+}
+
+/**
+ * Describes the HTTP request used to download a file's bytes from remote storage.
+ * Typically points at a presigned URL.
+ */
+export interface ExpoDownloadRequest {
+  /** Source URL (e.g. a presigned download URL). */
+  url: string;
+  /** Additional request headers. */
+  headers?: Record<string, string>;
+}
+
+/**
+ * Configuration for {@link ExpoFileSystemTransportAdapter}.
+ *
+ * The resolvers map an attachment to the request that transfers its bytes, keeping
+ * the transport agnostic of the remote storage backend (S3, Supabase, etc.).
+ */
+export interface ExpoFileSystemTransportAdapterOptions {
+  /** Resolves the upload request (e.g. a presigned URL) for an attachment. */
+  resolveUpload: (attachment: LocatedAttachmentRecord) => Promise<ExpoUploadRequest> | ExpoUploadRequest;
+  /** Resolves the download request (e.g. a presigned URL) for an attachment. */
+  resolveDownload: (attachment: LocatedAttachmentRecord) => Promise<ExpoDownloadRequest> | ExpoDownloadRequest;
+  /**
+   * Deletes the attachment's file from remote storage (e.g. a storage SDK call or a
+   * `DELETE` request). Delete is a plain remote operation, not a file transfer.
+   */
+  deleteFile: (attachment: AttachmentRecord) => Promise<void>;
+}
+
+/** The `expo-file-system` legacy API surface this adapter resolves and uses at runtime. */
+type ExpoUploadResult = { status: number; body?: string };
+type ExpoDownloadResult = { status: number; uri: string };
+
+interface ExpoFileSystemLegacy {
+  uploadAsync(url: string, fileUri: string, options?: Record<string, unknown>): Promise<ExpoUploadResult>;
+  downloadAsync(uri: string, fileUri: string, options?: Record<string, unknown>): Promise<ExpoDownloadResult>;
+  FileSystemUploadType?: { BINARY_CONTENT: number; MULTIPART: number };
+}
+
+/** Upload encodings, mirroring Expo's `FileSystemUploadType`. */
+const UPLOAD_TYPE_BINARY_CONTENT = 0;
+
+/**
+ * ExpoFileSystemTransportAdapter transfers attachment bytes directly between a local
+ * file URI and remote storage using Expo's native `uploadAsync` / `downloadAsync`.
+ *
+ * The bytes never enter the JS heap, so large files can be transferred without the
+ * memory pressure of the buffer-based transport. Requires `expo-file-system` (its
+ * `uploadAsync` / `downloadAsync` functions, available via `expo-file-system/legacy`
+ * on SDK 54+).
+ *
+ * @experimental
+ * @alpha This is currently experimental and may change without a major version bump.
+ */
+export class ExpoFileSystemTransportAdapter implements AttachmentTransportAdapter {
+  private fs: ExpoFileSystemLegacy;
+
+  constructor(private options: ExpoFileSystemTransportAdapterOptions) {
+    // `uploadAsync` / `downloadAsync` live in `expo-file-system/legacy` on SDK 54+,
+    // and in the main module on older SDKs. Metro only bundles static `require`
+    // string literals, so each candidate is required explicitly.
+    let resolved: ExpoFileSystemLegacy | undefined;
+    try {
+      const legacy = require('expo-file-system/legacy');
+      if (typeof legacy?.uploadAsync === 'function' && typeof legacy?.downloadAsync === 'function') {
+        resolved = legacy;
+      }
+    } catch {
+      // `/legacy` subpath not available on this SDK; fall back to the main module.
+    }
+    if (!resolved) {
+      try {
+        const main = require('expo-file-system');
+        if (typeof main?.uploadAsync === 'function' && typeof main?.downloadAsync === 'function') {
+          resolved = main;
+        }
+      } catch {
+        // expo-file-system not installed.
+      }
+    }
+
+    if (!resolved) {
+      throw new Error(`Could not resolve expo-file-system's uploadAsync/downloadAsync.
+To use the Expo File System transport please install expo-file-system.`);
+    }
+
+    this.fs = resolved;
+  }
+
+  async upload(attachment: LocatedAttachmentRecord): Promise<void> {
+    const request = await this.options.resolveUpload(attachment);
+    const binaryContent = this.fs.FileSystemUploadType?.BINARY_CONTENT ?? UPLOAD_TYPE_BINARY_CONTENT;
+
+    const result = await this.fs.uploadAsync(request.url, attachment.localUri, {
+      httpMethod: request.httpMethod ?? 'PUT',
+      uploadType: request.uploadType ?? binaryContent,
+      headers: request.headers,
+      fieldName: request.fieldName,
+      mimeType: request.mimeType ?? attachment.mediaType
+    });
+
+    if (!this.isOk(result.status)) {
+      throw new Error(`Upload for ${attachment.id} failed with status ${result.status}: ${result.body ?? ''}`);
+    }
+  }
+
+  async download(attachment: LocatedAttachmentRecord): Promise<void> {
+    const request = await this.options.resolveDownload(attachment);
+
+    const result = await this.fs.downloadAsync(request.url, attachment.localUri, {
+      headers: request.headers
+    });
+
+    if (!this.isOk(result.status)) {
+      throw new Error(`Download for ${attachment.id} failed with status ${result.status}`);
+    }
+  }
+
+  async delete(attachment: AttachmentRecord): Promise<void> {
+    await this.options.deleteFile(attachment);
+  }
+
+  private isOk(status: number): boolean {
+    return status >= 200 && status < 300;
+  }
+}