diff --git a/.changeset/attachment-streaming-transport.md b/.changeset/attachment-streaming-transport.md new file mode 100644 index 000000000..ecba13b2f --- /dev/null +++ b/.changeset/attachment-streaming-transport.md @@ -0,0 +1,7 @@ +--- +'@powersync/common': minor +'@powersync/attachments-storage-react-native': minor +'@powersync/node': minor +--- + +Add a streaming attachment transport (`AttachmentTransportAdapter` with a default `BufferedAttachmentTransport`, plus native `ExpoFileSystemTransportAdapter` and `ReactNativeFSTransportAdapter` implementations) and `AttachmentQueue.saveFileFromUri`, so large files can be uploaded and downloaded without being buffered in JS memory. diff --git a/packages/attachments-storage-react-native/README.md b/packages/attachments-storage-react-native/README.md index 0c41c39ee..8631ed192 100644 --- a/packages/attachments-storage-react-native/README.md +++ b/packages/attachments-storage-react-native/README.md @@ -5,9 +5,12 @@ > > Do not rely on this package for production use. -React Native file system storage adapters for [PowerSync](https://powersync.com) attachments. +React Native storage and transport adapters for [PowerSync](https://powersync.com) attachments. -This package provides `LocalStorageAdapter` implementations for React Native environments, allowing you to store and retrieve attachment files on device. +This package provides: + +- **Local storage adapters** (`LocalStorageAdapter`) — persist attachment files on device. +- **Streaming transport adapters** (`AttachmentTransportAdapter`) — move attachment bytes directly between the local file and remote storage using native APIs, without buffering the whole file in JS memory. Recommended for large files (recordings, videos) on lower-end devices. ## Installation @@ -19,7 +22,7 @@ pnpm add @powersync/attachments-storage-react-native yarn add @powersync/attachments-storage-react-native ``` -You'll also need to install one of the supported file system libraries: +You'll also need to install one of the supported file system libraries. The same library powers both the local storage adapter and the native transport adapter for that platform. ### For Expo projects @@ -36,7 +39,9 @@ npx expo install expo-file-system npm install @dr.pogodin/react-native-fs ``` -## Usage +## Local storage adapters + +The local storage adapter handles file persistence on the device. Pass it to the queue as `localStorage`. ### With Expo File System @@ -44,12 +49,13 @@ npm install @dr.pogodin/react-native-fs import { ExpoFileSystemStorageAdapter } from '@powersync/attachments-storage-react-native'; import { AttachmentQueue } from '@powersync/react-native'; -const storageAdapter = new ExpoFileSystemStorageAdapter(); +const localStorage = new ExpoFileSystemStorageAdapter(); const attachmentQueue = new AttachmentQueue({ - powersync: db, - storage: cloudStorage, - storageAdapter + db, + localStorage, + remoteStorage, // your RemoteStorageAdapter (buffered upload/download/delete) + watchAttachments }); ``` @@ -59,43 +65,137 @@ const attachmentQueue = new AttachmentQueue({ import { ReactNativeFileSystemStorageAdapter } from '@powersync/attachments-storage-react-native'; import { AttachmentQueue } from '@powersync/react-native'; -const storageAdapter = new ReactNativeFileSystemStorageAdapter(); +const localStorage = new ReactNativeFileSystemStorageAdapter(); + +const attachmentQueue = new AttachmentQueue({ + db, + localStorage, + remoteStorage, + watchAttachments +}); +``` + +### Custom storage directory + +Both local adapters accept an optional `storageDirectory` parameter: + +```typescript +const localStorage = new ExpoFileSystemStorageAdapter('/custom/path/to/attachments/'); +``` + +## Streaming transport adapters + +A transport adapter owns **all** remote operations (`upload` / `download` / `delete`) and transfers bytes natively — the file never enters the JS heap. Implement remote delete via the `deleteFile` callback. + +The queue takes **exactly one** remote mechanism: either `remoteStorage` or `transportAdapter`. A `transportAdapter` replaces `remoteStorage` entirely, so none is needed alongside it. + +Both transports are backend-agnostic: you supply resolver callbacks that map an attachment to a request (typically a presigned URL from your backend). + +### With Expo File System (`uploadAsync` / `downloadAsync`) + +```typescript +import { + ExpoFileSystemStorageAdapter, + ExpoFileSystemTransportAdapter +} from '@powersync/attachments-storage-react-native'; +import { AttachmentQueue } from '@powersync/react-native'; + +const localStorage = new ExpoFileSystemStorageAdapter(); + +const transportAdapter = new ExpoFileSystemTransportAdapter({ + resolveUpload: async (attachment) => ({ + url: await getSignedUploadUrl(attachment.filename), // from your backend + httpMethod: 'PUT', + mimeType: attachment.mediaType ?? 'application/octet-stream' + }), + resolveDownload: async (attachment) => ({ + url: await getSignedDownloadUrl(attachment.filename) + }), + deleteFile: async (attachment) => { + await deleteFromRemoteStorage(attachment.filename); // your SDK / DELETE call + } +}); const attachmentQueue = new AttachmentQueue({ - powersync: db, - storage: cloudStorage, - storageAdapter + db, + localStorage, + transportAdapter, // owns upload/download/delete — no remoteStorage needed + watchAttachments +}); +``` + +### With React Native FS (`uploadFiles` / `downloadFile`) + +Identical options shape. The upload is sent as a raw binary `PUT` (`binaryStreamOnly`), suitable for presigned S3/Supabase URLs. + +```typescript +import { + ReactNativeFileSystemStorageAdapter, + ReactNativeFSTransportAdapter +} from '@powersync/attachments-storage-react-native'; + +const localStorage = new ReactNativeFileSystemStorageAdapter(); + +const transportAdapter = new ReactNativeFSTransportAdapter({ + resolveUpload: async (attachment) => ({ + url: await getSignedUploadUrl(attachment.filename), + httpMethod: 'PUT', + mimeType: attachment.mediaType ?? 'application/octet-stream' + }), + resolveDownload: async (attachment) => ({ + url: await getSignedDownloadUrl(attachment.filename) + }), + deleteFile: async (attachment) => { + await deleteFromRemoteStorage(attachment.filename); + } }); ``` -### Custom Storage Directory +## Registering an on-disk file (buffer-free) -Both adapters accept an optional `storageDirectory` parameter: +For files already written to disk (recordings, camera/picker output), use `AttachmentQueue.saveFileFromUri` to register them without reading the bytes into memory — the local adapter's `moveFile` relocates the file into managed storage. ```typescript -const storageAdapter = new ExpoFileSystemStorageAdapter('/custom/path/to/attachments/'); +await attachmentQueue.saveFileFromUri({ + localUri, // path to the existing file + fileExtension: 'm4a', + mediaType: 'audio/m4a' +}); ``` ## API -Both adapters implement the `LocalStorageAdapter` interface from `@powersync/common`: +### Local storage adapters + +Implement the `LocalStorageAdapter` interface from `@powersync/common`: - `initialize()` - Create the storage directory if it doesn't exist - `clear()` - Remove all files from the storage directory - `getLocalUri(filename)` - Get the full path for a filename - `saveFile(filePath, data, options?)` - Save data to a file - `readFile(filePath, options?)` - Read a file as ArrayBuffer +- `moveFile(sourceUri, targetUri)` - Move a file into managed storage without buffering (enables `saveFileFromUri`) - `deleteFile(filePath)` - Delete a file - `fileExists(filePath)` - Check if a file exists - `makeDir(path)` - Create a directory - `rmDir(path)` - Remove a directory +### Transport adapters + +Implement the `AttachmentTransportAdapter` interface from `@powersync/common`: + +- `upload(attachment)` - Transfer the local file to remote storage +- `download(attachment)` - Transfer the remote file into `attachment.localUri` +- `delete(attachment)` - Delete the file from remote storage + ## Supported Versions -| Adapter | Library | Supported Versions | -| ------------------------------------- | ----------------------------- | ------------------ | -| `ExpoFileSystemStorageAdapter` | `expo-file-system` | >=19.0.0 (Expo 54+)| -| `ReactNativeFileSystemStorageAdapter` | `@dr.pogodin/react-native-fs` | ^2.25.0 | +| Adapter | Library | Supported Versions | +| ------------------------------------- | ----------------------------- | ------------------- | +| `ExpoFileSystemStorageAdapter` | `expo-file-system` | >=19.0.0 (Expo 54+) | +| `ExpoFileSystemTransportAdapter` | `expo-file-system` | >=19.0.0 (Expo 54+) | +| `ReactNativeFileSystemStorageAdapter` | `@dr.pogodin/react-native-fs` | ^2.25.0 | +| `ReactNativeFSTransportAdapter` | `@dr.pogodin/react-native-fs` | ^2.25.0 | ## License diff --git a/packages/attachments-storage-react-native/src/ExpoFileSystemStorageAdapter.ts b/packages/attachments-storage-react-native/src/ExpoFileSystemStorageAdapter.ts index c8cd0b94c..44030821b 100644 --- a/packages/attachments-storage-react-native/src/ExpoFileSystemStorageAdapter.ts +++ b/packages/attachments-storage-react-native/src/ExpoFileSystemStorageAdapter.ts @@ -1,5 +1,5 @@ import { decode as decodeBase64 } from 'base64-arraybuffer'; -import type { AttachmentData, LocalStorageAdapter } from '@powersync/common'; +import type { AttachmentData, StreamingLocalStorageAdapter } from '@powersync/common'; import type { File, Directory } from 'expo-file-system'; /** @@ -9,7 +9,7 @@ import type { File, Directory } from 'expo-file-system'; * @experimental * @alpha This is currently experimental and may change without a major version bump. */ -export class ExpoFileSystemStorageAdapter implements LocalStorageAdapter { +export class ExpoFileSystemStorageAdapter implements StreamingLocalStorageAdapter { private File: typeof File; private Directory: typeof Directory; private storageDir: Directory; @@ -81,6 +81,17 @@ To use the Expo File System attachment adapter please install expo-file-system ( return buffer; } + async moveFile(sourceUri: string, targetUri: string): Promise { + if (sourceUri !== targetUri) { + const target = new this.File(targetUri); + if (target.exists) { + target.delete(); + } + new this.File(sourceUri).move(target); + } + return new this.File(targetUri).size ?? 0; + } + async deleteFile(filePath: string): Promise { try { const file = new this.File(filePath); diff --git a/packages/attachments-storage-react-native/src/ExpoFileSystemTransportAdapter.ts b/packages/attachments-storage-react-native/src/ExpoFileSystemTransportAdapter.ts new file mode 100644 index 000000000..ce2bb1819 --- /dev/null +++ b/packages/attachments-storage-react-native/src/ExpoFileSystemTransportAdapter.ts @@ -0,0 +1,150 @@ +import type { AttachmentRecord, AttachmentTransportAdapter, LocatedAttachmentRecord } from '@powersync/common'; + +/** + * Describes the HTTP request used to upload a file's bytes to remote storage. + * Typically points at a presigned URL. + */ +export interface ExpoUploadRequest { + /** Destination URL (e.g. a presigned upload URL). */ + url: string; + /** HTTP method. Defaults to `PUT`. */ + httpMethod?: 'POST' | 'PUT' | 'PATCH'; + /** Additional request headers. */ + headers?: Record; + /** + * Upload encoding, matching Expo's `FileSystemUploadType` + * (`0` = binary content, `1` = multipart). Defaults to binary content. + */ + uploadType?: number; + /** Form field name, used only for multipart uploads. */ + fieldName?: string; + /** MIME type of the file. Defaults to the attachment's `mediaType`. */ + mimeType?: string; +} + +/** + * Describes the HTTP request used to download a file's bytes from remote storage. + * Typically points at a presigned URL. + */ +export interface ExpoDownloadRequest { + /** Source URL (e.g. a presigned download URL). */ + url: string; + /** Additional request headers. */ + headers?: Record; +} + +/** + * Configuration for {@link ExpoFileSystemTransportAdapter}. + * + * The resolvers map an attachment to the request that transfers its bytes, keeping + * the transport agnostic of the remote storage backend (S3, Supabase, etc.). + */ +export interface ExpoFileSystemTransportAdapterOptions { + /** Resolves the upload request (e.g. a presigned URL) for an attachment. */ + resolveUpload: (attachment: LocatedAttachmentRecord) => Promise | ExpoUploadRequest; + /** Resolves the download request (e.g. a presigned URL) for an attachment. */ + resolveDownload: (attachment: LocatedAttachmentRecord) => Promise | ExpoDownloadRequest; + /** + * Deletes the attachment's file from remote storage (e.g. a storage SDK call or a + * `DELETE` request). Delete is a plain remote operation, not a file transfer. + */ + deleteFile: (attachment: AttachmentRecord) => Promise; +} + +/** The `expo-file-system` legacy API surface this adapter resolves and uses at runtime. */ +type ExpoUploadResult = { status: number; body?: string }; +type ExpoDownloadResult = { status: number; uri: string }; + +interface ExpoFileSystemLegacy { + uploadAsync(url: string, fileUri: string, options?: Record): Promise; + downloadAsync(uri: string, fileUri: string, options?: Record): Promise; + FileSystemUploadType?: { BINARY_CONTENT: number; MULTIPART: number }; +} + +/** Upload encodings, mirroring Expo's `FileSystemUploadType`. */ +const UPLOAD_TYPE_BINARY_CONTENT = 0; + +/** + * ExpoFileSystemTransportAdapter transfers attachment bytes directly between a local + * file URI and remote storage using Expo's native `uploadAsync` / `downloadAsync`. + * + * The bytes never enter the JS heap, so large files can be transferred without the + * memory pressure of the buffer-based transport. Requires `expo-file-system` (its + * `uploadAsync` / `downloadAsync` functions, available via `expo-file-system/legacy` + * on SDK 54+). + * + * @experimental + * @alpha This is currently experimental and may change without a major version bump. + */ +export class ExpoFileSystemTransportAdapter implements AttachmentTransportAdapter { + private fs: ExpoFileSystemLegacy; + + constructor(private options: ExpoFileSystemTransportAdapterOptions) { + // `uploadAsync` / `downloadAsync` live in `expo-file-system/legacy` on SDK 54+, + // and in the main module on older SDKs. Metro only bundles static `require` + // string literals, so each candidate is required explicitly. + let resolved: ExpoFileSystemLegacy | undefined; + try { + const legacy = require('expo-file-system/legacy'); + if (typeof legacy?.uploadAsync === 'function' && typeof legacy?.downloadAsync === 'function') { + resolved = legacy; + } + } catch { + // `/legacy` subpath not available on this SDK; fall back to the main module. + } + if (!resolved) { + try { + const main = require('expo-file-system'); + if (typeof main?.uploadAsync === 'function' && typeof main?.downloadAsync === 'function') { + resolved = main; + } + } catch { + // expo-file-system not installed. + } + } + + if (!resolved) { + throw new Error(`Could not resolve expo-file-system's uploadAsync/downloadAsync. +To use the Expo File System transport please install expo-file-system.`); + } + + this.fs = resolved; + } + + async upload(attachment: LocatedAttachmentRecord): Promise { + const request = await this.options.resolveUpload(attachment); + const binaryContent = this.fs.FileSystemUploadType?.BINARY_CONTENT ?? UPLOAD_TYPE_BINARY_CONTENT; + + const result = await this.fs.uploadAsync(request.url, attachment.localUri, { + httpMethod: request.httpMethod ?? 'PUT', + uploadType: request.uploadType ?? binaryContent, + headers: request.headers, + fieldName: request.fieldName, + mimeType: request.mimeType ?? attachment.mediaType + }); + + if (!this.isOk(result.status)) { + throw new Error(`Upload for ${attachment.id} failed with status ${result.status}: ${result.body ?? ''}`); + } + } + + async download(attachment: LocatedAttachmentRecord): Promise { + const request = await this.options.resolveDownload(attachment); + + const result = await this.fs.downloadAsync(request.url, attachment.localUri, { + headers: request.headers + }); + + if (!this.isOk(result.status)) { + throw new Error(`Download for ${attachment.id} failed with status ${result.status}`); + } + } + + async delete(attachment: AttachmentRecord): Promise { + await this.options.deleteFile(attachment); + } + + private isOk(status: number): boolean { + return status >= 200 && status < 300; + } +} diff --git a/packages/attachments-storage-react-native/src/ReactNativeFSTransportAdapter.ts b/packages/attachments-storage-react-native/src/ReactNativeFSTransportAdapter.ts new file mode 100644 index 000000000..4c19320fa --- /dev/null +++ b/packages/attachments-storage-react-native/src/ReactNativeFSTransportAdapter.ts @@ -0,0 +1,132 @@ +import type { AttachmentRecord, AttachmentTransportAdapter, LocatedAttachmentRecord } from '@powersync/common'; + +/** + * Describes the HTTP request used to upload a file's bytes to remote storage. + * Typically points at a presigned URL. + */ +export interface ReactNativeFSUploadRequest { + /** Destination URL (e.g. a presigned upload URL). */ + url: string; + /** HTTP method. Defaults to `PUT`. */ + httpMethod?: 'POST' | 'PUT' | 'PATCH'; + /** Additional request headers. */ + headers?: Record; + /** + * Send the raw file bytes as the request body instead of a multipart form. + * Defaults to `true` — required for presigned `PUT` uploads (S3, Supabase, etc.). + */ + binaryStreamOnly?: boolean; + /** MIME type of the file. Defaults to the attachment's `mediaType`. */ + mimeType?: string; +} + +/** + * Describes the HTTP request used to download a file's bytes from remote storage. + * Typically points at a presigned URL. + */ +export interface ReactNativeFSDownloadRequest { + /** Source URL (e.g. a presigned download URL). */ + url: string; + /** Additional request headers. */ + headers?: Record; +} + +/** + * Configuration for {@link ReactNativeFSTransportAdapter}. + * + * The resolvers map an attachment to the request that transfers its bytes, keeping + * the transport agnostic of the remote storage backend (S3, Supabase, etc.). + */ +export interface ReactNativeFSTransportAdapterOptions { + /** Resolves the upload request (e.g. a presigned URL) for an attachment. */ + resolveUpload: (attachment: LocatedAttachmentRecord) => Promise | ReactNativeFSUploadRequest; + /** Resolves the download request (e.g. a presigned URL) for an attachment. */ + resolveDownload: ( + attachment: LocatedAttachmentRecord + ) => Promise | ReactNativeFSDownloadRequest; + /** + * Deletes the attachment's file from remote storage (e.g. a storage SDK call or a + * `DELETE` request). Delete is a plain remote operation, not a file transfer. + */ + deleteFile: (attachment: AttachmentRecord) => Promise; +} + +/** + * ReactNativeFSTransportAdapter transfers attachment bytes directly between a local + * file and remote storage using `@dr.pogodin/react-native-fs`'s native + * `uploadFiles` / `downloadFile`. + * + * The bytes never enter the JS heap, so large files can be transferred without the + * memory pressure of the buffer-based transport. Requires + * `@dr.pogodin/react-native-fs`. + * + * @experimental + * @alpha This is currently experimental and may change without a major version bump. + */ +export class ReactNativeFSTransportAdapter implements AttachmentTransportAdapter { + private rnfs: typeof import('@dr.pogodin/react-native-fs'); + + constructor(private options: ReactNativeFSTransportAdapterOptions) { + let rnfs: typeof import('@dr.pogodin/react-native-fs'); + try { + rnfs = require('@dr.pogodin/react-native-fs'); + } catch (e) { + throw new Error(`Could not resolve @dr.pogodin/react-native-fs. +To use the React Native File System transport please install @dr.pogodin/react-native-fs.`); + } + + this.rnfs = rnfs; + } + + async upload(attachment: LocatedAttachmentRecord): Promise { + const request = await this.options.resolveUpload(attachment); + + const { promise } = this.rnfs.uploadFiles({ + toUrl: request.url, + method: request.httpMethod ?? 'PUT', + binaryStreamOnly: request.binaryStreamOnly ?? true, + headers: request.headers, + files: [ + { + name: 'file', + filename: attachment.filename, + filepath: this.toPath(attachment.localUri), + filetype: request.mimeType ?? attachment.mediaType + } + ] + }); + + const result = await promise; + if (!this.isOk(result.statusCode)) { + throw new Error(`Upload for ${attachment.id} failed with status ${result.statusCode}: ${result.body ?? ''}`); + } + } + + async download(attachment: LocatedAttachmentRecord): Promise { + const request = await this.options.resolveDownload(attachment); + + const { promise } = this.rnfs.downloadFile({ + fromUrl: request.url, + toFile: this.toPath(attachment.localUri), + headers: request.headers + }); + + const result = await promise; + if (!this.isOk(result.statusCode)) { + throw new Error(`Download for ${attachment.id} failed with status ${result.statusCode}`); + } + } + + async delete(attachment: AttachmentRecord): Promise { + await this.options.deleteFile(attachment); + } + + /** react-native-fs expects plain filesystem paths, not `file://` URIs. */ + private toPath(uri: string): string { + return uri.startsWith('file://') ? uri.slice('file://'.length) : uri; + } + + private isOk(status: number): boolean { + return status >= 200 && status < 300; + } +} diff --git a/packages/attachments-storage-react-native/src/ReactNativeFileSystemStorageAdapter.ts b/packages/attachments-storage-react-native/src/ReactNativeFileSystemStorageAdapter.ts index 62e657386..14e052d03 100644 --- a/packages/attachments-storage-react-native/src/ReactNativeFileSystemStorageAdapter.ts +++ b/packages/attachments-storage-react-native/src/ReactNativeFileSystemStorageAdapter.ts @@ -1,5 +1,5 @@ import { decode as decodeBase64, encode as encodeBase64 } from 'base64-arraybuffer'; -import type { AttachmentData, LocalStorageAdapter } from '@powersync/common'; +import type { AttachmentData, StreamingLocalStorageAdapter } from '@powersync/common'; /** * ReactNativeFileSystemStorageAdapter implements LocalStorageAdapter using @dr.pogodin/react-native-fs. @@ -8,7 +8,7 @@ import type { AttachmentData, LocalStorageAdapter } from '@powersync/common'; * @experimental * @alpha This is currently experimental and may change without a major version bump. */ -export class ReactNativeFileSystemStorageAdapter implements LocalStorageAdapter { +export class ReactNativeFileSystemStorageAdapter implements StreamingLocalStorageAdapter { private rnfs: typeof import('@dr.pogodin/react-native-fs'); private storageDirectory: string; @@ -71,6 +71,17 @@ To use the React Native File System attachment adapter please install @dr.pogodi return decodeBase64(content); } + async moveFile(sourceUri: string, targetUri: string): Promise { + if (sourceUri !== targetUri) { + if (await this.rnfs.exists(targetUri)) { + await this.rnfs.unlink(targetUri); + } + await this.rnfs.moveFile(sourceUri, targetUri); + } + const stat = await this.rnfs.stat(targetUri); + return Number(stat.size); + } + async deleteFile(filePath: string): Promise { try { await this.rnfs.unlink(filePath); diff --git a/packages/attachments-storage-react-native/src/index.ts b/packages/attachments-storage-react-native/src/index.ts index a44ee674f..7f1917bb2 100644 --- a/packages/attachments-storage-react-native/src/index.ts +++ b/packages/attachments-storage-react-native/src/index.ts @@ -1,2 +1,4 @@ export * from './ExpoFileSystemStorageAdapter.js'; +export * from './ExpoFileSystemTransportAdapter.js'; export * from './ReactNativeFileSystemStorageAdapter.js'; +export * from './ReactNativeFSTransportAdapter.js'; diff --git a/packages/common/etc/common.api.md b/packages/common/etc/common.api.md index a464344b1..a3b8acea0 100644 --- a/packages/common/etc/common.api.md +++ b/packages/common/etc/common.api.md @@ -80,8 +80,8 @@ export interface AttachmentErrorHandler { export function attachmentFromSql(row: any): AttachmentRecord; // @alpha -export class AttachmentQueue { - constructor(input: AttachmentQueueOptions); +export class AttachmentQueue { + constructor(input: AttachmentQueueOptions); readonly archivedCacheLimit: number; // (undocumented) clearQueue(): Promise; @@ -94,16 +94,13 @@ export class AttachmentQueue { // (undocumented) expireCache(): Promise; generateAttachmentId(): Promise; - readonly localStorage: LocalStorageAdapter; + readonly localStorage: TLocal; readonly logger: PowerSyncLogger; - readonly remoteStorage: RemoteStorageAdapter; - saveFile(input: { + saveFile(options: SaveAttachmentOptions & { data: AttachmentData; - fileExtension: string; - mediaType?: string; - metaData?: string; - id?: string; - updateHook?: (transaction: Transaction, attachment: AttachmentRecord) => Promise; + }): Promise; + saveFileFromUri(this: AttachmentQueue, options: SaveAttachmentOptions & { + localUri: string; }): Promise; startSync(): Promise; stopSync(): Promise; @@ -116,19 +113,13 @@ export class AttachmentQueue { } // @alpha -export interface AttachmentQueueOptions { - archivedCacheLimit?: number; - db: CommonPowerSyncDatabase; - downloadAttachments?: boolean; - errorHandler?: AttachmentErrorHandler; - localStorage: LocalStorageAdapter; - logger?: PowerSyncLogger; +export type AttachmentQueueOptions = BaseAttachmentQueueOptions & ({ remoteStorage: RemoteStorageAdapter; - syncIntervalMs?: number; - syncThrottleDuration?: number; - tableName?: string; - watchAttachments: (onUpdate: (attachment: WatchedAttachmentItem[]) => Promise, signal: AbortSignal) => void; -} + transportAdapter?: never; +} | { + transportAdapter: AttachmentTransportAdapter; + remoteStorage?: never; +}); // @alpha export interface AttachmentRecord { @@ -178,6 +169,27 @@ export interface AttachmentTableOptions extends Omit; +// @alpha +export interface AttachmentTransportAdapter { + delete(attachment: AttachmentRecord): Promise; + download(attachment: LocatedAttachmentRecord): Promise; + upload(attachment: LocatedAttachmentRecord): Promise; +} + +// @alpha +export interface BaseAttachmentQueueOptions { + archivedCacheLimit?: number; + db: CommonPowerSyncDatabase; + downloadAttachments?: boolean; + errorHandler?: AttachmentErrorHandler; + localStorage: TLocal; + logger?: PowerSyncLogger; + syncIntervalMs?: number; + syncThrottleDuration?: number; + tableName?: string; + watchAttachments: (onUpdate: (attachment: WatchedAttachmentItem[]) => Promise, signal: AbortSignal) => void; +} + // @public (undocumented) export type BaseColumnType = { type: ColumnType; @@ -656,6 +668,11 @@ export interface LocalStorageAdapter { saveFile(filePath: string, data: AttachmentData): Promise; } +// @alpha +export type LocatedAttachmentRecord = AttachmentRecord & { + localUri: string; +}; + // @public (undocumented) export abstract class LockContext implements SqlExecutor, DBGetUtils { // (undocumented) @@ -911,6 +928,15 @@ export function sanitizeSQL(strings: TemplateStringsArray, ...values: any[]): st // @alpha export function sanitizeUUID(uuid: string): string; +// @alpha +export interface SaveAttachmentOptions { + fileExtension: string; + id?: string; + mediaType?: string; + metaData?: string; + updateHook?: (transaction: Transaction, attachment: AttachmentRecord) => Promise; +} + // Warning: (ae-forgotten-export) The symbol "SchemaType" needs to be exported by the entry point index.d.ts // // @public @@ -988,6 +1014,11 @@ export interface StandardWatchedQueryOptions extends WatchedQueryOption placeholderData?: RowType[]; } +// @alpha +export interface StreamingLocalStorageAdapter extends LocalStorageAdapter { + moveFile(sourceUri: string, targetUri: string): Promise; +} + // Warning: (ae-forgotten-export) The symbol "JSONValue" needs to be exported by the entry point index.d.ts // // @public (undocumented) diff --git a/packages/common/src/attachments/AttachmentQueue.ts b/packages/common/src/attachments/AttachmentQueue.ts index 438b3ce2f..98db917a1 100644 --- a/packages/common/src/attachments/AttachmentQueue.ts +++ b/packages/common/src/attachments/AttachmentQueue.ts @@ -5,7 +5,9 @@ import { Transaction } from '../db/DBAdapter.js'; import { AttachmentContext } from './AttachmentContext.js'; import { AttachmentErrorHandler } from './AttachmentErrorHandler.js'; import { AttachmentService } from './AttachmentService.js'; -import { AttachmentData, LocalStorageAdapter } from './LocalStorageAdapter.js'; +import { AttachmentTransportAdapter } from './AttachmentTransportAdapter.js'; +import { BufferedAttachmentTransport } from './BufferedAttachmentTransport.js'; +import { AttachmentData, LocalStorageAdapter, StreamingLocalStorageAdapter } from './LocalStorageAdapter.js'; import { RemoteStorageAdapter } from './RemoteStorageAdapter.js'; import { ATTACHMENT_TABLE, AttachmentRecord, AttachmentState } from './Schema.js'; import { SyncingService } from './SyncingService.js'; @@ -13,24 +15,22 @@ import { WatchedAttachmentItem } from './WatchedAttachmentItem.js'; import { CommonPowerSyncDatabase } from '../client/CommonPowerSyncDatabase.js'; /** - * Configuration options for {@link AttachmentQueue}. + * Fields common to every {@link AttachmentQueueOptions} variant. * * @experimental * @alpha This is currently experimental and may change without a major version bump. */ -export interface AttachmentQueueOptions { +export interface BaseAttachmentQueueOptions { /** * PowerSync database instance */ db: CommonPowerSyncDatabase; /** - * Remote storage adapter for upload/download operations - */ - remoteStorage: RemoteStorageAdapter; - /** - * Local storage adapter for file persistence + * Local storage adapter for file persistence. Its type determines whether + * {@link AttachmentQueue.saveFileFromUri} is available (a + * {@link StreamingLocalStorageAdapter} enables it). */ - localStorage: LocalStorageAdapter; + localStorage: TLocal; /** * Callback for monitoring attachment changes in your data model */ @@ -64,6 +64,52 @@ export interface AttachmentQueueOptions { errorHandler?: AttachmentErrorHandler; } +/** + * Configuration options for {@link AttachmentQueue}. + * + * Provide **exactly one** remote mechanism: + * - `remoteStorage` — a {@link RemoteStorageAdapter}, wrapped in the default + * default buffered transport that delegates upload/download/delete to it. + * - `transportAdapter` — an {@link AttachmentTransportAdapter} that owns all remote + * operations directly (e.g. a native file-URI implementation for buffer-free + * transfer of large files). No `remoteStorage` is needed in this case. + * + * Supplying both, or neither, is a type error. + * + * @experimental + * @alpha This is currently experimental and may change without a major version bump. + */ +export type AttachmentQueueOptions = + BaseAttachmentQueueOptions & + ( + | { remoteStorage: RemoteStorageAdapter; transportAdapter?: never } + | { transportAdapter: AttachmentTransportAdapter; remoteStorage?: never } + ); + +/** + * Fields shared by {@link AttachmentQueue.saveFile} and {@link AttachmentQueue.saveFileFromUri}. + * + * @alpha + */ +export interface SaveAttachmentOptions { + /** File extension (e.g., 'jpg', 'pdf') */ + fileExtension: string; + /** MIME type of the file (e.g., 'image/jpeg') */ + mediaType?: string; + /** Optional metadata to associate with the attachment */ + metaData?: string; + /** Optional custom ID. If not provided, a UUID will be generated */ + id?: string; + /** + * Optional callback to execute additional database operations within the same transaction as the + * attachment creation. + */ + updateHook?: (transaction: Transaction, attachment: AttachmentRecord) => Promise; +} + +/** How the file bytes reach managed storage when creating an upload attachment. */ +type AttachmentSource = { kind: 'data'; data: AttachmentData } | { kind: 'uri'; localUri: string }; + /** * AttachmentQueue manages the lifecycle and synchronization of attachments * between local and remote storage. @@ -73,7 +119,7 @@ export interface AttachmentQueueOptions { * @experimental * @alpha This is currently experimental and may change without a major version bump. */ -export class AttachmentQueue { +export class AttachmentQueue { /** Timer for periodic synchronization operations */ private periodicSyncTimer?: ReturnType; @@ -81,10 +127,7 @@ export class AttachmentQueue { private readonly syncingService: SyncingService; /** Adapter for local file storage operations */ - readonly localStorage: LocalStorageAdapter; - - /** Adapter for remote file storage operations */ - readonly remoteStorage: RemoteStorageAdapter; + readonly localStorage: TLocal; /** * Callback function to watch for changes in attachment references in your data model. @@ -159,6 +202,7 @@ export class AttachmentQueue { db, localStorage, remoteStorage, + transportAdapter, watchAttachments, logger, tableName = ATTACHMENT_TABLE, @@ -167,10 +211,9 @@ export class AttachmentQueue { downloadAttachments = true, archivedCacheLimit = 100, errorHandler - }: AttachmentQueueOptions) { + }: AttachmentQueueOptions) { this.db = db; this.syncLoopMutex = db.createMutex(); - this.remoteStorage = remoteStorage; this.localStorage = localStorage; this.watchAttachments = watchAttachments; this.tableName = tableName; @@ -180,10 +223,16 @@ export class AttachmentQueue { this.downloadAttachments = downloadAttachments; this.logger = logger ?? db.logger; this.attachmentService = new AttachmentService(db, this.logger, tableName, archivedCacheLimit); + + if (!transportAdapter && !remoteStorage) { + throw new Error('AttachmentQueue requires either a `remoteStorage` or a `transportAdapter`.'); + } + const transport = transportAdapter ?? new BufferedAttachmentTransport(localStorage, remoteStorage!); + this.syncingService = new SyncingService( this.attachmentService, localStorage, - remoteStorage, + transport, this.logger, errorHandler ); @@ -408,49 +457,30 @@ export class AttachmentQueue { return this.attachmentService.withContext(callback); } /** - * Saves a file to local storage and queues it for upload to remote storage. - * - * @param options - File save options - * @returns Promise resolving to the created attachment record + * Creates a `QUEUED_UPLOAD` attachment record, placing the file at the managed + * `localUri` from the given `source`, and persists the record + * alongside the caller's `updateHook` in a single transaction. */ - async saveFile({ - data, - fileExtension, - mediaType, - metaData, - id, - updateHook - }: { - /** - * The file data as ArrayBuffer, Blob, or base64 string - */ - data: AttachmentData; - /** - * File extension (e.g., 'jpg', 'pdf') - */ - fileExtension: string; - /** - * MIME type of the file (e.g., 'image/jpeg') - */ - mediaType?: string; - /** - * Optional metadata to associate with the attachment - */ - metaData?: string; - /** - * Optional custom ID. If not provided, a UUID will be generated - */ - id?: string; - /** - * Optional callback to execute additional database operations within the same transaction as the attachment - * creation. - */ - updateHook?: (transaction: Transaction, attachment: AttachmentRecord) => Promise; - }): Promise { + private async createUploadAttachment( + { fileExtension, mediaType, metaData, id, updateHook }: SaveAttachmentOptions, + source: AttachmentSource + ): Promise { const resolvedId = id ?? (await this.generateAttachmentId()); const filename = `${resolvedId}.${fileExtension}`; const localUri = this.localStorage.getLocalUri(filename); - const size = await this.localStorage.saveFile(localUri, data); + + let size: number; + if (source.kind === 'data') { + size = await this.localStorage.saveFile(localUri, source.data); + } else { + // saveFileFromUri is only exposed for streaming-capable local adapters; guard at + // runtime too for plain-JS callers. + const localStorage = this.localStorage as Partial; + if (!localStorage.moveFile) { + throw new Error('The configured local storage adapter does not support moveFile, required by saveFileFromUri.'); + } + size = await localStorage.moveFile(source.localUri, localUri); + } const attachment: AttachmentRecord = { id: resolvedId, @@ -474,6 +504,37 @@ export class AttachmentQueue { return attachment; } + /** + * Saves in-memory file data to local storage and queues it for upload. + * + * @param options - File data plus {@link SaveAttachmentOptions} + * @returns Promise resolving to the created attachment record + */ + async saveFile(options: SaveAttachmentOptions & { data: AttachmentData }): Promise { + return this.createUploadAttachment(options, { kind: 'data', data: options.data }); + } + + /** + * Registers a file that already exists on disk and queues it for upload, moving it + * into managed storage without loading it into memory. + * + * Prefer this over {@link AttachmentQueue.saveFile} for large, app-originated files + * (recordings, videos): it avoids reading the file into an `ArrayBuffer` just to write + * it back to disk. Requires the local storage adapter to implement `moveFile`. + * + * Only available when the queue is configured with a {@link StreamingLocalStorageAdapter} + * (one that implements `moveFile`). + * + * @param options - The existing file's `localUri` plus {@link SaveAttachmentOptions} + * @returns Promise resolving to the created attachment record + */ + async saveFileFromUri( + this: AttachmentQueue, + options: SaveAttachmentOptions & { localUri: string } + ): Promise { + return this.createUploadAttachment(options, { kind: 'uri', localUri: options.localUri }); + } + async deleteFile({ id, updateHook diff --git a/packages/common/src/attachments/AttachmentTransportAdapter.ts b/packages/common/src/attachments/AttachmentTransportAdapter.ts new file mode 100644 index 000000000..e38804cb9 --- /dev/null +++ b/packages/common/src/attachments/AttachmentTransportAdapter.ts @@ -0,0 +1,49 @@ +import { AttachmentRecord } from './Schema.js'; + +/** + * An {@link AttachmentRecord} that is guaranteed to have a `localUri`. + * + * The syncing service assigns `localUri` before invoking a transport download, + * so implementations always receive both the metadata and the destination path. + * + * @alpha + */ +export type LocatedAttachmentRecord = AttachmentRecord & { localUri: string }; + +/** + * AttachmentTransportAdapter owns all remote-side operations for an attachment — + * transfer (upload/download) and delete — as single operations. + * + * A transport owns the entire transfer, letting implementations pick the most + * efficient mechanism available (buffer, stream, or a platform-native file-URI + * upload/download API). On platforms like React Native this allows large files to + * be transferred without ever materializing them in the JS heap. + * + * The default transport composes the local and remote storage adapters. Provide a custom transport (via + * `AttachmentQueue`'s `transportAdapter` option) to own the whole remote side; in + * that case a separate `remoteStorage` is not required. + * + * @experimental + * @alpha This is currently experimental and may change without a major version bump. + */ +export interface AttachmentTransportAdapter { + /** + * Uploads the attachment's local file to remote storage. + * @param attachment - The attachment to upload. `localUri` points at the source file. + */ + upload(attachment: LocatedAttachmentRecord): Promise; + + /** + * Downloads the remote file into `attachment.localUri`. + * @param attachment - The attachment to download. `localUri` is the destination path, + * assigned by the syncing service before this call. + */ + download(attachment: LocatedAttachmentRecord): Promise; + + /** + * Deletes the attachment's file from remote storage. Local file removal is handled + * separately by the syncing service. + * @param attachment - The attachment to delete. + */ + delete(attachment: AttachmentRecord): Promise; +} diff --git a/packages/common/src/attachments/BufferedAttachmentTransport.ts b/packages/common/src/attachments/BufferedAttachmentTransport.ts new file mode 100644 index 000000000..2a85bfb7a --- /dev/null +++ b/packages/common/src/attachments/BufferedAttachmentTransport.ts @@ -0,0 +1,37 @@ +import { AttachmentTransportAdapter, LocatedAttachmentRecord } from './AttachmentTransportAdapter.js'; +import { LocalStorageAdapter } from './LocalStorageAdapter.js'; +import { RemoteStorageAdapter } from './RemoteStorageAdapter.js'; +import { AttachmentRecord } from './Schema.js'; + +/** + * Default {@link AttachmentTransportAdapter}, composing the local and remote + * storage adapters. + * + * The full file body is materialized as an `ArrayBuffer` in JS memory between the + * two adapter calls. This is fine for small files but can cause memory pressure on + * large ones — environments that support native transfer should provide a custom + * transport instead. + * + * @experimental + * @alpha This is currently experimental and may change without a major version bump. + */ +export class BufferedAttachmentTransport implements AttachmentTransportAdapter { + constructor( + private localStorage: LocalStorageAdapter, + private remoteStorage: RemoteStorageAdapter + ) {} + + async upload(attachment: LocatedAttachmentRecord): Promise { + const fileData = await this.localStorage.readFile(attachment.localUri); + await this.remoteStorage.uploadFile(fileData, attachment); + } + + async download(attachment: LocatedAttachmentRecord): Promise { + const fileData = await this.remoteStorage.downloadFile(attachment); + await this.localStorage.saveFile(attachment.localUri, fileData); + } + + async delete(attachment: AttachmentRecord): Promise { + await this.remoteStorage.deleteFile(attachment); + } +} diff --git a/packages/common/src/attachments/LocalStorageAdapter.ts b/packages/common/src/attachments/LocalStorageAdapter.ts index bd7713415..08afd0479 100644 --- a/packages/common/src/attachments/LocalStorageAdapter.ts +++ b/packages/common/src/attachments/LocalStorageAdapter.ts @@ -76,3 +76,23 @@ export interface LocalStorageAdapter { */ getLocalUri(filename: string): string; } + +/** + * A {@link LocalStorageAdapter} that can relocate a file into managed storage without + * loading it into memory. Required for {@link AttachmentQueue.saveFileFromUri}; only + * queues configured with a streaming-capable local adapter expose that method. + * + * @experimental + * @alpha This is currently experimental and may change without a major version bump. + */ +export interface StreamingLocalStorageAdapter extends LocalStorageAdapter { + /** + * Moves a file into managed storage without loading it into memory. + * Overwrites any existing file at the target. When source and target are the same + * path, this is a no-op that just reports the size. + * @param sourceUri - Path of the existing file + * @param targetUri - Destination path within managed storage + * @returns Number of bytes in the moved file + */ + moveFile(sourceUri: string, targetUri: string): Promise; +} diff --git a/packages/common/src/attachments/SyncingService.ts b/packages/common/src/attachments/SyncingService.ts index b6a36c6c3..0f9ee1f8f 100644 --- a/packages/common/src/attachments/SyncingService.ts +++ b/packages/common/src/attachments/SyncingService.ts @@ -1,7 +1,7 @@ import { LogLevels, PowerSyncLogger } from '../utils/Logger.js'; import { AttachmentService } from './AttachmentService.js'; +import { AttachmentTransportAdapter } from './AttachmentTransportAdapter.js'; import { LocalStorageAdapter } from './LocalStorageAdapter.js'; -import { RemoteStorageAdapter } from './RemoteStorageAdapter.js'; import { AttachmentRecord, AttachmentState } from './Schema.js'; import { AttachmentErrorHandler } from './AttachmentErrorHandler.js'; import { AttachmentContext } from './AttachmentContext.js'; @@ -10,25 +10,28 @@ import { AttachmentContext } from './AttachmentContext.js'; * Orchestrates attachment synchronization between local and remote storage. * Handles uploads, downloads, deletions, and state transitions. * + * Remote operations (upload/download/delete) go through the {@link AttachmentTransportAdapter}; + * local file operations use the {@link LocalStorageAdapter}. + * * @internal */ export class SyncingService { private attachmentService: AttachmentService; private localStorage: LocalStorageAdapter; - private remoteStorage: RemoteStorageAdapter; + private transport: AttachmentTransportAdapter; private logger: PowerSyncLogger; private errorHandler?: AttachmentErrorHandler; constructor( attachmentService: AttachmentService, localStorage: LocalStorageAdapter, - remoteStorage: RemoteStorageAdapter, + transport: AttachmentTransportAdapter, logger: PowerSyncLogger, errorHandler?: AttachmentErrorHandler ) { this.attachmentService = attachmentService; this.localStorage = localStorage; - this.remoteStorage = remoteStorage; + this.transport = transport; this.logger = logger; this.errorHandler = errorHandler; } @@ -114,8 +117,7 @@ export class SyncingService { throw new Error(`No localUri for attachment ${attachment.id}`); } - const fileBlob = await this.localStorage.readFile(attachment.localUri); - await this.remoteStorage.uploadFile(fileBlob, attachment); + await this.transport.upload({ ...attachment, localUri: attachment.localUri }); return { ...attachment, @@ -137,7 +139,7 @@ export class SyncingService { /** * Downloads an attachment from remote storage to local storage. - * Retrieves the file, converts to base64, and saves locally. + * The destination `localUri` is assigned here and the transport writes the file to it. * On success, marks as SYNCED. On failure, defers to error handler or archives. * * @param attachment - The attachment record to download @@ -146,10 +148,8 @@ export class SyncingService { async downloadAttachment(attachment: AttachmentRecord): Promise { this.logger.log({ level: LogLevels.info, message: `Downloading attachment ${attachment.filename}` }); try { - const fileData = await this.remoteStorage.downloadFile(attachment); - const localUri = this.localStorage.getLocalUri(attachment.filename); - await this.localStorage.saveFile(localUri, fileData); + await this.transport.download({ ...attachment, localUri }); return { ...attachment, @@ -181,7 +181,7 @@ export class SyncingService { */ async deleteAttachment(attachment: AttachmentRecord, context: AttachmentContext): Promise { try { - await this.remoteStorage.deleteFile(attachment); + await this.transport.delete(attachment); if (attachment.localUri) { await this.localStorage.deleteFile(attachment.localUri); } diff --git a/packages/common/src/index.ts b/packages/common/src/index.ts index b98726699..b703d87ff 100644 --- a/packages/common/src/index.ts +++ b/packages/common/src/index.ts @@ -1,6 +1,7 @@ export * from './attachments/AttachmentContext.js'; export * from './attachments/AttachmentErrorHandler.js'; export * from './attachments/AttachmentQueue.js'; +export * from './attachments/AttachmentTransportAdapter.js'; export * from './attachments/LocalStorageAdapter.js'; export * from './attachments/RemoteStorageAdapter.js'; export * from './attachments/Schema.js'; diff --git a/packages/node/src/attachments/NodeFileSystemAdapter.ts b/packages/node/src/attachments/NodeFileSystemAdapter.ts index 0d40ad393..9e270ebf1 100644 --- a/packages/node/src/attachments/NodeFileSystemAdapter.ts +++ b/packages/node/src/attachments/NodeFileSystemAdapter.ts @@ -1,12 +1,12 @@ import { promises as fs } from 'fs'; import * as path from 'path'; -import { AttachmentData, EncodingType, LocalStorageAdapter } from '@powersync/common'; +import { AttachmentData, EncodingType, StreamingLocalStorageAdapter } from '@powersync/common'; /** * NodeFileSystemAdapter implements LocalStorageAdapter using Node.js filesystem. * Suitable for Node.js environments and Electron applications. */ -export class NodeFileSystemAdapter implements LocalStorageAdapter { +export class NodeFileSystemAdapter implements StreamingLocalStorageAdapter { constructor(private storageDirectory: string = './user_data') {} async initialize(): Promise { @@ -60,6 +60,14 @@ export class NodeFileSystemAdapter implements LocalStorageAdapter { } } + async moveFile(sourceUri: string, targetUri: string): Promise { + if (sourceUri !== targetUri) { + await fs.rename(sourceUri, targetUri); + } + const stats = await fs.stat(targetUri); + return stats.size; + } + async deleteFile(path: string, options?: { filename?: string }): Promise { await fs.unlink(path).catch((err) => { if (err.code !== 'ENOENT') { diff --git a/packages/node/tests/attachments.test.ts b/packages/node/tests/attachments.test.ts index 024281c54..c6d9e11cb 100644 --- a/packages/node/tests/attachments.test.ts +++ b/packages/node/tests/attachments.test.ts @@ -13,7 +13,8 @@ import { AttachmentState, RemoteStorageAdapter, WatchedAttachmentItem, - AttachmentErrorHandler + AttachmentErrorHandler, + AttachmentTransportAdapter } from '../lib/index.js'; const MOCK_JPEG_U8A = [ @@ -38,7 +39,7 @@ const mockRemoteStorage: RemoteStorageAdapter = { const mockLocalStorage = new NodeFileSystemAdapter('./temp/attachments'); let db: AbstractPowerSyncDatabase; -let queue: AttachmentQueue; +let queue: AttachmentQueue; const schema = new Schema({ users: new Table({ name: column.text, @@ -467,7 +468,8 @@ describe('attachment queue', () => { filename: `${id}.jpg`, hasSynced: false, id: id, - localUri: null, + // The destination localUri is assigned before the transfer. + localUri: expect.any(String), mediaType: null, metaData: null, size: null, @@ -499,7 +501,7 @@ describe('attachment queue', () => { filename: `${id}.jpg`, hasSynced: false, id: id, - localUri: null, + localUri: expect.any(String), mediaType: null, metaData: null, size: null, @@ -755,3 +757,280 @@ describe('attachment queue', () => { } ); }); + +describe('attachment queue - transport', () => { + const linkUser = (id: string) => async (tx: any, attachment: AttachmentRecord) => { + await tx.execute('INSERT INTO users (id, name, email, photo_id) VALUES (uuid(), ?, ?, ?)', [ + 'transport', + 'transport@example.com', + attachment.id + ]); + }; + + it('default transport round-trips upload through localStorage.readFile', async () => { + const readFileSpy = vi.spyOn(mockLocalStorage, 'readFile'); + onTestFinished(() => readFileSpy.mockRestore()); + + await queue.startSync(); + + const record = await queue.saveFile({ + data: new Uint8Array(64).fill(9).buffer, + fileExtension: 'jpg', + updateHook: linkUser('') + }); + + await waitForMatchCondition( + () => watchAttachmentsTable(), + (results) => results.some((r) => r.id === record.id && r.state === AttachmentState.SYNCED), + 5 + ); + + expect(mockUploadFile).toHaveBeenCalled(); + expect(readFileSpy).toHaveBeenCalledWith(record.localUri); + + await queue.stopSync(); + }); + + it('custom transport handles upload and bypasses the buffer round-trip', async () => { + const readFileSpy = vi.spyOn(mockLocalStorage, 'readFile'); + const transportUpload = vi.fn().mockResolvedValue(undefined); + const transportAdapter: AttachmentTransportAdapter = { + upload: transportUpload, + download: vi.fn().mockResolvedValue(undefined), + delete: vi.fn().mockResolvedValue(undefined) + }; + + const q = new AttachmentQueue({ + db, + watchAttachments, + localStorage: mockLocalStorage, + transportAdapter, + syncIntervalMs: INTERVAL_MILLISECONDS, + archivedCacheLimit: 0 + }); + onTestFinished(async () => { + await q.stopSync(); + readFileSpy.mockRestore(); + }); + + await q.startSync(); + + const record = await q.saveFile({ + data: new Uint8Array(64).fill(9).buffer, + fileExtension: 'jpg', + updateHook: linkUser('') + }); + + await waitForMatchCondition( + () => watchAttachmentsTable(), + (results) => results.some((r) => r.id === record.id && r.state === AttachmentState.SYNCED), + 5 + ); + + // The transport owns the transfer; the buffer path is never touched. + expect(transportUpload).toHaveBeenCalled(); + expect(transportUpload.mock.calls[0][0].localUri).toBe(record.localUri); + expect(mockUploadFile).not.toHaveBeenCalled(); + expect(readFileSpy).not.toHaveBeenCalled(); + }); + + it('custom transport handles download with a pre-assigned localUri', async () => { + const transportDownload = vi.fn(async (attachment: AttachmentRecord & { localUri: string }) => { + // A native transport writes directly to the destination, no buffer returned. + await mockLocalStorage.saveFile(attachment.localUri, createMockJpegBuffer()); + }); + const transportAdapter: AttachmentTransportAdapter = { + upload: vi.fn().mockResolvedValue(undefined), + download: transportDownload, + delete: vi.fn().mockResolvedValue(undefined) + }; + + const q = new AttachmentQueue({ + db, + watchAttachments, + localStorage: mockLocalStorage, + transportAdapter, + syncIntervalMs: INTERVAL_MILLISECONDS, + archivedCacheLimit: 0 + }); + onTestFinished(() => q.stopSync()); + + await q.startSync(); + + const id = await q.generateAttachmentId(); + await db.execute('INSERT INTO users (id, name, email, photo_id) VALUES (uuid(), ?, ?, ?)', [ + 'downloader', + 'downloader@example.com', + id + ]); + + const attachments = await waitForMatchCondition( + () => watchAttachmentsTable(), + (results) => results.some((r) => r.id === id && r.state === AttachmentState.SYNCED), + 5 + ); + + // localUri is assigned before download and the buffer download path is skipped. + expect(transportDownload).toHaveBeenCalled(); + expect(transportDownload.mock.calls[0][0].localUri).toBeTruthy(); + expect(mockDownloadFile).not.toHaveBeenCalled(); + + const record = attachments.find((r) => r.id === id)!; + expect(await mockLocalStorage.fileExists(record.localUri!)).toBe(true); + }); + + // A custom transport replaces remoteStorage for ALL remote operations. Upload and + // download are covered by the two tests above; this covers the delete slice. + it('delete goes through the custom transport (not remoteStorage)', async () => { + const transportDelete = vi.fn().mockResolvedValue(undefined); + const transportAdapter: AttachmentTransportAdapter = { + upload: vi.fn().mockResolvedValue(undefined), + download: vi.fn(async (attachment: AttachmentRecord & { localUri: string }) => { + await mockLocalStorage.saveFile(attachment.localUri, createMockJpegBuffer()); + }), + delete: transportDelete + }; + + const q = new AttachmentQueue({ + db, + watchAttachments, + localStorage: mockLocalStorage, + transportAdapter, + syncIntervalMs: INTERVAL_MILLISECONDS, + archivedCacheLimit: 0 + }); + onTestFinished(() => q.stopSync()); + + await q.startSync(); + + const id = await q.generateAttachmentId(); + await db.execute('INSERT INTO users (id, name, email, photo_id) VALUES (uuid(), ?, ?, ?)', [ + 'deleter', + 'deleter@example.com', + id + ]); + await waitForMatchCondition( + () => watchAttachmentsTable(), + (results) => results.some((r) => r.id === id && r.state === AttachmentState.SYNCED), + 5 + ); + + await q.deleteFile({ + id, + updateHook: async (tx) => { + await tx.execute('UPDATE users SET photo_id = NULL WHERE photo_id = ?', [id]); + } + }); + + await waitForMatchCondition( + () => watchAttachmentsTable(), + (results) => !results.some((r) => r.id === id), + 5 + ); + + // With a custom transport present, it owns delete; the passed-alongside + // remoteStorage.deleteFile is not used. + expect(transportDelete).toHaveBeenCalled(); + expect(mockDeleteFile).not.toHaveBeenCalled(); + }); + + it('throws when neither remoteStorage nor transportAdapter is provided', () => { + // The options type makes this a compile error; cast to exercise the runtime guard + // that protects plain-JS callers. + expect( + () => + new AttachmentQueue({ + db, + watchAttachments, + localStorage: mockLocalStorage, + syncIntervalMs: INTERVAL_MILLISECONDS, + archivedCacheLimit: 0 + } as any) + ).toThrow(/remoteStorage/); + }); +}); + +describe('attachment queue - saveFileFromUri', () => { + it('registers an on-disk file into managed storage without buffering', async () => { + await mockLocalStorage.initialize(); + + // Simulate an app that has already written a recording to disk. + const sourceUri = mockLocalStorage.getLocalUri('incoming-recording.m4a'); + await mockLocalStorage.saveFile(sourceUri, new Uint8Array(256).fill(7).buffer); + + const readFileSpy = vi.spyOn(mockLocalStorage, 'readFile'); + const saveFileSpy = vi.spyOn(mockLocalStorage, 'saveFile'); + onTestFinished(() => { + readFileSpy.mockRestore(); + saveFileSpy.mockRestore(); + }); + + // No startSync, so nothing uploads: this isolates the registration step. + const record = await queue.saveFileFromUri({ + localUri: sourceUri, + fileExtension: 'm4a', + mediaType: 'audio/m4a' + }); + + expect(record.state).toBe(AttachmentState.QUEUED_UPLOAD); + expect(record.size).toBe(256); + expect(record.localUri).toBe(mockLocalStorage.getLocalUri(`${record.id}.m4a`)); + + // Registration never reads or rewrites the bytes. + expect(readFileSpy).not.toHaveBeenCalled(); + expect(saveFileSpy).not.toHaveBeenCalled(); + + // The file was moved from the source path to the managed path. + expect(await mockLocalStorage.fileExists(sourceUri)).toBe(false); + expect(await mockLocalStorage.fileExists(record.localUri!)).toBe(true); + }); + + it('queues the registered file for upload like any other attachment', async () => { + await mockLocalStorage.initialize(); + + const sourceUri = mockLocalStorage.getLocalUri('incoming-clip.m4a'); + await mockLocalStorage.saveFile(sourceUri, new Uint8Array(128).fill(3).buffer); + + await queue.startSync(); + + const record = await queue.saveFileFromUri({ + localUri: sourceUri, + fileExtension: 'm4a', + updateHook: async (tx, attachment) => { + await tx.execute('INSERT INTO users (id, name, email, photo_id) VALUES (uuid(), ?, ?, ?)', [ + 'recorder', + 'recorder@example.com', + attachment.id + ]); + } + }); + + await waitForMatchCondition( + () => watchAttachmentsTable(), + (results) => results.some((r) => r.id === record.id && r.state === AttachmentState.SYNCED), + 5 + ); + + expect(mockUploadFile).toHaveBeenCalled(); + expect(mockUploadFile.mock.calls[0][1].id).toBe(record.id); + + await queue.stopSync(); + }); + + it('throws when the local storage adapter cannot move files', async () => { + const noMove: any = Object.create(mockLocalStorage); + noMove.moveFile = undefined; + + const q = new AttachmentQueue({ + db, + watchAttachments, + remoteStorage: mockRemoteStorage, + localStorage: noMove, + syncIntervalMs: INTERVAL_MILLISECONDS, + archivedCacheLimit: 0 + }); + onTestFinished(() => q.stopSync()); + + await expect(q.saveFileFromUri({ localUri: 'anywhere.m4a', fileExtension: 'm4a' })).rejects.toThrow(/moveFile/); + }); +}); diff --git a/packages/react-native/android/.gitignore b/packages/react-native/android/.gitignore index d16386367..f7ab597e6 100644 --- a/packages/react-native/android/.gitignore +++ b/packages/react-native/android/.gitignore @@ -1 +1,2 @@ -build/ \ No newline at end of file +build/ +.gradle/ \ No newline at end of file