From 31ab61ffc934891f84c0e5a86b4def171e3b8351 Mon Sep 17 00:00:00 2001 From: Chris Constable Date: Wed, 17 Jun 2026 11:58:50 -0400 Subject: [PATCH 01/10] feat(extstore): add generic s3 driver interface. --- contrib/external-storage-s3/README.md | 155 ++++++++++++ contrib/external-storage-s3/package.json | 52 ++++ .../src/__tests__/test-driver.ts | 196 ++++++++++++++++ contrib/external-storage-s3/src/client.ts | 32 +++ contrib/external-storage-s3/src/driver.ts | 222 ++++++++++++++++++ contrib/external-storage-s3/src/index.ts | 6 + contrib/external-storage-s3/tsconfig.json | 9 + pnpm-lock.yaml | 13 + pnpm-workspace.yaml | 1 + tsconfig.prune.json | 2 + 10 files changed, 688 insertions(+) create mode 100644 contrib/external-storage-s3/README.md create mode 100644 contrib/external-storage-s3/package.json create mode 100644 contrib/external-storage-s3/src/__tests__/test-driver.ts create mode 100644 contrib/external-storage-s3/src/client.ts create mode 100644 contrib/external-storage-s3/src/driver.ts create mode 100644 contrib/external-storage-s3/src/index.ts create mode 100644 contrib/external-storage-s3/tsconfig.json diff --git a/contrib/external-storage-s3/README.md b/contrib/external-storage-s3/README.md new file mode 100644 index 000000000..bdc64ef0f --- /dev/null +++ b/contrib/external-storage-s3/README.md @@ -0,0 +1,155 @@ +# Amazon S3 External Storage Driver for the Temporal TypeScript SDK + +> ⚠️ **This package is experimental and may be subject to change.** ⚠️ + +`@temporalio/external-storage-s3` stores and retrieves Temporal payloads in Amazon S3 via the [External Storage](../../README.md) feature. + +This package has no AWS dependency: it defines the driver and the `S3StorageDriverClient` interface, and you supply the S3 client. Use the companion [`@temporalio/external-storage-s3-aws-sdk`](../external-storage-s3-aws-sdk) package for an [`@aws-sdk/client-s3`](https://www.npmjs.com/package/@aws-sdk/client-s3)-backed client, or implement the interface yourself. + +## Using the AWS SDK client + +Install the adapter package alongside this one: + + npm install @temporalio/external-storage-s3 @temporalio/external-storage-s3-aws-sdk @aws-sdk/client-s3 + +```ts +import { S3Client } from '@aws-sdk/client-s3'; +import { S3StorageDriver } from '@temporalio/external-storage-s3'; +import { AwsSdkS3StorageDriverClient } from '@temporalio/external-storage-s3-aws-sdk'; + +const s3Client = new S3Client({ region: 'us-east-1' }); +const driver = new S3StorageDriver({ + client: new AwsSdkS3StorageDriverClient(s3Client), + bucket: 'my-temporal-payloads', +}); +``` + +Register the resulting driver with the SDK's External Storage configuration so the +client and worker offload eligible payloads to it. + +## Custom S3 client implementations + +To use a different S3 library, implement `S3StorageDriverClient`. + +```ts +import type { S3StorageDriverClient } from '@temporalio/external-storage-s3'; + +const myClient: S3StorageDriverClient = { + async putObject(bucket, key, data, options) { + /* ... */ + }, + async objectExists(bucket, key, options) { + /* ... */ + return false; + }, + async getObject(bucket, key, options) { + /* ... */ + return new Uint8Array(); + }, +}; +``` +## Dynamic bucket selection + +Pass a callable as `bucket` to choose the destination per payload: + +```ts +const driver = new S3StorageDriver({ + client: new AwsSdkS3StorageDriverClient(s3Client), + bucket: (_context, payload) => + (payload.data?.length ?? 0) > 10 * 1024 * 1024 ? 'large-payloads' : 'small-payloads', +}); +``` + +## Required IAM permissions + +The credentials used by your S3 client must have these S3 permissions on the target bucket and its objects: + +```json +{ + "Effect": "Allow", + "Action": ["s3:PutObject", "s3:GetObject"], + "Resource": "arn:aws:s3:::my-temporal-payloads/*" +} +``` + +`s3:PutObject` is required by components that store payloads (typically the client and worker sending workflow/activity inputs); `s3:GetObject` is required by components that retrieve them. Components that only retrieve do not need `s3:PutObject`, and vice versa. + +## S3 Storage Key Specification + +All Temporal S3 drivers generate S3 keys in a consistent manner. + +### Key format + +Workflow key: +```text +v0/ns/{namespace}/wt/{workflow-type}/wi/{workflow-id}/ri/{run-id}/d/{hash-algorithm}/{hex-digest} +``` + +Activity key: +```text +v0/ns/{namespace}/at/{activity-type}/ai/{activity-id}/ri/{run-id}/d/{hash-algorithm}/{hex-digest} +``` + +Fallback key (unknown target): +```text +v0/d/{hash-algorithm}/{hex-digest} +``` + +- If no namespace, workflow, or activity information is available, the fallback is used. +- Dynamic path segments are percent-encoded (rules below). +- Missing values (including a missing `run-id`) are encoded as `null`. +- `hex-digest` is lower-case SHA-256 hex (64 characters). + +### Percent-encoding rules + +The Temporal SDKs escape anything that isn't listed in S3's safe character set: https://docs.aws.amazon.com/AmazonS3/latest/userguide/object-keys.html + +Safe Characters: +```text +Alphanumeric characters + 0-9 + a-z + A-Z + +Special characters + Exclamation point (!) + Hyphen (-) + Underscore (_) + Period (.) + Asterisk (*) + Single quotation mark (') + Opening parenthesis (() + Closing parenthesis ()) +``` + +### Examples + +Workflow key example: + +```text +input: + namespace=payments prod + workflow-type=ChargeWorkflow + workflow-id=order+123=abc + run-id=3f1d6c7a-8b2e-4f7a-9d0a-87a6f95e4d31 + hash-algorithm=sha256 + hex-digest=9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08 + +output: + v0/ns/payments%20prod/wt/ChargeWorkflow/wi/order%2B123%3Dabc/ri/3f1d6c7a-8b2e-4f7a-9d0a-87a6f95e4d31/d/sha256/9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08 +``` + +Activity key example: + +```text +input: + namespace=payments prod + activity-type=Capture/Charge + activity-id=activity id+42 + run-id=9e1d1fd9-2f8a-4c40-93e2-731f31b9268b + hash-algorithm=sha256 + hex-digest=2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824 + +output: + v0/ns/payments%20prod/at/Capture%2FCharge/ai/activity%20id%2B42/ri/9e1d1fd9-2f8a-4c40-93e2-731f31b9268b/d/sha256/2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824 +``` \ No newline at end of file diff --git a/contrib/external-storage-s3/package.json b/contrib/external-storage-s3/package.json new file mode 100644 index 000000000..2957f9cc9 --- /dev/null +++ b/contrib/external-storage-s3/package.json @@ -0,0 +1,52 @@ +{ + "name": "@temporalio/external-storage-s3", + "version": "1.17.2", + "description": "Amazon S3 external storage driver for the Temporal TypeScript SDK", + "main": "lib/index.js", + "types": "./lib/index.d.ts", + "keywords": [ + "temporal", + "workflow", + "external storage", + "payload", + "s3", + "aws" + ], + "author": "Temporal Technologies Inc. ", + "license": "MIT", + "scripts": { + "build": "tsc --build", + "test": "ava ./lib/__tests__/test-*.js" + }, + "ava": { + "timeout": "60s" + }, + "dependencies": { + "@temporalio/common": "workspace:*", + "@temporalio/proto": "workspace:*" + }, + "devDependencies": { + "ava": "^5.3.1" + }, + "engines": { + "node": ">= 20.3.0" + }, + "bugs": { + "url": "https://github.com/temporalio/sdk-typescript/issues" + }, + "repository": { + "type": "git", + "url": "git+https://github.com/temporalio/sdk-typescript.git", + "directory": "contrib/external-storage-s3" + }, + "homepage": "https://github.com/temporalio/sdk-typescript/tree/main/contrib/external-storage-s3", + "publishConfig": { + "access": "public" + }, + "files": [ + "src", + "lib", + "!src/__tests__", + "!lib/__tests__" + ] +} diff --git a/contrib/external-storage-s3/src/__tests__/test-driver.ts b/contrib/external-storage-s3/src/__tests__/test-driver.ts new file mode 100644 index 000000000..e5d060ac7 --- /dev/null +++ b/contrib/external-storage-s3/src/__tests__/test-driver.ts @@ -0,0 +1,196 @@ +/* eslint @typescript-eslint/no-non-null-assertion: 0 */ +import test from 'ava'; +import * as proto from '@temporalio/proto'; +import { ValueError } from '@temporalio/common'; +import type { Payload } from '@temporalio/common'; +import { StorageDriverClaim, type StorageDriverStoreContext } from '@temporalio/common/lib/converter/extstore'; +import { S3StorageDriver } from '../driver'; +import type { S3StorageDriverClient, S3RequestOptions } from '../client'; + +const PayloadProto = proto.temporal.api.common.v1.Payload; +const enc = (s: string): Uint8Array => new TextEncoder().encode(s); + +function makePayload(value: string): Payload { + return { metadata: { encoding: enc('json/plain') }, data: enc(value) }; +} + +function payloadBytes(p: Payload): Uint8Array { + return PayloadProto.encode(p).finish(); +} + +class FakeS3Client implements S3StorageDriverClient { + readonly objects = new Map(); + putCount = 0; + describeValue: Record = {}; + + async putObject(bucket: string, key: string, data: Uint8Array): Promise { + this.putCount += 1; + this.objects.set(`${bucket}/${key}`, data); + } + + async objectExists(bucket: string, key: string): Promise { + return this.objects.has(`${bucket}/${key}`); + } + + async getObject(bucket: string, key: string): Promise { + const value = this.objects.get(`${bucket}/${key}`); + if (!value) throw new Error(`missing object ${bucket}/${key}`); + return value; + } + + describe(): Record { + return this.describeValue; + } +} + +const workflowContext: StorageDriverStoreContext = { + target: { kind: 'workflow', namespace: 'my-ns', type: 'MyWorkflow', id: 'wf-1', runId: 'run-1' }, +}; + +test('store then retrieve round-trips the payload bytes', async (t) => { + const driver = new S3StorageDriver({ client: new FakeS3Client(), bucket: 'b' }); + const original = makePayload('"hello"'); + + const [claim] = await driver.store(workflowContext, [original]); + const [retrieved] = await driver.retrieve({}, [claim!]); + + t.deepEqual(payloadBytes(retrieved!), payloadBytes(original)); +}); + +test('key is content-addressed and segmented by the store context', async (t) => { + const driver = new S3StorageDriver({ client: new FakeS3Client(), bucket: 'b' }); + + const [claim] = await driver.store(workflowContext, [makePayload('"hello"')]); + + t.regex( + claim!.claimData.key!, + /^v0\/ns\/my-ns\/wt\/MyWorkflow\/wi\/wf-1\/ri\/run-1\/d\/sha256\/[0-9a-f]{64}$/ + ); + t.is(claim!.claimData.hashAlgorithm, 'sha256'); + t.is(claim!.claimData.bucket, 'b'); +}); + +test('key segments percent-encode anything outside the S3 safe set', async (t) => { + const driver = new S3StorageDriver({ client: new FakeS3Client(), bucket: 'b' }); + + const [claim] = await driver.store( + { + target: { + kind: 'workflow', + namespace: 'payments prod', + type: "Capture/Charge!*'()", + id: 'order+123=abc', + runId: 'r~1', + }, + }, + [makePayload('"x"')] + ); + + // space -> %20, / -> %2F, !*'() kept, + -> %2B, = -> %3D, ~ -> %7E + t.true( + claim!.claimData.key!.startsWith( + "v0/ns/payments%20prod/wt/Capture%2FCharge!*'()/wi/order%2B123%3Dabc/ri/r%7E1/d/sha256/" + ) + ); +}); + +test('a target with no identity falls back to a bare digest key', async (t) => { + const driver = new S3StorageDriver({ client: new FakeS3Client(), bucket: 'b' }); + + const [claim] = await driver.store({}, [makePayload('"x"')]); + + t.regex(claim!.claimData.key!, /^v0\/d\/sha256\/[0-9a-f]{64}$/); +}); + +test('identical payloads in the same scope deduplicate to one upload', async (t) => { + const client = new FakeS3Client(); + const driver = new S3StorageDriver({ client, bucket: 'b' }); + + await driver.store(workflowContext, [makePayload('"hello"')]); + await driver.store(workflowContext, [makePayload('"hello"')]); + + t.is(client.putCount, 1); +}); + +test('retrieve rejects when stored bytes fail the integrity check', async (t) => { + const client = new FakeS3Client(); + const driver = new S3StorageDriver({ client, bucket: 'b' }); + + const [claim] = await driver.store(workflowContext, [makePayload('"hello"')]); + client.objects.set(`${claim!.claimData.bucket}/${claim!.claimData.key}`, enc('tampered')); + + await t.throwsAsync(() => driver.retrieve({}, [claim!]), { + instanceOf: ValueError, + message: /integrity check failed/, + }); +}); + +test('store rejects payloads larger than maxPayloadSize', async (t) => { + const driver = new S3StorageDriver({ client: new FakeS3Client(), bucket: 'b', maxPayloadSize: 1 }); + + await t.throwsAsync(() => driver.store(workflowContext, [makePayload('"hello"')]), { + instanceOf: ValueError, + message: /exceeds the configured maxPayloadSize/, + }); +}); + +test('retrieve rejects a claim missing hash information', async (t) => { + const client = new FakeS3Client(); + const driver = new S3StorageDriver({ client, bucket: 'b' }); + client.objects.set('b/some-key', payloadBytes(makePayload('"hello"'))); + + const claim = new StorageDriverClaim({ bucket: 'b', key: 'some-key' }); + + await t.throwsAsync(() => driver.retrieve({}, [claim]), { + instanceOf: ValueError, + message: /missing required content hash information/, + }); +}); + +test('bucket selector chooses the destination per payload', async (t) => { + const client = new FakeS3Client(); + const driver = new S3StorageDriver({ + client, + bucket: (_ctx, payload) => ((payload.data?.length ?? 0) > 4 ? 'large' : 'small'), + }); + + const [big] = await driver.store(workflowContext, [makePayload('"a-long-value"')]); + const [small] = await driver.store(workflowContext, [makePayload('"x"')]); + + t.is(big!.claimData.bucket, 'large'); + t.is(small!.claimData.bucket, 'small'); +}); + +test('store aborts in-flight sibling uploads when one fails', async (t) => { + let siblingAborted = false; + const client: S3StorageDriverClient = { + async objectExists() { + return false; + }, + async putObject(bucket: string, _key: string, _data: Uint8Array, options?: S3RequestOptions): Promise { + if (bucket === 'fail') { + throw new Error('boom'); + } + // The surviving sibling resolves only once the shared signal aborts. + await new Promise((resolve) => { + options?.abortSignal?.addEventListener('abort', () => { + siblingAborted = true; + resolve(); + }); + }); + }, + async getObject(): Promise { + throw new Error('unused'); + }, + }; + const driver = new S3StorageDriver({ + client, + bucket: (_ctx, payload) => ((payload.data?.length ?? 0) < 5 ? 'fail' : 'ok'), + }); + + await t.throwsAsync( + () => driver.store(workflowContext, [makePayload('"x"'), makePayload('"a-much-longer-sibling-value"')]), + { message: /store failed/ } + ); + t.true(siblingAborted); +}); diff --git a/contrib/external-storage-s3/src/client.ts b/contrib/external-storage-s3/src/client.ts new file mode 100644 index 000000000..2c261ed99 --- /dev/null +++ b/contrib/external-storage-s3/src/client.ts @@ -0,0 +1,32 @@ +/** + * S3 object operations used by {@link S3StorageDriver}. + * + * @experimental + */ + +/** Per-request options. */ +export interface S3RequestOptions { + /** + * Aborts the in-flight request. + */ + abortSignal?: AbortSignal; +} + +/** + * S3 driver client operations + * + * @experimental + */ +export interface S3StorageDriverClient { + /** Upload `data` to the given bucket and key. */ + putObject(bucket: string, key: string, data: Uint8Array, options?: S3RequestOptions): Promise; + /** Resolve `true` if an object exists at the given bucket and key. */ + objectExists(bucket: string, key: string, options?: S3RequestOptions): Promise; + /** Download and resolve the bytes stored at the given bucket and key. */ + getObject(bucket: string, key: string, options?: S3RequestOptions): Promise; + /** + * Optional client-specific diagnostic metadata (e.g. region) that the driver + * appends to error messages to help diagnose common misconfigurations. + */ + describe?(): Record; +} diff --git a/contrib/external-storage-s3/src/driver.ts b/contrib/external-storage-s3/src/driver.ts new file mode 100644 index 000000000..79a9d5224 --- /dev/null +++ b/contrib/external-storage-s3/src/driver.ts @@ -0,0 +1,222 @@ +import { createHash } from 'node:crypto'; +import * as proto from '@temporalio/proto'; +import { ValueError } from '@temporalio/common'; +import type { Payload } from '@temporalio/common'; +import { + StorageDriverClaim, + type StorageDriver, + type StorageDriverStoreContext, + type StorageDriverRetrieveContext, + type StorageDriverTargetInfo, +} from '@temporalio/common/lib/converter/extstore'; +import type { S3StorageDriverClient } from './client'; + +const PayloadProto = proto.temporal.api.common.v1.Payload; + +const DRIVER_TYPE = 'aws.s3driver'; +const DEFAULT_MAX_PAYLOAD_SIZE = 50 * 1024 * 1024; + +/** Picks the destination bucket for a given payload. Enables dynamic per-payload routing. */ +export type BucketSelector = (context: StorageDriverStoreContext, payload: Payload) => string; + +/** @experimental */ +export interface S3StorageDriverOptions { + /** + * An {@link S3StorageDriverClient} that performs the underlying requests, + * e.g. an `AwsSdkS3StorageDriverClient`. + */ + client: S3StorageDriverClient; + /** Static bucket name or a {@link BucketSelector}. */ + bucket: string | BucketSelector; + /** + * Per-instance routing name written into the wire format. Defaults to + * `"aws.s3driver"`. Override only when registering multiple S3 drivers with + * distinct configurations under the same `ExternalStorage.drivers` list. + */ + driverName?: string; + /** + * Hard upper limit, in bytes, on the serialized size of a single payload. + * Defaults to 52428800 (50 MiB). Stores larger than this throw. + */ + maxPayloadSize?: number; +} + +/** + * Percent-encodes a key segment, escaping everything outside S3's safe-character + * set (alphanumerics and `! - _ . * ' ( )`). An empty or absent value becomes the + * literal `null`. + */ +function encodeKeySegment(value: string | undefined): string { + if (!value) return 'null'; + return encodeURIComponent(value).replace(/~/g, '%7E'); +} + +function buildContextSegments(target: StorageDriverTargetInfo | undefined): string { + if (target?.kind === 'workflow') { + return ( + `/ns/${encodeKeySegment(target.namespace)}` + + `/wt/${encodeKeySegment(target.type)}` + + `/wi/${encodeKeySegment(target.id)}` + + `/ri/${encodeKeySegment(target.runId)}` + ); + } + if (target?.kind === 'activity') { + return ( + `/ns/${encodeKeySegment(target.namespace)}` + + `/at/${encodeKeySegment(target.type)}` + + `/ai/${encodeKeySegment(target.id)}` + + `/ri/${encodeKeySegment(target.runId)}` + ); + } + return ''; +} + +/** + * Formats `describe()` output as ", k=v, k=v" for error messages. Returns an + * empty string when the client reports no metadata or `describe` itself throws. + */ +function formatClientContext(client: S3StorageDriverClient): string { + let info: Record; + try { + info = client.describe?.() ?? {}; + } catch { + return ''; + } + const entries = Object.entries(info); + return entries.map(([k, v]) => `, ${k}=${v}`).join(''); +} + +/** + * Runs the per-payload operations concurrently. On the first failure, aborts the + * shared signal so in-flight sibling requests are cancelled, then waits for them + * to settle before propagating the original error. + */ +async function gatherWithCancellation( + external: AbortSignal | undefined, + makeTasks: (signal: AbortSignal) => Promise[] +): Promise { + const controller = new AbortController(); + const signal = external ? AbortSignal.any([external, controller.signal]) : controller.signal; + const tasks = makeTasks(signal); + try { + return await Promise.all(tasks); + } catch (err) { + controller.abort(); + await Promise.allSettled(tasks); + throw err; + } +} + +/** + * Stores and retrieves Temporal payloads in Amazon S3. + * + * Payloads are content-addressed by a SHA-256 hash of their serialized bytes. S3 keys + * are created using the namespace, workflow/activity type, other metadata from the store + * context. Keys adhere to S3's safe-character set. + * + * @experimental + */ +export class S3StorageDriver implements StorageDriver { + readonly name: string; + readonly type = DRIVER_TYPE; + private readonly client: S3StorageDriverClient; + private readonly bucket: BucketSelector; + private readonly maxPayloadSize: number; + + constructor(options: S3StorageDriverOptions) { + const { client, bucket, driverName, maxPayloadSize = DEFAULT_MAX_PAYLOAD_SIZE } = options; + if (!Number.isFinite(maxPayloadSize) || maxPayloadSize <= 0) { + throw new ValueError(`maxPayloadSize must be a positive finite number, got ${String(maxPayloadSize)}`); + } + this.client = client; + this.bucket = (typeof bucket === 'string') ? () => bucket : bucket; + this.name = driverName || DRIVER_TYPE; + this.maxPayloadSize = maxPayloadSize; + } + + async store(context: StorageDriverStoreContext, payloads: Payload[]): Promise { + const contextSegments = buildContextSegments(context.target); + return gatherWithCancellation(context.abortSignal, (signal) => + payloads.map((payload) => this.upload(context, payload, contextSegments, signal)) + ); + } + + async retrieve(context: StorageDriverRetrieveContext, claims: StorageDriverClaim[]): Promise { + return gatherWithCancellation(context.abortSignal, (signal) => + claims.map((claim) => this.download(claim, signal)) + ); + } + + private async upload( + context: StorageDriverStoreContext, + payload: Payload, + contextSegments: string, + abortSignal: AbortSignal + ): Promise { + const bucket = this.bucket(context, payload); + + const payloadBytes = PayloadProto.encode(payload).finish(); + if (payloadBytes.length > this.maxPayloadSize) { + throw new ValueError( + `Payload size ${payloadBytes.length} bytes exceeds the configured maxPayloadSize of ${this.maxPayloadSize} bytes` + ); + } + + const hashValue = createHash('sha256').update(payloadBytes).digest('hex'); + const key = `v0${contextSegments}/d/sha256/${hashValue}`; + + try { + if (!(await this.client.objectExists(bucket, key, { abortSignal }))) { + await this.client.putObject(bucket, key, payloadBytes, { abortSignal }); + } + } catch (err) { + throw new Error( + `S3StorageDriver store failed [bucket=${bucket}, key=${key}${formatClientContext(this.client)}]`, + { cause: err } + ); + } + + return new StorageDriverClaim({ bucket, key, hashAlgorithm: 'sha256', hashValue }); + } + + private async download(claim: StorageDriverClaim, abortSignal: AbortSignal): Promise { + const { bucket, key, hashAlgorithm, hashValue: expectedHash } = claim.claimData; + if (!bucket || !key) { + throw new ValueError( + `S3StorageDriver claim is missing required location information: ` + + `claimData must contain 'bucket' and 'key'` + ); + } + if (!hashAlgorithm || !expectedHash) { + throw new ValueError( + `S3StorageDriver claim is missing required content hash information [bucket=${bucket}, key=${key}]: ` + + `claimData must contain 'hashAlgorithm' and 'hashValue'` + ); + } + if (hashAlgorithm !== 'sha256') { + throw new ValueError( + `S3StorageDriver unsupported hash algorithm [bucket=${bucket}, key=${key}]: expected sha256, got ${hashAlgorithm}` + ); + } + + let payloadBytes: Uint8Array; + try { + payloadBytes = await this.client.getObject(bucket, key, { abortSignal }); + } catch (err) { + throw new Error( + `S3StorageDriver retrieve failed [bucket=${bucket}, key=${key}${formatClientContext(this.client)}]`, + { cause: err } + ); + } + + const actualHash = createHash('sha256').update(payloadBytes).digest('hex'); + if (actualHash !== expectedHash) { + throw new ValueError( + `S3StorageDriver integrity check failed [bucket=${bucket}, key=${key}]: ` + + `expected ${hashAlgorithm}:${expectedHash}, got ${hashAlgorithm}:${actualHash}` + ); + } + + return PayloadProto.decode(payloadBytes); + } +} diff --git a/contrib/external-storage-s3/src/index.ts b/contrib/external-storage-s3/src/index.ts new file mode 100644 index 000000000..886e0837c --- /dev/null +++ b/contrib/external-storage-s3/src/index.ts @@ -0,0 +1,6 @@ +/** + * @experimental The External Storage S3 driver is an experimental feature and may be subject to change. + */ +export { S3StorageDriver } from './driver'; +export type { S3StorageDriverOptions, BucketSelector } from './driver'; +export type { S3StorageDriverClient, S3RequestOptions } from './client'; diff --git a/contrib/external-storage-s3/tsconfig.json b/contrib/external-storage-s3/tsconfig.json new file mode 100644 index 000000000..d8c9db5ce --- /dev/null +++ b/contrib/external-storage-s3/tsconfig.json @@ -0,0 +1,9 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "outDir": "./lib", + "rootDir": "./src" + }, + "references": [{ "path": "../../packages/common" }, { "path": "../../packages/proto" }], + "include": ["./src/**/*.ts"] +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 3ce2aa8e1..dfd38637b 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -161,6 +161,19 @@ importers: specifier: ^3.25.76 version: 3.25.76 + contrib/external-storage-s3: + dependencies: + '@temporalio/common': + specifier: workspace:* + version: link:../../packages/common + '@temporalio/proto': + specifier: workspace:* + version: link:../../packages/proto + devDependencies: + ava: + specifier: ^5.3.1 + version: 5.3.1 + contrib/interceptors-opentelemetry: dependencies: '@opentelemetry/api': diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 31b46cd3d..29a7fd8ea 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -20,6 +20,7 @@ packages: - contrib/ai-sdk - contrib/interceptors-opentelemetry - contrib/openai-agents + - contrib/external-storage-s3 - contrib/workflow-streams - scripts diff --git a/tsconfig.prune.json b/tsconfig.prune.json index 2944edb61..635e9fd4e 100644 --- a/tsconfig.prune.json +++ b/tsconfig.prune.json @@ -9,6 +9,7 @@ "@temporalio/openai-agents/workflow": ["contrib/openai-agents/src/workflow"], "@temporalio/openai-agents/workflow-interceptor": ["contrib/openai-agents/src/workflow/trace-interceptor"], "@temporalio/openai-agents/*": ["contrib/openai-agents/src/*"], + "@temporalio/external-storage-s3": ["contrib/external-storage-s3"], "@temporalio/*": ["packages/*"] } }, @@ -26,6 +27,7 @@ "./packages/create-project/src/index.ts", "./packages/envconfig/src/index.ts", "./contrib/interceptors-opentelemetry/src/index.ts", + "./contrib/external-storage-s3/src/index.ts", "./packages/nyc-test-coverage/src/index.ts", "./packages/meta/src/index.ts", "./packages/plugin/src/index.ts", From 346b9cf4f5d7c0830cd1776c41126dfdb2d8f0ce Mon Sep 17 00:00:00 2001 From: Chris Constable Date: Wed, 17 Jun 2026 11:59:45 -0400 Subject: [PATCH 02/10] feat(extstore): add aws-sdk s3 driver package. --- contrib/external-storage-s3-aws-sdk/README.md | 23 + .../external-storage-s3-aws-sdk/package.json | 55 ++ .../src/__tests__/test-aws-sdk-client.ts | 55 ++ .../src/aws-sdk-client.ts | 53 ++ .../external-storage-s3-aws-sdk/src/index.ts | 4 + .../external-storage-s3-aws-sdk/tsconfig.json | 9 + pnpm-lock.yaml | 547 +++++++++++++++++- pnpm-workspace.yaml | 1 + tsconfig.prune.json | 2 + 9 files changed, 731 insertions(+), 18 deletions(-) create mode 100644 contrib/external-storage-s3-aws-sdk/README.md create mode 100644 contrib/external-storage-s3-aws-sdk/package.json create mode 100644 contrib/external-storage-s3-aws-sdk/src/__tests__/test-aws-sdk-client.ts create mode 100644 contrib/external-storage-s3-aws-sdk/src/aws-sdk-client.ts create mode 100644 contrib/external-storage-s3-aws-sdk/src/index.ts create mode 100644 contrib/external-storage-s3-aws-sdk/tsconfig.json diff --git a/contrib/external-storage-s3-aws-sdk/README.md b/contrib/external-storage-s3-aws-sdk/README.md new file mode 100644 index 000000000..0154d395c --- /dev/null +++ b/contrib/external-storage-s3-aws-sdk/README.md @@ -0,0 +1,23 @@ +# AWS SDK Client for the Temporal S3 External Storage Driver + +> ⚠️ **This package is experimental and may be subject to change.** ⚠️ + +`@temporalio/external-storage-s3-aws-sdk` provides an [`@aws-sdk/client-s3`](https://www.npmjs.com/package/@aws-sdk/client-s3)-backed `S3StorageDriverClient` for [`@temporalio/external-storage-s3`](../external-storage-s3). + +`@aws-sdk/client-s3` is a peer dependency, so the driver uses the same `S3Client` (and version) your application already configures. + +## Usage + + npm install @temporalio/external-storage-s3 @temporalio/external-storage-s3-aws-sdk @aws-sdk/client-s3 + +```ts +import { S3Client } from '@aws-sdk/client-s3'; +import { S3StorageDriver } from '@temporalio/external-storage-s3'; +import { AwsSdkS3StorageDriverClient } from '@temporalio/external-storage-s3-aws-sdk'; + +const s3Client = new S3Client({ region: 'us-east-1' }); +const driver = new S3StorageDriver({ + client: new AwsSdkS3StorageDriverClient(s3Client), + bucket: 'my-temporal-payloads', +}); +``` diff --git a/contrib/external-storage-s3-aws-sdk/package.json b/contrib/external-storage-s3-aws-sdk/package.json new file mode 100644 index 000000000..a19506188 --- /dev/null +++ b/contrib/external-storage-s3-aws-sdk/package.json @@ -0,0 +1,55 @@ +{ + "name": "@temporalio/external-storage-s3-aws-sdk", + "version": "1.17.2", + "description": "AWS SDK (@aws-sdk/client-s3) client for the Temporal S3 external storage driver", + "main": "lib/index.js", + "types": "./lib/index.d.ts", + "keywords": [ + "temporal", + "workflow", + "external storage", + "payload", + "s3", + "aws" + ], + "author": "Temporal Technologies Inc. ", + "license": "MIT", + "scripts": { + "build": "tsc --build", + "test": "ava ./lib/__tests__/test-*.js" + }, + "ava": { + "timeout": "60s" + }, + "dependencies": { + "@temporalio/external-storage-s3": "workspace:*" + }, + "peerDependencies": { + "@aws-sdk/client-s3": "^3.0.0" + }, + "devDependencies": { + "@aws-sdk/client-s3": "^3.0.0", + "ava": "^5.3.1" + }, + "engines": { + "node": ">= 20.3.0" + }, + "bugs": { + "url": "https://github.com/temporalio/sdk-typescript/issues" + }, + "repository": { + "type": "git", + "url": "git+https://github.com/temporalio/sdk-typescript.git", + "directory": "contrib/external-storage-s3-aws-sdk" + }, + "homepage": "https://github.com/temporalio/sdk-typescript/tree/main/contrib/external-storage-s3-aws-sdk", + "publishConfig": { + "access": "public" + }, + "files": [ + "src", + "lib", + "!src/__tests__", + "!lib/__tests__" + ] +} diff --git a/contrib/external-storage-s3-aws-sdk/src/__tests__/test-aws-sdk-client.ts b/contrib/external-storage-s3-aws-sdk/src/__tests__/test-aws-sdk-client.ts new file mode 100644 index 000000000..96f2569a4 --- /dev/null +++ b/contrib/external-storage-s3-aws-sdk/src/__tests__/test-aws-sdk-client.ts @@ -0,0 +1,55 @@ +import test from 'ava'; +import type { S3Client } from '@aws-sdk/client-s3'; +import { AwsSdkS3StorageDriverClient } from '../aws-sdk-client'; + +/** Minimal S3Client stand-in: only `send` and `config.region` are exercised. */ +function fakeS3Client(send: (command: unknown) => Promise, region?: string): S3Client { + return { send, config: { region } } as unknown as S3Client; +} + +test('objectExists maps a NotFound error to false', async (t) => { + const client = new AwsSdkS3StorageDriverClient( + fakeS3Client(() => Promise.reject(Object.assign(new Error('not found'), { name: 'NotFound' }))) + ); + t.false(await client.objectExists('b', 'k')); +}); + +test('objectExists maps a 404 status to false', async (t) => { + const client = new AwsSdkS3StorageDriverClient( + fakeS3Client(() => Promise.reject(Object.assign(new Error('nope'), { $metadata: { httpStatusCode: 404 } }))) + ); + t.false(await client.objectExists('b', 'k')); +}); + +test('objectExists rethrows non-404 errors', async (t) => { + const client = new AwsSdkS3StorageDriverClient( + fakeS3Client(() => Promise.reject(Object.assign(new Error('denied'), { $metadata: { httpStatusCode: 403 } }))) + ); + await t.throwsAsync(() => client.objectExists('b', 'k'), { message: 'denied' }); +}); + +test('objectExists returns true when the head succeeds', async (t) => { + const client = new AwsSdkS3StorageDriverClient(fakeS3Client(() => Promise.resolve({}))); + t.true(await client.objectExists('b', 'k')); +}); + +test('getObject reads the response body as bytes', async (t) => { + const bytes = new Uint8Array([1, 2, 3]); + const client = new AwsSdkS3StorageDriverClient(fakeS3Client(() => Promise.resolve({ Body: { transformToByteArray: async () => bytes } }))); + t.deepEqual(await client.getObject('b', 'k'), bytes); +}); + +test('getObject throws when the response has no body', async (t) => { + const client = new AwsSdkS3StorageDriverClient(fakeS3Client(() => Promise.resolve({}))); + await t.throwsAsync(() => client.getObject('b', 'k'), { message: /empty body/ }); +}); + +test('describe surfaces a plain-string region', (t) => { + const client = new AwsSdkS3StorageDriverClient(fakeS3Client(() => Promise.resolve({}), 'us-west-2')); + t.deepEqual(client.describe?.(), { clientRegion: 'us-west-2' }); +}); + +test('describe omits region when unavailable', (t) => { + const client = new AwsSdkS3StorageDriverClient(fakeS3Client(() => Promise.resolve({}))); + t.deepEqual(client.describe?.(), {}); +}); diff --git a/contrib/external-storage-s3-aws-sdk/src/aws-sdk-client.ts b/contrib/external-storage-s3-aws-sdk/src/aws-sdk-client.ts new file mode 100644 index 000000000..aa98470eb --- /dev/null +++ b/contrib/external-storage-s3-aws-sdk/src/aws-sdk-client.ts @@ -0,0 +1,53 @@ +import { PutObjectCommand, GetObjectCommand, HeadObjectCommand } from '@aws-sdk/client-s3'; +import type { S3Client as AwsS3Client } from '@aws-sdk/client-s3'; +import type { S3StorageDriverClient, S3RequestOptions } from '@temporalio/external-storage-s3'; + +/** + * An {@link S3StorageDriverClient} backed by an `@aws-sdk/client-s3` `S3Client`, + * for use with `S3StorageDriver`. + * + * @experimental + */ +export class AwsSdkS3StorageDriverClient implements S3StorageDriverClient { + constructor(private readonly client: AwsS3Client) {} + + describe(): Record { + // v3 normalizes `region` to an async provider, so it is usually unavailable + // synchronously here; surface it only when it was supplied as a plain string. + const region = this.client.config?.region; + return typeof region === 'string' && region ? { clientRegion: region } : {}; + } + + async objectExists(bucket: string, key: string, options?: S3RequestOptions): Promise { + try { + await this.client.send(new HeadObjectCommand({ Bucket: bucket, Key: key }), { abortSignal: options?.abortSignal }); + return true; + } catch (err) { + if (isNotFound(err)) { + return false; + } + throw err; + } + } + + async putObject(bucket: string, key: string, data: Uint8Array, options?: S3RequestOptions): Promise { + await this.client.send(new PutObjectCommand({ Bucket: bucket, Key: key, Body: data }), { + abortSignal: options?.abortSignal, + }); + } + + async getObject(bucket: string, key: string, options?: S3RequestOptions): Promise { + const response = await this.client.send(new GetObjectCommand({ Bucket: bucket, Key: key }), { + abortSignal: options?.abortSignal, + }); + if (!response.Body) { + throw new Error(`S3 GetObject returned an empty body [bucket=${bucket}, key=${key}]`); + } + return response.Body.transformToByteArray(); + } +} + +function isNotFound(err: unknown): boolean { + const e = err as { name?: string; $metadata?: { httpStatusCode?: number } }; + return e?.name === 'NotFound' || e?.$metadata?.httpStatusCode === 404; +} diff --git a/contrib/external-storage-s3-aws-sdk/src/index.ts b/contrib/external-storage-s3-aws-sdk/src/index.ts new file mode 100644 index 000000000..98e2a0cf7 --- /dev/null +++ b/contrib/external-storage-s3-aws-sdk/src/index.ts @@ -0,0 +1,4 @@ +/** + * @experimental The External Storage S3 driver is an experimental feature and may be subject to change. + */ +export { AwsSdkS3StorageDriverClient } from './aws-sdk-client'; diff --git a/contrib/external-storage-s3-aws-sdk/tsconfig.json b/contrib/external-storage-s3-aws-sdk/tsconfig.json new file mode 100644 index 000000000..2da50c9be --- /dev/null +++ b/contrib/external-storage-s3-aws-sdk/tsconfig.json @@ -0,0 +1,9 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "outDir": "./lib", + "rootDir": "./src" + }, + "references": [{ "path": "../external-storage-s3" }], + "include": ["./src/**/*.ts"] +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index dfd38637b..88152ce5d 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -174,6 +174,19 @@ importers: specifier: ^5.3.1 version: 5.3.1 + contrib/external-storage-s3-aws-sdk: + dependencies: + '@temporalio/external-storage-s3': + specifier: workspace:* + version: link:../external-storage-s3 + devDependencies: + '@aws-sdk/client-s3': + specifier: ^3.0.0 + version: 3.1060.0 + ava: + specifier: ^5.3.1 + version: 5.3.1 + contrib/interceptors-opentelemetry: dependencies: '@opentelemetry/api': @@ -1003,6 +1016,29 @@ packages: resolution: {integrity: sha512-lFMjJTrFL3j7L9yBxwYfCq2k6qqwHyzuUl/XBnif78PWTJYyL/dfowQHWE3sp6U6ZzqWiiIZnpTMO96zhkjwtg==} engines: {node: '>=6.0.0'} + '@aws-crypto/crc32@5.2.0': + resolution: {integrity: sha512-nLbCWqQNgUiwwtFsen1AdzAtvuLRsQS8rYgMuxCrdKf9kOssamGLuPwyTY9wyYblNr9+1XM8v6zoDTPPSIeANg==} + engines: {node: '>=16.0.0'} + + '@aws-crypto/crc32c@5.2.0': + resolution: {integrity: sha512-+iWb8qaHLYKrNvGRbiYRHSdKRWhto5XlZUEBwDjYNf+ly5SVYG6zEoYIdxvf5R3zyeP16w4PLBn3rH1xc74Rag==} + + '@aws-crypto/sha1-browser@5.2.0': + resolution: {integrity: sha512-OH6lveCFfcDjX4dbAvCFSYUjJZjDr/3XJ3xHtjn3Oj5b9RjojQo8npoLeA/bNwkOkrSQ0wgrHzXk4tDRxGKJeg==} + + '@aws-crypto/sha256-browser@5.2.0': + resolution: {integrity: sha512-AXfN/lGotSQwu6HNcEsIASo7kWXZ5HYWvfOmSNKDsEqC4OashTp8alTmaz+F7TC2L083SFv5RdB+qU3Vs1kZqw==} + + '@aws-crypto/sha256-js@5.2.0': + resolution: {integrity: sha512-FFQQyu7edu4ufvIZ+OadFpHHOt+eSTBaYaki44c+akjg7qZg9oOQeLlk77F6tSYqjDAFClrHJk9tMf0HdVyOvA==} + engines: {node: '>=16.0.0'} + + '@aws-crypto/supports-web-crypto@5.2.0': + resolution: {integrity: sha512-iAvUotm021kM33eCdNfwIN//F77/IADDSs58i+MDaOqFrVjZo9bAal0NK7HurRuWLLpF1iLX7gbWrjHjeo+YFg==} + + '@aws-crypto/util@5.2.0': + resolution: {integrity: sha512-4RkU9EsI6ZpBve5fseQlGNUWKMa1RLPQ1dnjnQoe07ldfIzcsGb5hC5W0Dm7u423KWzawlrpbjXBrXCEv9zazQ==} + '@aws-lambda-powertools/commons@2.32.0': resolution: {integrity: sha512-vsdakJDZu/KkJ5+1WHkawQ5R06aCK0XYB3nc3tpcBfO1YmOHkn+QpuGiZtkmpzIykGPAO9lnBjK5joAeim2o4A==} @@ -1017,6 +1053,98 @@ packages: '@middy/core': optional: true + '@aws-sdk/client-s3@3.1060.0': + resolution: {integrity: sha512-lYdSUOE965Cz/kb3YVDMKz7C4icH0yJxkwB5M0KKAu1nGWT3L78Ty5g2wP3AhZEKH5VzNhPUo8AEcspWOfAGCw==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/core@3.974.17': + resolution: {integrity: sha512-r8o4h2K7j6P9ngno+8ei0aK0U/4JwDb7A2fMMxGVoSqDN8AFlIzSDeZHME9LcVLR2codyhtr1WAAg+/nmkeeMA==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/crc64-nvme@3.972.10': + resolution: {integrity: sha512-QsyJJlx+bSgApcd6kkloZ+nHg2nWJTwUA39/KiDcNRYjz9UOReQcNJRlJBImK+eF9EWl2LG5SW7LaVFuYUE8HQ==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/credential-provider-env@3.972.43': + resolution: {integrity: sha512-g0XVQKzaA/4cq1vz1IvCQwYM+1Pkv01J9yHDpCTXekVuGZRDEz0wqBQ1AuYTq7FM6uik4uBGH8Tb5d9YvgeA7g==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/credential-provider-http@3.972.45': + resolution: {integrity: sha512-w9PuOoKCt6+xoESvY+zlV0u3PKQ0mVL259PcsVR6a3S/uYJJHnIi4r1NxdJHEcNldUVRIciltWnFMGBR4YEm3g==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/credential-provider-ini@3.972.48': + resolution: {integrity: sha512-+6BQ6Lrnc+EyAGElLRW6j+Sa+RirPHnIJsobvYO6nnyK+oGKmz1ne/ieclbLWyjyDKEU3/JVJWcWY3VLFPvGtQ==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/credential-provider-login@3.972.47': + resolution: {integrity: sha512-Iy2ebWVgrZBH05464uJiQYu6HSSiROnwVZptthEFXx2gWjo1ORCxEAFZB5Cr2MdfrSnZ+0QUPkZ1ZpCqpkUrLQ==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/credential-provider-node@3.972.50': + resolution: {integrity: sha512-b05Aelq5cqAvCCDQjCYacl0XmR8QhBNSqLbsdISkQmlQBa5oPS66zYPteWcSp5LswbpoIe552EUGjluKiadBig==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/credential-provider-process@3.972.43': + resolution: {integrity: sha512-GPokLNyvTfCmuaHk+v3GKVs4ZT3cMu5kgS2a+NPkOMt96cq6fSIK0g+mZHpGS6Cd4QGrPKesANEaLUKgOskTzg==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/credential-provider-sso@3.972.47': + resolution: {integrity: sha512-0AzvLrzlvJs0DzbeWGvNj+bX3Uzd7VNS6vDqCOdZzBlCGKGd78uxctJSW9iK/Rt/nxiJqpTvrYQlVJ4guVM2Dw==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/credential-provider-web-identity@3.972.47': + resolution: {integrity: sha512-eksfbUErOejUAGWBAcNqaP7IX21oUOEo73d9R56k9Ua4d57qS90NEYkWJsuSGzTXMFulCu17qXJI/qGmM7hvoA==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/middleware-bucket-endpoint@3.972.19': + resolution: {integrity: sha512-BkjsoevWdtXyfmItfvNW693XO/T2ooXAz3wx3fX1Y7YUHJB+Pvj7XM6Mu9n6lCQ32tF88NzguCOlL8G7e62SOA==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/middleware-expect-continue@3.972.15': + resolution: {integrity: sha512-GFxHAXAO2iV4EIYZ97NcIiJiMATEjCm9sWS0VaRvHgxE9EDsL2tF0J08si74IT/YqFsdOgF/GWtoI2LkgbGj/Q==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/middleware-flexible-checksums@3.974.25': + resolution: {integrity: sha512-u4EmdygVkPTO0UjNcXqqXR5eG5WWzU2bGan1ZsujTqgC1PLDtgXqqK8LbySJ7i1gefAggHfUztZ5B67NLhmKJQ==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/middleware-location-constraint@3.972.12': + resolution: {integrity: sha512-9fXwH5lPa4M9lD6KhKOWZX2sXefuJX0PR/vxZ2u/ZYVrgr/tEUiFTdEBJ88y3/psuocWQ5IZmf5+T1hsa1qDUA==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/middleware-sdk-s3@3.972.46': + resolution: {integrity: sha512-ziGg3WIaAyRb8SO5fdoHBg+u6ikOhDN8QOagRKvZtDkfxFizHdDufCSoQkaOfvlpIxXRvTlFUaHpylMih4/KCw==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/middleware-ssec@3.972.12': + resolution: {integrity: sha512-WiuUb6fqkMAAM3b1+2M2B74Zobh4JUsoS0s2gE792IRJCYaGp2BJzMK9rhfniNijxg1PhVppUeufpiEdDLNy+w==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/nested-clients@3.997.15': + resolution: {integrity: sha512-Fpri1/PXKMKveORZ7E00VLTlWS5DkfZkW70PUE+bOnpWpAeHAQLoiDHhkzN3kNWbbSsGg64+IZYiq/EZgME3Mg==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/signature-v4-multi-region@3.996.31': + resolution: {integrity: sha512-Kn2up9SlG1KC6wRtwf0d7waTGF6rvp9DxYqB54x6UCKdQ6kyaXCqHL4WGb5vUJga5kS8FxnjhY0LqM28aMvnNQ==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/token-providers@3.1060.0': + resolution: {integrity: sha512-6NZaMKkFhpaNiwLpHi1sZaYjidL/lCJE6ME6NxwA8gv9vQna+Kr0j4OFwVoz6tANRWM3WbGz6jiPsGX/Vkjwow==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/types@3.973.10': + resolution: {integrity: sha512-992QrTO7G9qCvKD0fx1rMlqcL14plUcRAbwmqqYVsuF3GrqcvlAL9qxR+baMafarEZ+l7DUQ5lCMmt5mbMhF7g==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/util-locate-window@3.965.5': + resolution: {integrity: sha512-WhlJNNINQB+9qtLtZJcpQdgZw3SCDCpXdUJP7cToGwHbCWCnRckGlc6Bx/OhWwIYFNAn+FIydY8SZ0QmVu3xTQ==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/xml-builder@3.972.27': + resolution: {integrity: sha512-hpsCXCOI436kxWpjtRuIHVvuPP81MOw8f18jzfZeg+UOiiOvlqWcmWChzEhJEu16cOC6+ku4ncBN+7rdt+DZ9g==} + engines: {node: '>=20.0.0'} + '@aws/lambda-invoke-store@0.2.4': resolution: {integrity: sha512-iY8yvjE0y651BixKNPgmv1WrQc+GZ142sb0z4gYnChDDY2YqI4P/jsSopBWrKfAt7LOJAkOXt7rC/hms+WclQQ==} engines: {node: '>=18.0.0'} @@ -1474,6 +1602,9 @@ packages: '@cfworker/json-schema': optional: true + '@nodable/entities@2.1.1': + resolution: {integrity: sha512-Pig3HxDIoMgjdEH8OCf/dkcTmLFjJRjWuq8jSnklu284/TKOPibSRERmOykiwmyXTtv61mP+44f3GMx0tLAyjg==} + '@nodelib/fs.scandir@2.1.5': resolution: {integrity: sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==} engines: {node: '>= 8'} @@ -1675,6 +1806,42 @@ packages: resolution: {integrity: sha512-TV7t8GKYaJWsn00tFDqBw8+Uqmr8A0fRU1tvTQhyZzGv0sJCGRQL3JGMI3ucuKo3XIZdUP+Lx7/gh2t3lewy7g==} engines: {node: '>=14.16'} + '@smithy/core@3.24.6': + resolution: {integrity: sha512-wBXDRup6UU97VKyaiRo8AssnfStPtG0oAAfpq/bC0a1YYau8pM86YB4kM6ccoVi1mS8l/UHbn9oDM+7uozr/ug==} + engines: {node: '>=18.0.0'} + + '@smithy/credential-provider-imds@4.3.7': + resolution: {integrity: sha512-xj8gq/bjFABAh6qWPSDCYcY3kzQIm4b561C+YnHH4zGq8rOgzQ3Shk+JGlpUxSd41UGiO6FkLdUCtNX1FAeHgg==} + engines: {node: '>=18.0.0'} + + '@smithy/fetch-http-handler@5.4.6': + resolution: {integrity: sha512-FEwEYJ1jlBKdhe9TPzfghEi1bP55ZeEImlDkEa62bBBYzUcnB6RUCyuiS2mqKt6ZVjUbBgcNhzfIctH+Hevx9g==} + engines: {node: '>=18.0.0'} + + '@smithy/is-array-buffer@2.2.0': + resolution: {integrity: sha512-GGP3O9QFD24uGeAXYUjwSTXARoqpZykHadOmA8G5vfJPK0/DC67qa//0qvqrJzL1xc8WQWX7/yc7fwudjPHPhA==} + engines: {node: '>=14.0.0'} + + '@smithy/node-http-handler@4.7.6': + resolution: {integrity: sha512-3fya8i7GrJilQouk4cZJKdy5k8MWQBpjfXrRNaXDedH8r779tr0jcxyH3+yoTmsluc2+vF4S343yFbnvu8ExDQ==} + engines: {node: '>=18.0.0'} + + '@smithy/signature-v4@5.4.6': + resolution: {integrity: sha512-Ojg4B6oIDlIr1R86xCDJt1zJWnYa0VINmqdjfe9qxWjdRivHalZ3iSlQgVqYbW0MdpFOC5XfHEWsnbmdnpIILQ==} + engines: {node: '>=18.0.0'} + + '@smithy/types@4.14.3': + resolution: {integrity: sha512-YupL0ZWmFtJexUN2cHzkvvF/b9pKrtAIfT1o7/oY/Ppu8IYeZ+lDPM5vZdQJaSeA132dJCqojjGC9NhXeF71VQ==} + engines: {node: '>=18.0.0'} + + '@smithy/util-buffer-from@2.2.0': + resolution: {integrity: sha512-IJdWBbTcMQ6DA0gdNhh/BwrLkDR+ADW5Kr1aZmd4k3DIF6ezMV4R2NIAmT08wQJ3yUK82thHWmC/TnK/wpMMIA==} + engines: {node: '>=14.0.0'} + + '@smithy/util-utf8@2.3.0': + resolution: {integrity: sha512-R8Rdn8Hy72KKcebgLiv8jQcQkXoLMOGGv5uI1/k0l+snqkOzQ1R0ChUBCxWMlBsFMekWjq0wRudIweFs7sKT5A==} + engines: {node: '>=14.0.0'} + '@standard-schema/spec@1.1.0': resolution: {integrity: sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w==} @@ -2152,11 +2319,6 @@ packages: resolution: {integrity: sha512-TgUZgYvqZprrl7YldZNoa9OciCAyZR+Ejm9eXzKCmjsF5IKp/wgQ7Z/ZpjpGTIUPwrHQIcYeI8qDh4PsEwxMbw==} engines: {node: '>=0.4.0'} - acorn@8.15.0: - resolution: {integrity: sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg==} - engines: {node: '>=0.4.0'} - hasBin: true - acorn@8.16.0: resolution: {integrity: sha512-UVJyE9MttOsBQIDKw1skb9nAwQuR5wuGD3+82K6JgJlm/Y+KI92oNsMNGZCYdDsVtRHSak0pcV5Dno5+4jh9sw==} engines: {node: '>=0.4.0'} @@ -2375,6 +2537,9 @@ packages: resolution: {integrity: sha512-oP5VkATKlNwcgvxi0vM0p/D3n2C3EReYVX+DNYs5TjZFn/oQt2j+4sVJtSMr18pdRr8wjTcBl6LoV+FUwzPmNA==} engines: {node: '>=18'} + bowser@2.14.1: + resolution: {integrity: sha512-tzPjzCxygAKWFOJP011oxFHs57HzIhOEracIgAePE4pqB3LikALKnSzUyU4MGs9/iCEUuHlAJTjTc5M+u7YEGg==} + brace-expansion@1.1.12: resolution: {integrity: sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg==} @@ -3101,6 +3266,13 @@ packages: fast-uri@3.1.0: resolution: {integrity: sha512-iPeeDKJSWf4IEOasVVrknXpaBV0IApz/gp7S2bb7Z4Lljbl2MGJRqInZiUrQwV16cpzw/D3S5j5Julj/gT52AA==} + fast-xml-builder@1.2.0: + resolution: {integrity: sha512-00aAWieqff+ZJhsXA4g1g7M8k+7AYoMUUHF+/zFb5U6Uv/P0Vl4QZo84/IcufzYalLuEj9928bXN9PbbFzMF0Q==} + + fast-xml-parser@5.7.3: + resolution: {integrity: sha512-C0AaNuC+mscy6vrAQKAc/rMq+zAPHodfHGZu4sGVehvAQt/JLG1O5zEcYcXSY5zSqr4YVgxsB+pHXTq0i7eDlg==} + hasBin: true + fastq@1.16.0: resolution: {integrity: sha512-ifCoaXsDrsdkWTtiNJX5uzHDsrck5TzfKKDcuFFTIrrc/BS076qgEIfoIy1VeZqViznfKiysPYTh/QeHtnIsYA==} @@ -4242,6 +4414,10 @@ packages: resolution: {integrity: sha512-RjhtfwJOxzcFmNOi6ltcbcu4Iu+FL3zEj83dk4kAS+fVpTxXLO1b38RvJgT/0QwvV/L3aY9TAnyv0EOqW4GoMQ==} engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} + path-expression-matcher@1.5.0: + resolution: {integrity: sha512-cbrerZV+6rvdQrrD+iGMcZFEiiSrbv9Tfdkvnusy6y0x0GKBXREFg/Y65GhIfm0tnLntThhzCnfKwp1WRjeCyQ==} + engines: {node: '>=14.0.0'} + path-key@3.1.1: resolution: {integrity: sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==} engines: {node: '>=8'} @@ -4762,6 +4938,9 @@ packages: resolution: {integrity: sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==} engines: {node: '>=8'} + strnum@2.3.0: + resolution: {integrity: sha512-ums3KNd42PGyx5xaoVTO1mjU1bH3NpY4vsrVlnv9PNGqQj8wd7rJ6nEypLrJ7z5vxK5RP0yMLo6J/Gsm62DI5Q==} + supertap@3.0.1: resolution: {integrity: sha512-u1ZpIBCawJnO+0QePsEiOknOfCRq0yERxiAchT0i4li0WHNUJbf0evXXSXOcCAR4M8iMDoajXYmstm/qO81Isw==} engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} @@ -5173,6 +5352,10 @@ packages: resolution: {integrity: sha512-+QU2zd6OTD8XWIJCbffaiQeH9U73qIqafo1x6V1snCWYGJf6cVE0cDR4D8xRzcEnfI21IFrUPzPGtcPf8AC+Rw==} engines: {node: ^14.17.0 || ^16.13.0 || >=18.0.0} + xml-naming@0.1.0: + resolution: {integrity: sha512-k8KO9hrMyNk6tUWqUfkTEZbezRRpONVOzUTnc97VnCvyj6Tf9lyUR9EDAIeiVLv56jsMcoXEwjW8Kv5yPY52lw==} + engines: {node: '>=16.0.0'} + xmlcreate@2.0.4: resolution: {integrity: sha512-nquOebG4sngPmGPICTS5EnxqhKbCmz5Ox5hsszI2T6U5qdrJizBc+0ilYSEjTSzU0yZcmvppztXe/5Al5fUwdg==} @@ -5262,6 +5445,53 @@ snapshots: '@jridgewell/gen-mapping': 0.3.3 '@jridgewell/trace-mapping': 0.3.31 + '@aws-crypto/crc32@5.2.0': + dependencies: + '@aws-crypto/util': 5.2.0 + '@aws-sdk/types': 3.973.10 + tslib: 2.6.2 + + '@aws-crypto/crc32c@5.2.0': + dependencies: + '@aws-crypto/util': 5.2.0 + '@aws-sdk/types': 3.973.10 + tslib: 2.6.2 + + '@aws-crypto/sha1-browser@5.2.0': + dependencies: + '@aws-crypto/supports-web-crypto': 5.2.0 + '@aws-crypto/util': 5.2.0 + '@aws-sdk/types': 3.973.10 + '@aws-sdk/util-locate-window': 3.965.5 + '@smithy/util-utf8': 2.3.0 + tslib: 2.6.2 + + '@aws-crypto/sha256-browser@5.2.0': + dependencies: + '@aws-crypto/sha256-js': 5.2.0 + '@aws-crypto/supports-web-crypto': 5.2.0 + '@aws-crypto/util': 5.2.0 + '@aws-sdk/types': 3.973.10 + '@aws-sdk/util-locate-window': 3.965.5 + '@smithy/util-utf8': 2.3.0 + tslib: 2.6.2 + + '@aws-crypto/sha256-js@5.2.0': + dependencies: + '@aws-crypto/util': 5.2.0 + '@aws-sdk/types': 3.973.10 + tslib: 2.6.2 + + '@aws-crypto/supports-web-crypto@5.2.0': + dependencies: + tslib: 2.6.2 + + '@aws-crypto/util@5.2.0': + dependencies: + '@aws-sdk/types': 3.973.10 + '@smithy/util-utf8': 2.3.0 + tslib: 2.6.2 + '@aws-lambda-powertools/commons@2.32.0': dependencies: '@aws/lambda-invoke-store': 0.2.4 @@ -5271,6 +5501,219 @@ snapshots: '@aws-lambda-powertools/commons': 2.32.0 '@aws/lambda-invoke-store': 0.2.4 + '@aws-sdk/client-s3@3.1060.0': + dependencies: + '@aws-crypto/sha1-browser': 5.2.0 + '@aws-crypto/sha256-browser': 5.2.0 + '@aws-crypto/sha256-js': 5.2.0 + '@aws-sdk/core': 3.974.17 + '@aws-sdk/credential-provider-node': 3.972.50 + '@aws-sdk/middleware-bucket-endpoint': 3.972.19 + '@aws-sdk/middleware-expect-continue': 3.972.15 + '@aws-sdk/middleware-flexible-checksums': 3.974.25 + '@aws-sdk/middleware-location-constraint': 3.972.12 + '@aws-sdk/middleware-sdk-s3': 3.972.46 + '@aws-sdk/middleware-ssec': 3.972.12 + '@aws-sdk/signature-v4-multi-region': 3.996.31 + '@aws-sdk/types': 3.973.10 + '@smithy/core': 3.24.6 + '@smithy/fetch-http-handler': 5.4.6 + '@smithy/node-http-handler': 4.7.6 + '@smithy/types': 4.14.3 + tslib: 2.6.2 + + '@aws-sdk/core@3.974.17': + dependencies: + '@aws-sdk/types': 3.973.10 + '@aws-sdk/xml-builder': 3.972.27 + '@aws/lambda-invoke-store': 0.2.4 + '@smithy/core': 3.24.6 + '@smithy/signature-v4': 5.4.6 + '@smithy/types': 4.14.3 + bowser: 2.14.1 + tslib: 2.6.2 + + '@aws-sdk/crc64-nvme@3.972.10': + dependencies: + '@smithy/types': 4.14.3 + tslib: 2.6.2 + + '@aws-sdk/credential-provider-env@3.972.43': + dependencies: + '@aws-sdk/core': 3.974.17 + '@aws-sdk/types': 3.973.10 + '@smithy/core': 3.24.6 + '@smithy/types': 4.14.3 + tslib: 2.6.2 + + '@aws-sdk/credential-provider-http@3.972.45': + dependencies: + '@aws-sdk/core': 3.974.17 + '@aws-sdk/types': 3.973.10 + '@smithy/core': 3.24.6 + '@smithy/fetch-http-handler': 5.4.6 + '@smithy/node-http-handler': 4.7.6 + '@smithy/types': 4.14.3 + tslib: 2.6.2 + + '@aws-sdk/credential-provider-ini@3.972.48': + dependencies: + '@aws-sdk/core': 3.974.17 + '@aws-sdk/credential-provider-env': 3.972.43 + '@aws-sdk/credential-provider-http': 3.972.45 + '@aws-sdk/credential-provider-login': 3.972.47 + '@aws-sdk/credential-provider-process': 3.972.43 + '@aws-sdk/credential-provider-sso': 3.972.47 + '@aws-sdk/credential-provider-web-identity': 3.972.47 + '@aws-sdk/nested-clients': 3.997.15 + '@aws-sdk/types': 3.973.10 + '@smithy/core': 3.24.6 + '@smithy/credential-provider-imds': 4.3.7 + '@smithy/types': 4.14.3 + tslib: 2.6.2 + + '@aws-sdk/credential-provider-login@3.972.47': + dependencies: + '@aws-sdk/core': 3.974.17 + '@aws-sdk/nested-clients': 3.997.15 + '@aws-sdk/types': 3.973.10 + '@smithy/core': 3.24.6 + '@smithy/types': 4.14.3 + tslib: 2.6.2 + + '@aws-sdk/credential-provider-node@3.972.50': + dependencies: + '@aws-sdk/credential-provider-env': 3.972.43 + '@aws-sdk/credential-provider-http': 3.972.45 + '@aws-sdk/credential-provider-ini': 3.972.48 + '@aws-sdk/credential-provider-process': 3.972.43 + '@aws-sdk/credential-provider-sso': 3.972.47 + '@aws-sdk/credential-provider-web-identity': 3.972.47 + '@aws-sdk/types': 3.973.10 + '@smithy/core': 3.24.6 + '@smithy/credential-provider-imds': 4.3.7 + '@smithy/types': 4.14.3 + tslib: 2.6.2 + + '@aws-sdk/credential-provider-process@3.972.43': + dependencies: + '@aws-sdk/core': 3.974.17 + '@aws-sdk/types': 3.973.10 + '@smithy/core': 3.24.6 + '@smithy/types': 4.14.3 + tslib: 2.6.2 + + '@aws-sdk/credential-provider-sso@3.972.47': + dependencies: + '@aws-sdk/core': 3.974.17 + '@aws-sdk/nested-clients': 3.997.15 + '@aws-sdk/token-providers': 3.1060.0 + '@aws-sdk/types': 3.973.10 + '@smithy/core': 3.24.6 + '@smithy/types': 4.14.3 + tslib: 2.6.2 + + '@aws-sdk/credential-provider-web-identity@3.972.47': + dependencies: + '@aws-sdk/core': 3.974.17 + '@aws-sdk/nested-clients': 3.997.15 + '@aws-sdk/types': 3.973.10 + '@smithy/core': 3.24.6 + '@smithy/types': 4.14.3 + tslib: 2.6.2 + + '@aws-sdk/middleware-bucket-endpoint@3.972.19': + dependencies: + '@aws-sdk/core': 3.974.17 + '@aws-sdk/types': 3.973.10 + '@smithy/core': 3.24.6 + '@smithy/types': 4.14.3 + tslib: 2.6.2 + + '@aws-sdk/middleware-expect-continue@3.972.15': + dependencies: + '@aws-sdk/types': 3.973.10 + '@smithy/core': 3.24.6 + '@smithy/types': 4.14.3 + tslib: 2.6.2 + + '@aws-sdk/middleware-flexible-checksums@3.974.25': + dependencies: + '@aws-crypto/crc32': 5.2.0 + '@aws-crypto/crc32c': 5.2.0 + '@aws-crypto/util': 5.2.0 + '@aws-sdk/core': 3.974.17 + '@aws-sdk/crc64-nvme': 3.972.10 + '@aws-sdk/types': 3.973.10 + '@smithy/core': 3.24.6 + '@smithy/types': 4.14.3 + tslib: 2.6.2 + + '@aws-sdk/middleware-location-constraint@3.972.12': + dependencies: + '@aws-sdk/types': 3.973.10 + '@smithy/types': 4.14.3 + tslib: 2.6.2 + + '@aws-sdk/middleware-sdk-s3@3.972.46': + dependencies: + '@aws-sdk/core': 3.974.17 + '@aws-sdk/signature-v4-multi-region': 3.996.31 + '@aws-sdk/types': 3.973.10 + '@smithy/core': 3.24.6 + '@smithy/types': 4.14.3 + tslib: 2.6.2 + + '@aws-sdk/middleware-ssec@3.972.12': + dependencies: + '@aws-sdk/types': 3.973.10 + '@smithy/types': 4.14.3 + tslib: 2.6.2 + + '@aws-sdk/nested-clients@3.997.15': + dependencies: + '@aws-crypto/sha256-browser': 5.2.0 + '@aws-crypto/sha256-js': 5.2.0 + '@aws-sdk/core': 3.974.17 + '@aws-sdk/signature-v4-multi-region': 3.996.31 + '@aws-sdk/types': 3.973.10 + '@smithy/core': 3.24.6 + '@smithy/fetch-http-handler': 5.4.6 + '@smithy/node-http-handler': 4.7.6 + '@smithy/types': 4.14.3 + tslib: 2.6.2 + + '@aws-sdk/signature-v4-multi-region@3.996.31': + dependencies: + '@aws-sdk/types': 3.973.10 + '@smithy/signature-v4': 5.4.6 + '@smithy/types': 4.14.3 + tslib: 2.6.2 + + '@aws-sdk/token-providers@3.1060.0': + dependencies: + '@aws-sdk/core': 3.974.17 + '@aws-sdk/nested-clients': 3.997.15 + '@aws-sdk/types': 3.973.10 + '@smithy/core': 3.24.6 + '@smithy/types': 4.14.3 + tslib: 2.6.2 + + '@aws-sdk/types@3.973.10': + dependencies: + '@smithy/types': 4.14.3 + tslib: 2.6.2 + + '@aws-sdk/util-locate-window@3.965.5': + dependencies: + tslib: 2.6.2 + + '@aws-sdk/xml-builder@3.972.27': + dependencies: + '@smithy/types': 4.14.3 + fast-xml-parser: 5.7.3 + tslib: 2.6.2 + '@aws/lambda-invoke-store@0.2.4': {} '@babel/code-frame@7.27.1': @@ -5778,6 +6221,8 @@ snapshots: - supports-color optional: true + '@nodable/entities@2.1.1': {} + '@nodelib/fs.scandir@2.1.5': dependencies: '@nodelib/fs.stat': 2.0.5 @@ -6008,6 +6453,54 @@ snapshots: '@sindresorhus/is@5.6.0': {} + '@smithy/core@3.24.6': + dependencies: + '@aws-crypto/crc32': 5.2.0 + '@smithy/types': 4.14.3 + tslib: 2.6.2 + + '@smithy/credential-provider-imds@4.3.7': + dependencies: + '@smithy/core': 3.24.6 + '@smithy/types': 4.14.3 + tslib: 2.6.2 + + '@smithy/fetch-http-handler@5.4.6': + dependencies: + '@smithy/core': 3.24.6 + '@smithy/types': 4.14.3 + tslib: 2.6.2 + + '@smithy/is-array-buffer@2.2.0': + dependencies: + tslib: 2.6.2 + + '@smithy/node-http-handler@4.7.6': + dependencies: + '@smithy/core': 3.24.6 + '@smithy/types': 4.14.3 + tslib: 2.6.2 + + '@smithy/signature-v4@5.4.6': + dependencies: + '@smithy/core': 3.24.6 + '@smithy/types': 4.14.3 + tslib: 2.6.2 + + '@smithy/types@4.14.3': + dependencies: + tslib: 2.6.2 + + '@smithy/util-buffer-from@2.2.0': + dependencies: + '@smithy/is-array-buffer': 2.2.0 + tslib: 2.6.2 + + '@smithy/util-utf8@2.3.0': + dependencies: + '@smithy/util-buffer-from': 2.2.0 + tslib: 2.6.2 + '@standard-schema/spec@1.1.0': {} '@swc/core-darwin-arm64@1.3.102': @@ -6618,22 +7111,20 @@ snapshots: mime-types: 3.0.2 negotiator: 1.0.0 - acorn-import-attributes@1.9.5(acorn@8.15.0): + acorn-import-attributes@1.9.5(acorn@8.16.0): dependencies: - acorn: 8.15.0 + acorn: 8.16.0 acorn-import-phases@1.0.4(acorn@8.16.0): dependencies: acorn: 8.16.0 - acorn-jsx@5.3.2(acorn@8.15.0): + acorn-jsx@5.3.2(acorn@8.16.0): dependencies: - acorn: 8.15.0 + acorn: 8.16.0 acorn-walk@8.3.1: {} - acorn@8.15.0: {} - acorn@8.16.0: {} agent-base@6.0.2: @@ -6794,7 +7285,7 @@ snapshots: ava@5.3.1: dependencies: - acorn: 8.15.0 + acorn: 8.16.0 acorn-walk: 8.3.1 ansi-styles: 6.2.3 arrgv: 1.0.2 @@ -6901,6 +7392,8 @@ snapshots: transitivePeerDependencies: - supports-color + bowser@2.14.1: {} + brace-expansion@1.1.12: dependencies: balanced-match: 1.0.2 @@ -7626,14 +8119,14 @@ snapshots: espree@10.4.0: dependencies: - acorn: 8.15.0 - acorn-jsx: 5.3.2(acorn@8.15.0) + acorn: 8.16.0 + acorn-jsx: 5.3.2(acorn@8.16.0) eslint-visitor-keys: 4.2.1 espree@9.6.1: dependencies: - acorn: 8.15.0 - acorn-jsx: 5.3.2(acorn@8.15.0) + acorn: 8.16.0 + acorn-jsx: 5.3.2(acorn@8.16.0) eslint-visitor-keys: 3.4.3 esprima@4.0.1: {} @@ -7770,6 +8263,18 @@ snapshots: fast-uri@3.1.0: {} + fast-xml-builder@1.2.0: + dependencies: + path-expression-matcher: 1.5.0 + xml-naming: 0.1.0 + + fast-xml-parser@5.7.3: + dependencies: + '@nodable/entities': 2.1.1 + fast-xml-builder: 1.2.0 + path-expression-matcher: 1.5.0 + strnum: 2.3.0 + fastq@1.16.0: dependencies: reusify: 1.0.4 @@ -8153,8 +8658,8 @@ snapshots: import-in-the-middle@1.15.0: dependencies: - acorn: 8.15.0 - acorn-import-attributes: 1.9.5(acorn@8.15.0) + acorn: 8.16.0 + acorn-import-attributes: 1.9.5(acorn@8.16.0) cjs-module-lexer: 1.2.3 module-details-from-path: 1.0.3 @@ -8867,6 +9372,8 @@ snapshots: path-exists@5.0.0: {} + path-expression-matcher@1.5.0: {} + path-key@3.1.1: {} path-parse@1.0.7: {} @@ -9492,6 +9999,8 @@ snapshots: strip-json-comments@3.1.1: {} + strnum@2.3.0: {} + supertap@3.0.1: dependencies: indent-string: 5.0.0 @@ -10026,6 +10535,8 @@ snapshots: imurmurhash: 0.1.4 signal-exit: 4.1.0 + xml-naming@0.1.0: {} + xmlcreate@2.0.4: {} xtend@4.0.2: {} diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 29a7fd8ea..a90feb787 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -21,6 +21,7 @@ packages: - contrib/interceptors-opentelemetry - contrib/openai-agents - contrib/external-storage-s3 + - contrib/external-storage-s3-aws-sdk - contrib/workflow-streams - scripts diff --git a/tsconfig.prune.json b/tsconfig.prune.json index 635e9fd4e..153314820 100644 --- a/tsconfig.prune.json +++ b/tsconfig.prune.json @@ -10,6 +10,7 @@ "@temporalio/openai-agents/workflow-interceptor": ["contrib/openai-agents/src/workflow/trace-interceptor"], "@temporalio/openai-agents/*": ["contrib/openai-agents/src/*"], "@temporalio/external-storage-s3": ["contrib/external-storage-s3"], + "@temporalio/external-storage-s3-aws-sdk": ["contrib/external-storage-s3-aws-sdk"], "@temporalio/*": ["packages/*"] } }, @@ -28,6 +29,7 @@ "./packages/envconfig/src/index.ts", "./contrib/interceptors-opentelemetry/src/index.ts", "./contrib/external-storage-s3/src/index.ts", + "./contrib/external-storage-s3-aws-sdk/src/index.ts", "./packages/nyc-test-coverage/src/index.ts", "./packages/meta/src/index.ts", "./packages/plugin/src/index.ts", From 17bd40472453572ac23294e3d4bf860e00c3b049 Mon Sep 17 00:00:00 2001 From: Chris Constable Date: Wed, 17 Jun 2026 13:06:36 -0400 Subject: [PATCH 03/10] fix formatting issues --- .../src/__tests__/test-aws-sdk-client.ts | 4 +++- .../src/aws-sdk-client.ts | 4 +++- contrib/external-storage-s3/README.md | 14 +++++++++----- .../src/__tests__/test-driver.ts | 5 +---- contrib/external-storage-s3/src/driver.ts | 9 +++------ 5 files changed, 19 insertions(+), 17 deletions(-) diff --git a/contrib/external-storage-s3-aws-sdk/src/__tests__/test-aws-sdk-client.ts b/contrib/external-storage-s3-aws-sdk/src/__tests__/test-aws-sdk-client.ts index 96f2569a4..5ad965b34 100644 --- a/contrib/external-storage-s3-aws-sdk/src/__tests__/test-aws-sdk-client.ts +++ b/contrib/external-storage-s3-aws-sdk/src/__tests__/test-aws-sdk-client.ts @@ -35,7 +35,9 @@ test('objectExists returns true when the head succeeds', async (t) => { test('getObject reads the response body as bytes', async (t) => { const bytes = new Uint8Array([1, 2, 3]); - const client = new AwsSdkS3StorageDriverClient(fakeS3Client(() => Promise.resolve({ Body: { transformToByteArray: async () => bytes } }))); + const client = new AwsSdkS3StorageDriverClient( + fakeS3Client(() => Promise.resolve({ Body: { transformToByteArray: async () => bytes } })) + ); t.deepEqual(await client.getObject('b', 'k'), bytes); }); diff --git a/contrib/external-storage-s3-aws-sdk/src/aws-sdk-client.ts b/contrib/external-storage-s3-aws-sdk/src/aws-sdk-client.ts index aa98470eb..5de7834d3 100644 --- a/contrib/external-storage-s3-aws-sdk/src/aws-sdk-client.ts +++ b/contrib/external-storage-s3-aws-sdk/src/aws-sdk-client.ts @@ -20,7 +20,9 @@ export class AwsSdkS3StorageDriverClient implements S3StorageDriverClient { async objectExists(bucket: string, key: string, options?: S3RequestOptions): Promise { try { - await this.client.send(new HeadObjectCommand({ Bucket: bucket, Key: key }), { abortSignal: options?.abortSignal }); + await this.client.send(new HeadObjectCommand({ Bucket: bucket, Key: key }), { + abortSignal: options?.abortSignal, + }); return true; } catch (err) { if (isNotFound(err)) { diff --git a/contrib/external-storage-s3/README.md b/contrib/external-storage-s3/README.md index bdc64ef0f..136c480db 100644 --- a/contrib/external-storage-s3/README.md +++ b/contrib/external-storage-s3/README.md @@ -48,6 +48,7 @@ const myClient: S3StorageDriverClient = { }, }; ``` + ## Dynamic bucket selection Pass a callable as `bucket` to choose the destination per payload: @@ -55,8 +56,7 @@ Pass a callable as `bucket` to choose the destination per payload: ```ts const driver = new S3StorageDriver({ client: new AwsSdkS3StorageDriverClient(s3Client), - bucket: (_context, payload) => - (payload.data?.length ?? 0) > 10 * 1024 * 1024 ? 'large-payloads' : 'small-payloads', + bucket: (_context, payload) => ((payload.data?.length ?? 0) > 10 * 1024 * 1024 ? 'large-payloads' : 'small-payloads'), }); ``` @@ -81,16 +81,19 @@ All Temporal S3 drivers generate S3 keys in a consistent manner. ### Key format Workflow key: + ```text v0/ns/{namespace}/wt/{workflow-type}/wi/{workflow-id}/ri/{run-id}/d/{hash-algorithm}/{hex-digest} ``` Activity key: + ```text v0/ns/{namespace}/at/{activity-type}/ai/{activity-id}/ri/{run-id}/d/{hash-algorithm}/{hex-digest} ``` Fallback key (unknown target): + ```text v0/d/{hash-algorithm}/{hex-digest} ``` @@ -105,13 +108,14 @@ v0/d/{hash-algorithm}/{hex-digest} The Temporal SDKs escape anything that isn't listed in S3's safe character set: https://docs.aws.amazon.com/AmazonS3/latest/userguide/object-keys.html Safe Characters: + ```text -Alphanumeric characters +Alphanumeric characters 0-9 a-z A-Z -Special characters +Special characters Exclamation point (!) Hyphen (-) Underscore (_) @@ -152,4 +156,4 @@ input: output: v0/ns/payments%20prod/at/Capture%2FCharge/ai/activity%20id%2B42/ri/9e1d1fd9-2f8a-4c40-93e2-731f31b9268b/d/sha256/2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824 -``` \ No newline at end of file +``` diff --git a/contrib/external-storage-s3/src/__tests__/test-driver.ts b/contrib/external-storage-s3/src/__tests__/test-driver.ts index e5d060ac7..0ed103274 100644 --- a/contrib/external-storage-s3/src/__tests__/test-driver.ts +++ b/contrib/external-storage-s3/src/__tests__/test-driver.ts @@ -62,10 +62,7 @@ test('key is content-addressed and segmented by the store context', async (t) => const [claim] = await driver.store(workflowContext, [makePayload('"hello"')]); - t.regex( - claim!.claimData.key!, - /^v0\/ns\/my-ns\/wt\/MyWorkflow\/wi\/wf-1\/ri\/run-1\/d\/sha256\/[0-9a-f]{64}$/ - ); + t.regex(claim!.claimData.key!, /^v0\/ns\/my-ns\/wt\/MyWorkflow\/wi\/wf-1\/ri\/run-1\/d\/sha256\/[0-9a-f]{64}$/); t.is(claim!.claimData.hashAlgorithm, 'sha256'); t.is(claim!.claimData.bucket, 'b'); }); diff --git a/contrib/external-storage-s3/src/driver.ts b/contrib/external-storage-s3/src/driver.ts index 79a9d5224..bfccb5244 100644 --- a/contrib/external-storage-s3/src/driver.ts +++ b/contrib/external-storage-s3/src/driver.ts @@ -129,7 +129,7 @@ export class S3StorageDriver implements StorageDriver { throw new ValueError(`maxPayloadSize must be a positive finite number, got ${String(maxPayloadSize)}`); } this.client = client; - this.bucket = (typeof bucket === 'string') ? () => bucket : bucket; + this.bucket = typeof bucket === 'string' ? () => bucket : bucket; this.name = driverName || DRIVER_TYPE; this.maxPayloadSize = maxPayloadSize; } @@ -142,9 +142,7 @@ export class S3StorageDriver implements StorageDriver { } async retrieve(context: StorageDriverRetrieveContext, claims: StorageDriverClaim[]): Promise { - return gatherWithCancellation(context.abortSignal, (signal) => - claims.map((claim) => this.download(claim, signal)) - ); + return gatherWithCancellation(context.abortSignal, (signal) => claims.map((claim) => this.download(claim, signal))); } private async upload( @@ -183,8 +181,7 @@ export class S3StorageDriver implements StorageDriver { const { bucket, key, hashAlgorithm, hashValue: expectedHash } = claim.claimData; if (!bucket || !key) { throw new ValueError( - `S3StorageDriver claim is missing required location information: ` + - `claimData must contain 'bucket' and 'key'` + `S3StorageDriver claim is missing required location information: ` + `claimData must contain 'bucket' and 'key'` ); } if (!hashAlgorithm || !expectedHash) { From ebccb71a13b733571c0707f7584fbf781521926b Mon Sep 17 00:00:00 2001 From: Chris Constable Date: Thu, 18 Jun 2026 11:08:10 -0400 Subject: [PATCH 04/10] improve s3 driver utility function names. --- contrib/external-storage-s3/src/driver.ts | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/contrib/external-storage-s3/src/driver.ts b/contrib/external-storage-s3/src/driver.ts index bfccb5244..5516ea06a 100644 --- a/contrib/external-storage-s3/src/driver.ts +++ b/contrib/external-storage-s3/src/driver.ts @@ -91,7 +91,7 @@ function formatClientContext(client: S3StorageDriverClient): string { * shared signal so in-flight sibling requests are cancelled, then waits for them * to settle before propagating the original error. */ -async function gatherWithCancellation( +async function runAllWithAbortOnError( external: AbortSignal | undefined, makeTasks: (signal: AbortSignal) => Promise[] ): Promise { @@ -136,16 +136,18 @@ export class S3StorageDriver implements StorageDriver { async store(context: StorageDriverStoreContext, payloads: Payload[]): Promise { const contextSegments = buildContextSegments(context.target); - return gatherWithCancellation(context.abortSignal, (signal) => - payloads.map((payload) => this.upload(context, payload, contextSegments, signal)) + return runAllWithAbortOnError(context.abortSignal, (signal) => + payloads.map((payload) => this.storePayload(context, payload, contextSegments, signal)) ); } async retrieve(context: StorageDriverRetrieveContext, claims: StorageDriverClaim[]): Promise { - return gatherWithCancellation(context.abortSignal, (signal) => claims.map((claim) => this.download(claim, signal))); + return runAllWithAbortOnError(context.abortSignal, (signal) => + claims.map((claim) => this.retrievePayload(claim, signal)) + ); } - private async upload( + private async storePayload( context: StorageDriverStoreContext, payload: Payload, contextSegments: string, @@ -177,7 +179,7 @@ export class S3StorageDriver implements StorageDriver { return new StorageDriverClaim({ bucket, key, hashAlgorithm: 'sha256', hashValue }); } - private async download(claim: StorageDriverClaim, abortSignal: AbortSignal): Promise { + private async retrievePayload(claim: StorageDriverClaim, abortSignal: AbortSignal): Promise { const { bucket, key, hashAlgorithm, hashValue: expectedHash } = claim.claimData; if (!bucket || !key) { throw new ValueError( From dc7b2360a595979c3f93ea1caf98c757674d0600 Mon Sep 17 00:00:00 2001 From: Chris Constable Date: Thu, 18 Jun 2026 11:43:09 -0400 Subject: [PATCH 05/10] make extstore driver packages private --- contrib/external-storage-s3-aws-sdk/package.json | 3 ++- .../src/__tests__/test-aws-sdk-client.ts | 1 - contrib/external-storage-s3-aws-sdk/src/aws-sdk-client.ts | 2 -- contrib/external-storage-s3/package.json | 3 ++- 4 files changed, 4 insertions(+), 5 deletions(-) diff --git a/contrib/external-storage-s3-aws-sdk/package.json b/contrib/external-storage-s3-aws-sdk/package.json index a19506188..0c28bd2ab 100644 --- a/contrib/external-storage-s3-aws-sdk/package.json +++ b/contrib/external-storage-s3-aws-sdk/package.json @@ -1,6 +1,7 @@ { "name": "@temporalio/external-storage-s3-aws-sdk", - "version": "1.17.2", + "version": "1.18.1", + "private": true, "description": "AWS SDK (@aws-sdk/client-s3) client for the Temporal S3 external storage driver", "main": "lib/index.js", "types": "./lib/index.d.ts", diff --git a/contrib/external-storage-s3-aws-sdk/src/__tests__/test-aws-sdk-client.ts b/contrib/external-storage-s3-aws-sdk/src/__tests__/test-aws-sdk-client.ts index 5ad965b34..794c45d14 100644 --- a/contrib/external-storage-s3-aws-sdk/src/__tests__/test-aws-sdk-client.ts +++ b/contrib/external-storage-s3-aws-sdk/src/__tests__/test-aws-sdk-client.ts @@ -2,7 +2,6 @@ import test from 'ava'; import type { S3Client } from '@aws-sdk/client-s3'; import { AwsSdkS3StorageDriverClient } from '../aws-sdk-client'; -/** Minimal S3Client stand-in: only `send` and `config.region` are exercised. */ function fakeS3Client(send: (command: unknown) => Promise, region?: string): S3Client { return { send, config: { region } } as unknown as S3Client; } diff --git a/contrib/external-storage-s3-aws-sdk/src/aws-sdk-client.ts b/contrib/external-storage-s3-aws-sdk/src/aws-sdk-client.ts index 5de7834d3..07575521d 100644 --- a/contrib/external-storage-s3-aws-sdk/src/aws-sdk-client.ts +++ b/contrib/external-storage-s3-aws-sdk/src/aws-sdk-client.ts @@ -12,8 +12,6 @@ export class AwsSdkS3StorageDriverClient implements S3StorageDriverClient { constructor(private readonly client: AwsS3Client) {} describe(): Record { - // v3 normalizes `region` to an async provider, so it is usually unavailable - // synchronously here; surface it only when it was supplied as a plain string. const region = this.client.config?.region; return typeof region === 'string' && region ? { clientRegion: region } : {}; } diff --git a/contrib/external-storage-s3/package.json b/contrib/external-storage-s3/package.json index 2957f9cc9..497afc4d8 100644 --- a/contrib/external-storage-s3/package.json +++ b/contrib/external-storage-s3/package.json @@ -1,6 +1,7 @@ { "name": "@temporalio/external-storage-s3", - "version": "1.17.2", + "version": "1.18.1", + "private": true, "description": "Amazon S3 external storage driver for the Temporal TypeScript SDK", "main": "lib/index.js", "types": "./lib/index.d.ts", From a6c963f252f8ff0eebacd9610854ecb146e80ecd Mon Sep 17 00:00:00 2001 From: Chris Constable Date: Thu, 18 Jun 2026 13:05:39 -0400 Subject: [PATCH 06/10] tighten extstore s3 driver tests. --- .../src/__tests__/test-aws-sdk-client.ts | 5 ----- .../src/__tests__/test-driver.ts | 20 ++++++++++++++++--- 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/contrib/external-storage-s3-aws-sdk/src/__tests__/test-aws-sdk-client.ts b/contrib/external-storage-s3-aws-sdk/src/__tests__/test-aws-sdk-client.ts index 794c45d14..52382272b 100644 --- a/contrib/external-storage-s3-aws-sdk/src/__tests__/test-aws-sdk-client.ts +++ b/contrib/external-storage-s3-aws-sdk/src/__tests__/test-aws-sdk-client.ts @@ -49,8 +49,3 @@ test('describe surfaces a plain-string region', (t) => { const client = new AwsSdkS3StorageDriverClient(fakeS3Client(() => Promise.resolve({}), 'us-west-2')); t.deepEqual(client.describe?.(), { clientRegion: 'us-west-2' }); }); - -test('describe omits region when unavailable', (t) => { - const client = new AwsSdkS3StorageDriverClient(fakeS3Client(() => Promise.resolve({}))); - t.deepEqual(client.describe?.(), {}); -}); diff --git a/contrib/external-storage-s3/src/__tests__/test-driver.ts b/contrib/external-storage-s3/src/__tests__/test-driver.ts index 0ed103274..8ce2a4ae5 100644 --- a/contrib/external-storage-s3/src/__tests__/test-driver.ts +++ b/contrib/external-storage-s3/src/__tests__/test-driver.ts @@ -1,4 +1,5 @@ /* eslint @typescript-eslint/no-non-null-assertion: 0 */ +import { createHash } from 'node:crypto'; import test from 'ava'; import * as proto from '@temporalio/proto'; import { ValueError } from '@temporalio/common'; @@ -18,6 +19,10 @@ function payloadBytes(p: Payload): Uint8Array { return PayloadProto.encode(p).finish(); } +function sha256Hex(bytes: Uint8Array): string { + return createHash('sha256').update(bytes).digest('hex'); +} + class FakeS3Client implements S3StorageDriverClient { readonly objects = new Map(); putCount = 0; @@ -59,12 +64,22 @@ test('store then retrieve round-trips the payload bytes', async (t) => { test('key is content-addressed and segmented by the store context', async (t) => { const driver = new S3StorageDriver({ client: new FakeS3Client(), bucket: 'b' }); + const payload = makePayload('"hello"'); - const [claim] = await driver.store(workflowContext, [makePayload('"hello"')]); + const [claim] = await driver.store(workflowContext, [payload]); - t.regex(claim!.claimData.key!, /^v0\/ns\/my-ns\/wt\/MyWorkflow\/wi\/wf-1\/ri\/run-1\/d\/sha256\/[0-9a-f]{64}$/); + // Content-addressed: the digest segment is the SHA-256 of the serialized payload + // bytes, and the store context fields form the preceding path segments in order. + const digest = sha256Hex(payloadBytes(payload)); + t.is(claim!.claimData.key, `v0/ns/my-ns/wt/MyWorkflow/wi/wf-1/ri/run-1/d/sha256/${digest}`); + t.is(claim!.claimData.hashValue, digest); t.is(claim!.claimData.hashAlgorithm, 'sha256'); t.is(claim!.claimData.bucket, 'b'); + + // Different content under the same context changes only the trailing digest segment. + const [other] = await driver.store(workflowContext, [makePayload('"world"')]); + t.not(other!.claimData.key, claim!.claimData.key); + t.is(other!.claimData.key!.replace(/[0-9a-f]{64}$/, ''), claim!.claimData.key!.replace(/[0-9a-f]{64}$/, '')); }); test('key segments percent-encode anything outside the S3 safe set', async (t) => { @@ -168,7 +183,6 @@ test('store aborts in-flight sibling uploads when one fails', async (t) => { if (bucket === 'fail') { throw new Error('boom'); } - // The surviving sibling resolves only once the shared signal aborts. await new Promise((resolve) => { options?.abortSignal?.addEventListener('abort', () => { siblingAborted = true; From 5a15795fe1791db69f89b8ee148eb8082c0574e3 Mon Sep 17 00:00:00 2001 From: Chris Constable Date: Tue, 23 Jun 2026 13:31:34 -0400 Subject: [PATCH 07/10] remove explicit unwrapping in extstore tests. --- .../src/__tests__/test-driver.ts | 39 ++++++++++++------- 1 file changed, 24 insertions(+), 15 deletions(-) diff --git a/contrib/external-storage-s3/src/__tests__/test-driver.ts b/contrib/external-storage-s3/src/__tests__/test-driver.ts index 8ce2a4ae5..a822b9379 100644 --- a/contrib/external-storage-s3/src/__tests__/test-driver.ts +++ b/contrib/external-storage-s3/src/__tests__/test-driver.ts @@ -1,4 +1,4 @@ -/* eslint @typescript-eslint/no-non-null-assertion: 0 */ +import assert from 'node:assert'; import { createHash } from 'node:crypto'; import test from 'ava'; import * as proto from '@temporalio/proto'; @@ -57,9 +57,11 @@ test('store then retrieve round-trips the payload bytes', async (t) => { const original = makePayload('"hello"'); const [claim] = await driver.store(workflowContext, [original]); - const [retrieved] = await driver.retrieve({}, [claim!]); + assert(claim); + const [retrieved] = await driver.retrieve({}, [claim]); + assert(retrieved); - t.deepEqual(payloadBytes(retrieved!), payloadBytes(original)); + t.deepEqual(payloadBytes(retrieved), payloadBytes(original)); }); test('key is content-addressed and segmented by the store context', async (t) => { @@ -67,19 +69,21 @@ test('key is content-addressed and segmented by the store context', async (t) => const payload = makePayload('"hello"'); const [claim] = await driver.store(workflowContext, [payload]); + assert(claim?.claimData.key); // Content-addressed: the digest segment is the SHA-256 of the serialized payload // bytes, and the store context fields form the preceding path segments in order. const digest = sha256Hex(payloadBytes(payload)); - t.is(claim!.claimData.key, `v0/ns/my-ns/wt/MyWorkflow/wi/wf-1/ri/run-1/d/sha256/${digest}`); - t.is(claim!.claimData.hashValue, digest); - t.is(claim!.claimData.hashAlgorithm, 'sha256'); - t.is(claim!.claimData.bucket, 'b'); + t.is(claim.claimData.key, `v0/ns/my-ns/wt/MyWorkflow/wi/wf-1/ri/run-1/d/sha256/${digest}`); + t.is(claim.claimData.hashValue, digest); + t.is(claim.claimData.hashAlgorithm, 'sha256'); + t.is(claim.claimData.bucket, 'b'); // Different content under the same context changes only the trailing digest segment. const [other] = await driver.store(workflowContext, [makePayload('"world"')]); - t.not(other!.claimData.key, claim!.claimData.key); - t.is(other!.claimData.key!.replace(/[0-9a-f]{64}$/, ''), claim!.claimData.key!.replace(/[0-9a-f]{64}$/, '')); + assert(other?.claimData.key); + t.not(other.claimData.key, claim.claimData.key); + t.is(other.claimData.key.replace(/[0-9a-f]{64}$/, ''), claim.claimData.key.replace(/[0-9a-f]{64}$/, '')); }); test('key segments percent-encode anything outside the S3 safe set', async (t) => { @@ -97,10 +101,11 @@ test('key segments percent-encode anything outside the S3 safe set', async (t) = }, [makePayload('"x"')] ); + assert(claim?.claimData.key); // space -> %20, / -> %2F, !*'() kept, + -> %2B, = -> %3D, ~ -> %7E t.true( - claim!.claimData.key!.startsWith( + claim.claimData.key.startsWith( "v0/ns/payments%20prod/wt/Capture%2FCharge!*'()/wi/order%2B123%3Dabc/ri/r%7E1/d/sha256/" ) ); @@ -110,8 +115,9 @@ test('a target with no identity falls back to a bare digest key', async (t) => { const driver = new S3StorageDriver({ client: new FakeS3Client(), bucket: 'b' }); const [claim] = await driver.store({}, [makePayload('"x"')]); + assert(claim?.claimData.key); - t.regex(claim!.claimData.key!, /^v0\/d\/sha256\/[0-9a-f]{64}$/); + t.regex(claim.claimData.key, /^v0\/d\/sha256\/[0-9a-f]{64}$/); }); test('identical payloads in the same scope deduplicate to one upload', async (t) => { @@ -129,9 +135,10 @@ test('retrieve rejects when stored bytes fail the integrity check', async (t) => const driver = new S3StorageDriver({ client, bucket: 'b' }); const [claim] = await driver.store(workflowContext, [makePayload('"hello"')]); - client.objects.set(`${claim!.claimData.bucket}/${claim!.claimData.key}`, enc('tampered')); + assert(claim); + client.objects.set(`${claim.claimData.bucket}/${claim.claimData.key}`, enc('tampered')); - await t.throwsAsync(() => driver.retrieve({}, [claim!]), { + await t.throwsAsync(() => driver.retrieve({}, [claim]), { instanceOf: ValueError, message: /integrity check failed/, }); @@ -168,9 +175,11 @@ test('bucket selector chooses the destination per payload', async (t) => { const [big] = await driver.store(workflowContext, [makePayload('"a-long-value"')]); const [small] = await driver.store(workflowContext, [makePayload('"x"')]); + assert(big); + assert(small); - t.is(big!.claimData.bucket, 'large'); - t.is(small!.claimData.bucket, 'small'); + t.is(big.claimData.bucket, 'large'); + t.is(small.claimData.bucket, 'small'); }); test('store aborts in-flight sibling uploads when one fails', async (t) => { From 165d454fa12e55ae619ec80d0f7e353f40612ee0 Mon Sep 17 00:00:00 2001 From: Chris Constable Date: Tue, 23 Jun 2026 13:32:03 -0400 Subject: [PATCH 08/10] remove unnecessary extstore s3client alias. --- contrib/external-storage-s3-aws-sdk/src/aws-sdk-client.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/contrib/external-storage-s3-aws-sdk/src/aws-sdk-client.ts b/contrib/external-storage-s3-aws-sdk/src/aws-sdk-client.ts index 07575521d..dacdf72f2 100644 --- a/contrib/external-storage-s3-aws-sdk/src/aws-sdk-client.ts +++ b/contrib/external-storage-s3-aws-sdk/src/aws-sdk-client.ts @@ -1,5 +1,5 @@ import { PutObjectCommand, GetObjectCommand, HeadObjectCommand } from '@aws-sdk/client-s3'; -import type { S3Client as AwsS3Client } from '@aws-sdk/client-s3'; +import type { S3Client } from '@aws-sdk/client-s3'; import type { S3StorageDriverClient, S3RequestOptions } from '@temporalio/external-storage-s3'; /** @@ -9,7 +9,7 @@ import type { S3StorageDriverClient, S3RequestOptions } from '@temporalio/extern * @experimental */ export class AwsSdkS3StorageDriverClient implements S3StorageDriverClient { - constructor(private readonly client: AwsS3Client) {} + constructor(private readonly client: S3Client) {} describe(): Record { const region = this.client.config?.region; From cfd9c1f42a6b1cf630edb6b4a9f350c3145a0cb5 Mon Sep 17 00:00:00 2001 From: Chris Constable Date: Thu, 25 Jun 2026 11:27:26 -0400 Subject: [PATCH 09/10] Address small nits and update tests. --- contrib/external-storage-s3/README.md | 2 +- .../src/__tests__/test-driver.ts | 12 +++++++++--- contrib/external-storage-s3/src/driver.ts | 14 ++++++++++---- 3 files changed, 20 insertions(+), 8 deletions(-) diff --git a/contrib/external-storage-s3/README.md b/contrib/external-storage-s3/README.md index 136c480db..73b205255 100644 --- a/contrib/external-storage-s3/README.md +++ b/contrib/external-storage-s3/README.md @@ -2,7 +2,7 @@ > ⚠️ **This package is experimental and may be subject to change.** ⚠️ -`@temporalio/external-storage-s3` stores and retrieves Temporal payloads in Amazon S3 via the [External Storage](../../README.md) feature. +`@temporalio/external-storage-s3` stores and retrieves Temporal payloads in Amazon S3 via the [External Storage](https://docs.temporal.io/external-storage) feature. This package has no AWS dependency: it defines the driver and the `S3StorageDriverClient` interface, and you supply the S3 client. Use the companion [`@temporalio/external-storage-s3-aws-sdk`](../external-storage-s3-aws-sdk) package for an [`@aws-sdk/client-s3`](https://www.npmjs.com/package/@aws-sdk/client-s3)-backed client, or implement the interface yourself. diff --git a/contrib/external-storage-s3/src/__tests__/test-driver.ts b/contrib/external-storage-s3/src/__tests__/test-driver.ts index a822b9379..cc0e83b52 100644 --- a/contrib/external-storage-s3/src/__tests__/test-driver.ts +++ b/contrib/external-storage-s3/src/__tests__/test-driver.ts @@ -71,15 +71,12 @@ test('key is content-addressed and segmented by the store context', async (t) => const [claim] = await driver.store(workflowContext, [payload]); assert(claim?.claimData.key); - // Content-addressed: the digest segment is the SHA-256 of the serialized payload - // bytes, and the store context fields form the preceding path segments in order. const digest = sha256Hex(payloadBytes(payload)); t.is(claim.claimData.key, `v0/ns/my-ns/wt/MyWorkflow/wi/wf-1/ri/run-1/d/sha256/${digest}`); t.is(claim.claimData.hashValue, digest); t.is(claim.claimData.hashAlgorithm, 'sha256'); t.is(claim.claimData.bucket, 'b'); - // Different content under the same context changes only the trailing digest segment. const [other] = await driver.store(workflowContext, [makePayload('"world"')]); assert(other?.claimData.key); t.not(other.claimData.key, claim.claimData.key); @@ -120,6 +117,15 @@ test('a target with no identity falls back to a bare digest key', async (t) => { t.regex(claim.claimData.key, /^v0\/d\/sha256\/[0-9a-f]{64}$/); }); +test('missing context segments are encoded as the literal "null"', async (t) => { + const driver = new S3StorageDriver({ client: new FakeS3Client(), bucket: 'b' }); + + const [claim] = await driver.store({ target: { kind: 'workflow', namespace: 'my-ns' } }, [makePayload('"x"')]); + assert(claim?.claimData.key); + + t.true(claim.claimData.key.startsWith('v0/ns/my-ns/wt/null/wi/null/ri/null/d/sha256/')); +}); + test('identical payloads in the same scope deduplicate to one upload', async (t) => { const client = new FakeS3Client(); const driver = new S3StorageDriver({ client, bucket: 'b' }); diff --git a/contrib/external-storage-s3/src/driver.ts b/contrib/external-storage-s3/src/driver.ts index 5516ea06a..518bc966d 100644 --- a/contrib/external-storage-s3/src/driver.ts +++ b/contrib/external-storage-s3/src/driver.ts @@ -15,11 +15,17 @@ const PayloadProto = proto.temporal.api.common.v1.Payload; const DRIVER_TYPE = 'aws.s3driver'; const DEFAULT_MAX_PAYLOAD_SIZE = 50 * 1024 * 1024; +/** Key segment written when a context value is empty or absent. */ +const NULL_SEGMENT = 'null'; /** Picks the destination bucket for a given payload. Enables dynamic per-payload routing. */ export type BucketSelector = (context: StorageDriverStoreContext, payload: Payload) => string; -/** @experimental */ +/** + * Configuration for an {@link S3StorageDriver}. + * + * @experimental + */ export interface S3StorageDriverOptions { /** * An {@link S3StorageDriverClient} that performs the underlying requests, @@ -47,7 +53,7 @@ export interface S3StorageDriverOptions { * literal `null`. */ function encodeKeySegment(value: string | undefined): string { - if (!value) return 'null'; + if (!value) return NULL_SEGMENT; return encodeURIComponent(value).replace(/~/g, '%7E'); } @@ -124,13 +130,13 @@ export class S3StorageDriver implements StorageDriver { private readonly maxPayloadSize: number; constructor(options: S3StorageDriverOptions) { - const { client, bucket, driverName, maxPayloadSize = DEFAULT_MAX_PAYLOAD_SIZE } = options; + const { client, bucket, driverName = DRIVER_TYPE, maxPayloadSize = DEFAULT_MAX_PAYLOAD_SIZE } = options; if (!Number.isFinite(maxPayloadSize) || maxPayloadSize <= 0) { throw new ValueError(`maxPayloadSize must be a positive finite number, got ${String(maxPayloadSize)}`); } this.client = client; this.bucket = typeof bucket === 'string' ? () => bucket : bucket; - this.name = driverName || DRIVER_TYPE; + this.name = driverName; this.maxPayloadSize = maxPayloadSize; } From e5ae6655962a0699aadb6c6830a97d8a98813cfa Mon Sep 17 00:00:00 2001 From: Chris Constable Date: Thu, 25 Jun 2026 11:37:07 -0400 Subject: [PATCH 10/10] Add upload deduping to avoid race. Uploads are idempotent but races could still occur and waste time/bandwidth. --- .../src/__tests__/test-driver.ts | 12 ++++++++ contrib/external-storage-s3/src/driver.ts | 29 +++++++++++++++---- 2 files changed, 35 insertions(+), 6 deletions(-) diff --git a/contrib/external-storage-s3/src/__tests__/test-driver.ts b/contrib/external-storage-s3/src/__tests__/test-driver.ts index cc0e83b52..87fb84978 100644 --- a/contrib/external-storage-s3/src/__tests__/test-driver.ts +++ b/contrib/external-storage-s3/src/__tests__/test-driver.ts @@ -136,6 +136,18 @@ test('identical payloads in the same scope deduplicate to one upload', async (t) t.is(client.putCount, 1); }); +test('concurrent identical payloads in one batch upload once', async (t) => { + const client = new FakeS3Client(); + const driver = new S3StorageDriver({ client, bucket: 'b' }); + + const [first, second] = await driver.store(workflowContext, [makePayload('"hello"'), makePayload('"hello"')]); + assert(first); + assert(second); + + t.is(client.putCount, 1); + t.is(first.claimData.key, second.claimData.key); +}); + test('retrieve rejects when stored bytes fail the integrity check', async (t) => { const client = new FakeS3Client(); const driver = new S3StorageDriver({ client, bucket: 'b' }); diff --git a/contrib/external-storage-s3/src/driver.ts b/contrib/external-storage-s3/src/driver.ts index 518bc966d..953728d3d 100644 --- a/contrib/external-storage-s3/src/driver.ts +++ b/contrib/external-storage-s3/src/driver.ts @@ -142,9 +142,10 @@ export class S3StorageDriver implements StorageDriver { async store(context: StorageDriverStoreContext, payloads: Payload[]): Promise { const contextSegments = buildContextSegments(context.target); - return runAllWithAbortOnError(context.abortSignal, (signal) => - payloads.map((payload) => this.storePayload(context, payload, contextSegments, signal)) - ); + return runAllWithAbortOnError(context.abortSignal, (signal) => { + const uploads = new Map>(); + return payloads.map((payload) => this.storePayload(context, payload, contextSegments, signal, uploads)); + }); } async retrieve(context: StorageDriverRetrieveContext, claims: StorageDriverClaim[]): Promise { @@ -157,7 +158,8 @@ export class S3StorageDriver implements StorageDriver { context: StorageDriverStoreContext, payload: Payload, contextSegments: string, - abortSignal: AbortSignal + abortSignal: AbortSignal, + uploads: Map> ): Promise { const bucket = this.bucket(context, payload); @@ -172,9 +174,13 @@ export class S3StorageDriver implements StorageDriver { const key = `v0${contextSegments}/d/sha256/${hashValue}`; try { - if (!(await this.client.objectExists(bucket, key, { abortSignal }))) { - await this.client.putObject(bucket, key, payloadBytes, { abortSignal }); + const dedupeKey = `${bucket} ${key}`; + let upload = uploads.get(dedupeKey); + if (!upload) { + upload = this.uploadIfAbsent(bucket, key, payloadBytes, abortSignal); + uploads.set(dedupeKey, upload); } + await upload; } catch (err) { throw new Error( `S3StorageDriver store failed [bucket=${bucket}, key=${key}${formatClientContext(this.client)}]`, @@ -185,6 +191,17 @@ export class S3StorageDriver implements StorageDriver { return new StorageDriverClaim({ bucket, key, hashAlgorithm: 'sha256', hashValue }); } + private async uploadIfAbsent( + bucket: string, + key: string, + payloadBytes: Uint8Array, + abortSignal: AbortSignal + ): Promise { + if (!(await this.client.objectExists(bucket, key, { abortSignal }))) { + await this.client.putObject(bucket, key, payloadBytes, { abortSignal }); + } + } + private async retrievePayload(claim: StorageDriverClaim, abortSignal: AbortSignal): Promise { const { bucket, key, hashAlgorithm, hashValue: expectedHash } = claim.claimData; if (!bucket || !key) {