feat(extstore): add extstore s3 driver (#2129)

Author: cconstable

contrib/external-storage-s3-aws-sdk/README.md

@@ -0,0 +1,23 @@
+# AWS SDK Client for the Temporal S3 External Storage Driver
+
+> ⚠️ **This package is experimental and may be subject to change.** ⚠️
+
+`@temporalio/external-storage-s3-aws-sdk` provides an [`@aws-sdk/client-s3`](https://www.npmjs.com/package/@aws-sdk/client-s3)-backed `S3StorageDriverClient` for [`@temporalio/external-storage-s3`](../external-storage-s3).
+
+`@aws-sdk/client-s3` is a peer dependency, so the driver uses the same `S3Client` (and version) your application already configures.
+
+## Usage
+
+    npm install @temporalio/external-storage-s3 @temporalio/external-storage-s3-aws-sdk @aws-sdk/client-s3
+
+```ts
+import { S3Client } from '@aws-sdk/client-s3';
+import { S3StorageDriver } from '@temporalio/external-storage-s3';
+import { AwsSdkS3StorageDriverClient } from '@temporalio/external-storage-s3-aws-sdk';
+
+const s3Client = new S3Client({ region: 'us-east-1' });
+const driver = new S3StorageDriver({
+  client: new AwsSdkS3StorageDriverClient(s3Client),
+  bucket: 'my-temporal-payloads',
+});
+```

contrib/external-storage-s3-aws-sdk/package.json

@@ -0,0 +1,56 @@
+{
+  "name": "@temporalio/external-storage-s3-aws-sdk",
+  "version": "1.18.1",
+  "private": true,
+  "description": "AWS SDK (@aws-sdk/client-s3) client for the Temporal S3 external storage driver",
+  "main": "lib/index.js",
+  "types": "./lib/index.d.ts",
+  "keywords": [
+    "temporal",
+    "workflow",
+    "external storage",
+    "payload",
+    "s3",
+    "aws"
+  ],
+  "author": "Temporal Technologies Inc. <sdk@temporal.io>",
+  "license": "MIT",
+  "scripts": {
+    "build": "tsc --build",
+    "test": "ava ./lib/__tests__/test-*.js"
+  },
+  "ava": {
+    "timeout": "60s"
+  },
+  "dependencies": {
+    "@temporalio/external-storage-s3": "workspace:*"
+  },
+  "peerDependencies": {
+    "@aws-sdk/client-s3": "^3.0.0"
+  },
+  "devDependencies": {
+    "@aws-sdk/client-s3": "^3.0.0",
+    "ava": "^5.3.1"
+  },
+  "engines": {
+    "node": ">= 20.3.0"
+  },
+  "bugs": {
+    "url": "https://github.com/temporalio/sdk-typescript/issues"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/temporalio/sdk-typescript.git",
+    "directory": "contrib/external-storage-s3-aws-sdk"
+  },
+  "homepage": "https://github.com/temporalio/sdk-typescript/tree/main/contrib/external-storage-s3-aws-sdk",
+  "publishConfig": {
+    "access": "public"
+  },
+  "files": [
+    "src",
+    "lib",
+    "!src/__tests__",
+    "!lib/__tests__"
+  ]
+}

contrib/external-storage-s3-aws-sdk/src/__tests__/test-aws-sdk-client.ts

@@ -0,0 +1,51 @@
+import test from 'ava';
+import type { S3Client } from '@aws-sdk/client-s3';
+import { AwsSdkS3StorageDriverClient } from '../aws-sdk-client';
+
+function fakeS3Client(send: (command: unknown) => Promise<unknown>, region?: string): S3Client {
+  return { send, config: { region } } as unknown as S3Client;
+}
+
+test('objectExists maps a NotFound error to false', async (t) => {
+  const client = new AwsSdkS3StorageDriverClient(
+    fakeS3Client(() => Promise.reject(Object.assign(new Error('not found'), { name: 'NotFound' })))
+  );
+  t.false(await client.objectExists('b', 'k'));
+});
+
+test('objectExists maps a 404 status to false', async (t) => {
+  const client = new AwsSdkS3StorageDriverClient(
+    fakeS3Client(() => Promise.reject(Object.assign(new Error('nope'), { $metadata: { httpStatusCode: 404 } })))
+  );
+  t.false(await client.objectExists('b', 'k'));
+});
+
+test('objectExists rethrows non-404 errors', async (t) => {
+  const client = new AwsSdkS3StorageDriverClient(
+    fakeS3Client(() => Promise.reject(Object.assign(new Error('denied'), { $metadata: { httpStatusCode: 403 } })))
+  );
+  await t.throwsAsync(() => client.objectExists('b', 'k'), { message: 'denied' });
+});
+
+test('objectExists returns true when the head succeeds', async (t) => {
+  const client = new AwsSdkS3StorageDriverClient(fakeS3Client(() => Promise.resolve({})));
+  t.true(await client.objectExists('b', 'k'));
+});
+
+test('getObject reads the response body as bytes', async (t) => {
+  const bytes = new Uint8Array([1, 2, 3]);
+  const client = new AwsSdkS3StorageDriverClient(
+    fakeS3Client(() => Promise.resolve({ Body: { transformToByteArray: async () => bytes } }))
+  );
+  t.deepEqual(await client.getObject('b', 'k'), bytes);
+});
+
+test('getObject throws when the response has no body', async (t) => {
+  const client = new AwsSdkS3StorageDriverClient(fakeS3Client(() => Promise.resolve({})));
+  await t.throwsAsync(() => client.getObject('b', 'k'), { message: /empty body/ });
+});
+
+test('describe surfaces a plain-string region', (t) => {
+  const client = new AwsSdkS3StorageDriverClient(fakeS3Client(() => Promise.resolve({}), 'us-west-2'));
+  t.deepEqual(client.describe?.(), { clientRegion: 'us-west-2' });
+});

contrib/external-storage-s3-aws-sdk/src/aws-sdk-client.ts

@@ -0,0 +1,53 @@
+import { PutObjectCommand, GetObjectCommand, HeadObjectCommand } from '@aws-sdk/client-s3';
+import type { S3Client } from '@aws-sdk/client-s3';
+import type { S3StorageDriverClient, S3RequestOptions } from '@temporalio/external-storage-s3';
+
+/**
+ * An {@link S3StorageDriverClient} backed by an `@aws-sdk/client-s3` `S3Client`,
+ * for use with `S3StorageDriver`.
+ *
+ * @experimental
+ */
+export class AwsSdkS3StorageDriverClient implements S3StorageDriverClient {
+  constructor(private readonly client: S3Client) {}
+
+  describe(): Record<string, string> {
+    const region = this.client.config?.region;
+    return typeof region === 'string' && region ? { clientRegion: region } : {};
+  }
+
+  async objectExists(bucket: string, key: string, options?: S3RequestOptions): Promise<boolean> {
+    try {
+      await this.client.send(new HeadObjectCommand({ Bucket: bucket, Key: key }), {
+        abortSignal: options?.abortSignal,
+      });
+      return true;
+    } catch (err) {
+      if (isNotFound(err)) {
+        return false;
+      }
+      throw err;
+    }
+  }
+
+  async putObject(bucket: string, key: string, data: Uint8Array, options?: S3RequestOptions): Promise<void> {
+    await this.client.send(new PutObjectCommand({ Bucket: bucket, Key: key, Body: data }), {
+      abortSignal: options?.abortSignal,
+    });
+  }
+
+  async getObject(bucket: string, key: string, options?: S3RequestOptions): Promise<Uint8Array> {
+    const response = await this.client.send(new GetObjectCommand({ Bucket: bucket, Key: key }), {
+      abortSignal: options?.abortSignal,
+    });
+    if (!response.Body) {
+      throw new Error(`S3 GetObject returned an empty body [bucket=${bucket}, key=${key}]`);
+    }
+    return response.Body.transformToByteArray();
+  }
+}
+
+function isNotFound(err: unknown): boolean {
+  const e = err as { name?: string; $metadata?: { httpStatusCode?: number } };
+  return e?.name === 'NotFound' || e?.$metadata?.httpStatusCode === 404;
+}

contrib/external-storage-s3-aws-sdk/src/index.ts

@@ -0,0 +1,4 @@
+/**
+ * @experimental The External Storage S3 driver is an experimental feature and may be subject to change.
+ */
+export { AwsSdkS3StorageDriverClient } from './aws-sdk-client';

contrib/external-storage-s3-aws-sdk/tsconfig.json

@@ -0,0 +1,9 @@
+{
+  "extends": "../../tsconfig.base.json",
+  "compilerOptions": {
+    "outDir": "./lib",
+    "rootDir": "./src"
+  },
+  "references": [{ "path": "../external-storage-s3" }],
+  "include": ["./src/**/*.ts"]
+}

contrib/external-storage-s3/README.md

@@ -0,0 +1,159 @@
+# Amazon S3 External Storage Driver for the Temporal TypeScript SDK
+
+> ⚠️ **This package is experimental and may be subject to change.** ⚠️
+
+`@temporalio/external-storage-s3` stores and retrieves Temporal payloads in Amazon S3 via the [External Storage](https://docs.temporal.io/external-storage) feature.
+
+This package has no AWS dependency: it defines the driver and the `S3StorageDriverClient` interface, and you supply the S3 client. Use the companion [`@temporalio/external-storage-s3-aws-sdk`](../external-storage-s3-aws-sdk) package for an [`@aws-sdk/client-s3`](https://www.npmjs.com/package/@aws-sdk/client-s3)-backed client, or implement the interface yourself.
+
+## Using the AWS SDK client
+
+Install the adapter package alongside this one:
+
+    npm install @temporalio/external-storage-s3 @temporalio/external-storage-s3-aws-sdk @aws-sdk/client-s3
+
+```ts
+import { S3Client } from '@aws-sdk/client-s3';
+import { S3StorageDriver } from '@temporalio/external-storage-s3';
+import { AwsSdkS3StorageDriverClient } from '@temporalio/external-storage-s3-aws-sdk';
+
+const s3Client = new S3Client({ region: 'us-east-1' });
+const driver = new S3StorageDriver({
+  client: new AwsSdkS3StorageDriverClient(s3Client),
+  bucket: 'my-temporal-payloads',
+});
+```
+
+Register the resulting driver with the SDK's External Storage configuration so the
+client and worker offload eligible payloads to it.
+
+## Custom S3 client implementations
+
+To use a different S3 library, implement `S3StorageDriverClient`.
+
+```ts
+import type { S3StorageDriverClient } from '@temporalio/external-storage-s3';
+
+const myClient: S3StorageDriverClient = {
+  async putObject(bucket, key, data, options) {
+    /* ... */
+  },
+  async objectExists(bucket, key, options) {
+    /* ... */
+    return false;
+  },
+  async getObject(bucket, key, options) {
+    /* ... */
+    return new Uint8Array();
+  },
+};
+```
+
+## Dynamic bucket selection
+
+Pass a callable as `bucket` to choose the destination per payload:
+
+```ts
+const driver = new S3StorageDriver({
+  client: new AwsSdkS3StorageDriverClient(s3Client),
+  bucket: (_context, payload) => ((payload.data?.length ?? 0) > 10 * 1024 * 1024 ? 'large-payloads' : 'small-payloads'),
+});
+```
+
+## Required IAM permissions
+
+The credentials used by your S3 client must have these S3 permissions on the target bucket and its objects:
+
+```json
+{
+  "Effect": "Allow",
+  "Action": ["s3:PutObject", "s3:GetObject"],
+  "Resource": "arn:aws:s3:::my-temporal-payloads/*"
+}
+```
+
+`s3:PutObject` is required by components that store payloads (typically the client and worker sending workflow/activity inputs); `s3:GetObject` is required by components that retrieve them. Components that only retrieve do not need `s3:PutObject`, and vice versa.
+
+## S3 Storage Key Specification
+
+All Temporal S3 drivers generate S3 keys in a consistent manner.
+
+### Key format
+
+Workflow key:
+
+```text
+v0/ns/{namespace}/wt/{workflow-type}/wi/{workflow-id}/ri/{run-id}/d/{hash-algorithm}/{hex-digest}
+```
+
+Activity key:
+
+```text
+v0/ns/{namespace}/at/{activity-type}/ai/{activity-id}/ri/{run-id}/d/{hash-algorithm}/{hex-digest}
+```
+
+Fallback key (unknown target):
+
+```text
+v0/d/{hash-algorithm}/{hex-digest}
+```
+
+- If no namespace, workflow, or activity information is available, the fallback is used.
+- Dynamic path segments are percent-encoded (rules below).
+- Missing values (including a missing `run-id`) are encoded as `null`.
+- `hex-digest` is lower-case SHA-256 hex (64 characters).
+
+### Percent-encoding rules
+
+The Temporal SDKs escape anything that isn't listed in S3's safe character set: https://docs.aws.amazon.com/AmazonS3/latest/userguide/object-keys.html
+
+Safe Characters:
+
+```text
+Alphanumeric characters
+  0-9
+  a-z
+  A-Z
+
+Special characters
+  Exclamation point (!)
+  Hyphen (-)
+  Underscore (_)
+  Period (.)
+  Asterisk (*)
+  Single quotation mark (')
+  Opening parenthesis (()
+  Closing parenthesis ())
+```
+
+### Examples
+
+Workflow key example:
+
+```text
+input:
+  namespace=payments prod
+  workflow-type=ChargeWorkflow
+  workflow-id=order+123=abc
+  run-id=3f1d6c7a-8b2e-4f7a-9d0a-87a6f95e4d31
+  hash-algorithm=sha256
+  hex-digest=9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08
+
+output:
+  v0/ns/payments%20prod/wt/ChargeWorkflow/wi/order%2B123%3Dabc/ri/3f1d6c7a-8b2e-4f7a-9d0a-87a6f95e4d31/d/sha256/9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08
+```
+
+Activity key example:
+
+```text
+input:
+  namespace=payments prod
+  activity-type=Capture/Charge
+  activity-id=activity id+42
+  run-id=9e1d1fd9-2f8a-4c40-93e2-731f31b9268b
+  hash-algorithm=sha256
+  hex-digest=2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824
+
+output:
+  v0/ns/payments%20prod/at/Capture%2FCharge/ai/activity%20id%2B42/ri/9e1d1fd9-2f8a-4c40-93e2-731f31b9268b/d/sha256/2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824
+```