Add read-only Amazon S3 Tables support via icebird/s3tables subpath

Adds an optional `icebird/s3tables` subpath that connects to the S3 Tables
Iceberg REST catalog and reads table data. The core `icebird` package stays
AWS-free: requests are signed with icebird's own SigV4 implementation, and the
only optional peer dependency is `@aws-sdk/credential-providers` (imported
lazily, and only when resolving the default credential chain — explicit
credentials need no AWS SDK at all).

- New `icebird/s3tables` subpath export and `src/aws/*` modules.
- Generic `signRequest` hook on the REST catalog (`restCatalogConnect` /
  `restFetch`) so signing stays cloud-agnostic.
- Extract the hand-rolled SigV4 primitives into `src/sigv4.js` and reuse them
  for both S3 data reads (service `s3`) and the S3 Tables catalog (`s3tables`).

Fixes found while reading a live S3 Tables bucket:
- double-URI-encode the canonical path for non-`s3` services (S3 Tables 403'd
  on catalog paths containing the URL-encoded warehouse prefix)
- use the regional S3 endpoint so non us-east-1 buckets don't 301 redirect
- decode boxed primitives, maps, and enums in the Avro reader, which S3 Tables
  manifests use

Signed-off-by: Karl Pietrzak <karl@medplum.com>

Author: The-Alchemist

README.md

@@ -116,6 +116,8 @@ const { metadata } = await restCatalogLoadTable(ctx, { namespace: 'analytics', t
 const data = await icebergRead({ tableUrl: metadata.location, metadata })
 ```
 
+For Amazon S3 Tables, use the optional [`icebird/s3tables`](#amazon-s3-tables) subpath (read-only in this release).
+
 ## SQL
 
 Icebird ships a SQL engine on top of [squirreling](https://github.com/hyparam/squirreling). `icebergQuery` runs a SQL query across one or more iceberg tables. Rows are streamed lazily. Multi-segment namespaces in the SQL `FROM` clause must be dot-separated and quoted: `FROM "analytics.orders"` resolves to namespace `analytics`, table `orders`.
@@ -131,6 +133,53 @@ const result = await icebergQuery({
 const rows = await collect(result)
 ```
 
+## Amazon S3 Tables
+
+Read-only support for [Amazon S3 Tables](https://docs.aws.amazon.com/AmazonS3/latest/userguide/s3-tables-tables.html) lives in the optional `icebird/s3tables` subpath. The main `icebird` export has **no AWS dependency**, and requests are signed with icebird's own SigV4 implementation. The only optional dependency is `@aws-sdk/credential-providers`, used to resolve the default AWS credential chain — and even that is imported lazily, so passing explicit credentials needs no AWS SDK at all.
+
+Install the peer dependency only if you rely on the default credential chain:
+
+```bash
+npm install icebird @aws-sdk/credential-providers
+```
+
+Connect with the default AWS credential chain (env vars, shared config, IAM role on Lambda/EC2):
+
+```javascript
+import { icebergRead } from 'icebird'
+import { loadS3TablesTable, s3TablesCatalogConnectFromEnv } from 'icebird/s3tables'
+
+const catalog = await s3TablesCatalogConnectFromEnv({
+  region: 'us-east-1',
+  tableBucketArn: 'arn:aws:s3tables:us-east-1:111122223333:bucket/my-bucket',
+})
+const { metadata, tableUrl, resolver } = await loadS3TablesTable({
+  catalog, namespace: 'analytics', table: 'orders',
+})
+const rows = await icebergRead({ tableUrl, metadata, resolver })
+```
+
+Or pass explicit credentials:
+
+```javascript
+import { icebergRead, restCatalogLoadTable } from 'icebird'
+import { loadS3TablesTable, s3TablesCatalogConnect, s3TablesResolver } from 'icebird/s3tables'
+
+const creds = { region: 'us-east-1', accessKeyId, secretAccessKey }
+const catalog = await s3TablesCatalogConnect({
+  ...creds,
+  tableBucketArn: 'arn:aws:s3tables:us-east-1:111122223333:bucket/my-bucket',
+})
+
+const resolver = await s3TablesResolver(creds)
+const { metadata } = await restCatalogLoadTable(catalog, { namespace: 'analytics', table: 'orders' })
+const rows = await icebergRead({ tableUrl: metadata.location, metadata, resolver })
+```
+
+**IAM (read-only):** grant `s3tables:GetTableBucket`, `s3tables:ListNamespaces`, `s3tables:GetNamespace`, `s3tables:ListTables`, `s3tables:GetTable`, `s3tables:GetTableMetadataLocation`, and `s3tables:GetTableData` on your table bucket and tables.
+
+**Limitations:** S3 Tables namespaces are single-level only. `s3Lister` does not work on table-bucket warehouse paths (use the REST catalog to load metadata). Writes, Glue REST endpoint, and OAuth are not supported via this subpath yet.
+
 ## Writing
 
 Icebird has experimental write support for Iceberg v2 (and v3 deletion vectors). All write functions take a `Catalog` and dispatch internally — the same call works against `fileCatalog({ resolver })` or a REST catalog context returned by `restCatalogConnect`.

package.json

@@ -34,6 +34,10 @@
       "types": "./types/avro/index.d.ts",
       "import": "./src/avro/index.js"
     },
+    "./s3tables": {
+      "types": "./types/aws/s3tables.d.ts",
+      "import": "./src/aws/s3tables.js"
+    },
     "./src/*.js": {
       "types": "./types/*.d.ts",
       "import": "./src/*.js"
@@ -54,7 +58,16 @@
     "hyparquet-writer": "0.16.1",
     "squirreling": "0.12.24"
   },
+  "peerDependencies": {
+    "@aws-sdk/credential-providers": "^3.0.0"
+  },
+  "peerDependenciesMeta": {
+    "@aws-sdk/credential-providers": {
+      "optional": true
+    }
+  },
   "devDependencies": {
+    "@aws-sdk/credential-providers": "3.1079.0",
     "@types/node": "26.0.0",
     "@vitest/coverage-v8": "4.1.9",
     "eslint": "9.39.4",

scripts/s3tables-smoke.mjs

@@ -0,0 +1,42 @@
+import { icebergRead, restCatalogListNamespaces, restCatalogListTables } from '../src/index.js'
+import { loadS3TablesTable, s3TablesCatalogConnectFromEnv } from '../src/aws/s3tables.js'
+
+const region = process.env.AWS_REGION ?? 'us-east-1'
+const tableBucketArn = process.env.S3TABLES_BUCKET_ARN
+const namespace = process.env.S3TABLES_NAMESPACE ?? 'default'
+const table = process.env.S3TABLES_TABLE ?? 'orders'
+
+if (!tableBucketArn) {
+  console.error('Set S3TABLES_BUCKET_ARN=arn:aws:s3tables:...')
+  process.exit(1)
+}
+
+console.log(`region=${region} bucket=${tableBucketArn}`)
+console.log(`namespace=${namespace} table=${table}\n`)
+
+const catalog = await s3TablesCatalogConnectFromEnv({ region, tableBucketArn })
+console.log('connected. prefix:', catalog.prefix)
+
+const namespaces = await restCatalogListNamespaces(catalog)
+const tables = await restCatalogListTables(catalog, { namespace })
+console.log('namespaces:', JSON.stringify(namespaces))
+console.log(`tables in "${namespace}": ${tables.length}`)
+
+const { metadata, tableUrl, resolver } = await loadS3TablesTable({ catalog, namespace, table })
+console.log('metadata.location:', metadata.location)
+console.log('schema fields:', metadata.schemas?.[metadata['current-schema-id'] ?? 0]?.fields?.map(f => f.name))
+
+const rows = await icebergRead({ tableUrl, metadata, resolver, rowEnd: 10 })
+
+console.log(`\nfirst ${rows.length} row(s) of ${namespace}.${table}:\n`)
+const preview = rows.map(row => {
+  /** @type {Record<string, unknown>} */
+  const out = {}
+  for (const [key, value] of Object.entries(row)) {
+    if (value instanceof Date) out[key] = value.toISOString()
+    else if (typeof value === 'string' && value.length > 80) out[key] = `${value.slice(0, 77)}...`
+    else out[key] = value
+  }
+  return out
+})
+console.table(preview)

src/avro/avro.read.js

@@ -96,7 +96,26 @@ function readType(reader, type) {
       }
     }
     return arr
-  } else if (typeof type === 'object' && type.logicalType) {
+  } else if (typeof type === 'object' && type.type === 'map') {
+    // Avro map: repeated blocks of (count, [string key, value]...) ending with 0.
+    /** @type {Record<string, any>} */
+    const map = {}
+    while (true) {
+      let count = readZigZag(reader)
+      if (count === 0) break
+      if (count < 0) {
+        count = -count
+        readZigZag(reader) // block size in bytes
+      }
+      for (let i = 0; i < count; i++) {
+        const key = readType(reader, 'string')
+        map[key] = readType(reader, type.values)
+      }
+    }
+    return map
+  } else if (typeof type === 'object' && type.type === 'enum') {
+    return type.symbols[readZigZag(reader)]
+  } else if (typeof type === 'object' && 'logicalType' in type && type.logicalType) {
     if (type.logicalType === 'date' && type.type === 'int') {
       const value = readZigZag(reader)
       return new Date(value * 86400000)
@@ -159,9 +178,11 @@ function readType(reader, type) {
     const text = new TextDecoder().decode(bytes)
     reader.offset += length
     return text
+  } else if (typeof type === 'object' && typeof type.type === 'string') {
+    // Boxed primitive/named type, e.g. { "type": "string" } or { "type": "long" }.
+    return readType(reader, type.type)
   } else {
-    // enum, fixed, null, map
-    throw new Error(`unsupported type: ${type}`)
+    throw new Error(`unsupported type: ${JSON.stringify(type)}`)
   }
 }
 

src/avro/avro.write.js

@@ -67,7 +67,8 @@ function writeType(writer, schema, value) {
       // annotation (e.g. an Iceberg array with logicalType=map is still an array).
       const tag = typeof s === 'string' ? s
         : s.type === 'record' || s.type === 'array' || s.type === 'fixed' ? s.type
-          : s.logicalType
+          : 'logicalType' in s && s.logicalType ? s.logicalType
+            : s.type
 
       if (value == null) return tag === 'null'
       if (tag === 'boolean') return typeof value === 'boolean'

src/aws/credentials.js

@@ -0,0 +1,38 @@
+/**
+ * @typedef {object} ResolvedAwsCredentials
+ * @property {string} accessKeyId
+ * @property {string} secretAccessKey
+ * @property {string} [sessionToken]
+ * @property {string} region
+ */
+
+/**
+ * Resolve AWS credentials from explicit keys or the default Node provider chain.
+ *
+ * The optional peer dependency `@aws-sdk/credential-providers` is imported
+ * lazily and only when falling back to the chain, so passing explicit keys
+ * keeps this module free of the AWS SDK (and browser-compatible).
+ *
+ * @param {object} options
+ * @param {string} options.region
+ * @param {string} [options.accessKeyId]
+ * @param {string} [options.secretAccessKey]
+ * @param {string} [options.sessionToken]
+ * @returns {Promise<ResolvedAwsCredentials>}
+ */
+export async function resolveAwsCredentials({
+  region, accessKeyId, secretAccessKey, sessionToken,
+}) {
+  if (accessKeyId && secretAccessKey) {
+    return { accessKeyId, secretAccessKey, sessionToken, region }
+  }
+  const { fromNodeProviderChain } = await import('@aws-sdk/credential-providers')
+  const provider = fromNodeProviderChain({ clientConfig: { region } })
+  const creds = await provider()
+  return {
+    accessKeyId: creds.accessKeyId,
+    secretAccessKey: creds.secretAccessKey,
+    sessionToken: creds.sessionToken,
+    region,
+  }
+}

src/aws/s3tables.js

@@ -0,0 +1,102 @@
+import { loadTable } from '../catalog/loadTable.js'
+import { restCatalogConnect } from '../catalog/rest.js'
+import { s3SignedResolver } from '../s3.js'
+import { createSigV4SignRequest } from '../sigv4.js'
+import { resolveAwsCredentials } from './credentials.js'
+
+/**
+ * @import {Resolver, RestCatalogContext} from '../types.js'
+ * @import {ResolvedAwsCredentials} from './credentials.js'
+ */
+
+/** @typedef {RestCatalogContext & { s3TablesCreds?: ResolvedAwsCredentials }} S3TablesCatalogContext */
+
+/**
+ * @typedef {object} S3TablesConnectOptions
+ * @property {string} region - AWS region, e.g. `us-east-1`
+ * @property {string} tableBucketArn - e.g. `arn:aws:s3tables:us-east-1:111122223333:bucket/my-bucket`
+ * @property {string} [accessKeyId] - Omit to use the default AWS credential chain
+ * @property {string} [secretAccessKey]
+ * @property {string} [sessionToken]
+ */
+
+/**
+ * Iceberg REST endpoint URL for Amazon S3 Tables in a region.
+ *
+ * @param {string} region
+ * @returns {string}
+ */
+export function s3TablesEndpoint(region) {
+  return `https://s3tables.${region}.amazonaws.com/iceberg`
+}
+
+/**
+ * Connect to the Amazon S3 Tables Iceberg REST catalog for a table bucket.
+ *
+ * When credentials are resolved from the default chain, the optional peer
+ * dependency `@aws-sdk/credential-providers` is required (pass explicit keys to
+ * avoid it). Catalog requests are SigV4-signed with service name `s3tables`.
+ * Use {@link s3TablesResolver} with the same credentials to read table data
+ * files (SigV4 with service name `s3`).
+ *
+ * @param {S3TablesConnectOptions} options
+ * @returns {Promise<S3TablesCatalogContext>}
+ */
+export async function s3TablesCatalogConnect({
+  region, tableBucketArn, accessKeyId, secretAccessKey, sessionToken,
+}) {
+  const creds = await resolveAwsCredentials({ region, accessKeyId, secretAccessKey, sessionToken })
+  const ctx = await restCatalogConnect({
+    url: s3TablesEndpoint(region),
+    warehouse: tableBucketArn,
+    signRequest: createSigV4SignRequest({
+      accessKeyId: creds.accessKeyId,
+      secretAccessKey: creds.secretAccessKey,
+      sessionToken: creds.sessionToken,
+      region,
+      service: 's3tables',
+    }),
+  })
+  return Object.freeze({ ...ctx, s3TablesCreds: creds })
+}
+
+/**
+ * Connect using the default AWS credential chain (env vars, shared config, IAM role).
+ *
+ * @param {object} options
+ * @param {string} options.region
+ * @param {string} options.tableBucketArn
+ * @returns {Promise<S3TablesCatalogContext>}
+ */
+export function s3TablesCatalogConnectFromEnv({ region, tableBucketArn }) {
+  return s3TablesCatalogConnect({ region, tableBucketArn })
+}
+
+/**
+ * Build a SigV4 `Resolver` for reading S3 Tables data files (`s3://…--table-s3/…`).
+ *
+ * @param {S3TablesConnectOptions} options
+ * @returns {Promise<Resolver>}
+ */
+export async function s3TablesResolver({ region, accessKeyId, secretAccessKey, sessionToken }) {
+  const creds = await resolveAwsCredentials({ region, accessKeyId, secretAccessKey, sessionToken })
+  return s3SignedResolver(creds)
+}
+
+/**
+ * Load a table from an S3 Tables catalog context, wiring a resolver from stored
+ * credentials when none is supplied.
+ *
+ * @param {object} options
+ * @param {S3TablesCatalogContext} options.catalog
+ * @param {string | string[]} options.namespace
+ * @param {string} options.table
+ * @param {Resolver} [options.resolver]
+ * @returns {ReturnType<typeof loadTable>}
+ */
+export function loadS3TablesTable({ catalog, namespace, table, resolver }) {
+  const eff = resolver ?? (catalog.s3TablesCreds
+    ? s3SignedResolver(catalog.s3TablesCreds)
+    : undefined)
+  return loadTable({ catalog, namespace, table, resolver: eff })
+}

src/catalog/rest.js

@@ -20,14 +20,17 @@ import { parseIcebergJson } from '../json.js'
  * @param {string} options.url - catalog base URL, with or without trailing slash
  * @param {string} [options.warehouse] - optional warehouse query param sent to /v1/config
  * @param {RequestInit} [options.requestInit] - fetch options (e.g. Authorization header)
+ * @param {(url: string, init?: RequestInit) => Promise<RequestInit>} [options.signRequest] - per-request auth hook
  * @returns {Promise<RestCatalogContext>}
  */
-export async function restCatalogConnect({ url, warehouse, requestInit }) {
+export async function restCatalogConnect({ url, warehouse, requestInit, signRequest }) {
   const base = url.replace(/\/$/, '')
   const configUrl = warehouse
     ? `${base}/v1/config?warehouse=${encodeURIComponent(warehouse)}`
     : `${base}/v1/config`
-  const res = await fetch(configUrl, requestInit)
+  let init = requestInit
+  if (signRequest) init = await signRequest(configUrl, init)
+  const res = await fetch(configUrl, init)
   if (!res.ok) await throwRestError(res)
   const body = parseIcebergJson(await res.text())
   const defaults = body.defaults ?? {}
@@ -36,14 +39,17 @@ export async function restCatalogConnect({ url, warehouse, requestInit }) {
   // config — overrides wins over defaults. Cloudflare R2 Data Catalog returns
   // it via `overrides.prefix`.
   const prefix = overrides.prefix ?? defaults.prefix ?? ''
-  return Object.freeze({
+  /** @type {RestCatalogContext} */
+  const ctx = {
     type: 'rest',
     url: base,
     prefix: typeof prefix === 'string' ? prefix : '',
     defaults,
     overrides,
     requestInit,
-  })
+  }
+  if (signRequest) ctx.signRequest = signRequest
+  return Object.freeze(ctx)
 }
 
 /**
@@ -339,7 +345,8 @@ function encodeNamespace(namespace) {
 async function restFetch(ctx, path, init) {
   const prefixSegment = ctx.prefix ? `${ctx.prefix.replace(/^\/|\/$/g, '')}/` : ''
   const fullUrl = `${ctx.url}/v1/${prefixSegment}${path}`
-  const merged = mergeRequestInit(ctx.requestInit, init)
+  let merged = mergeRequestInit(ctx.requestInit, init)
+  if (ctx.signRequest) merged = await ctx.signRequest(fullUrl, merged)
   const res = await fetch(fullUrl, merged)
   if (!res.ok) await throwRestError(res)
   return res