From d0a785e58a8f380eba52f8d7ace0d26ebea62733 Mon Sep 17 00:00:00 2001 From: Yostra Date: Tue, 24 Mar 2026 01:55:18 +0100 Subject: [PATCH 01/19] delete current --- packages/source-stripe/src/openapi/types.ts | 81 --------------------- 1 file changed, 81 deletions(-) delete mode 100644 packages/source-stripe/src/openapi/types.ts diff --git a/packages/source-stripe/src/openapi/types.ts b/packages/source-stripe/src/openapi/types.ts deleted file mode 100644 index 386b9e268..000000000 --- a/packages/source-stripe/src/openapi/types.ts +++ /dev/null @@ -1,81 +0,0 @@ -export type OpenApiSchemaObject = { - type?: string - format?: string - nullable?: boolean - properties?: Record - items?: OpenApiSchemaOrReference - oneOf?: OpenApiSchemaOrReference[] - anyOf?: OpenApiSchemaOrReference[] - allOf?: OpenApiSchemaOrReference[] - enum?: unknown[] - additionalProperties?: boolean | OpenApiSchemaOrReference - 'x-resourceId'?: string - 'x-expandableFields'?: string[] - 'x-expansionResources'?: { - oneOf?: OpenApiSchemaOrReference[] - } -} - -export type OpenApiReferenceObject = { - $ref: string -} - -export type OpenApiSchemaOrReference = OpenApiSchemaObject | OpenApiReferenceObject - -export type OpenApiSpec = { - openapi: string - info?: { - version?: string - } - components?: { - schemas?: Record - } -} - -export type ScalarType = 'text' | 'boolean' | 'bigint' | 'numeric' | 'json' | 'timestamptz' - -export type ParsedColumn = { - name: string - type: ScalarType - nullable: boolean - expandableReference?: boolean -} - -export type ParsedResourceTable = { - tableName: string - resourceId: string - sourceSchemaName: string - columns: ParsedColumn[] -} - -export type ParsedOpenApiSpec = { - apiVersion: string - tables: ParsedResourceTable[] -} - -export type ParseSpecOptions = { - /** - * Map Stripe x-resourceId values to concrete Postgres table names. - * Entries are matched case-sensitively. - */ - resourceAliases?: Record - /** - * Restrict parsing to these table names. - * If omitted, all resolvable x-resourceId entries are parsed. - */ - allowedTables?: string[] -} - -export type ResolveSpecConfig = { - apiVersion: string - openApiSpecPath?: string - cacheDir?: string -} - -export type ResolvedOpenApiSpec = { - apiVersion: string - spec: OpenApiSpec - source: 'explicit_path' | 'cache' | 'github' - cachePath?: string - commitSha?: string -} From 521be6068466d1c61f0de4ba19deb2980f188447 Mon Sep 17 00:00:00 2001 From: Yostra Date: Tue, 24 Mar 2026 01:55:36 +0100 Subject: [PATCH 02/19] open api from dynamic --- .../openapi/__tests__/fixtures/minimalSpec.ts | 196 ++++++++ .../openapi/__tests__/listFnResolver.test.ts | 144 ++++++ .../openapi/__tests__/postgresAdapter.test.ts | 89 ++++ .../openapi/__tests__/specFetchHelper.test.ts | 106 +++++ packages/openapi/__tests__/specParser.test.ts | 365 +++++++++++++++ .../__tests__/writePathPlanner.test.ts | 39 ++ packages/openapi/dialectAdapter.ts | 14 + packages/openapi/index.ts | 24 + packages/openapi/jsonSchemaConverter.ts | 35 ++ packages/openapi/listFnResolver.ts | 357 +++++++++++++++ packages/openapi/package.json | 28 ++ packages/openapi/postgresAdapter.ts | 126 ++++++ packages/openapi/runtimeMappings.ts | 74 +++ packages/openapi/specFetchHelper.ts | 181 ++++++++ packages/openapi/specParser.ts | 422 ++++++++++++++++++ packages/openapi/tsconfig.json | 19 + packages/openapi/types.ts | 118 +++++ packages/openapi/writePathPlanner.ts | 43 ++ 18 files changed, 2380 insertions(+) create mode 100644 packages/openapi/__tests__/fixtures/minimalSpec.ts create mode 100644 packages/openapi/__tests__/listFnResolver.test.ts create mode 100644 packages/openapi/__tests__/postgresAdapter.test.ts create mode 100644 packages/openapi/__tests__/specFetchHelper.test.ts create mode 100644 packages/openapi/__tests__/specParser.test.ts create mode 100644 packages/openapi/__tests__/writePathPlanner.test.ts create mode 100644 packages/openapi/dialectAdapter.ts create mode 100644 packages/openapi/index.ts create mode 100644 packages/openapi/jsonSchemaConverter.ts create mode 100644 packages/openapi/listFnResolver.ts create mode 100644 packages/openapi/package.json create mode 100644 packages/openapi/postgresAdapter.ts create mode 100644 packages/openapi/runtimeMappings.ts create mode 100644 packages/openapi/specFetchHelper.ts create mode 100644 packages/openapi/specParser.ts create mode 100644 packages/openapi/tsconfig.json create mode 100644 packages/openapi/types.ts create mode 100644 packages/openapi/writePathPlanner.ts diff --git a/packages/openapi/__tests__/fixtures/minimalSpec.ts b/packages/openapi/__tests__/fixtures/minimalSpec.ts new file mode 100644 index 000000000..6a8ab5ff3 --- /dev/null +++ b/packages/openapi/__tests__/fixtures/minimalSpec.ts @@ -0,0 +1,196 @@ +import type { OpenApiSpec, OpenApiPathItem } from '../../types' + +function listPath( + schemaRef: string, + opts: { supportsCreatedFilter?: boolean; supportsLimit?: boolean } = {} +): OpenApiPathItem { + const parameters: { name: string; in: string }[] = [] + if (opts.supportsCreatedFilter) { + parameters.push({ name: 'created', in: 'query' }) + } + if (opts.supportsLimit !== false) { + parameters.push({ name: 'limit', in: 'query' }) + } + return { + get: { + parameters, + responses: { + '200': { + content: { + 'application/json': { + schema: { + type: 'object', + properties: { + object: { type: 'string', enum: ['list'] }, + data: { type: 'array', items: { $ref: `#/components/schemas/${schemaRef}` } }, + has_more: { type: 'boolean' }, + url: { type: 'string' }, + }, + }, + }, + }, + }, + }, + }, + } +} + +function v2ListPath(schemaRef: string): OpenApiPathItem { + return { + get: { + responses: { + '200': { + content: { + 'application/json': { + schema: { + type: 'object', + properties: { + data: { type: 'array', items: { $ref: `#/components/schemas/${schemaRef}` } }, + next_page_url: { type: 'string', nullable: true }, + previous_page_url: { type: 'string', nullable: true }, + }, + }, + }, + }, + }, + }, + }, + } +} + +export const minimalStripeOpenApiSpec: OpenApiSpec = { + openapi: '3.0.0', + info: { + version: '2020-08-27', + }, + paths: { + '/v1/customers': listPath('customer', { supportsCreatedFilter: true }), + '/v1/plans': listPath('plan', { supportsCreatedFilter: true }), + '/v1/prices': listPath('price', { supportsCreatedFilter: true }), + '/v1/products': listPath('product', { supportsCreatedFilter: true }), + '/v1/subscription_items': listPath('subscription_item'), + '/v1/checkout/sessions': listPath('checkout_session', { supportsCreatedFilter: true }), + '/v1/radar/early_fraud_warnings': listPath('early_fraud_warning', { + supportsCreatedFilter: true, + }), + '/v1/entitlements/active_entitlements': listPath('active_entitlement'), + '/v1/entitlements/features': listPath('entitlements_feature'), + '/v2/core/accounts': v2ListPath('v2_core_account'), + '/v2/core/event_destinations': v2ListPath('v2_core_event_destination'), + }, + components: { + schemas: { + customer: { + 'x-resourceId': 'customer', + oneOf: [ + { + type: 'object', + properties: { + id: { type: 'string' }, + object: { type: 'string' }, + created: { type: 'integer' }, + }, + }, + { + type: 'object', + properties: { + id: { type: 'string' }, + deleted: { type: 'boolean' }, + }, + }, + ], + }, + plan: { + 'x-resourceId': 'plan', + type: 'object', + properties: { + id: { type: 'string' }, + active: { type: 'boolean' }, + amount: { type: 'integer' }, + }, + }, + price: { + 'x-resourceId': 'price', + type: 'object', + properties: { + id: { type: 'string' }, + product: { type: 'string' }, + unit_amount: { type: 'integer' }, + metadata: { type: 'object', additionalProperties: true }, + }, + }, + product: { + 'x-resourceId': 'product', + type: 'object', + properties: { + id: { type: 'string' }, + name: { type: 'string' }, + }, + }, + subscription_item: { + 'x-resourceId': 'subscription_item', + type: 'object', + properties: { + id: { type: 'string' }, + deleted: { type: 'boolean' }, + subscription: { type: 'string' }, + quantity: { type: 'integer' }, + }, + }, + checkout_session: { + 'x-resourceId': 'checkout.session', + type: 'object', + properties: { + id: { type: 'string' }, + amount_total: { type: 'integer' }, + customer: { type: 'string', nullable: true }, + }, + }, + early_fraud_warning: { + 'x-resourceId': 'radar.early_fraud_warning', + type: 'object', + properties: { + id: { type: 'string' }, + charge: { type: 'string' }, + }, + }, + active_entitlement: { + 'x-resourceId': 'entitlements.active_entitlement', + type: 'object', + properties: { + id: { type: 'string' }, + customer: { type: 'string' }, + feature: { type: 'string' }, + }, + }, + entitlements_feature: { + 'x-resourceId': 'entitlements.feature', + type: 'object', + properties: { + id: { type: 'string' }, + lookup_key: { type: 'string' }, + }, + }, + v2_core_account: { + 'x-resourceId': 'v2.core.account', + type: 'object', + properties: { + id: { type: 'string' }, + display_name: { type: 'string' }, + contact_email: { type: 'string', nullable: true }, + created: { type: 'string', format: 'date-time' }, + }, + }, + v2_core_event_destination: { + 'x-resourceId': 'v2.core.event_destination', + type: 'object', + properties: { + id: { type: 'string' }, + name: { type: 'string' }, + enabled_events: { type: 'array', items: { type: 'string' } }, + livemode: { type: 'boolean' }, + }, + }, + }, + }, +} diff --git a/packages/openapi/__tests__/listFnResolver.test.ts b/packages/openapi/__tests__/listFnResolver.test.ts new file mode 100644 index 000000000..6162797a8 --- /dev/null +++ b/packages/openapi/__tests__/listFnResolver.test.ts @@ -0,0 +1,144 @@ +import { describe, expect, it, vi } from 'vitest' +import { discoverListEndpoints, buildListFn } from '../listFnResolver' +import { minimalStripeOpenApiSpec } from './fixtures/minimalSpec' +import type Stripe from 'stripe' + +function mockStripe() { + const listFn = vi.fn().mockResolvedValue({ data: [], has_more: false }) + return { + customers: { list: listFn }, + plans: { list: listFn }, + products: { list: listFn }, + subscriptionItems: { list: listFn }, + checkout: { sessions: { list: listFn } }, + radar: { earlyFraudWarnings: { list: listFn } }, + entitlements: { + activeEntitlements: { list: listFn }, + features: { list: listFn }, + }, + _listFn: listFn, + } +} + +describe('discoverListEndpoints', () => { + it('maps table names to their API paths', () => { + const endpoints = discoverListEndpoints(minimalStripeOpenApiSpec) + + expect(endpoints.get('customers')).toEqual({ + tableName: 'customers', + resourceId: 'customer', + apiPath: '/v1/customers', + supportsCreatedFilter: true, + supportsLimit: true, + }) + expect(endpoints.get('checkout_sessions')).toEqual({ + tableName: 'checkout_sessions', + resourceId: 'checkout.session', + apiPath: '/v1/checkout/sessions', + supportsCreatedFilter: true, + supportsLimit: true, + }) + expect(endpoints.get('early_fraud_warnings')).toEqual({ + tableName: 'early_fraud_warnings', + resourceId: 'radar.early_fraud_warning', + apiPath: '/v1/radar/early_fraud_warnings', + supportsCreatedFilter: true, + supportsLimit: true, + }) + }) + + it('discovers v2 list endpoints using next_page_url format', () => { + const endpoints = discoverListEndpoints(minimalStripeOpenApiSpec) + + expect(endpoints.get('v2_core_accounts')).toEqual({ + tableName: 'v2_core_accounts', + resourceId: 'v2.core.account', + apiPath: '/v2/core/accounts', + supportsCreatedFilter: false, + supportsLimit: false, + }) + expect(endpoints.get('v2_core_event_destinations')).toEqual({ + tableName: 'v2_core_event_destinations', + resourceId: 'v2.core.event_destination', + apiPath: '/v2/core/event_destinations', + supportsCreatedFilter: false, + supportsLimit: false, + }) + }) + + it('skips paths with path parameters', () => { + const spec = { + ...minimalStripeOpenApiSpec, + paths: { + ...minimalStripeOpenApiSpec.paths, + '/v1/customers/{customer}/sources': { + get: { + responses: { + '200': { + content: { + 'application/json': { + schema: { + type: 'object' as const, + properties: { + object: { type: 'string' as const, enum: ['list'] }, + data: { + type: 'array' as const, + items: { $ref: '#/components/schemas/customer' }, + }, + has_more: { type: 'boolean' as const }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + } + const endpoints = discoverListEndpoints(spec) + const paths = Array.from(endpoints.values()).map((e) => e.apiPath) + expect(paths).not.toContain('/v1/customers/{customer}/sources') + }) + + it('returns empty map when spec has no paths', () => { + const endpoints = discoverListEndpoints({ openapi: '3.0.0' }) + expect(endpoints.size).toBe(0) + }) +}) + +describe('buildListFn', () => { + it('resolves a simple top-level path', async () => { + const mock = mockStripe() + const listFn = buildListFn(mock as unknown as Stripe, '/v1/customers') + await listFn({ limit: 10 }) + expect(mock._listFn).toHaveBeenCalledWith({ limit: 10 }) + }) + + it('resolves a nested namespace path', async () => { + const mock = mockStripe() + const listFn = buildListFn(mock as unknown as Stripe, '/v1/checkout/sessions') + await listFn({ limit: 5 }) + expect(mock._listFn).toHaveBeenCalledWith({ limit: 5 }) + }) + + it('converts snake_case segments to camelCase', async () => { + const mock = mockStripe() + const listFn = buildListFn(mock as unknown as Stripe, '/v1/subscription_items') + await listFn({ limit: 1 }) + expect(mock._listFn).toHaveBeenCalled() + }) + + it('resolves deeply nested snake_case paths', async () => { + const mock = mockStripe() + const listFn = buildListFn(mock as unknown as Stripe, '/v1/radar/early_fraud_warnings') + await listFn({ limit: 1 }) + expect(mock._listFn).toHaveBeenCalled() + }) + + it('throws when a path segment does not exist on the SDK', async () => { + const mock = mockStripe() + const listFn = buildListFn(mock as unknown as Stripe, '/v1/nonexistent_resource') + await expect(() => listFn({ limit: 1 })).toThrow(/Stripe SDK has no property/) + }) +}) diff --git a/packages/openapi/__tests__/postgresAdapter.test.ts b/packages/openapi/__tests__/postgresAdapter.test.ts new file mode 100644 index 000000000..cda4d691f --- /dev/null +++ b/packages/openapi/__tests__/postgresAdapter.test.ts @@ -0,0 +1,89 @@ +import { describe, expect, it } from 'vitest' +import { PostgresAdapter } from '../postgresAdapter' +import type { ParsedResourceTable } from '../types' + +const SAMPLE_TABLE: ParsedResourceTable = { + tableName: 'customers', + resourceId: 'customer', + sourceSchemaName: 'customer', + columns: [ + { name: 'created', type: 'bigint', nullable: false }, + { name: 'deleted', type: 'boolean', nullable: true }, + { name: 'metadata', type: 'json', nullable: true }, + { name: 'expires_at', type: 'timestamptz', nullable: true }, + ], +} + +const EXPANDABLE_REFERENCE_TABLE: ParsedResourceTable = { + tableName: 'charges', + resourceId: 'charge', + sourceSchemaName: 'charge', + columns: [{ name: 'customer', type: 'json', nullable: true, expandableReference: true }], +} + +describe('PostgresAdapter', () => { + it('emits deterministic DDL statements with runtime-required metadata columns', () => { + const adapter = new PostgresAdapter({ schemaName: 'stripe' }) + const statements = adapter.buildAllStatements([SAMPLE_TABLE]) + + expect(statements).toHaveLength(9) + expect(statements[0]).toContain('CREATE TABLE "stripe"."customers"') + expect(statements[0]).toContain('"_raw_data" jsonb NOT NULL') + expect(statements[0]).toContain('"_account_id" text NOT NULL') + expect(statements[0]).toContain( + '"id" text GENERATED ALWAYS AS ((_raw_data->>\'id\')::text) STORED' + ) + expect(statements[0]).toContain( + '"metadata" jsonb GENERATED ALWAYS AS ((_raw_data->\'metadata\')::jsonb) STORED' + ) + // Temporal columns are stored as text generated columns for immutability safety. + expect(statements[0]).toContain( + '"expires_at" text GENERATED ALWAYS AS ((_raw_data->>\'expires_at\')::text) STORED' + ) + expect( + statements.some((stmt) => stmt.includes('ADD COLUMN IF NOT EXISTS "created" bigint')) + ).toBe(true) + expect( + statements.some((stmt) => stmt.includes('ADD COLUMN IF NOT EXISTS "deleted" boolean')) + ).toBe(true) + expect( + statements.some((stmt) => stmt.includes('ADD COLUMN IF NOT EXISTS "metadata" jsonb')) + ).toBe(true) + expect( + statements.some((stmt) => stmt.includes('ADD COLUMN IF NOT EXISTS "expires_at" text')) + ).toBe(true) + expect(statements[5]).toContain( + 'FOREIGN KEY ("_account_id") REFERENCES "stripe"."_accounts" (id)' + ) + expect(statements[7]).toContain('DROP TRIGGER IF EXISTS handle_updated_at') + expect(statements[8]).toContain('EXECUTE FUNCTION set_updated_at()') + }) + + it('produces stable output across repeated calls', () => { + const adapter = new PostgresAdapter({ schemaName: 'stripe' }) + const first = adapter.buildAllStatements([SAMPLE_TABLE]) + const second = adapter.buildAllStatements([SAMPLE_TABLE]) + expect(second).toEqual(first) + }) + + it('materializes expandable reference columns as text ids for compatibility', () => { + const adapter = new PostgresAdapter({ schemaName: 'stripe' }) + const statements = adapter.buildAllStatements([EXPANDABLE_REFERENCE_TABLE]) + + expect(statements[0]).toContain('"customer" text GENERATED ALWAYS AS (CASE') + expect(statements[0]).toContain("WHEN jsonb_typeof(_raw_data->'customer') = 'object'") + expect(statements[0]).toContain("THEN (_raw_data->'customer'->>'id')") + }) + + it('uses accountSchema for FK when provided (split schema)', () => { + const adapter = new PostgresAdapter({ + schemaName: 'stripe_data', + accountSchema: 'stripe_sync', + }) + const statements = adapter.buildAllStatements([SAMPLE_TABLE]) + expect(statements[0]).toContain('CREATE TABLE "stripe_data"."customers"') + expect(statements[5]).toContain( + 'FOREIGN KEY ("_account_id") REFERENCES "stripe_sync"."_accounts" (id)' + ) + }) +}) diff --git a/packages/openapi/__tests__/specFetchHelper.test.ts b/packages/openapi/__tests__/specFetchHelper.test.ts new file mode 100644 index 000000000..109b10852 --- /dev/null +++ b/packages/openapi/__tests__/specFetchHelper.test.ts @@ -0,0 +1,106 @@ +import fs from 'node:fs/promises' +import os from 'node:os' +import path from 'node:path' +import { afterEach, describe, expect, it, vi } from 'vitest' +import { resolveOpenApiSpec } from '../specFetchHelper' +import { minimalStripeOpenApiSpec } from './fixtures/minimalSpec' + +async function createTempDir(prefix: string): Promise { + return fs.mkdtemp(path.join(os.tmpdir(), `${prefix}-`)) +} + +describe('resolveOpenApiSpec', () => { + afterEach(() => { + vi.unstubAllGlobals() + vi.restoreAllMocks() + }) + + it('prefers explicit local spec path over cache and network', async () => { + const tempDir = await createTempDir('openapi-explicit') + const specPath = path.join(tempDir, 'spec3.json') + await fs.writeFile(specPath, JSON.stringify(minimalStripeOpenApiSpec), 'utf8') + const fetchMock = vi.fn() + vi.stubGlobal('fetch', fetchMock) + + const result = await resolveOpenApiSpec({ + apiVersion: '2020-08-27', + openApiSpecPath: specPath, + cacheDir: tempDir, + }) + + expect(result.source).toBe('explicit_path') + expect(fetchMock).not.toHaveBeenCalled() + await fs.rm(tempDir, { recursive: true, force: true }) + }) + + it('uses cache by api version when available', async () => { + const tempDir = await createTempDir('openapi-cache') + const cachePath = path.join(tempDir, '2020-08-27.spec3.sdk.json') + await fs.writeFile(cachePath, JSON.stringify(minimalStripeOpenApiSpec), 'utf8') + const fetchMock = vi.fn() + vi.stubGlobal('fetch', fetchMock) + + const result = await resolveOpenApiSpec({ + apiVersion: '2020-08-27', + cacheDir: tempDir, + }) + + expect(result.source).toBe('cache') + expect(result.cachePath).toBe(cachePath) + expect(fetchMock).not.toHaveBeenCalled() + await fs.rm(tempDir, { recursive: true, force: true }) + }) + + it('fetches from GitHub when cache misses and persists cache', async () => { + const tempDir = await createTempDir('openapi-fetch') + const fetchMock = vi.fn(async (input: URL | string) => { + const url = String(input) + if (url.includes('/commits')) { + return new Response(JSON.stringify([{ sha: 'abc123def456' }]), { status: 200 }) + } + return new Response(JSON.stringify(minimalStripeOpenApiSpec), { status: 200 }) + }) + vi.stubGlobal('fetch', fetchMock) + + const result = await resolveOpenApiSpec({ + apiVersion: '2020-08-27', + cacheDir: tempDir, + }) + + expect(result.source).toBe('github') + expect(result.commitSha).toBe('abc123def456') + + const cached = await fs.readFile(path.join(tempDir, '2020-08-27.spec3.sdk.json'), 'utf8') + expect(JSON.parse(cached)).toMatchObject({ openapi: '3.0.0' }) + expect(fetchMock).toHaveBeenCalledTimes(2) + await fs.rm(tempDir, { recursive: true, force: true }) + }) + + it('throws for malformed explicit spec files', async () => { + const tempDir = await createTempDir('openapi-malformed') + const specPath = path.join(tempDir, 'spec3.json') + await fs.writeFile(specPath, JSON.stringify({ openapi: '3.0.0' }), 'utf8') + + await expect( + resolveOpenApiSpec({ + apiVersion: '2020-08-27', + openApiSpecPath: specPath, + }) + ).rejects.toThrow(/components|schemas/i) + await fs.rm(tempDir, { recursive: true, force: true }) + }) + + it('fails fast when GitHub resolution fails and no explicit spec path is set', async () => { + const tempDir = await createTempDir('openapi-fail-fast') + const fetchMock = vi.fn(async () => new Response('boom', { status: 500 })) + vi.stubGlobal('fetch', fetchMock) + + await expect( + resolveOpenApiSpec({ + apiVersion: '2020-08-27', + cacheDir: tempDir, + }) + ).rejects.toThrow(/Failed to resolve Stripe OpenAPI commit/) + await fs.rm(tempDir, { recursive: true, force: true }) + }) +}) diff --git a/packages/openapi/__tests__/specParser.test.ts b/packages/openapi/__tests__/specParser.test.ts new file mode 100644 index 000000000..e9cab1b47 --- /dev/null +++ b/packages/openapi/__tests__/specParser.test.ts @@ -0,0 +1,365 @@ +import { describe, expect, it } from 'vitest' +import { SpecParser } from '../specParser' +import { minimalStripeOpenApiSpec } from './fixtures/minimalSpec' +import type { OpenApiSpec } from '../../types' + +describe('SpecParser', () => { + it('parses aliased resources into deterministic tables and column types', () => { + const parser = new SpecParser() + const parsed = parser.parse(minimalStripeOpenApiSpec, { + allowedTables: ['checkout_sessions', 'customers', 'early_fraud_warnings'], + }) + + expect(parsed.tables.map((table) => table.tableName)).toEqual([ + 'checkout_sessions', + 'customers', + 'early_fraud_warnings', + ]) + + const customers = parsed.tables.find((table) => table.tableName === 'customers') + expect(customers?.columns).toEqual([ + { name: 'created', type: 'bigint', nullable: false }, + { name: 'deleted', type: 'boolean', nullable: false }, + { name: 'object', type: 'text', nullable: false }, + ]) + + const checkoutSessions = parsed.tables.find((table) => table.tableName === 'checkout_sessions') + expect(checkoutSessions?.columns).toContainEqual({ + name: 'amount_total', + type: 'bigint', + nullable: false, + }) + }) + + it('injects compatibility columns for runtime-critical tables', () => { + const parser = new SpecParser() + const parsed = parser.parse( + { + ...minimalStripeOpenApiSpec, + components: { schemas: {} }, + }, + { allowedTables: ['active_entitlements', 'subscription_items'] } + ) + + const activeEntitlements = parsed.tables.find( + (table) => table.tableName === 'active_entitlements' + ) + expect(activeEntitlements?.columns).toContainEqual({ + name: 'customer', + type: 'text', + nullable: true, + }) + + const subscriptionItems = parsed.tables.find( + (table) => table.tableName === 'subscription_items' + ) + expect(subscriptionItems?.columns).toContainEqual({ + name: 'deleted', + type: 'boolean', + nullable: true, + }) + expect(subscriptionItems?.columns).toContainEqual({ + name: 'subscription', + type: 'text', + nullable: true, + }) + }) + + it('is deterministic regardless of schema key order', () => { + const parser = new SpecParser() + const normal = parser.parse(minimalStripeOpenApiSpec, { + allowedTables: ['customers', 'plans', 'prices'], + }) + + const reversedSchemas = Object.fromEntries( + Object.entries(minimalStripeOpenApiSpec.components?.schemas ?? {}).reverse() + ) + const reversed = parser.parse( + { + ...minimalStripeOpenApiSpec, + components: { + schemas: reversedSchemas, + }, + }, + { allowedTables: ['customers', 'plans', 'prices'] } + ) + + expect(reversed).toEqual(normal) + }) + + it('marks expandable references from x-expansionResources metadata', () => { + const parser = new SpecParser() + const parsed = parser.parse( + { + ...minimalStripeOpenApiSpec, + components: { + schemas: { + charge: { + 'x-resourceId': 'charge', + type: 'object', + properties: { + id: { type: 'string' }, + customer: { + anyOf: [{ type: 'string' }, { $ref: '#/components/schemas/customer' }], + 'x-expansionResources': { + oneOf: [{ $ref: '#/components/schemas/customer' }], + }, + }, + }, + }, + customer: { + 'x-resourceId': 'customer', + type: 'object', + properties: { + id: { type: 'string' }, + }, + }, + }, + }, + }, + { allowedTables: ['charges'] } + ) + + const charges = parsed.tables.find((table) => table.tableName === 'charges') + expect(charges?.columns).toContainEqual({ + name: 'customer', + type: 'json', + nullable: false, + expandableReference: true, + }) + }) + + describe('discoverListableResourceIds', () => { + it('discovers resource ids from list endpoints in paths', () => { + const parser = new SpecParser() + const ids = parser.discoverListableResourceIds(minimalStripeOpenApiSpec) + + expect(ids).toEqual( + new Set([ + 'customer', + 'plan', + 'price', + 'product', + 'subscription_item', + 'checkout.session', + 'radar.early_fraud_warning', + 'entitlements.active_entitlement', + 'entitlements.feature', + 'v2.core.account', + 'v2.core.event_destination', + ]) + ) + }) + + it('optionally includes nested list endpoints', () => { + const parser = new SpecParser() + const spec: OpenApiSpec = { + ...minimalStripeOpenApiSpec, + paths: { + ...minimalStripeOpenApiSpec.paths, + '/v1/accounts/{account}/persons': { + get: { + responses: { + '200': { + content: { + 'application/json': { + schema: { + type: 'object' as const, + properties: { + object: { + type: 'string' as const, + enum: ['list'], + }, + data: { + type: 'array' as const, + items: { + $ref: '#/components/schemas/person', + }, + }, + has_more: { + type: 'boolean' as const, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + components: { + ...minimalStripeOpenApiSpec.components, + schemas: { + ...minimalStripeOpenApiSpec.components.schemas, + person: { + 'x-resourceId': 'person', + type: 'object', + properties: { + id: { type: 'string' }, + }, + }, + }, + }, + } + + const ids = parser.discoverListableResourceIds(spec, { includeNested: true }) + expect(ids).toContain('person') + }) + + it('returns empty set when spec has no paths', () => { + const parser = new SpecParser() + const specWithoutPaths: OpenApiSpec = { + ...minimalStripeOpenApiSpec, + paths: undefined, + } + expect(parser.discoverListableResourceIds(specWithoutPaths)).toEqual(new Set()) + }) + + it('ignores non-list GET endpoints', () => { + const parser = new SpecParser() + const spec: OpenApiSpec = { + openapi: '3.0.0', + paths: { + '/v1/customers/{customer}': { + get: { + responses: { + '200': { + content: { + 'application/json': { + schema: { + type: 'object', + properties: { + id: { type: 'string' }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + components: { + schemas: { + customer: { + 'x-resourceId': 'customer', + type: 'object', + properties: { id: { type: 'string' } }, + }, + }, + }, + } + expect(parser.discoverListableResourceIds(spec)).toEqual(new Set()) + }) + }) + + describe('auto-discovery via paths (no allowedTables)', () => { + it('creates tables only for resources with list endpoints', () => { + const parser = new SpecParser() + const parsed = parser.parse(minimalStripeOpenApiSpec) + + const tableNames = parsed.tables.map((t) => t.tableName) + expect(tableNames).toEqual([ + 'active_entitlements', + 'checkout_sessions', + 'customers', + 'early_fraud_warnings', + 'features', + 'plans', + 'prices', + 'products', + 'subscription_items', + 'v2_core_accounts', + 'v2_core_event_destinations', + ]) + }) + + it('includes nested listables when they appear in the OpenAPI paths', () => { + const parser = new SpecParser() + const spec: OpenApiSpec = { + ...minimalStripeOpenApiSpec, + paths: { + ...minimalStripeOpenApiSpec.paths, + '/v1/accounts/{account}/persons': { + get: { + responses: { + '200': { + content: { + 'application/json': { + schema: { + type: 'object' as const, + properties: { + object: { + type: 'string' as const, + enum: ['list'], + }, + data: { + type: 'array' as const, + items: { + $ref: '#/components/schemas/person', + }, + }, + has_more: { + type: 'boolean' as const, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + components: { + ...minimalStripeOpenApiSpec.components, + schemas: { + ...minimalStripeOpenApiSpec.components.schemas, + person: { + 'x-resourceId': 'person', + type: 'object', + properties: { + id: { type: 'string' }, + }, + }, + }, + }, + } + + const parsed = parser.parse(spec) + const tableNames = parsed.tables.map((table) => table.tableName) + expect(tableNames).toContain('persons') + }) + + it('excludes schemas that have no list endpoint', () => { + const parser = new SpecParser() + const specWithLimitedPaths: OpenApiSpec = { + ...minimalStripeOpenApiSpec, + paths: { + '/v1/customers': minimalStripeOpenApiSpec.paths!['/v1/customers'], + '/v1/products': minimalStripeOpenApiSpec.paths!['/v1/products'], + }, + } + const parsed = parser.parse(specWithLimitedPaths) + + const tableNames = parsed.tables.map((t) => t.tableName) + expect(tableNames).toEqual(['customers', 'products']) + expect(tableNames).not.toContain('plans') + expect(tableNames).not.toContain('subscription_items') + }) + + it('resolves table name aliases from x-resourceId during discovery', () => { + const parser = new SpecParser() + const parsed = parser.parse(minimalStripeOpenApiSpec) + + const earlyFraud = parsed.tables.find((t) => t.tableName === 'early_fraud_warnings') + expect(earlyFraud).toBeDefined() + expect(earlyFraud?.resourceId).toBe('radar.early_fraud_warning') + + const checkout = parsed.tables.find((t) => t.tableName === 'checkout_sessions') + expect(checkout).toBeDefined() + expect(checkout?.resourceId).toBe('checkout.session') + }) + }) +}) diff --git a/packages/openapi/__tests__/writePathPlanner.test.ts b/packages/openapi/__tests__/writePathPlanner.test.ts new file mode 100644 index 000000000..9d337906a --- /dev/null +++ b/packages/openapi/__tests__/writePathPlanner.test.ts @@ -0,0 +1,39 @@ +import { describe, expect, it } from 'vitest' +import { WritePathPlanner } from '../writePathPlanner' +import type { ParsedResourceTable } from '../types' + +describe('WritePathPlanner', () => { + it('builds deterministic write plans aligned to raw json upsert assumptions', () => { + const planner = new WritePathPlanner() + const tables: ParsedResourceTable[] = [ + { + tableName: 'customers', + resourceId: 'customer', + sourceSchemaName: 'customer', + columns: [ + { name: 'deleted', type: 'boolean', nullable: true }, + { name: 'created', type: 'bigint', nullable: false }, + ], + }, + { + tableName: 'plans', + resourceId: 'plan', + sourceSchemaName: 'plan', + columns: [{ name: 'active', type: 'boolean', nullable: false }], + }, + ] + + const plans = planner.buildPlans(tables) + expect(plans.map((plan) => plan.tableName)).toEqual(['customers', 'plans']) + expect(plans[0]).toMatchObject({ + tableName: 'customers', + conflictTarget: ['id'], + extraColumns: [], + metadataColumns: ['_raw_data', '_last_synced_at', '_account_id'], + }) + expect(plans[0].generatedColumns).toEqual([ + { column: 'created', pgType: 'bigint' }, + { column: 'deleted', pgType: 'boolean' }, + ]) + }) +}) diff --git a/packages/openapi/dialectAdapter.ts b/packages/openapi/dialectAdapter.ts new file mode 100644 index 000000000..9d684d399 --- /dev/null +++ b/packages/openapi/dialectAdapter.ts @@ -0,0 +1,14 @@ +import type { ParsedResourceTable } from './types' + +export interface DialectAdapter { + /** + * Create all statements needed to materialize a single parsed table. + */ + buildTableStatements(table: ParsedResourceTable): string[] + + /** + * Create all statements needed to materialize all parsed tables. + * Implementations must be deterministic for a given input. + */ + buildAllStatements(tables: ParsedResourceTable[]): string[] +} diff --git a/packages/openapi/index.ts b/packages/openapi/index.ts new file mode 100644 index 000000000..78569f356 --- /dev/null +++ b/packages/openapi/index.ts @@ -0,0 +1,24 @@ +export type * from './types' +export { + SpecParser, + OPENAPI_RESOURCE_TABLE_ALIASES, + RUNTIME_RESOURCE_ALIASES, +} from './specParser' +export { OPENAPI_COMPATIBILITY_COLUMNS } from './runtimeMappings' +export { PostgresAdapter } from './postgresAdapter' +export { WritePathPlanner } from './writePathPlanner' +export { resolveOpenApiSpec } from './specFetchHelper' +export type { DialectAdapter } from './dialectAdapter' +export { + buildListFn, + buildRetrieveFn, + buildV2ListFn, + buildV2RetrieveFn, + discoverListEndpoints, + discoverNestedEndpoints, + canResolveSdkResource, + isV2Path, +} from './listFnResolver' +export type { NestedEndpoint } from './listFnResolver' +export { parsedTableToJsonSchema } from './jsonSchemaConverter' +export { RUNTIME_REQUIRED_TABLES } from './runtimeMappings' diff --git a/packages/openapi/jsonSchemaConverter.ts b/packages/openapi/jsonSchemaConverter.ts new file mode 100644 index 000000000..15b400409 --- /dev/null +++ b/packages/openapi/jsonSchemaConverter.ts @@ -0,0 +1,35 @@ +import type { ParsedResourceTable, ScalarType } from './types' + +const SCALAR_TYPE_TO_JSON_SCHEMA: Record = { + text: { type: 'string' }, + boolean: { type: 'boolean' }, + bigint: { type: 'integer' }, + numeric: { type: 'number' }, + json: { type: 'object' }, + timestamptz: { type: 'string', format: 'date-time' }, +} + +export function parsedTableToJsonSchema( + table: ParsedResourceTable +): Record { + const properties: Record = { + id: { type: 'string' }, + } + const required: string[] = ['id'] + + for (const col of table.columns) { + const mapped = SCALAR_TYPE_TO_JSON_SCHEMA[col.type] ?? { type: 'string' } + if (col.nullable) { + properties[col.name] = { oneOf: [mapped, { type: 'null' }] } + } else { + properties[col.name] = mapped + required.push(col.name) + } + } + + return { + type: 'object', + properties, + required, + } +} diff --git a/packages/openapi/listFnResolver.ts b/packages/openapi/listFnResolver.ts new file mode 100644 index 000000000..04a05e096 --- /dev/null +++ b/packages/openapi/listFnResolver.ts @@ -0,0 +1,357 @@ +import type Stripe from 'stripe' +import type { OpenApiSchemaObject, OpenApiSpec } from './types' +import { OPENAPI_RESOURCE_TABLE_ALIASES } from './runtimeMappings' + +const SCHEMA_REF_PREFIX = '#/components/schemas/' + +type ListFn = ( + params: Stripe.PaginationParams & { created?: Stripe.RangeQueryParam } +) => Promise<{ data: unknown[]; has_more: boolean; pageCursor?: string }> + +export type ListEndpoint = { + tableName: string + resourceId: string + apiPath: string + supportsCreatedFilter: boolean + supportsLimit: boolean +} + +export type NestedEndpoint = { + tableName: string + resourceId: string + apiPath: string + parentTableName: string + parentParamName: string + supportsPagination: boolean +} + +function snakeToCamel(s: string): string { + return s.replace(/_([a-z])/g, (_, c) => c.toUpperCase()) +} + +function resolveTableName(resourceId: string, aliases: Record): string { + const alias = aliases[resourceId] + if (alias) return alias + const normalized = resourceId.toLowerCase().replace(/[.]/g, '_') + return normalized.endsWith('s') ? normalized : `${normalized}s` +} + +/** + * Detect whether a response schema describes a list endpoint. + * v1 lists have `object: enum ["list"]` with a `data` array. + * v2 lists have a `data` array with `next_page_url`. + */ +function isListResponseSchema(schema: OpenApiSchemaObject): boolean { + const dataProp = schema.properties?.data + if (!dataProp || !('type' in dataProp) || dataProp.type !== 'array') return false + + const objectProp = schema.properties?.object + if (objectProp && 'enum' in objectProp && objectProp.enum?.includes('list')) return true + + if (schema.properties?.next_page_url) return true + + return false +} + +/** + * Scan the spec for list endpoints (GET paths that return a Stripe list object) + * and return one entry per table. Prefers top-level paths over nested ones. + * Supports both v1 (object: "list") and v2 (next_page_url) response formats. + */ +export function discoverListEndpoints( + spec: OpenApiSpec, + aliases: Record = OPENAPI_RESOURCE_TABLE_ALIASES +): Map { + const endpoints = new Map() + const paths = spec.paths + if (!paths) return endpoints + + for (const [apiPath, pathItem] of Object.entries(paths)) { + if (apiPath.includes('{')) continue + + const getOp = pathItem.get + if (!getOp?.responses) continue + + const responseSchema = getOp.responses['200']?.content?.['application/json']?.schema + if (!responseSchema) continue + + if (!isListResponseSchema(responseSchema)) continue + + const dataProp = responseSchema.properties?.data + if (!dataProp || !('type' in dataProp) || dataProp.type !== 'array') continue + + const itemsRef = dataProp.items + if (!itemsRef || !('$ref' in itemsRef) || typeof itemsRef.$ref !== 'string') continue + if (!itemsRef.$ref.startsWith(SCHEMA_REF_PREFIX)) continue + + const schemaName = itemsRef.$ref.slice(SCHEMA_REF_PREFIX.length) + const schema = spec.components?.schemas?.[schemaName] + if (!schema || '$ref' in schema) continue + + const resourceId = schema['x-resourceId'] + if (!resourceId || typeof resourceId !== 'string') continue + + const tableName = resolveTableName(resourceId, aliases) + if (!endpoints.has(tableName)) { + const params = getOp.parameters ?? [] + const PAGINATION_PARAMS = new Set([ + 'limit', + 'starting_after', + 'ending_before', + 'created', + 'expand', + ]) + const hasRequiredQueryParams = params.some( + (p: { name?: string; in?: string; required?: boolean }) => + p.required === true && p.in === 'query' && !PAGINATION_PARAMS.has(p.name ?? '') + ) + if (hasRequiredQueryParams) continue + + const supportsCreatedFilter = params.some( + (p: { name?: string; in?: string }) => p.name === 'created' && p.in === 'query' + ) + const supportsLimit = params.some( + (p: { name?: string; in?: string }) => p.name === 'limit' && p.in === 'query' + ) + endpoints.set(tableName, { + tableName, + resourceId, + apiPath, + supportsCreatedFilter, + supportsLimit, + }) + } + } + + return endpoints +} + +/** + * Scan the spec for nested list endpoints (paths with `{param}` segments that + * return a Stripe list object) and map each to its parent resource. + */ +export function discoverNestedEndpoints( + spec: OpenApiSpec, + topLevelEndpoints: Map, + aliases: Record = OPENAPI_RESOURCE_TABLE_ALIASES +): NestedEndpoint[] { + const nested: NestedEndpoint[] = [] + const paths = spec.paths + if (!paths) return nested + + const topLevelByPath = new Map() + for (const endpoint of topLevelEndpoints.values()) { + topLevelByPath.set(endpoint.apiPath, endpoint) + } + + for (const [apiPath, pathItem] of Object.entries(paths)) { + if (!apiPath.includes('{')) continue + + const getOp = pathItem.get + if (!getOp?.responses) continue + + const responseSchema = getOp.responses['200']?.content?.['application/json']?.schema + if (!responseSchema) continue + + if (!isListResponseSchema(responseSchema)) continue + + const dataProp = responseSchema.properties?.data + if (!dataProp || !('type' in dataProp) || dataProp.type !== 'array') continue + + const itemsRef = dataProp.items + if (!itemsRef || !('$ref' in itemsRef) || typeof itemsRef.$ref !== 'string') continue + if (!itemsRef.$ref.startsWith(SCHEMA_REF_PREFIX)) continue + + const schemaName = itemsRef.$ref.slice(SCHEMA_REF_PREFIX.length) + const schema = spec.components?.schemas?.[schemaName] + if (!schema || '$ref' in schema) continue + + const resourceId = schema['x-resourceId'] + if (!resourceId || typeof resourceId !== 'string') continue + + const paramMatch = apiPath.match(/\{([^}]+)\}/) + if (!paramMatch) continue + const parentParamName = paramMatch[1] + + const parentPath = apiPath.slice(0, apiPath.indexOf('/{')) + const parentEndpoint = topLevelByPath.get(parentPath) + if (!parentEndpoint) continue + + const params = getOp.parameters ?? [] + const supportsPagination = params.some((p: { name?: string }) => p.name === 'limit') + + nested.push({ + tableName: resolveTableName(resourceId, aliases), + resourceId, + apiPath, + parentTableName: parentEndpoint.tableName, + parentParamName, + supportsPagination, + }) + } + + return nested +} + +export function isV2Path(apiPath: string): boolean { + return apiPath.startsWith('/v2/') +} + +function pathToSdkSegments(apiPath: string): string[] { + if (isV2Path(apiPath)) { + return [ + 'v2', + ...apiPath + .replace(/^\/v2\//, '') + .split('/') + .filter((s) => !s.startsWith('{')) + .map(snakeToCamel), + ] + } + return apiPath + .replace(/^\/v[12]\//, '') + .split('/') + .filter((s) => !s.startsWith('{')) + .map(snakeToCamel) +} + +// eslint-disable-next-line @typescript-eslint/no-explicit-any +function resolveStripeResource(stripe: Stripe, segments: string[], apiPath: string): any { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + let resource: any = stripe + for (const segment of segments) { + resource = resource?.[segment] + if (!resource) { + throw new Error(`Stripe SDK has no property "${segment}" when resolving path "${apiPath}"`) + } + } + return resource +} + +/** + * Check whether an API path can be resolved to a Stripe SDK resource. + * v1 requires both `.list()` and `.retrieve()`. + * v2 only requires `.list()` (retrieve may not be available on all v2 resources). + */ +export function canResolveSdkResource(stripe: Stripe, apiPath: string): boolean { + try { + const segments = pathToSdkSegments(apiPath) + const resource = resolveStripeResource(stripe, segments, apiPath) + if (isV2Path(apiPath)) { + return typeof resource.list === 'function' + } + return typeof resource.list === 'function' && typeof resource.retrieve === 'function' + } catch { + return false + } +} + +/** + * Build a callable list function by navigating the Stripe SDK object using + * the API path segments converted from snake_case to camelCase. + * Path parameters (e.g. `{customer}`) are stripped automatically. + */ +export function buildListFn(stripe: Stripe, apiPath: string, apiKey: string = ''): ListFn { + const v2 = isV2Path(apiPath) + if (v2) { + return buildV2ListFn(apiKey, apiPath) + } + const segments = pathToSdkSegments(apiPath) + return (params) => { + const resource = resolveStripeResource(stripe, segments, apiPath) + if (typeof resource.list !== 'function') { + throw new Error(`Stripe SDK resource at "${apiPath}" has no list() method`) + } + return resource.list(params) + } +} + +type RetrieveFn = (id: string) => Promise> + +/** + * Build a callable retrieve function by navigating the Stripe SDK object using + * the API path segments converted from snake_case to camelCase. + * Path parameters (e.g. `{customer}`) are stripped automatically. + */ +export function buildRetrieveFn(stripe: Stripe, apiPath: string, apiKey: string): RetrieveFn { + const v2 = isV2Path(apiPath) + if (v2) { + return buildV2RetrieveFn(apiKey, apiPath) + } + const segments = pathToSdkSegments(apiPath) + return (id: string) => { + const resource = resolveStripeResource(stripe, segments, apiPath) + if (typeof resource.retrieve !== 'function') { + throw new Error(`Stripe SDK resource at "${apiPath}" has no retrieve() method`) + } + return resource.retrieve(id) + } +} + +/** + * Build a list function that calls Stripe rawRequest directly for a fixed endpoint. + * Useful when the Stripe SDK does not expose a matching namespace. + */ +export function buildRawRequestListFn(stripe: Stripe, apiPath: string): ListFn { + return (params) => + stripe.rawRequest('GET', apiPath, { ...params }) as unknown as Promise<{ + data: unknown[] + has_more: boolean + }> +} + +function extractPageToken(nextPageUrl: string | null | undefined): string | undefined { + if (!nextPageUrl) return undefined + try { + const url = new URL(nextPageUrl, 'https://api.stripe.com') + return url.searchParams.get('page') ?? undefined + } catch { + return undefined + } +} + +/** + * Build a list function for v2 API endpoints. + * V2 uses `page` token pagination and returns `next_page_url` instead of `has_more`. + * The response is normalized to the v1 shape so the sync worker can process it uniformly. + */ +export function buildV2ListFn(apiKey: string, apiPath: string): ListFn { + return async (params) => { + const qs = new URLSearchParams() + qs.set('limit', String(Math.min(params.limit ?? 20, 20))) + if (params.starting_after) qs.set('page', params.starting_after) + const url = `https://api.stripe.com${apiPath}?${qs.toString()}` + + const response = await fetch(url, { + headers: { + Authorization: `Bearer ${apiKey}`, + 'Stripe-Version': '2026-02-25.clover', + }, + }) + + const raw = await response.text() + + const body = JSON.parse(raw) as { + data: unknown[] + next_page_url?: string | null + } + const nextToken = extractPageToken(body.next_page_url) + return { + data: body.data ?? [], + has_more: !!body.next_page_url, + pageCursor: nextToken, + } + } +} + +export function buildV2RetrieveFn(apiKey: string, apiPath: string): RetrieveFn { + return async (id: string) => { + const response = await fetch(`https://api.stripe.com${apiPath}/${id}`, { + headers: { + Authorization: `Bearer ${apiKey}`, + 'Stripe-Version': '2026-02-25.clover', + }, + }) + return (await response.json()) as Stripe.Response + } +} diff --git a/packages/openapi/package.json b/packages/openapi/package.json new file mode 100644 index 000000000..aef37391a --- /dev/null +++ b/packages/openapi/package.json @@ -0,0 +1,28 @@ +{ + "name": "@stripe/openapi", + "version": "0.1.0", + "private": true, + "type": "module", + "main": "./dist/index.cjs", + "exports": { + ".": { + "types": "./index.ts", + "import": "./dist/index.js", + "require": "./dist/index.cjs" + } + }, + "scripts": { + "build": "tsup index.ts --format esm,cjs --dts", + "test": "vitest --passWithNoTests" + }, + "files": [ + "dist" + ], + "dependencies": { + "stripe": "^17.7.0" + }, + "devDependencies": { + "@types/node": "^24.5.0", + "vitest": "^3.2.4" + } +} diff --git a/packages/openapi/postgresAdapter.ts b/packages/openapi/postgresAdapter.ts new file mode 100644 index 000000000..bbb38cf45 --- /dev/null +++ b/packages/openapi/postgresAdapter.ts @@ -0,0 +1,126 @@ +import { Buffer } from 'node:buffer' +import { createHash } from 'node:crypto' +import type { DialectAdapter } from './dialectAdapter' +import type { ParsedColumn, ParsedResourceTable, ScalarType } from './types' + +const PG_IDENTIFIER_MAX_BYTES = 63 + +type PostgresAdapterOptions = { + schemaName?: string + /** Schema for _accounts table (FK target). Defaults to schemaName when not provided. */ + accountSchema?: string + materializeTemporalAsText?: boolean +} + +export class PostgresAdapter implements DialectAdapter { + private readonly schemaName: string + private readonly accountSchema: string + private readonly materializeTemporalAsText: boolean + + constructor(options: PostgresAdapterOptions = {}) { + this.schemaName = options.schemaName ?? 'stripe' + this.accountSchema = options.accountSchema ?? this.schemaName + this.materializeTemporalAsText = options.materializeTemporalAsText ?? true + } + + buildAllStatements(tables: ParsedResourceTable[]): string[] { + return [...tables] + .sort((a, b) => a.tableName.localeCompare(b.tableName)) + .flatMap((table) => this.buildTableStatements(table)) + } + + buildTableStatements(table: ParsedResourceTable): string[] { + const quotedSchema = this.quoteIdent(this.schemaName) + const quotedTable = this.quoteIdent(table.tableName) + const generatedColumns = table.columns.map((column) => this.buildGeneratedColumn(column)) + const generatedColumnAlters = generatedColumns.map( + (columnDef) => + `ALTER TABLE ${quotedSchema}.${quotedTable} ADD COLUMN IF NOT EXISTS ${columnDef};` + ) + const columnDefs = [ + '"_raw_data" jsonb NOT NULL', + '"_last_synced_at" timestamptz', + '"_updated_at" timestamptz NOT NULL DEFAULT now()', + '"_account_id" text NOT NULL', + '"id" text GENERATED ALWAYS AS ((_raw_data->>\'id\')::text) STORED', + ...generatedColumns, + 'PRIMARY KEY ("id")', + ] + + const fkName = this.safeIdentifier(`fk_${table.tableName}_account`) + const accountIdxName = this.safeIdentifier(`idx_${table.tableName}_account_id`) + const quotedAccountSchema = this.quoteIdent(this.accountSchema) + + return [ + `CREATE TABLE ${quotedSchema}.${quotedTable} (\n ${columnDefs.join(',\n ')}\n);`, + ...generatedColumnAlters, + `ALTER TABLE ${quotedSchema}.${quotedTable} ADD CONSTRAINT ${this.quoteIdent( + fkName + )} FOREIGN KEY ("_account_id") REFERENCES ${quotedAccountSchema}."_accounts" (id);`, + `CREATE INDEX ${this.quoteIdent(accountIdxName)} ON ${quotedSchema}.${quotedTable} ("_account_id");`, + `DROP TRIGGER IF EXISTS handle_updated_at ON ${quotedSchema}.${quotedTable};`, + `CREATE TRIGGER handle_updated_at BEFORE UPDATE ON ${quotedSchema}.${quotedTable} FOR EACH ROW EXECUTE FUNCTION set_updated_at();`, + ] + } + + private buildGeneratedColumn(column: ParsedColumn): string { + const forceReferenceText = column.expandableReference === true + const pgType = forceReferenceText ? 'text' : this.pgType(column.type) + const escapedPath = column.name.replace(/'/g, "''") + const expression = forceReferenceText + ? this.buildExpandableReferenceTextExpression(escapedPath) + : pgType === 'jsonb' + ? `(_raw_data->'${escapedPath}')::jsonb` + : pgType === 'text' + ? `(_raw_data->>'${escapedPath}')::text` + : `(NULLIF(_raw_data->>'${escapedPath}', ''))::${pgType}` + + return `${this.quoteIdent(column.name)} ${pgType} GENERATED ALWAYS AS (${expression}) STORED` + } + + private buildExpandableReferenceTextExpression(escapedPath: string): string { + const jsonPath = `_raw_data->'${escapedPath}'` + return `CASE + WHEN jsonb_typeof(${jsonPath}) = 'object' AND ${jsonPath} ? 'id' + THEN (${jsonPath}->>'id') + ELSE (_raw_data->>'${escapedPath}') + END` + } + + private pgType(type: ScalarType): string { + if (type === 'timestamptz' && this.materializeTemporalAsText) { + return 'text' + } + + switch (type) { + case 'text': + return 'text' + case 'boolean': + return 'boolean' + case 'bigint': + return 'bigint' + case 'numeric': + return 'numeric' + case 'json': + return 'jsonb' + case 'timestamptz': + return 'timestamptz' + } + } + + private quoteIdent(value: string): string { + return `"${value.replaceAll('"', '""')}"` + } + + private safeIdentifier(name: string): string { + if (Buffer.byteLength(name) <= PG_IDENTIFIER_MAX_BYTES) { + return name + } + + const hash = createHash('sha1').update(name).digest('hex').slice(0, 8) + const suffix = `_h${hash}` + const maxBaseBytes = PG_IDENTIFIER_MAX_BYTES - Buffer.byteLength(suffix) + const truncatedBase = Buffer.from(name).subarray(0, maxBaseBytes).toString('utf8') + return `${truncatedBase}${suffix}` + } +} diff --git a/packages/openapi/runtimeMappings.ts b/packages/openapi/runtimeMappings.ts new file mode 100644 index 000000000..b88c69c60 --- /dev/null +++ b/packages/openapi/runtimeMappings.ts @@ -0,0 +1,74 @@ +import type { ParsedColumn } from './types' + +/** + * Table names that must exist for runtime sync and webhook processing. + * Includes all default sync objects plus child tables. + */ +export const RUNTIME_REQUIRED_TABLES: ReadonlyArray = [ + 'products', + 'coupons', + 'prices', + 'plans', + 'customers', + 'subscriptions', + 'subscription_schedules', + 'invoices', + 'charges', + 'setup_intents', + 'payment_methods', + 'payment_intents', + 'tax_ids', + 'credit_notes', + 'disputes', + 'early_fraud_warnings', + 'refunds', + 'checkout_sessions', + 'subscription_items', + 'checkout_session_line_items', + 'features', +] + +/** + * Overrides for x-resourceId values whose table name cannot be inferred by the + * default pluralisation / dot-to-underscore rule in SpecParser.resolveTableName. + */ +export const OPENAPI_RESOURCE_TABLE_ALIASES: Record = { + 'radar.early_fraud_warning': 'early_fraud_warnings', + 'entitlements.active_entitlement': 'active_entitlements', + 'entitlements.feature': 'features', + item: 'checkout_session_line_items', +} + +/** + * Compatibility columns that should exist even if not present in the current OpenAPI shape. + * This preserves backwards compatibility for existing queries and write paths. + * todo: Remove this + */ +export const OPENAPI_COMPATIBILITY_COLUMNS: Record = { + active_entitlements: [ + { name: 'customer', type: 'text', nullable: true }, + { name: 'object', type: 'text', nullable: true }, + { name: 'feature', type: 'text', nullable: true }, + { name: 'livemode', type: 'boolean', nullable: true }, + { name: 'lookup_key', type: 'text', nullable: true }, + ], + checkout_session_line_items: [ + { name: 'checkout_session', type: 'text', nullable: true }, + { name: 'amount_discount', type: 'bigint', nullable: true }, + { name: 'amount_tax', type: 'bigint', nullable: true }, + ], + customers: [{ name: 'deleted', type: 'boolean', nullable: true }], + early_fraud_warnings: [{ name: 'payment_intent', type: 'text', nullable: true }], + features: [ + { name: 'object', type: 'text', nullable: true }, + { name: 'name', type: 'text', nullable: true }, + { name: 'lookup_key', type: 'text', nullable: true }, + { name: 'active', type: 'boolean', nullable: true }, + { name: 'livemode', type: 'boolean', nullable: true }, + { name: 'metadata', type: 'json', nullable: true }, + ], + subscription_items: [ + { name: 'deleted', type: 'boolean', nullable: true }, + { name: 'subscription', type: 'text', nullable: true }, + ], +} diff --git a/packages/openapi/specFetchHelper.ts b/packages/openapi/specFetchHelper.ts new file mode 100644 index 000000000..d3aafbe25 --- /dev/null +++ b/packages/openapi/specFetchHelper.ts @@ -0,0 +1,181 @@ +import os from 'node:os' +import fs from 'node:fs/promises' +import path from 'node:path' +import type { OpenApiSpec, ResolveSpecConfig, ResolvedOpenApiSpec } from './types' + +const DEFAULT_CACHE_DIR = path.join(os.tmpdir(), 'stripe-sync-openapi-cache') + +export async function resolveOpenApiSpec(config: ResolveSpecConfig): Promise { + const apiVersion = config.apiVersion + if (!apiVersion || !/^\d{4}-\d{2}-\d{2}(\.\w+)?$/.test(apiVersion)) { + throw new Error( + `Invalid Stripe API version "${apiVersion}". Expected YYYY-MM-DD or YYYY-MM-DD.codename.` + ) + } + + if (config.openApiSpecPath) { + const explicitSpec = await readSpecFromPath(config.openApiSpecPath) + return { + apiVersion, + spec: explicitSpec, + source: 'explicit_path', + cachePath: config.openApiSpecPath, + } + } + + const cacheDir = config.cacheDir ?? DEFAULT_CACHE_DIR + const cachePath = getCachePath(cacheDir, apiVersion) + const cachedSpec = await tryReadCachedSpec(cachePath) + if (cachedSpec) { + return { + apiVersion, + spec: cachedSpec, + source: 'cache', + cachePath, + } + } + + let commitSha = await resolveCommitShaForApiVersion(apiVersion) + if (!commitSha) { + commitSha = await resolveLatestCommitSha() + } + if (!commitSha) { + throw new Error( + `Could not resolve Stripe OpenAPI commit for API version ${apiVersion} and no local spec path was provided.` + ) + } + + const spec = await fetchSpecForCommit(commitSha) + validateOpenApiSpec(spec) + await tryWriteCache(cachePath, spec) + + return { + apiVersion, + spec, + source: 'github', + cachePath, + commitSha, + } +} + +async function readSpecFromPath(openApiSpecPath: string): Promise { + const raw = await fs.readFile(openApiSpecPath, 'utf8') + let parsed: unknown + try { + parsed = JSON.parse(raw) + } catch (error) { + throw new Error( + `Failed to parse OpenAPI spec at ${openApiSpecPath}: ${error instanceof Error ? error.message : String(error)}` + ) + } + validateOpenApiSpec(parsed) + return parsed +} + +async function tryReadCachedSpec(cachePath: string): Promise { + try { + const raw = await fs.readFile(cachePath, 'utf8') + const parsed = JSON.parse(raw) as unknown + validateOpenApiSpec(parsed) + return parsed + } catch { + return null + } +} + +async function tryWriteCache(cachePath: string, spec: OpenApiSpec): Promise { + try { + await fs.mkdir(path.dirname(cachePath), { recursive: true }) + await fs.writeFile(cachePath, JSON.stringify(spec), 'utf8') + } catch { + // Best effort only. Cache writes should never block migration flow. + } +} + +function getCachePath(cacheDir: string, apiVersion: string): string { + const safeVersion = apiVersion.replace(/[^0-9a-zA-Z_-]/g, '_') + return path.join(cacheDir, `${safeVersion}.spec3.sdk.json`) +} + +function extractDatePart(apiVersion: string): string { + const match = apiVersion.match(/^(\d{4}-\d{2}-\d{2})/) + return match ? match[1] : apiVersion +} + +async function resolveLatestCommitSha(): Promise { + const url = new URL('https://api.github.com/repos/stripe/openapi/commits') + url.searchParams.set('path', 'latest/openapi.spec3.sdk.json') + url.searchParams.set('per_page', '1') + + const response = await fetch(url, { headers: githubHeaders() }) + if (!response.ok) { + throw new Error( + `Failed to resolve latest Stripe OpenAPI commit (${response.status} ${response.statusText})` + ) + } + + const json = (await response.json()) as Array<{ sha?: string }> + const commitSha = json[0]?.sha + return typeof commitSha === 'string' && commitSha.length > 0 ? commitSha : null +} + +async function resolveCommitShaForApiVersion(apiVersion: string): Promise { + const until = `${extractDatePart(apiVersion)}T23:59:59Z` + const url = new URL('https://api.github.com/repos/stripe/openapi/commits') + url.searchParams.set('path', 'latest/openapi.spec3.sdk.json') + url.searchParams.set('until', until) + url.searchParams.set('per_page', '1') + + const response = await fetch(url, { headers: githubHeaders() }) + if (!response.ok) { + throw new Error( + `Failed to resolve Stripe OpenAPI commit (${response.status} ${response.statusText})` + ) + } + + const json = (await response.json()) as Array<{ sha?: string }> + const commitSha = json[0]?.sha + return typeof commitSha === 'string' && commitSha.length > 0 ? commitSha : null +} + +async function fetchSpecForCommit(commitSha: string): Promise { + const url = `https://raw.githubusercontent.com/stripe/openapi/${commitSha}/latest/openapi.spec3.sdk.json` + const response = await fetch(url, { headers: githubHeaders() }) + if (!response.ok) { + throw new Error( + `Failed to download Stripe OpenAPI spec for commit ${commitSha} (${response.status} ${response.statusText})` + ) + } + + const spec = (await response.json()) as unknown + validateOpenApiSpec(spec) + return spec +} + +function validateOpenApiSpec(spec: unknown): asserts spec is OpenApiSpec { + if (!spec || typeof spec !== 'object') { + throw new Error('OpenAPI spec is not an object') + } + const candidate = spec as Partial + if (typeof candidate.openapi !== 'string' || candidate.openapi.trim().length === 0) { + throw new Error('OpenAPI spec is missing the "openapi" field') + } + if (!candidate.components || typeof candidate.components !== 'object') { + throw new Error('OpenAPI spec is missing "components"') + } + if (!candidate.components.schemas || typeof candidate.components.schemas !== 'object') { + throw new Error('OpenAPI spec is missing "components.schemas"') + } +} + +function githubHeaders(): HeadersInit { + const headers: Record = { + Accept: 'application/vnd.github+json', + 'User-Agent': 'stripe-sync-engine-openapi', + } + const token = process.env.GITHUB_TOKEN || process.env.GH_TOKEN + if (token) { + headers.Authorization = `Bearer ${token}` + } + return headers +} diff --git a/packages/openapi/specParser.ts b/packages/openapi/specParser.ts new file mode 100644 index 000000000..2203d5bb5 --- /dev/null +++ b/packages/openapi/specParser.ts @@ -0,0 +1,422 @@ +import type { + OpenApiSchemaObject, + OpenApiSchemaOrReference, + OpenApiSpec, + ParseSpecOptions, + ParsedColumn, + ParsedOpenApiSpec, + ScalarType, +} from './types' +import { + RUNTIME_REQUIRED_TABLES, + OPENAPI_COMPATIBILITY_COLUMNS, + OPENAPI_RESOURCE_TABLE_ALIASES, +} from './runtimeMappings' + +const SCHEMA_REF_PREFIX = '#/components/schemas/' + +const RESERVED_COLUMNS = new Set([ + 'id', + '_raw_data', + '_last_synced_at', + '_updated_at', + '_account_id', +]) + +export { RUNTIME_REQUIRED_TABLES, OPENAPI_RESOURCE_TABLE_ALIASES } +/** @deprecated Use OPENAPI_RESOURCE_TABLE_ALIASES instead. */ +export const RUNTIME_RESOURCE_ALIASES = OPENAPI_RESOURCE_TABLE_ALIASES + +type ColumnAccumulator = { + type: ScalarType + nullable: boolean + expandableReference: boolean +} + +export class SpecParser { + parse(spec: OpenApiSpec, options: ParseSpecOptions = {}): ParsedOpenApiSpec { + const schemas = spec.components?.schemas + if (!schemas || typeof schemas !== 'object') { + throw new Error('OpenAPI spec is missing components.schemas') + } + + const aliases = { ...OPENAPI_RESOURCE_TABLE_ALIASES, ...(options.resourceAliases ?? {}) } + const excluded = new Set(options.excludedTables ?? []) + const allowedTables = options.allowedTables + ? new Set(options.allowedTables.filter((t) => !excluded.has(t))) + : this.discoverAllowedTables(spec, aliases, excluded) + const tableMap = new Map< + string, + { + resourceId: string + sourceSchemaName: string + columns: Map + } + >() + + for (const schemaName of Object.keys(schemas).sort((a, b) => a.localeCompare(b))) { + const schema = this.resolveSchema({ $ref: `#/components/schemas/${schemaName}` }, spec) + const resourceId = schema['x-resourceId'] + if (!resourceId || typeof resourceId !== 'string') { + continue + } + + const tableName = this.resolveTableName(resourceId, aliases) + if (!allowedTables.has(tableName)) { + continue + } + + const propCandidates = this.collectPropertyCandidates( + { $ref: `#/components/schemas/${schemaName}` }, + spec + ) + const parsedColumns = this.parseColumns(propCandidates, spec) + + const existing = + tableMap.get(tableName) ?? + ({ + resourceId, + sourceSchemaName: schemaName, + columns: new Map(), + } as const) + + for (const column of parsedColumns) { + const current = existing.columns.get(column.name) + if (!current) { + existing.columns.set(column.name, { + type: column.type, + nullable: column.nullable, + expandableReference: column.expandableReference ?? false, + }) + continue + } + existing.columns.set(column.name, { + type: this.mergeTypes(current.type, column.type), + nullable: current.nullable || column.nullable, + expandableReference: current.expandableReference || (column.expandableReference ?? false), + }) + } + + tableMap.set(tableName, existing) + } + + for (const tableName of Array.from(allowedTables).sort((a, b) => a.localeCompare(b))) { + const current = + tableMap.get(tableName) ?? + ({ + resourceId: tableName, + sourceSchemaName: 'compatibility_fallback', + columns: new Map(), + } as const) + for (const compatibilityColumn of OPENAPI_COMPATIBILITY_COLUMNS[tableName] ?? []) { + const existing = current.columns.get(compatibilityColumn.name) + if (!existing) { + current.columns.set(compatibilityColumn.name, { + type: compatibilityColumn.type, + nullable: compatibilityColumn.nullable, + expandableReference: compatibilityColumn.expandableReference ?? false, + }) + } + } + tableMap.set(tableName, current) + } + + const tables = Array.from(tableMap.entries()) + .sort(([a], [b]) => a.localeCompare(b)) + .map(([tableName, table]) => ({ + tableName, + resourceId: table.resourceId, + sourceSchemaName: table.sourceSchemaName, + columns: Array.from(table.columns.entries()) + .map(([name, value]) => ({ + name, + type: value.type, + nullable: value.nullable, + ...(value.expandableReference ? { expandableReference: true } : {}), + })) + .sort((a, b) => a.name.localeCompare(b.name)), + })) + + return { + apiVersion: spec.info?.version ?? spec.openapi ?? 'unknown', + tables, + } + } + + /** + * Scan the spec's `paths` for GET endpoints that return Stripe list objects, + * and resolve each listed resource's x-resourceId into a table name. + */ + private discoverAllowedTables( + spec: OpenApiSpec, + aliases: Record, + excluded: Set + ): Set { + const resourceIds = this.discoverListableResourceIds(spec, { + includeNested: true, + }) + const tables = new Set() + for (const resourceId of resourceIds) { + const tableName = this.resolveTableName(resourceId, aliases) + if (!excluded.has(tableName)) { + tables.add(tableName) + } + } + return tables + } + + /** + * Extract x-resourceId values for every schema that is returned by a list + * endpoint. Supports both v1 (object: "list") and v2 (next_page_url) formats. + */ + discoverListableResourceIds( + spec: OpenApiSpec, + options: { includeNested: boolean } = { includeNested: false } + ): Set { + const resourceIds = new Set() + const paths = spec.paths + if (!paths) { + return resourceIds + } + + for (const [apiPath, pathItem] of Object.entries(paths)) { + if (!options.includeNested && apiPath.includes('{')) continue + + const getOp = pathItem.get + if (!getOp?.responses) continue + + const responseSchema = getOp.responses['200']?.content?.['application/json']?.schema + if (!responseSchema) continue + + if (!this.isListResponseSchema(responseSchema)) continue + + const dataProp = responseSchema.properties?.data + if (!dataProp || !('type' in dataProp) || dataProp.type !== 'array') continue + + const itemsRef = dataProp.items + if (!itemsRef || !this.isReference(itemsRef)) continue + if (!itemsRef.$ref.startsWith(SCHEMA_REF_PREFIX)) continue + + const schemaName = itemsRef.$ref.slice(SCHEMA_REF_PREFIX.length) + const schema = spec.components?.schemas?.[schemaName] + if (!schema || '$ref' in schema) continue + + const resourceId = schema['x-resourceId'] + if (resourceId && typeof resourceId === 'string') { + resourceIds.add(resourceId) + } + } + + return resourceIds + } + + /** + * Detect whether a response schema describes a list endpoint. + * v1 lists have `object: enum ["list"]` with a `data` array. + * v2 lists have a `data` array with `next_page_url`. + */ + private isListResponseSchema(schema: OpenApiSchemaObject): boolean { + const dataProp = schema.properties?.data + if (!dataProp || !('type' in dataProp) || dataProp.type !== 'array') return false + + const objectProp = schema.properties?.object + if (objectProp && 'enum' in objectProp && objectProp.enum?.includes('list')) return true + + if (schema.properties?.next_page_url) return true + + return false + } + + private resolveTableName(resourceId: string, aliases: Record): string { + const alias = aliases[resourceId] + if (alias) { + return alias + } + + const normalized = resourceId.toLowerCase().replace(/[.]/g, '_') + return normalized.endsWith('s') ? normalized : `${normalized}s` + } + + private parseColumns( + propCandidates: Map, + spec: OpenApiSpec + ): ParsedColumn[] { + const columns: ParsedColumn[] = [] + for (const [propertyName, candidates] of Array.from(propCandidates.entries()).sort(([a], [b]) => + a.localeCompare(b) + )) { + if (RESERVED_COLUMNS.has(propertyName)) { + continue + } + const inferred = this.inferFromCandidates(candidates, spec) + columns.push({ + name: propertyName, + type: inferred.type, + nullable: inferred.nullable, + ...(inferred.expandableReference ? { expandableReference: true } : {}), + }) + } + return columns + } + + private inferFromCandidates( + candidates: OpenApiSchemaOrReference[], + spec: OpenApiSpec + ): { type: ScalarType; nullable: boolean; expandableReference: boolean } { + if (candidates.length === 0) { + return { type: 'text', nullable: true, expandableReference: false } + } + + let mergedType: ScalarType | null = null + let nullable = false + let expandableReference = false + for (const candidate of candidates) { + const inferred = this.inferType(candidate, spec) + mergedType = mergedType ? this.mergeTypes(mergedType, inferred.type) : inferred.type + nullable = nullable || inferred.nullable + expandableReference = + expandableReference || this.isExpandableReferenceCandidate(candidate, spec) + } + + return { type: mergedType ?? 'text', nullable, expandableReference } + } + + private mergeTypes(left: ScalarType, right: ScalarType): ScalarType { + if (left === right) return left + if (left === 'json' || right === 'json') return 'json' + if ((left === 'numeric' && right === 'bigint') || (left === 'bigint' && right === 'numeric')) { + return 'numeric' + } + if (left === 'timestamptz' && right === 'text') return 'text' + if (left === 'text' && right === 'timestamptz') return 'text' + return 'text' + } + + private inferType( + schemaOrRef: OpenApiSchemaOrReference, + spec: OpenApiSpec + ): { type: ScalarType; nullable: boolean } { + const schema = this.resolveSchema(schemaOrRef, spec) + const nullable = Boolean(schema.nullable) + + if (schema.oneOf?.length) { + const merged = this.inferFromCandidates(schema.oneOf, spec) + return { type: merged.type, nullable: nullable || merged.nullable } + } + if (schema.anyOf?.length) { + const merged = this.inferFromCandidates(schema.anyOf, spec) + return { type: merged.type, nullable: nullable || merged.nullable } + } + if (schema.allOf?.length) { + const merged = this.inferFromCandidates(schema.allOf, spec) + return { type: merged.type, nullable: nullable || merged.nullable } + } + + if (schema.type === 'boolean') return { type: 'boolean', nullable } + if (schema.type === 'integer') return { type: 'bigint', nullable } + if (schema.type === 'number') return { type: 'numeric', nullable } + if (schema.type === 'string') { + if (schema.format === 'date-time') { + return { type: 'timestamptz', nullable } + } + return { type: 'text', nullable } + } + if (schema.type === 'array') return { type: 'json', nullable } + if (schema.type === 'object') return { type: 'json', nullable } + if (schema.properties || schema.additionalProperties) return { type: 'json', nullable } + + if (schema.enum && schema.enum.length > 0) { + const values = schema.enum + if (values.every((value) => typeof value === 'boolean')) { + return { type: 'boolean', nullable } + } + if (values.every((value) => typeof value === 'number' && Number.isInteger(value))) { + return { type: 'bigint', nullable } + } + if (values.every((value) => typeof value === 'number')) { + return { type: 'numeric', nullable } + } + } + + return { type: 'text', nullable: true } + } + + private isExpandableReferenceCandidate( + schemaOrRef: OpenApiSchemaOrReference, + spec: OpenApiSpec + ): boolean { + const schema = this.resolveSchema(schemaOrRef, spec) + return Boolean(schema['x-expansionResources']) + } + + private collectPropertyCandidates( + schemaOrRef: OpenApiSchemaOrReference, + spec: OpenApiSpec, + seenRefs = new Set(), + seenSchemas = new Set() + ): Map { + if (this.isReference(schemaOrRef)) { + if (seenRefs.has(schemaOrRef.$ref)) { + return new Map() + } + seenRefs.add(schemaOrRef.$ref) + } + + const schema = this.resolveSchema(schemaOrRef, spec) + if (seenSchemas.has(schema)) { + return new Map() + } + seenSchemas.add(schema) + + const merged = new Map() + const pushProp = (name: string, value: OpenApiSchemaOrReference) => { + const existing = merged.get(name) ?? [] + existing.push(value) + merged.set(name, existing) + } + + for (const [name, value] of Object.entries(schema.properties ?? {})) { + pushProp(name, value) + } + + for (const composed of [schema.allOf, schema.oneOf, schema.anyOf]) { + if (!composed) continue + for (const subSchema of composed) { + const subProps = this.collectPropertyCandidates(subSchema, spec, seenRefs, seenSchemas) + for (const [name, candidates] of subProps.entries()) { + for (const candidate of candidates) { + pushProp(name, candidate) + } + } + } + } + + return merged + } + + private resolveSchema( + schemaOrRef: OpenApiSchemaOrReference, + spec: OpenApiSpec + ): OpenApiSchemaObject { + if (!this.isReference(schemaOrRef)) { + return schemaOrRef + } + + if (!schemaOrRef.$ref.startsWith(SCHEMA_REF_PREFIX)) { + throw new Error(`Unsupported OpenAPI reference: ${schemaOrRef.$ref}`) + } + const schemaName = schemaOrRef.$ref.slice(SCHEMA_REF_PREFIX.length) + const resolved = spec.components?.schemas?.[schemaName] + if (!resolved) { + throw new Error(`Failed to resolve OpenAPI schema reference: ${schemaOrRef.$ref}`) + } + if (this.isReference(resolved)) { + return this.resolveSchema(resolved, spec) + } + return resolved + } + + private isReference(schemaOrRef: OpenApiSchemaOrReference): schemaOrRef is { $ref: string } { + return typeof (schemaOrRef as { $ref?: string }).$ref === 'string' + } +} diff --git a/packages/openapi/tsconfig.json b/packages/openapi/tsconfig.json new file mode 100644 index 000000000..df859faad --- /dev/null +++ b/packages/openapi/tsconfig.json @@ -0,0 +1,19 @@ +{ + "compilerOptions": { + "target": "esnext", + "module": "esnext", + "moduleResolution": "bundler", + "declaration": true, + "outDir": "dist", + "esModuleInterop": true, + "skipLibCheck": true, + "strict": true, + "resolveJsonModule": true + }, + "include": [ + "*.ts" + ], + "exclude": [ + "__tests__" + ] +} \ No newline at end of file diff --git a/packages/openapi/types.ts b/packages/openapi/types.ts new file mode 100644 index 000000000..14c5657ed --- /dev/null +++ b/packages/openapi/types.ts @@ -0,0 +1,118 @@ +export type OpenApiSchemaObject = { + type?: string + format?: string + nullable?: boolean + properties?: Record + items?: OpenApiSchemaOrReference + oneOf?: OpenApiSchemaOrReference[] + anyOf?: OpenApiSchemaOrReference[] + allOf?: OpenApiSchemaOrReference[] + enum?: unknown[] + additionalProperties?: boolean | OpenApiSchemaOrReference + 'x-resourceId'?: string + 'x-expandableFields'?: string[] + 'x-expansionResources'?: { + oneOf?: OpenApiSchemaOrReference[] + } +} + +export type OpenApiReferenceObject = { + $ref: string +} + +export type OpenApiSchemaOrReference = OpenApiSchemaObject | OpenApiReferenceObject + +export type OpenApiResponseContent = { + schema?: OpenApiSchemaObject +} + +export type OpenApiResponse = { + content?: { + 'application/json'?: OpenApiResponseContent + } +} + +export type OpenApiOperationObject = { + operationId?: string + parameters?: { + name?: string + in?: string + required?: boolean + schema?: OpenApiSchemaOrReference + }[] + responses?: Record +} + +export type OpenApiPathItem = Record + +export type OpenApiSpec = { + openapi: string + info?: { + version?: string + } + paths?: Record + components?: { + schemas?: Record + } +} + +export type ScalarType = 'text' | 'boolean' | 'bigint' | 'numeric' | 'json' | 'timestamptz' + +export type ParsedColumn = { + name: string + type: ScalarType + nullable: boolean + expandableReference?: boolean +} + +export type ParsedResourceTable = { + tableName: string + resourceId: string + sourceSchemaName: string + columns: ParsedColumn[] +} + +export type ParsedOpenApiSpec = { + apiVersion: string + tables: ParsedResourceTable[] +} + +export type ParseSpecOptions = { + /** + * Map Stripe x-resourceId values to concrete Postgres table names. + * Entries are matched case-sensitively. + */ + resourceAliases?: Record + /** + * Restrict parsing to these table names. + * If omitted, listable resources are discovered from the spec's paths. + */ + allowedTables?: string[] + /** + * Table names to exclude from parsing, even if discovered or allowed. + * Used to avoid collisions with tables managed outside of the OpenAPI adapter + * (e.g. the bootstrap `_accounts` table). + */ + excludedTables?: string[] +} + +export type ResolveSpecConfig = { + apiVersion: string + openApiSpecPath?: string + cacheDir?: string +} + +export type ResolvedOpenApiSpec = { + apiVersion: string + spec: OpenApiSpec + source: 'explicit_path' | 'cache' | 'github' + cachePath?: string + commitSha?: string +} + +export type WritePlan = { + tableName: string + conflictTarget: string[] + extraColumns: Array<{ column: string; pgType: string; entryKey: string }> + metadataColumns: ['_raw_data', '_last_synced_at', '_account_id'] +} diff --git a/packages/openapi/writePathPlanner.ts b/packages/openapi/writePathPlanner.ts new file mode 100644 index 000000000..3121a5d3c --- /dev/null +++ b/packages/openapi/writePathPlanner.ts @@ -0,0 +1,43 @@ +import type { ParsedResourceTable, ScalarType, WritePlan } from './types' + +type TableWritePlan = WritePlan & { + generatedColumns: Array<{ column: string; pgType: string }> +} + +export class WritePathPlanner { + buildPlans(tables: ParsedResourceTable[]): TableWritePlan[] { + return [...tables] + .sort((a, b) => a.tableName.localeCompare(b.tableName)) + .map((table) => this.buildPlan(table)) + } + + buildPlan(table: ParsedResourceTable): TableWritePlan { + return { + tableName: table.tableName, + conflictTarget: ['id'], + extraColumns: [], + metadataColumns: ['_raw_data', '_last_synced_at', '_account_id'], + generatedColumns: table.columns + .map((column) => ({ column: column.name, pgType: this.scalarTypeToPgType(column.type) })) + .sort((a, b) => a.column.localeCompare(b.column)), + } + } + + private scalarTypeToPgType(type: ScalarType): string { + switch (type) { + case 'boolean': + return 'boolean' + case 'bigint': + return 'bigint' + case 'numeric': + return 'numeric' + case 'json': + return 'jsonb' + case 'timestamptz': + return 'timestamptz' + case 'text': + default: + return 'text' + } + } +} From 7d788992c24dd7f0320ef22fd6a08cf2b61f1e11 Mon Sep 17 00:00:00 2001 From: Yostra Date: Tue, 24 Mar 2026 01:55:54 +0100 Subject: [PATCH 03/19] glue source-stripe --- packages/openapi/index.ts | 22 +- packages/source-stripe/package.json | 1 + packages/source-stripe/src/catalog.ts | 4 +- packages/source-stripe/src/index.ts | 11 +- .../source-stripe/src/resourceRegistry.ts | 479 ++++++------------ packages/source-stripe/src/src-list-api.ts | 36 +- packages/source-stripe/src/types.ts | 21 +- 7 files changed, 215 insertions(+), 359 deletions(-) diff --git a/packages/openapi/index.ts b/packages/openapi/index.ts index 78569f356..d8300b9da 100644 --- a/packages/openapi/index.ts +++ b/packages/openapi/index.ts @@ -1,14 +1,14 @@ -export type * from './types' +export type * from './types.js' export { SpecParser, OPENAPI_RESOURCE_TABLE_ALIASES, RUNTIME_RESOURCE_ALIASES, -} from './specParser' -export { OPENAPI_COMPATIBILITY_COLUMNS } from './runtimeMappings' -export { PostgresAdapter } from './postgresAdapter' -export { WritePathPlanner } from './writePathPlanner' -export { resolveOpenApiSpec } from './specFetchHelper' -export type { DialectAdapter } from './dialectAdapter' +} from './specParser.js' +export { OPENAPI_COMPATIBILITY_COLUMNS } from './runtimeMappings.js' +export { PostgresAdapter } from './postgresAdapter.js' +export { WritePathPlanner } from './writePathPlanner.js' +export { resolveOpenApiSpec } from './specFetchHelper.js' +export type { DialectAdapter } from './dialectAdapter.js' export { buildListFn, buildRetrieveFn, @@ -18,7 +18,7 @@ export { discoverNestedEndpoints, canResolveSdkResource, isV2Path, -} from './listFnResolver' -export type { NestedEndpoint } from './listFnResolver' -export { parsedTableToJsonSchema } from './jsonSchemaConverter' -export { RUNTIME_REQUIRED_TABLES } from './runtimeMappings' +} from './listFnResolver.js' +export type { NestedEndpoint } from './listFnResolver.js' +export { parsedTableToJsonSchema } from './jsonSchemaConverter.js' +export { RUNTIME_REQUIRED_TABLES } from './runtimeMappings.js' diff --git a/packages/source-stripe/package.json b/packages/source-stripe/package.json index 25dd61ea8..8c0804be6 100644 --- a/packages/source-stripe/package.json +++ b/packages/source-stripe/package.json @@ -27,6 +27,7 @@ "src" ], "dependencies": { + "@stripe/openapi": "workspace:*", "@stripe/sync-protocol": "workspace:*", "stripe": "^17.7.0", "ws": "^8.18.0", diff --git a/packages/source-stripe/src/catalog.ts b/packages/source-stripe/src/catalog.ts index d754968c9..e915fb617 100644 --- a/packages/source-stripe/src/catalog.ts +++ b/packages/source-stripe/src/catalog.ts @@ -1,7 +1,7 @@ import type { CatalogMessage, Stream } from '@stripe/sync-protocol' import type { ResourceConfig } from './types.js' -import type { ParsedResourceTable } from './openapi/types.js' -import { parsedTableToJsonSchema } from './openapi/jsonSchemaConverter.js' +import type { ParsedResourceTable } from '@stripe/openapi' +import { parsedTableToJsonSchema } from '@stripe/openapi' /** Derive a CatalogMessage from the existing resource registry (no json_schema). */ export function catalogFromRegistry(registry: Record): CatalogMessage { diff --git a/packages/source-stripe/src/index.ts b/packages/source-stripe/src/index.ts index 065e3fb6e..b749c35a6 100644 --- a/packages/source-stripe/src/index.ts +++ b/packages/source-stripe/src/index.ts @@ -119,9 +119,11 @@ const source = { }, async discover({ config }) { - const registry = buildResourceRegistry(makeClient(config)) + const resolved = await resolveOpenApiSpec({ + apiVersion: config.api_version ?? '2020-08-27', + }) + const registry = buildResourceRegistry(makeClient(config), resolved.spec, config.api_key) try { - const resolved = await resolveOpenApiSpec({ apiVersion: '2020-08-27' }) const parser = new SpecParser() const parsed = parser.parse(resolved.spec, { resourceAliases: OPENAPI_RESOURCE_TABLE_ALIASES, @@ -172,8 +174,11 @@ const source = { }, async *read({ config, catalog, state }, $stdin?) { - const registry = buildResourceRegistry(makeClient(config)) const stripe = makeClient(config) + const resolved = await resolveOpenApiSpec({ + apiVersion: config.api_version ?? '2020-08-27', + }) + const registry = buildResourceRegistry(stripe, resolved.spec, config.api_key) const streamNames = new Set(catalog.streams.map((s) => s.stream.name)) // Event-driven mode: iterate over incoming webhook inputs diff --git a/packages/source-stripe/src/resourceRegistry.ts b/packages/source-stripe/src/resourceRegistry.ts index f16bd898b..43e5cdbee 100644 --- a/packages/source-stripe/src/resourceRegistry.ts +++ b/packages/source-stripe/src/resourceRegistry.ts @@ -1,367 +1,178 @@ import Stripe from 'stripe' -import type { ResourceConfig, StripeListResourceConfig } from './types.js' +import type { ResourceConfig } from './types.js' +import type { OpenApiSpec, NestedEndpoint } from '@stripe/openapi' +import { + discoverListEndpoints, + discoverNestedEndpoints, + buildListFn, + buildRetrieveFn, + canResolveSdkResource, + isV2Path, + RUNTIME_REQUIRED_TABLES as OPENAPI_RUNTIME_REQUIRED_TABLES, +} from '@stripe/openapi' + +/** + * The default set of table names synced when no explicit selection is made. + * These correspond to the resources that were previously hardcoded with sync: true. + */ +export const DEFAULT_SYNC_OBJECTS: readonly string[] = [ + 'products', + 'coupons', + 'prices', + 'plans', + 'customers', + 'subscriptions', + 'subscription_schedules', + 'invoices', + 'charges', + 'setup_intents', + 'payment_methods', + 'payment_intents', + 'tax_ids', + 'credit_notes', + 'disputes', + 'early_fraud_warnings', + 'refunds', + 'checkout_sessions', +] + +export type StripeObject = string + +export const CORE_SYNC_OBJECTS = DEFAULT_SYNC_OBJECTS as readonly string[] + +export type CoreSyncObject = string + +export const SYNC_OBJECTS = ['all', 'customer_with_entitlements', ...DEFAULT_SYNC_OBJECTS] as const + +export type SyncObjectName = string -interface ResourceDef { - /** Backfill sequencing order. Lower numbers sync first, ensuring dependencies are populated before dependents. */ - readonly order: number - /** Destination table name (e.g. 'payment_intents'). */ - readonly tableName: string - /** Resource keys that must be backfilled before this one (e.g. ['customer', 'invoice']). */ - readonly dependencies?: readonly string[] - /** Curried list endpoint: `list(stripe)(params) → page`. */ - readonly list: ( - s: Stripe - ) => ( - p: Stripe.PaginationParams & { created?: Stripe.RangeQueryParam } - ) => Promise<{ data: unknown[]; has_more: boolean }> - /** Curried retrieve endpoint: `retrieve(stripe)(id) → object`. */ - readonly retrieve: (s: Stripe) => (id: string) => Promise> - /** Whether the list API accepts a `created` range filter for incremental backfill. */ - readonly supportsCreatedFilter: boolean - /** Whether this resource is included in a default full sync. */ - readonly sync: boolean - /** Returns true when the object has reached a terminal state and won't change again. */ - // eslint-disable-next-line @typescript-eslint/no-explicit-any - readonly isFinalState?: (entity: any) => boolean - /** Tables created from nested data (like line items) that don't have their own top-level list API. */ - readonly childTables?: readonly string[] - /** Sub-list expansions fetched per parent object (e.g. subscription items, invoice line items). */ - readonly listExpands?: readonly Record< - string, - (s: Stripe) => (id: string) => Promise> - >[] -} - -const RESOURCE_MAP: Record = { - product: { - order: 1, - tableName: 'products', - list: (s) => (p) => s.products.list(p), - retrieve: (s) => (id) => s.products.retrieve(id), - supportsCreatedFilter: true, - sync: true, - }, - coupon: { - order: 1, - tableName: 'coupons', - list: (s) => (p) => s.coupons.list(p), - retrieve: (s) => (id) => s.coupons.retrieve(id), - supportsCreatedFilter: true, - sync: true, - isFinalState: (c: Stripe.Coupon | Stripe.DeletedCoupon) => 'deleted' in c && c.deleted === true, - }, - price: { - order: 2, - tableName: 'prices', - dependencies: ['product'], - list: (s) => (p) => s.prices.list(p), - retrieve: (s) => (id) => s.prices.retrieve(id), - supportsCreatedFilter: true, - sync: true, - }, - plan: { - order: 3, - tableName: 'plans', - dependencies: ['product'], - list: (s) => (p) => s.plans.list(p), - retrieve: (s) => (id) => s.plans.retrieve(id), - supportsCreatedFilter: true, - sync: true, - }, - customer: { - order: 4, - tableName: 'customers', - list: (s) => (p) => s.customers.list(p), - retrieve: (s) => (id) => s.customers.retrieve(id), - supportsCreatedFilter: true, - sync: true, - isFinalState: (c: Stripe.Customer | Stripe.DeletedCustomer) => - 'deleted' in c && c.deleted === true, - }, - subscription: { - order: 5, - tableName: 'subscriptions', - dependencies: ['customer', 'price'], - list: (s) => (p) => s.subscriptions.list(p), - retrieve: (s) => (id) => s.subscriptions.retrieve(id), - listExpands: [ - { items: (s) => (id) => s.subscriptionItems.list({ subscription: id, limit: 100 }) }, - ], - supportsCreatedFilter: true, - sync: true, - childTables: ['subscription_items'], - isFinalState: (s: Stripe.Subscription) => - s.status === 'canceled' || s.status === 'incomplete_expired', - }, - subscription_schedules: { - order: 6, - tableName: 'subscription_schedules', - dependencies: ['customer'], - list: (s) => (p) => s.subscriptionSchedules.list(p), - retrieve: (s) => (id) => s.subscriptionSchedules.retrieve(id), - supportsCreatedFilter: true, - sync: true, - isFinalState: (s: Stripe.SubscriptionSchedule) => - s.status === 'canceled' || s.status === 'completed', - }, - invoice: { - order: 7, - tableName: 'invoices', - dependencies: ['customer', 'subscription'], - list: (s) => (p) => s.invoices.list(p), - retrieve: (s) => (id) => s.invoices.retrieve(id), - listExpands: [{ lines: (s) => (id) => s.invoices.listLineItems(id, { limit: 100 }) }], - supportsCreatedFilter: true, - sync: true, - isFinalState: (i: Stripe.Invoice) => i.status === 'void', - }, - charge: { - order: 8, - tableName: 'charges', - dependencies: ['customer', 'invoice'], - list: (s) => (p) => s.charges.list(p), - retrieve: (s) => (id) => s.charges.retrieve(id), - listExpands: [{ refunds: (s) => (id) => s.refunds.list({ charge: id, limit: 100 }) }], - supportsCreatedFilter: true, - sync: true, - isFinalState: (c: Stripe.Charge) => c.status === 'failed' || c.status === 'succeeded', - }, - setup_intent: { - order: 9, - tableName: 'setup_intents', - dependencies: ['customer'], - list: (s) => (p) => s.setupIntents.list(p), - retrieve: (s) => (id) => s.setupIntents.retrieve(id), - supportsCreatedFilter: true, - sync: true, - isFinalState: (s: Stripe.SetupIntent) => s.status === 'canceled' || s.status === 'succeeded', - }, - payment_method: { - order: 10, - tableName: 'payment_methods', - dependencies: ['customer'], - list: (s) => (p) => s.paymentMethods.list(p), - retrieve: (s) => (id) => s.paymentMethods.retrieve(id), - supportsCreatedFilter: false, - sync: true, - }, - payment_intent: { - order: 11, - tableName: 'payment_intents', - dependencies: ['customer', 'invoice'], - list: (s) => (p) => s.paymentIntents.list(p), - retrieve: (s) => (id) => s.paymentIntents.retrieve(id), - supportsCreatedFilter: true, - sync: true, - isFinalState: (p: Stripe.PaymentIntent) => p.status === 'canceled' || p.status === 'succeeded', - }, - tax_id: { - order: 12, - tableName: 'tax_ids', - dependencies: ['customer'], - list: (s) => (p) => s.taxIds.list(p), - retrieve: (s) => (id) => s.taxIds.retrieve(id), - supportsCreatedFilter: false, - sync: true, - }, - credit_note: { - order: 13, - tableName: 'credit_notes', - dependencies: ['customer', 'invoice'], - list: (s) => (p) => s.creditNotes.list(p), - retrieve: (s) => (id) => s.creditNotes.retrieve(id), - listExpands: [{ lines: (s) => (id) => s.creditNotes.listLineItems(id, { limit: 100 }) }], - supportsCreatedFilter: true, - sync: true, - isFinalState: (c: Stripe.CreditNote) => c.status === 'void', - }, - dispute: { - order: 14, - tableName: 'disputes', - dependencies: ['charge'], - list: (s) => (p) => s.disputes.list(p), - retrieve: (s) => (id) => s.disputes.retrieve(id), - supportsCreatedFilter: true, - sync: true, - isFinalState: (d: Stripe.Dispute) => d.status === 'won' || d.status === 'lost', - }, - early_fraud_warning: { - order: 15, - tableName: 'early_fraud_warnings', - dependencies: ['payment_intent', 'charge'], - list: (s) => (p) => s.radar.earlyFraudWarnings.list(p), - retrieve: (s) => (id) => s.radar.earlyFraudWarnings.retrieve(id), - supportsCreatedFilter: true, - sync: true, - }, - refund: { - order: 16, - tableName: 'refunds', - dependencies: ['payment_intent', 'charge'], - list: (s) => (p) => s.refunds.list(p), - retrieve: (s) => (id) => s.refunds.retrieve(id), - supportsCreatedFilter: true, - sync: true, - }, - checkout_sessions: { - order: 17, - tableName: 'checkout_sessions', - dependencies: ['customer', 'subscription', 'payment_intent', 'invoice'], - list: (s) => (p) => s.checkout.sessions.list(p), - retrieve: (s) => (id) => s.checkout.sessions.retrieve(id), - listExpands: [{ lines: (s) => (id) => s.checkout.sessions.listLineItems(id, { limit: 100 }) }], - supportsCreatedFilter: true, - sync: true, - childTables: ['checkout_session_line_items'], - }, - active_entitlements: { - order: 18, - tableName: 'active_entitlements', - dependencies: ['customer'], - list: (s) => (p) => - s.entitlements.activeEntitlements.list(p as Stripe.Entitlements.ActiveEntitlementListParams), - retrieve: (s) => (id) => s.entitlements.activeEntitlements.retrieve(id), - supportsCreatedFilter: true, - sync: false, - }, - review: { - order: 19, - tableName: 'reviews', - dependencies: ['payment_intent', 'charge'], - list: (s) => (p) => s.reviews.list(p), - retrieve: (s) => (id) => s.reviews.retrieve(id), - supportsCreatedFilter: true, - sync: false, - }, -} satisfies Record - -// Union of all object keys defined in RESOURCE_MAP. Used as the canonical object-name type across sync and registry helpers. -export type StripeObject = keyof typeof RESOURCE_MAP - -// Sync-enabled objects derived from RESOURCE_MAP metadata. -// Used for default full-sync selection and SyncObjectName composition. -export const CORE_SYNC_OBJECTS = Object.keys(RESOURCE_MAP).filter( - (k) => RESOURCE_MAP[k].sync -) as StripeObject[] - -// Type for one sync-enabled object key (excludes pseudo objects). -// Used where callers must operate on concrete sync resources only. -export type CoreSyncObject = (typeof CORE_SYNC_OBJECTS)[number] - -// Public sync object options including pseudo entries like "all". -// Used by sync input typing/validation for object selection. -export const SYNC_OBJECTS = ['all', 'customer_with_entitlements', ...CORE_SYNC_OBJECTS] as const - -// Type of valid sync object input values. -// Used by exported config/types and CLI/object selection paths. -export type SyncObjectName = (typeof SYNC_OBJECTS)[number] - -// Entity names accepted for webhook revalidation overrides. -// Used by StripeSyncConfig.revalidateObjectsViaStripeApi typing. export const REVALIDATE_ENTITIES = [ - ...Object.keys(RESOURCE_MAP), + ...DEFAULT_SYNC_OBJECTS, 'radar.early_fraud_warning', 'subscription_schedule', 'entitlements', ] as const -// Type for a single revalidation entity name. -// Used by RevalidateEntity in shared sync config types. export type RevalidateEntityName = (typeof REVALIDATE_ENTITIES)[number] -// Tables that must exist for runtime sync and webhook processing. -// Used by migration/spec filtering to assert required schema coverage. -export const RUNTIME_REQUIRED_TABLES: ReadonlyArray = Array.from( - new Set([ - ...Object.values(RESOURCE_MAP).map((r) => r.tableName), - ...Object.values(RESOURCE_MAP).flatMap((r) => r.childTables ?? []), - 'features', // from customer_with_entitlements - ]) -) +export const RUNTIME_REQUIRED_TABLES: ReadonlyArray = OPENAPI_RUNTIME_REQUIRED_TABLES -// Canonical table names for each RESOURCE_MAP object key. -// Used by OpenAPI/runtime adapters to avoid duplicating table mappings. -export const RESOURCE_TABLE_NAME_MAP = Object.fromEntries( - Object.entries(RESOURCE_MAP).map(([objectName, def]) => [objectName, def.tableName]) -) as Record +export const RESOURCE_TABLE_NAME_MAP: Record = Object.fromEntries( + DEFAULT_SYNC_OBJECTS.map((t) => [t, t]) +) -// Builds runtime ResourceConfig objects from RESOURCE_MAP + Stripe client. -// Used by StripeSync constructor to initialize this.resourceRegistry. -export function buildResourceRegistry(stripe: Stripe): Record { - return Object.fromEntries( - Object.entries(RESOURCE_MAP).map(([key, def]) => { - const config: StripeListResourceConfig = { - order: def.order, - tableName: def.tableName, - supportsCreatedFilter: def.supportsCreatedFilter, - sync: def.sync, - dependencies: def.dependencies ? [...def.dependencies] : [], - isFinalState: def.isFinalState, - listFn: def.list(stripe), - retrieveFn: def.retrieve(stripe), - listExpands: def.listExpands?.map((expand) => - Object.fromEntries(Object.entries(expand).map(([prop, fn]) => [prop, fn(stripe)])) - ), - } - return [key, config] - }) - ) as Record +/** + * Build a ResourceConfig for every listable resource discovered in the OpenAPI spec. + * All resources get list + retrieve functions derived dynamically from the spec paths. + */ +export function buildResourceRegistry( + stripe: Stripe, + spec: OpenApiSpec, + apiKey: string +): Record { + const endpoints = discoverListEndpoints(spec) + const nestedEndpoints = discoverNestedEndpoints(spec, endpoints) + const registry: Record = {} + const seenNested = new Set() + + for (const [tableName, endpoint] of endpoints) { + const v2 = isV2Path(endpoint.apiPath) + if (!v2 && !canResolveSdkResource(stripe, endpoint.apiPath)) continue + + const children = nestedEndpoints + .filter((n: NestedEndpoint) => n.parentTableName === tableName) + .map((n: NestedEndpoint) => ({ + tableName: n.tableName, + resourceId: n.resourceId, + apiPath: n.apiPath, + parentParamName: n.parentParamName, + supportsPagination: n.supportsPagination, + })) + + const config: ResourceConfig = { + order: 1, + tableName, + supportsCreatedFilter: !v2 && endpoint.supportsCreatedFilter, + supportsLimit: endpoint.supportsLimit, + sync: true, + dependencies: [], + listFn: buildListFn(stripe, endpoint.apiPath, apiKey), + retrieveFn: buildRetrieveFn(stripe, endpoint.apiPath, apiKey), + nestedResources: children.length > 0 ? children : undefined, + } + registry[tableName] = config + registry[endpoint.resourceId] = config + } + + for (const nested of nestedEndpoints) { + if (!nested.parentTableName || registry[nested.tableName] || registry[nested.resourceId]) { + continue + } + if (seenNested.has(nested.tableName)) { + continue + } + seenNested.add(nested.tableName) + + const config: ResourceConfig = { + order: 2, + tableName: nested.tableName, + supportsCreatedFilter: false, + supportsLimit: nested.supportsPagination, + sync: false, + dependencies: [], + listFn: undefined, + retrieveFn: undefined, + nestedResources: undefined, + parentParamName: nested.parentParamName, + } + + registry[nested.tableName] = config + registry[nested.resourceId] = config + } + + return registry } -// Alias map from Stripe event object names to internal registry keys. -// Used by normalizeStripeObjectName during webhook/upsert ingestion. -export const STRIPE_OBJECT_TO_SYNC_OBJECT_ALIASES: Record = { +export const STRIPE_OBJECT_TO_SYNC_OBJECT_ALIASES: Record = { 'checkout.session': 'checkout_sessions', - 'radar.early_fraud_warning': 'early_fraud_warning', + 'radar.early_fraud_warning': 'early_fraud_warnings', 'entitlements.active_entitlement': 'active_entitlements', 'entitlements.feature': 'active_entitlements', subscription_schedule: 'subscription_schedules', } -// Converts Stripe object names into canonical RESOURCE_MAP keys. -// Used before config/table lookups in webhook and sync flows. -export function normalizeStripeObjectName(stripeObjectName: string): StripeObject { - const normalizedObjectName = - STRIPE_OBJECT_TO_SYNC_OBJECT_ALIASES[stripeObjectName] ?? stripeObjectName - return normalizedObjectName as StripeObject +export function normalizeStripeObjectName(stripeObjectName: string): string { + return STRIPE_OBJECT_TO_SYNC_OBJECT_ALIASES[stripeObjectName] ?? stripeObjectName } -// Maps Stripe ID prefixes (e.g. cus_) to registry object names. -// Used when we only have an ID and need to resolve resource type. -export const PREFIX_RESOURCE_MAP: Record = { - cus_: 'customer', - gcus_: 'customer', - in_: 'invoice', - price_: 'price', - prod_: 'product', - sub_: 'subscription', - seti_: 'setup_intent', - pm_: 'payment_method', - dp_: 'dispute', - du_: 'dispute', - ch_: 'charge', - pi_: 'payment_intent', - txi_: 'tax_id', - cn_: 'credit_note', - issfr_: 'early_fraud_warning', - prv_: 'review', - re_: 'refund', +export const PREFIX_RESOURCE_MAP: Record = { + cus_: 'customers', + gcus_: 'customers', + in_: 'invoices', + price_: 'prices', + prod_: 'products', + sub_: 'subscriptions', + seti_: 'setup_intents', + pm_: 'payment_methods', + dp_: 'disputes', + du_: 'disputes', + ch_: 'charges', + pi_: 'payment_intents', + txi_: 'tax_ids', + cn_: 'credit_notes', + issfr_: 'early_fraud_warnings', + prv_: 'reviews', + re_: 'refunds', feat_: 'active_entitlements', cs_: 'checkout_sessions', } -// Prefixes sorted longest-first to avoid partial-prefix collisions. -// Used by getResourceFromPrefix for deterministic prefix matching. const SORTED_PREFIXES = Object.keys(PREFIX_RESOURCE_MAP).sort((a, b) => b.length - a.length) -// Resolves a Stripe ID string to a registry object key by prefix. -// Used by getResourceConfigFromId and single-entity sync routing. export function getResourceFromPrefix(stripeId: string): string | undefined { const prefix = SORTED_PREFIXES.find((p) => stripeId.startsWith(p)) - return prefix ? (PREFIX_RESOURCE_MAP[prefix] as string) : undefined + return prefix ? PREFIX_RESOURCE_MAP[prefix] : undefined } -// Gets ResourceConfig for a raw Stripe ID like cus_/ch_/pi_. -// Used by StripeSync.syncSingleEntity to pick retrieve/upsert behavior. export function getResourceConfigFromId( stripeId: string, registry: Record @@ -370,8 +181,6 @@ export function getResourceConfigFromId( return resourceName ? registry[resourceName] : undefined } -// Resolves table name for a canonical object key in a registry. -// Used by webhook and worker paths before writing to Postgres. export function getTableName(object: string, registry: Record): string { const config = registry[object] if (!config) throw new Error(`No resource config found for object type: ${object}`) diff --git a/packages/source-stripe/src/src-list-api.ts b/packages/source-stripe/src/src-list-api.ts index 96f75990c..0aca0dafe 100644 --- a/packages/source-stripe/src/src-list-api.ts +++ b/packages/source-stripe/src/src-list-api.ts @@ -7,6 +7,19 @@ import type { import { toRecordMessage } from '@stripe/sync-protocol' import type { ResourceConfig } from './types.js' +const SKIPPABLE_ERROR_PATTERNS = [ + 'only available in testmode', + 'not in live mode', + 'Must provide customer', + 'Must provide ', + 'Missing required param', +] + +function isSkippableError(err: unknown): boolean { + const msg = err instanceof Error ? err.message : String(err) + return SKIPPABLE_ERROR_PATTERNS.some((p) => msg.includes(p)) +} + function findConfigByTableName( registry: Record, tableName: string @@ -36,6 +49,8 @@ export async function* listApiBackfill(opts: { continue } + if (!resourceConfig.listFn) continue + // Skip already-complete streams (e.g. resuming after full backfill for events polling) const streamState = state?.[stream.name] if (streamState?.status === 'complete') continue @@ -56,12 +71,17 @@ export async function* listApiBackfill(opts: { // Drain any queued events before each page if (drainQueue) yield* drainQueue() - const params: { limit: number; starting_after?: string } = { limit: 100 } + const params: Record = {} + if (resourceConfig.supportsLimit !== false) { + params.limit = 100 + } if (pageCursor) { params.starting_after = pageCursor } - const response = await resourceConfig.listFn(params) + const response = await resourceConfig.listFn( + params as Parameters[0] + ) for (const item of response.data) { yield toRecordMessage(stream.name, item as Record) @@ -69,7 +89,9 @@ export async function* listApiBackfill(opts: { } hasMore = response.has_more - if (response.data.length > 0) { + if (response.pageCursor) { + pageCursor = response.pageCursor + } else if (response.data.length > 0) { pageCursor = (response.data[response.data.length - 1] as { id: string }).id } @@ -95,6 +117,14 @@ export async function* listApiBackfill(opts: { status: 'complete', } satisfies StreamStatusMessage } catch (err) { + if (isSkippableError(err)) { + yield { + type: 'stream_status', + stream: stream.name, + status: 'complete', + } satisfies StreamStatusMessage + continue + } const isRateLimit = err instanceof Error && err.message.includes('Rate limit') yield { type: 'error', diff --git a/packages/source-stripe/src/types.ts b/packages/source-stripe/src/types.ts index 56106c9b7..c8acfbcb3 100644 --- a/packages/source-stripe/src/types.ts +++ b/packages/source-stripe/src/types.ts @@ -29,21 +29,32 @@ export type BaseResourceConfig = { isFinalState?: (entity: any) => boolean } -export type StripeListResourceConfig = BaseResourceConfig & { +export type ResourceConfig = BaseResourceConfig & { /** Function to list items from Stripe API */ - listFn: (params: Stripe.PaginationParams & { created?: Stripe.RangeQueryParam }) => Promise<{ + listFn?: (params: Stripe.PaginationParams & { created?: Stripe.RangeQueryParam }) => Promise<{ data: unknown[] has_more: boolean + pageCursor?: string }> /** Function to retrieve a single item by ID from Stripe API */ // eslint-disable-next-line @typescript-eslint/no-explicit-any - retrieveFn: (id: string) => Promise> + retrieveFn?: (id: string) => Promise> + /** Whether the list API supports the `limit` parameter */ + supportsLimit?: boolean /** Optional list of sub-resources to expand during upsert/fetching (e.g. 'refunds', 'listLineItems') */ listExpands?: Record Promise>>[] + /** Nested child resources discovered from the spec (e.g. subscription items under subscriptions) */ + nestedResources?: { + tableName: string + resourceId: string + apiPath: string + parentParamName: string + supportsPagination: boolean + }[] + /** For nested resources, the parent path parameter name */ + parentParamName?: string } -/** Union of all resource configuration types */ -export type ResourceConfig = StripeListResourceConfig export type RevalidateEntity = RevalidateEntityName From 6334d98483ad75d78919da0db1725c5f82951226 Mon Sep 17 00:00:00 2001 From: Yostra Date: Tue, 24 Mar 2026 01:56:32 +0100 Subject: [PATCH 04/19] package lock --- pnpm-lock.yaml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index ec7da7796..36b49b672 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -345,6 +345,19 @@ importers: specifier: ^3.2.4 version: 3.2.4(@types/node@25.5.0)(jiti@2.6.1)(lightningcss@1.32.0)(terser@5.46.1)(tsx@4.21.0)(yaml@2.8.1) + packages/openapi: + dependencies: + stripe: + specifier: ^17.7.0 + version: 17.7.0 + devDependencies: + '@types/node': + specifier: ^24.5.0 + version: 24.10.1 + vitest: + specifier: ^3.2.4 + version: 3.2.4(@types/node@24.10.1)(tsx@4.21.0)(yaml@2.8.1) + packages/protocol: dependencies: citty: @@ -363,6 +376,9 @@ importers: packages/source-stripe: dependencies: + '@stripe/openapi': + specifier: workspace:* + version: link:../openapi '@stripe/sync-protocol': specifier: workspace:* version: link:../protocol From 50efdbd61d25a2cfffad494d8feea64742446448 Mon Sep 17 00:00:00 2001 From: Yostra Date: Tue, 24 Mar 2026 05:22:19 +0100 Subject: [PATCH 05/19] plugin loading --- e2e/connector-loading.test.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/e2e/connector-loading.test.sh b/e2e/connector-loading.test.sh index 88b0c9286..81e084a7c 100755 --- a/e2e/connector-loading.test.sh +++ b/e2e/connector-loading.test.sh @@ -44,6 +44,7 @@ echo "" echo "--- Step 1: Packing packages ---" PROTOCOL_TGZ=$(cd "$REPO_ROOT" && pnpm --filter @stripe/sync-protocol pack 2>/dev/null | tail -1) +OPENAPI_TGZ=$(cd "$REPO_ROOT" && pnpm --filter @stripe/openapi pack 2>/dev/null | tail -1) ENGINE_TGZ=$(cd "$REPO_ROOT" && pnpm --filter @stripe/sync-engine pack 2>/dev/null | tail -1) SOURCE_TGZ=$(cd "$REPO_ROOT" && pnpm --filter @stripe/sync-source-stripe pack 2>/dev/null | tail -1) DEST_TGZ=$(cd "$REPO_ROOT" && pnpm --filter @stripe/sync-destination-postgres pack 2>/dev/null | tail -1) @@ -52,7 +53,7 @@ STATE_PG_TGZ=$(cd "$REPO_ROOT" && pnpm --filter @stripe/sync-state-postgres pack UTIL_PG_TGZ=$(cd "$REPO_ROOT" && pnpm --filter @stripe/sync-util-postgres pack 2>/dev/null | tail -1) TSCLI_TGZ=$(cd "$REPO_ROOT" && pnpm --filter @stripe/sync-ts-cli pack 2>/dev/null | tail -1) -for tgz in "$PROTOCOL_TGZ" "$ENGINE_TGZ" "$SOURCE_TGZ" "$DEST_TGZ" "$DEST_SHEETS_TGZ" \ +for tgz in "$PROTOCOL_TGZ" "$OPENAPI_TGZ" "$ENGINE_TGZ" "$SOURCE_TGZ" "$DEST_TGZ" "$DEST_SHEETS_TGZ" \ "$STATE_PG_TGZ" "$UTIL_PG_TGZ" "$TSCLI_TGZ"; do if [ ! -f "$tgz" ]; then echo "FAIL: tarball not found: $tgz" @@ -84,6 +85,7 @@ cat > package.json < package.json <&1 | tail -5 echo "" From 07b794bfcb4adbdbb0c31e2a86f78a2c04e4d783 Mon Sep 17 00:00:00 2001 From: Yostra Date: Tue, 24 Mar 2026 05:26:50 +0100 Subject: [PATCH 06/19] remove pg artifacts --- .../openapi/__tests__/postgresAdapter.test.ts | 89 ------------- packages/openapi/dialectAdapter.ts | 14 -- packages/openapi/index.ts | 2 - packages/openapi/postgresAdapter.ts | 126 ------------------ 4 files changed, 231 deletions(-) delete mode 100644 packages/openapi/__tests__/postgresAdapter.test.ts delete mode 100644 packages/openapi/dialectAdapter.ts delete mode 100644 packages/openapi/postgresAdapter.ts diff --git a/packages/openapi/__tests__/postgresAdapter.test.ts b/packages/openapi/__tests__/postgresAdapter.test.ts deleted file mode 100644 index cda4d691f..000000000 --- a/packages/openapi/__tests__/postgresAdapter.test.ts +++ /dev/null @@ -1,89 +0,0 @@ -import { describe, expect, it } from 'vitest' -import { PostgresAdapter } from '../postgresAdapter' -import type { ParsedResourceTable } from '../types' - -const SAMPLE_TABLE: ParsedResourceTable = { - tableName: 'customers', - resourceId: 'customer', - sourceSchemaName: 'customer', - columns: [ - { name: 'created', type: 'bigint', nullable: false }, - { name: 'deleted', type: 'boolean', nullable: true }, - { name: 'metadata', type: 'json', nullable: true }, - { name: 'expires_at', type: 'timestamptz', nullable: true }, - ], -} - -const EXPANDABLE_REFERENCE_TABLE: ParsedResourceTable = { - tableName: 'charges', - resourceId: 'charge', - sourceSchemaName: 'charge', - columns: [{ name: 'customer', type: 'json', nullable: true, expandableReference: true }], -} - -describe('PostgresAdapter', () => { - it('emits deterministic DDL statements with runtime-required metadata columns', () => { - const adapter = new PostgresAdapter({ schemaName: 'stripe' }) - const statements = adapter.buildAllStatements([SAMPLE_TABLE]) - - expect(statements).toHaveLength(9) - expect(statements[0]).toContain('CREATE TABLE "stripe"."customers"') - expect(statements[0]).toContain('"_raw_data" jsonb NOT NULL') - expect(statements[0]).toContain('"_account_id" text NOT NULL') - expect(statements[0]).toContain( - '"id" text GENERATED ALWAYS AS ((_raw_data->>\'id\')::text) STORED' - ) - expect(statements[0]).toContain( - '"metadata" jsonb GENERATED ALWAYS AS ((_raw_data->\'metadata\')::jsonb) STORED' - ) - // Temporal columns are stored as text generated columns for immutability safety. - expect(statements[0]).toContain( - '"expires_at" text GENERATED ALWAYS AS ((_raw_data->>\'expires_at\')::text) STORED' - ) - expect( - statements.some((stmt) => stmt.includes('ADD COLUMN IF NOT EXISTS "created" bigint')) - ).toBe(true) - expect( - statements.some((stmt) => stmt.includes('ADD COLUMN IF NOT EXISTS "deleted" boolean')) - ).toBe(true) - expect( - statements.some((stmt) => stmt.includes('ADD COLUMN IF NOT EXISTS "metadata" jsonb')) - ).toBe(true) - expect( - statements.some((stmt) => stmt.includes('ADD COLUMN IF NOT EXISTS "expires_at" text')) - ).toBe(true) - expect(statements[5]).toContain( - 'FOREIGN KEY ("_account_id") REFERENCES "stripe"."_accounts" (id)' - ) - expect(statements[7]).toContain('DROP TRIGGER IF EXISTS handle_updated_at') - expect(statements[8]).toContain('EXECUTE FUNCTION set_updated_at()') - }) - - it('produces stable output across repeated calls', () => { - const adapter = new PostgresAdapter({ schemaName: 'stripe' }) - const first = adapter.buildAllStatements([SAMPLE_TABLE]) - const second = adapter.buildAllStatements([SAMPLE_TABLE]) - expect(second).toEqual(first) - }) - - it('materializes expandable reference columns as text ids for compatibility', () => { - const adapter = new PostgresAdapter({ schemaName: 'stripe' }) - const statements = adapter.buildAllStatements([EXPANDABLE_REFERENCE_TABLE]) - - expect(statements[0]).toContain('"customer" text GENERATED ALWAYS AS (CASE') - expect(statements[0]).toContain("WHEN jsonb_typeof(_raw_data->'customer') = 'object'") - expect(statements[0]).toContain("THEN (_raw_data->'customer'->>'id')") - }) - - it('uses accountSchema for FK when provided (split schema)', () => { - const adapter = new PostgresAdapter({ - schemaName: 'stripe_data', - accountSchema: 'stripe_sync', - }) - const statements = adapter.buildAllStatements([SAMPLE_TABLE]) - expect(statements[0]).toContain('CREATE TABLE "stripe_data"."customers"') - expect(statements[5]).toContain( - 'FOREIGN KEY ("_account_id") REFERENCES "stripe_sync"."_accounts" (id)' - ) - }) -}) diff --git a/packages/openapi/dialectAdapter.ts b/packages/openapi/dialectAdapter.ts deleted file mode 100644 index 9d684d399..000000000 --- a/packages/openapi/dialectAdapter.ts +++ /dev/null @@ -1,14 +0,0 @@ -import type { ParsedResourceTable } from './types' - -export interface DialectAdapter { - /** - * Create all statements needed to materialize a single parsed table. - */ - buildTableStatements(table: ParsedResourceTable): string[] - - /** - * Create all statements needed to materialize all parsed tables. - * Implementations must be deterministic for a given input. - */ - buildAllStatements(tables: ParsedResourceTable[]): string[] -} diff --git a/packages/openapi/index.ts b/packages/openapi/index.ts index d8300b9da..43399c784 100644 --- a/packages/openapi/index.ts +++ b/packages/openapi/index.ts @@ -5,10 +5,8 @@ export { RUNTIME_RESOURCE_ALIASES, } from './specParser.js' export { OPENAPI_COMPATIBILITY_COLUMNS } from './runtimeMappings.js' -export { PostgresAdapter } from './postgresAdapter.js' export { WritePathPlanner } from './writePathPlanner.js' export { resolveOpenApiSpec } from './specFetchHelper.js' -export type { DialectAdapter } from './dialectAdapter.js' export { buildListFn, buildRetrieveFn, diff --git a/packages/openapi/postgresAdapter.ts b/packages/openapi/postgresAdapter.ts deleted file mode 100644 index bbb38cf45..000000000 --- a/packages/openapi/postgresAdapter.ts +++ /dev/null @@ -1,126 +0,0 @@ -import { Buffer } from 'node:buffer' -import { createHash } from 'node:crypto' -import type { DialectAdapter } from './dialectAdapter' -import type { ParsedColumn, ParsedResourceTable, ScalarType } from './types' - -const PG_IDENTIFIER_MAX_BYTES = 63 - -type PostgresAdapterOptions = { - schemaName?: string - /** Schema for _accounts table (FK target). Defaults to schemaName when not provided. */ - accountSchema?: string - materializeTemporalAsText?: boolean -} - -export class PostgresAdapter implements DialectAdapter { - private readonly schemaName: string - private readonly accountSchema: string - private readonly materializeTemporalAsText: boolean - - constructor(options: PostgresAdapterOptions = {}) { - this.schemaName = options.schemaName ?? 'stripe' - this.accountSchema = options.accountSchema ?? this.schemaName - this.materializeTemporalAsText = options.materializeTemporalAsText ?? true - } - - buildAllStatements(tables: ParsedResourceTable[]): string[] { - return [...tables] - .sort((a, b) => a.tableName.localeCompare(b.tableName)) - .flatMap((table) => this.buildTableStatements(table)) - } - - buildTableStatements(table: ParsedResourceTable): string[] { - const quotedSchema = this.quoteIdent(this.schemaName) - const quotedTable = this.quoteIdent(table.tableName) - const generatedColumns = table.columns.map((column) => this.buildGeneratedColumn(column)) - const generatedColumnAlters = generatedColumns.map( - (columnDef) => - `ALTER TABLE ${quotedSchema}.${quotedTable} ADD COLUMN IF NOT EXISTS ${columnDef};` - ) - const columnDefs = [ - '"_raw_data" jsonb NOT NULL', - '"_last_synced_at" timestamptz', - '"_updated_at" timestamptz NOT NULL DEFAULT now()', - '"_account_id" text NOT NULL', - '"id" text GENERATED ALWAYS AS ((_raw_data->>\'id\')::text) STORED', - ...generatedColumns, - 'PRIMARY KEY ("id")', - ] - - const fkName = this.safeIdentifier(`fk_${table.tableName}_account`) - const accountIdxName = this.safeIdentifier(`idx_${table.tableName}_account_id`) - const quotedAccountSchema = this.quoteIdent(this.accountSchema) - - return [ - `CREATE TABLE ${quotedSchema}.${quotedTable} (\n ${columnDefs.join(',\n ')}\n);`, - ...generatedColumnAlters, - `ALTER TABLE ${quotedSchema}.${quotedTable} ADD CONSTRAINT ${this.quoteIdent( - fkName - )} FOREIGN KEY ("_account_id") REFERENCES ${quotedAccountSchema}."_accounts" (id);`, - `CREATE INDEX ${this.quoteIdent(accountIdxName)} ON ${quotedSchema}.${quotedTable} ("_account_id");`, - `DROP TRIGGER IF EXISTS handle_updated_at ON ${quotedSchema}.${quotedTable};`, - `CREATE TRIGGER handle_updated_at BEFORE UPDATE ON ${quotedSchema}.${quotedTable} FOR EACH ROW EXECUTE FUNCTION set_updated_at();`, - ] - } - - private buildGeneratedColumn(column: ParsedColumn): string { - const forceReferenceText = column.expandableReference === true - const pgType = forceReferenceText ? 'text' : this.pgType(column.type) - const escapedPath = column.name.replace(/'/g, "''") - const expression = forceReferenceText - ? this.buildExpandableReferenceTextExpression(escapedPath) - : pgType === 'jsonb' - ? `(_raw_data->'${escapedPath}')::jsonb` - : pgType === 'text' - ? `(_raw_data->>'${escapedPath}')::text` - : `(NULLIF(_raw_data->>'${escapedPath}', ''))::${pgType}` - - return `${this.quoteIdent(column.name)} ${pgType} GENERATED ALWAYS AS (${expression}) STORED` - } - - private buildExpandableReferenceTextExpression(escapedPath: string): string { - const jsonPath = `_raw_data->'${escapedPath}'` - return `CASE - WHEN jsonb_typeof(${jsonPath}) = 'object' AND ${jsonPath} ? 'id' - THEN (${jsonPath}->>'id') - ELSE (_raw_data->>'${escapedPath}') - END` - } - - private pgType(type: ScalarType): string { - if (type === 'timestamptz' && this.materializeTemporalAsText) { - return 'text' - } - - switch (type) { - case 'text': - return 'text' - case 'boolean': - return 'boolean' - case 'bigint': - return 'bigint' - case 'numeric': - return 'numeric' - case 'json': - return 'jsonb' - case 'timestamptz': - return 'timestamptz' - } - } - - private quoteIdent(value: string): string { - return `"${value.replaceAll('"', '""')}"` - } - - private safeIdentifier(name: string): string { - if (Buffer.byteLength(name) <= PG_IDENTIFIER_MAX_BYTES) { - return name - } - - const hash = createHash('sha1').update(name).digest('hex').slice(0, 8) - const suffix = `_h${hash}` - const maxBaseBytes = PG_IDENTIFIER_MAX_BYTES - Buffer.byteLength(suffix) - const truncatedBase = Buffer.from(name).subarray(0, maxBaseBytes).toString('utf8') - return `${truncatedBase}${suffix}` - } -} From f0df8f64fbb6f6407f62c637fb6ea1231137c231 Mon Sep 17 00:00:00 2001 From: Yostra Date: Tue, 24 Mar 2026 05:34:09 +0100 Subject: [PATCH 07/19] format --- packages/openapi/jsonSchemaConverter.ts | 4 +--- packages/openapi/tsconfig.json | 10 +++------- packages/source-stripe/src/types.ts | 1 - 3 files changed, 4 insertions(+), 11 deletions(-) diff --git a/packages/openapi/jsonSchemaConverter.ts b/packages/openapi/jsonSchemaConverter.ts index 15b400409..31d9b3924 100644 --- a/packages/openapi/jsonSchemaConverter.ts +++ b/packages/openapi/jsonSchemaConverter.ts @@ -9,9 +9,7 @@ const SCALAR_TYPE_TO_JSON_SCHEMA: Record { +export function parsedTableToJsonSchema(table: ParsedResourceTable): Record { const properties: Record = { id: { type: 'string' }, } diff --git a/packages/openapi/tsconfig.json b/packages/openapi/tsconfig.json index df859faad..362cfc683 100644 --- a/packages/openapi/tsconfig.json +++ b/packages/openapi/tsconfig.json @@ -10,10 +10,6 @@ "strict": true, "resolveJsonModule": true }, - "include": [ - "*.ts" - ], - "exclude": [ - "__tests__" - ] -} \ No newline at end of file + "include": ["*.ts"], + "exclude": ["__tests__"] +} diff --git a/packages/source-stripe/src/types.ts b/packages/source-stripe/src/types.ts index c8acfbcb3..f67be9d3c 100644 --- a/packages/source-stripe/src/types.ts +++ b/packages/source-stripe/src/types.ts @@ -55,7 +55,6 @@ export type ResourceConfig = BaseResourceConfig & { parentParamName?: string } - export type RevalidateEntity = RevalidateEntityName export const SUPPORTED_WEBHOOK_EVENTS: Stripe.WebhookEndpointCreateParams.EnabledEvent[] = [ From da06823d65106ac3c726bf2d7cc6f008d98a7cb5 Mon Sep 17 00:00:00 2001 From: Yostra Date: Tue, 24 Mar 2026 06:02:26 +0100 Subject: [PATCH 08/19] don't use sdk --- .../openapi/__tests__/listFnResolver.test.ts | 58 +---- .../__tests__/writePathPlanner.test.ts | 39 ---- packages/openapi/index.ts | 22 +- packages/openapi/listFnResolver.ts | 207 ++++++------------ packages/openapi/package.json | 4 +- packages/openapi/specParser.ts | 2 - packages/openapi/types.ts | 7 - packages/openapi/writePathPlanner.ts | 43 ---- 8 files changed, 81 insertions(+), 301 deletions(-) delete mode 100644 packages/openapi/__tests__/writePathPlanner.test.ts delete mode 100644 packages/openapi/writePathPlanner.ts diff --git a/packages/openapi/__tests__/listFnResolver.test.ts b/packages/openapi/__tests__/listFnResolver.test.ts index 6162797a8..dd3cd6361 100644 --- a/packages/openapi/__tests__/listFnResolver.test.ts +++ b/packages/openapi/__tests__/listFnResolver.test.ts @@ -1,24 +1,6 @@ -import { describe, expect, it, vi } from 'vitest' -import { discoverListEndpoints, buildListFn } from '../listFnResolver' +import { describe, expect, it } from 'vitest' +import { discoverListEndpoints } from '../listFnResolver' import { minimalStripeOpenApiSpec } from './fixtures/minimalSpec' -import type Stripe from 'stripe' - -function mockStripe() { - const listFn = vi.fn().mockResolvedValue({ data: [], has_more: false }) - return { - customers: { list: listFn }, - plans: { list: listFn }, - products: { list: listFn }, - subscriptionItems: { list: listFn }, - checkout: { sessions: { list: listFn } }, - radar: { earlyFraudWarnings: { list: listFn } }, - entitlements: { - activeEntitlements: { list: listFn }, - features: { list: listFn }, - }, - _listFn: listFn, - } -} describe('discoverListEndpoints', () => { it('maps table names to their API paths', () => { @@ -106,39 +88,3 @@ describe('discoverListEndpoints', () => { expect(endpoints.size).toBe(0) }) }) - -describe('buildListFn', () => { - it('resolves a simple top-level path', async () => { - const mock = mockStripe() - const listFn = buildListFn(mock as unknown as Stripe, '/v1/customers') - await listFn({ limit: 10 }) - expect(mock._listFn).toHaveBeenCalledWith({ limit: 10 }) - }) - - it('resolves a nested namespace path', async () => { - const mock = mockStripe() - const listFn = buildListFn(mock as unknown as Stripe, '/v1/checkout/sessions') - await listFn({ limit: 5 }) - expect(mock._listFn).toHaveBeenCalledWith({ limit: 5 }) - }) - - it('converts snake_case segments to camelCase', async () => { - const mock = mockStripe() - const listFn = buildListFn(mock as unknown as Stripe, '/v1/subscription_items') - await listFn({ limit: 1 }) - expect(mock._listFn).toHaveBeenCalled() - }) - - it('resolves deeply nested snake_case paths', async () => { - const mock = mockStripe() - const listFn = buildListFn(mock as unknown as Stripe, '/v1/radar/early_fraud_warnings') - await listFn({ limit: 1 }) - expect(mock._listFn).toHaveBeenCalled() - }) - - it('throws when a path segment does not exist on the SDK', async () => { - const mock = mockStripe() - const listFn = buildListFn(mock as unknown as Stripe, '/v1/nonexistent_resource') - await expect(() => listFn({ limit: 1 })).toThrow(/Stripe SDK has no property/) - }) -}) diff --git a/packages/openapi/__tests__/writePathPlanner.test.ts b/packages/openapi/__tests__/writePathPlanner.test.ts deleted file mode 100644 index 9d337906a..000000000 --- a/packages/openapi/__tests__/writePathPlanner.test.ts +++ /dev/null @@ -1,39 +0,0 @@ -import { describe, expect, it } from 'vitest' -import { WritePathPlanner } from '../writePathPlanner' -import type { ParsedResourceTable } from '../types' - -describe('WritePathPlanner', () => { - it('builds deterministic write plans aligned to raw json upsert assumptions', () => { - const planner = new WritePathPlanner() - const tables: ParsedResourceTable[] = [ - { - tableName: 'customers', - resourceId: 'customer', - sourceSchemaName: 'customer', - columns: [ - { name: 'deleted', type: 'boolean', nullable: true }, - { name: 'created', type: 'bigint', nullable: false }, - ], - }, - { - tableName: 'plans', - resourceId: 'plan', - sourceSchemaName: 'plan', - columns: [{ name: 'active', type: 'boolean', nullable: false }], - }, - ] - - const plans = planner.buildPlans(tables) - expect(plans.map((plan) => plan.tableName)).toEqual(['customers', 'plans']) - expect(plans[0]).toMatchObject({ - tableName: 'customers', - conflictTarget: ['id'], - extraColumns: [], - metadataColumns: ['_raw_data', '_last_synced_at', '_account_id'], - }) - expect(plans[0].generatedColumns).toEqual([ - { column: 'created', pgType: 'bigint' }, - { column: 'deleted', pgType: 'boolean' }, - ]) - }) -}) diff --git a/packages/openapi/index.ts b/packages/openapi/index.ts index 43399c784..fb25b6ed1 100644 --- a/packages/openapi/index.ts +++ b/packages/openapi/index.ts @@ -1,22 +1,20 @@ export type * from './types.js' -export { - SpecParser, - OPENAPI_RESOURCE_TABLE_ALIASES, - RUNTIME_RESOURCE_ALIASES, -} from './specParser.js' +export { SpecParser, OPENAPI_RESOURCE_TABLE_ALIASES } from './specParser.js' export { OPENAPI_COMPATIBILITY_COLUMNS } from './runtimeMappings.js' -export { WritePathPlanner } from './writePathPlanner.js' export { resolveOpenApiSpec } from './specFetchHelper.js' export { - buildListFn, - buildRetrieveFn, - buildV2ListFn, - buildV2RetrieveFn, discoverListEndpoints, discoverNestedEndpoints, - canResolveSdkResource, isV2Path, + buildListFn, + buildRetrieveFn, +} from './listFnResolver.js' +export type { + ListEndpoint, + NestedEndpoint, + ListFn, + RetrieveFn, + ListParams, } from './listFnResolver.js' -export type { NestedEndpoint } from './listFnResolver.js' export { parsedTableToJsonSchema } from './jsonSchemaConverter.js' export { RUNTIME_REQUIRED_TABLES } from './runtimeMappings.js' diff --git a/packages/openapi/listFnResolver.ts b/packages/openapi/listFnResolver.ts index 04a05e096..1b1b4f34d 100644 --- a/packages/openapi/listFnResolver.ts +++ b/packages/openapi/listFnResolver.ts @@ -1,12 +1,20 @@ -import type Stripe from 'stripe' import type { OpenApiSchemaObject, OpenApiSpec } from './types' import { OPENAPI_RESOURCE_TABLE_ALIASES } from './runtimeMappings' const SCHEMA_REF_PREFIX = '#/components/schemas/' -type ListFn = ( - params: Stripe.PaginationParams & { created?: Stripe.RangeQueryParam } -) => Promise<{ data: unknown[]; has_more: boolean; pageCursor?: string }> +export type ListParams = { + limit?: number + starting_after?: string + ending_before?: string + created?: { gt?: number; gte?: number; lt?: number; lte?: number } +} + +export type ListResult = { data: unknown[]; has_more: boolean; pageCursor?: string } + +export type ListFn = (params: ListParams) => Promise + +export type RetrieveFn = (id: string) => Promise export type ListEndpoint = { tableName: string @@ -25,10 +33,6 @@ export type NestedEndpoint = { supportsPagination: boolean } -function snakeToCamel(s: string): string { - return s.replace(/_([a-z])/g, (_, c) => c.toUpperCase()) -} - function resolveTableName(resourceId: string, aliases: Record): string { const alias = aliases[resourceId] if (alias) return alias @@ -197,161 +201,86 @@ export function isV2Path(apiPath: string): boolean { return apiPath.startsWith('/v2/') } -function pathToSdkSegments(apiPath: string): string[] { - if (isV2Path(apiPath)) { - return [ - 'v2', - ...apiPath - .replace(/^\/v2\//, '') - .split('/') - .filter((s) => !s.startsWith('{')) - .map(snakeToCamel), - ] - } - return apiPath - .replace(/^\/v[12]\//, '') - .split('/') - .filter((s) => !s.startsWith('{')) - .map(snakeToCamel) -} +// --------------------------------------------------------------------------- +// HTTP-based list / retrieve builders (no Stripe SDK dependency) +// --------------------------------------------------------------------------- -// eslint-disable-next-line @typescript-eslint/no-explicit-any -function resolveStripeResource(stripe: Stripe, segments: string[], apiPath: string): any { - // eslint-disable-next-line @typescript-eslint/no-explicit-any - let resource: any = stripe - for (const segment of segments) { - resource = resource?.[segment] - if (!resource) { - throw new Error(`Stripe SDK has no property "${segment}" when resolving path "${apiPath}"`) - } - } - return resource +const STRIPE_API_BASE = 'https://api.stripe.com' +const V2_STRIPE_VERSION = '2026-02-25.clover' + +function authHeaders(apiKey: string): Record { + return { Authorization: `Bearer ${apiKey}` } } /** - * Check whether an API path can be resolved to a Stripe SDK resource. - * v1 requires both `.list()` and `.retrieve()`. - * v2 only requires `.list()` (retrieve may not be available on all v2 resources). + * Build a callable list function that hits the Stripe HTTP API directly. + * Supports both v1 (has_more pagination) and v2 (next_page_url pagination). */ -export function canResolveSdkResource(stripe: Stripe, apiPath: string): boolean { - try { - const segments = pathToSdkSegments(apiPath) - const resource = resolveStripeResource(stripe, segments, apiPath) - if (isV2Path(apiPath)) { - return typeof resource.list === 'function' +export function buildListFn(apiKey: string, apiPath: string): ListFn { + if (isV2Path(apiPath)) { + return async (params) => { + const qs = new URLSearchParams() + qs.set('limit', String(Math.min(params.limit ?? 20, 20))) + if (params.starting_after) qs.set('page', params.starting_after) + + const response = await fetch(`${STRIPE_API_BASE}${apiPath}?${qs}`, { + headers: { ...authHeaders(apiKey), 'Stripe-Version': V2_STRIPE_VERSION }, + }) + const body = (await response.json()) as { + data: unknown[] + next_page_url?: string | null + } + const pageCursor = extractPageToken(body.next_page_url) + return { data: body.data ?? [], has_more: !!body.next_page_url, pageCursor } } - return typeof resource.list === 'function' && typeof resource.retrieve === 'function' - } catch { - return false } -} -/** - * Build a callable list function by navigating the Stripe SDK object using - * the API path segments converted from snake_case to camelCase. - * Path parameters (e.g. `{customer}`) are stripped automatically. - */ -export function buildListFn(stripe: Stripe, apiPath: string, apiKey: string = ''): ListFn { - const v2 = isV2Path(apiPath) - if (v2) { - return buildV2ListFn(apiKey, apiPath) - } - const segments = pathToSdkSegments(apiPath) - return (params) => { - const resource = resolveStripeResource(stripe, segments, apiPath) - if (typeof resource.list !== 'function') { - throw new Error(`Stripe SDK resource at "${apiPath}" has no list() method`) + return async (params) => { + const qs = new URLSearchParams() + if (params.limit != null) qs.set('limit', String(params.limit)) + if (params.starting_after) qs.set('starting_after', params.starting_after) + if (params.ending_before) qs.set('ending_before', params.ending_before) + if (params.created) { + for (const [op, val] of Object.entries(params.created)) { + if (val != null) qs.set(`created[${op}]`, String(val)) + } } - return resource.list(params) + + const response = await fetch(`${STRIPE_API_BASE}${apiPath}?${qs}`, { + headers: authHeaders(apiKey), + }) + const body = (await response.json()) as { data: unknown[]; has_more: boolean } + return { data: body.data ?? [], has_more: body.has_more } } } -type RetrieveFn = (id: string) => Promise> - /** - * Build a callable retrieve function by navigating the Stripe SDK object using - * the API path segments converted from snake_case to camelCase. - * Path parameters (e.g. `{customer}`) are stripped automatically. + * Build a callable retrieve function that hits the Stripe HTTP API directly. */ -export function buildRetrieveFn(stripe: Stripe, apiPath: string, apiKey: string): RetrieveFn { - const v2 = isV2Path(apiPath) - if (v2) { - return buildV2RetrieveFn(apiKey, apiPath) - } - const segments = pathToSdkSegments(apiPath) - return (id: string) => { - const resource = resolveStripeResource(stripe, segments, apiPath) - if (typeof resource.retrieve !== 'function') { - throw new Error(`Stripe SDK resource at "${apiPath}" has no retrieve() method`) +export function buildRetrieveFn(apiKey: string, apiPath: string): RetrieveFn { + if (isV2Path(apiPath)) { + return async (id) => { + const response = await fetch(`${STRIPE_API_BASE}${apiPath}/${id}`, { + headers: { ...authHeaders(apiKey), 'Stripe-Version': V2_STRIPE_VERSION }, + }) + return await response.json() } - return resource.retrieve(id) } -} -/** - * Build a list function that calls Stripe rawRequest directly for a fixed endpoint. - * Useful when the Stripe SDK does not expose a matching namespace. - */ -export function buildRawRequestListFn(stripe: Stripe, apiPath: string): ListFn { - return (params) => - stripe.rawRequest('GET', apiPath, { ...params }) as unknown as Promise<{ - data: unknown[] - has_more: boolean - }> + return async (id) => { + const response = await fetch(`${STRIPE_API_BASE}${apiPath}/${id}`, { + headers: authHeaders(apiKey), + }) + return await response.json() + } } function extractPageToken(nextPageUrl: string | null | undefined): string | undefined { if (!nextPageUrl) return undefined try { - const url = new URL(nextPageUrl, 'https://api.stripe.com') + const url = new URL(nextPageUrl, STRIPE_API_BASE) return url.searchParams.get('page') ?? undefined } catch { return undefined } } - -/** - * Build a list function for v2 API endpoints. - * V2 uses `page` token pagination and returns `next_page_url` instead of `has_more`. - * The response is normalized to the v1 shape so the sync worker can process it uniformly. - */ -export function buildV2ListFn(apiKey: string, apiPath: string): ListFn { - return async (params) => { - const qs = new URLSearchParams() - qs.set('limit', String(Math.min(params.limit ?? 20, 20))) - if (params.starting_after) qs.set('page', params.starting_after) - const url = `https://api.stripe.com${apiPath}?${qs.toString()}` - - const response = await fetch(url, { - headers: { - Authorization: `Bearer ${apiKey}`, - 'Stripe-Version': '2026-02-25.clover', - }, - }) - - const raw = await response.text() - - const body = JSON.parse(raw) as { - data: unknown[] - next_page_url?: string | null - } - const nextToken = extractPageToken(body.next_page_url) - return { - data: body.data ?? [], - has_more: !!body.next_page_url, - pageCursor: nextToken, - } - } -} - -export function buildV2RetrieveFn(apiKey: string, apiPath: string): RetrieveFn { - return async (id: string) => { - const response = await fetch(`https://api.stripe.com${apiPath}/${id}`, { - headers: { - Authorization: `Bearer ${apiKey}`, - 'Stripe-Version': '2026-02-25.clover', - }, - }) - return (await response.json()) as Stripe.Response - } -} diff --git a/packages/openapi/package.json b/packages/openapi/package.json index aef37391a..ef99149ec 100644 --- a/packages/openapi/package.json +++ b/packages/openapi/package.json @@ -18,9 +18,7 @@ "files": [ "dist" ], - "dependencies": { - "stripe": "^17.7.0" - }, + "dependencies": {}, "devDependencies": { "@types/node": "^24.5.0", "vitest": "^3.2.4" diff --git a/packages/openapi/specParser.ts b/packages/openapi/specParser.ts index 2203d5bb5..438f4c8f7 100644 --- a/packages/openapi/specParser.ts +++ b/packages/openapi/specParser.ts @@ -24,8 +24,6 @@ const RESERVED_COLUMNS = new Set([ ]) export { RUNTIME_REQUIRED_TABLES, OPENAPI_RESOURCE_TABLE_ALIASES } -/** @deprecated Use OPENAPI_RESOURCE_TABLE_ALIASES instead. */ -export const RUNTIME_RESOURCE_ALIASES = OPENAPI_RESOURCE_TABLE_ALIASES type ColumnAccumulator = { type: ScalarType diff --git a/packages/openapi/types.ts b/packages/openapi/types.ts index 14c5657ed..cbd697c45 100644 --- a/packages/openapi/types.ts +++ b/packages/openapi/types.ts @@ -109,10 +109,3 @@ export type ResolvedOpenApiSpec = { cachePath?: string commitSha?: string } - -export type WritePlan = { - tableName: string - conflictTarget: string[] - extraColumns: Array<{ column: string; pgType: string; entryKey: string }> - metadataColumns: ['_raw_data', '_last_synced_at', '_account_id'] -} diff --git a/packages/openapi/writePathPlanner.ts b/packages/openapi/writePathPlanner.ts deleted file mode 100644 index 3121a5d3c..000000000 --- a/packages/openapi/writePathPlanner.ts +++ /dev/null @@ -1,43 +0,0 @@ -import type { ParsedResourceTable, ScalarType, WritePlan } from './types' - -type TableWritePlan = WritePlan & { - generatedColumns: Array<{ column: string; pgType: string }> -} - -export class WritePathPlanner { - buildPlans(tables: ParsedResourceTable[]): TableWritePlan[] { - return [...tables] - .sort((a, b) => a.tableName.localeCompare(b.tableName)) - .map((table) => this.buildPlan(table)) - } - - buildPlan(table: ParsedResourceTable): TableWritePlan { - return { - tableName: table.tableName, - conflictTarget: ['id'], - extraColumns: [], - metadataColumns: ['_raw_data', '_last_synced_at', '_account_id'], - generatedColumns: table.columns - .map((column) => ({ column: column.name, pgType: this.scalarTypeToPgType(column.type) })) - .sort((a, b) => a.column.localeCompare(b.column)), - } - } - - private scalarTypeToPgType(type: ScalarType): string { - switch (type) { - case 'boolean': - return 'boolean' - case 'bigint': - return 'bigint' - case 'numeric': - return 'numeric' - case 'json': - return 'jsonb' - case 'timestamptz': - return 'timestamptz' - case 'text': - default: - return 'text' - } - } -} From 9d3b6f3c20c424be6e358adfa574f3ebd8eb2eb7 Mon Sep 17 00:00:00 2001 From: Yostra Date: Tue, 24 Mar 2026 06:04:08 +0100 Subject: [PATCH 09/19] openapi doesn't use sdk --- packages/source-stripe/src/index.ts | 4 ++-- packages/source-stripe/src/resourceRegistry.ts | 10 +++------- packages/source-stripe/src/types.ts | 16 ++++------------ 3 files changed, 9 insertions(+), 21 deletions(-) diff --git a/packages/source-stripe/src/index.ts b/packages/source-stripe/src/index.ts index b749c35a6..fc942cbb9 100644 --- a/packages/source-stripe/src/index.ts +++ b/packages/source-stripe/src/index.ts @@ -122,7 +122,7 @@ const source = { const resolved = await resolveOpenApiSpec({ apiVersion: config.api_version ?? '2020-08-27', }) - const registry = buildResourceRegistry(makeClient(config), resolved.spec, config.api_key) + const registry = buildResourceRegistry(resolved.spec, config.api_key) try { const parser = new SpecParser() const parsed = parser.parse(resolved.spec, { @@ -178,7 +178,7 @@ const source = { const resolved = await resolveOpenApiSpec({ apiVersion: config.api_version ?? '2020-08-27', }) - const registry = buildResourceRegistry(stripe, resolved.spec, config.api_key) + const registry = buildResourceRegistry(resolved.spec, config.api_key) const streamNames = new Set(catalog.streams.map((s) => s.stream.name)) // Event-driven mode: iterate over incoming webhook inputs diff --git a/packages/source-stripe/src/resourceRegistry.ts b/packages/source-stripe/src/resourceRegistry.ts index 43e5cdbee..626f1d63f 100644 --- a/packages/source-stripe/src/resourceRegistry.ts +++ b/packages/source-stripe/src/resourceRegistry.ts @@ -1,13 +1,11 @@ -import Stripe from 'stripe' import type { ResourceConfig } from './types.js' import type { OpenApiSpec, NestedEndpoint } from '@stripe/openapi' import { discoverListEndpoints, discoverNestedEndpoints, + isV2Path, buildListFn, buildRetrieveFn, - canResolveSdkResource, - isV2Path, RUNTIME_REQUIRED_TABLES as OPENAPI_RUNTIME_REQUIRED_TABLES, } from '@stripe/openapi' @@ -65,7 +63,6 @@ export const RESOURCE_TABLE_NAME_MAP: Record = Object.fromEntrie * All resources get list + retrieve functions derived dynamically from the spec paths. */ export function buildResourceRegistry( - stripe: Stripe, spec: OpenApiSpec, apiKey: string ): Record { @@ -76,7 +73,6 @@ export function buildResourceRegistry( for (const [tableName, endpoint] of endpoints) { const v2 = isV2Path(endpoint.apiPath) - if (!v2 && !canResolveSdkResource(stripe, endpoint.apiPath)) continue const children = nestedEndpoints .filter((n: NestedEndpoint) => n.parentTableName === tableName) @@ -95,8 +91,8 @@ export function buildResourceRegistry( supportsLimit: endpoint.supportsLimit, sync: true, dependencies: [], - listFn: buildListFn(stripe, endpoint.apiPath, apiKey), - retrieveFn: buildRetrieveFn(stripe, endpoint.apiPath, apiKey), + listFn: buildListFn(apiKey, endpoint.apiPath), + retrieveFn: buildRetrieveFn(apiKey, endpoint.apiPath), nestedResources: children.length > 0 ? children : undefined, } registry[tableName] = config diff --git a/packages/source-stripe/src/types.ts b/packages/source-stripe/src/types.ts index f67be9d3c..bb549066c 100644 --- a/packages/source-stripe/src/types.ts +++ b/packages/source-stripe/src/types.ts @@ -1,4 +1,5 @@ -import Stripe from 'stripe' +import type Stripe from 'stripe' +import type { ListFn, RetrieveFn } from '@stripe/openapi' import type { RevalidateEntityName } from './resourceRegistry.js' /** @@ -30,19 +31,10 @@ export type BaseResourceConfig = { } export type ResourceConfig = BaseResourceConfig & { - /** Function to list items from Stripe API */ - listFn?: (params: Stripe.PaginationParams & { created?: Stripe.RangeQueryParam }) => Promise<{ - data: unknown[] - has_more: boolean - pageCursor?: string - }> - /** Function to retrieve a single item by ID from Stripe API */ - // eslint-disable-next-line @typescript-eslint/no-explicit-any - retrieveFn?: (id: string) => Promise> + listFn?: ListFn + retrieveFn?: RetrieveFn /** Whether the list API supports the `limit` parameter */ supportsLimit?: boolean - /** Optional list of sub-resources to expand during upsert/fetching (e.g. 'refunds', 'listLineItems') */ - listExpands?: Record Promise>>[] /** Nested child resources discovered from the spec (e.g. subscription items under subscriptions) */ nestedResources?: { tableName: string From 72ca19bee260d33b05d718823253f810396dbd47 Mon Sep 17 00:00:00 2001 From: Yostra Date: Wed, 25 Mar 2026 03:34:27 +0100 Subject: [PATCH 10/19] remove hard coded v2 & runtime mapping --- packages/openapi/index.ts | 3 +- packages/openapi/listFnResolver.ts | 21 +++++++------- packages/openapi/runtimeMappings.ts | 28 ------------------- packages/openapi/specParser.ts | 8 ++---- packages/source-stripe/src/index.ts | 12 ++------ .../source-stripe/src/resourceRegistry.ts | 12 ++++---- 6 files changed, 24 insertions(+), 60 deletions(-) diff --git a/packages/openapi/index.ts b/packages/openapi/index.ts index fb25b6ed1..8976e3589 100644 --- a/packages/openapi/index.ts +++ b/packages/openapi/index.ts @@ -1,6 +1,7 @@ export type * from './types.js' export { SpecParser, OPENAPI_RESOURCE_TABLE_ALIASES } from './specParser.js' export { OPENAPI_COMPATIBILITY_COLUMNS } from './runtimeMappings.js' + export { resolveOpenApiSpec } from './specFetchHelper.js' export { discoverListEndpoints, @@ -8,6 +9,7 @@ export { isV2Path, buildListFn, buildRetrieveFn, + resolveTableName, } from './listFnResolver.js' export type { ListEndpoint, @@ -17,4 +19,3 @@ export type { ListParams, } from './listFnResolver.js' export { parsedTableToJsonSchema } from './jsonSchemaConverter.js' -export { RUNTIME_REQUIRED_TABLES } from './runtimeMappings.js' diff --git a/packages/openapi/listFnResolver.ts b/packages/openapi/listFnResolver.ts index 1b1b4f34d..d919ab76d 100644 --- a/packages/openapi/listFnResolver.ts +++ b/packages/openapi/listFnResolver.ts @@ -33,7 +33,7 @@ export type NestedEndpoint = { supportsPagination: boolean } -function resolveTableName(resourceId: string, aliases: Record): string { +export function resolveTableName(resourceId: string, aliases: Record): string { const alias = aliases[resourceId] if (alias) return alias const normalized = resourceId.toLowerCase().replace(/[.]/g, '_') @@ -206,7 +206,6 @@ export function isV2Path(apiPath: string): boolean { // --------------------------------------------------------------------------- const STRIPE_API_BASE = 'https://api.stripe.com' -const V2_STRIPE_VERSION = '2026-02-25.clover' function authHeaders(apiKey: string): Record { return { Authorization: `Bearer ${apiKey}` } @@ -216,16 +215,17 @@ function authHeaders(apiKey: string): Record { * Build a callable list function that hits the Stripe HTTP API directly. * Supports both v1 (has_more pagination) and v2 (next_page_url pagination). */ -export function buildListFn(apiKey: string, apiPath: string): ListFn { +export function buildListFn(apiKey: string, apiPath: string, apiVersion?: string): ListFn { if (isV2Path(apiPath)) { return async (params) => { const qs = new URLSearchParams() qs.set('limit', String(Math.min(params.limit ?? 20, 20))) if (params.starting_after) qs.set('page', params.starting_after) - const response = await fetch(`${STRIPE_API_BASE}${apiPath}?${qs}`, { - headers: { ...authHeaders(apiKey), 'Stripe-Version': V2_STRIPE_VERSION }, - }) + const headers = authHeaders(apiKey) + if (apiVersion) headers['Stripe-Version'] = apiVersion + + const response = await fetch(`${STRIPE_API_BASE}${apiPath}?${qs}`, { headers }) const body = (await response.json()) as { data: unknown[] next_page_url?: string | null @@ -257,12 +257,13 @@ export function buildListFn(apiKey: string, apiPath: string): ListFn { /** * Build a callable retrieve function that hits the Stripe HTTP API directly. */ -export function buildRetrieveFn(apiKey: string, apiPath: string): RetrieveFn { +export function buildRetrieveFn(apiKey: string, apiPath: string, apiVersion?: string): RetrieveFn { if (isV2Path(apiPath)) { return async (id) => { - const response = await fetch(`${STRIPE_API_BASE}${apiPath}/${id}`, { - headers: { ...authHeaders(apiKey), 'Stripe-Version': V2_STRIPE_VERSION }, - }) + const headers = authHeaders(apiKey) + if (apiVersion) headers['Stripe-Version'] = apiVersion + + const response = await fetch(`${STRIPE_API_BASE}${apiPath}/${id}`, { headers }) return await response.json() } } diff --git a/packages/openapi/runtimeMappings.ts b/packages/openapi/runtimeMappings.ts index b88c69c60..c5ee8d336 100644 --- a/packages/openapi/runtimeMappings.ts +++ b/packages/openapi/runtimeMappings.ts @@ -1,33 +1,5 @@ import type { ParsedColumn } from './types' -/** - * Table names that must exist for runtime sync and webhook processing. - * Includes all default sync objects plus child tables. - */ -export const RUNTIME_REQUIRED_TABLES: ReadonlyArray = [ - 'products', - 'coupons', - 'prices', - 'plans', - 'customers', - 'subscriptions', - 'subscription_schedules', - 'invoices', - 'charges', - 'setup_intents', - 'payment_methods', - 'payment_intents', - 'tax_ids', - 'credit_notes', - 'disputes', - 'early_fraud_warnings', - 'refunds', - 'checkout_sessions', - 'subscription_items', - 'checkout_session_line_items', - 'features', -] - /** * Overrides for x-resourceId values whose table name cannot be inferred by the * default pluralisation / dot-to-underscore rule in SpecParser.resolveTableName. diff --git a/packages/openapi/specParser.ts b/packages/openapi/specParser.ts index 438f4c8f7..8d24cf890 100644 --- a/packages/openapi/specParser.ts +++ b/packages/openapi/specParser.ts @@ -7,11 +7,7 @@ import type { ParsedOpenApiSpec, ScalarType, } from './types' -import { - RUNTIME_REQUIRED_TABLES, - OPENAPI_COMPATIBILITY_COLUMNS, - OPENAPI_RESOURCE_TABLE_ALIASES, -} from './runtimeMappings' +import { OPENAPI_COMPATIBILITY_COLUMNS, OPENAPI_RESOURCE_TABLE_ALIASES } from './runtimeMappings' const SCHEMA_REF_PREFIX = '#/components/schemas/' @@ -23,7 +19,7 @@ const RESERVED_COLUMNS = new Set([ '_account_id', ]) -export { RUNTIME_REQUIRED_TABLES, OPENAPI_RESOURCE_TABLE_ALIASES } +export { OPENAPI_RESOURCE_TABLE_ALIASES } type ColumnAccumulator = { type: ScalarType diff --git a/packages/source-stripe/src/index.ts b/packages/source-stripe/src/index.ts index fc942cbb9..b72a7c045 100644 --- a/packages/source-stripe/src/index.ts +++ b/packages/source-stripe/src/index.ts @@ -8,12 +8,7 @@ import Stripe from 'stripe' import { z } from 'zod' import { buildResourceRegistry } from './resourceRegistry.js' import { catalogFromRegistry, catalogFromOpenApi } from './catalog.js' -import { resolveOpenApiSpec } from './openapi/specFetchHelper.js' -import { - SpecParser, - RUNTIME_REQUIRED_TABLES, - OPENAPI_RESOURCE_TABLE_ALIASES, -} from './openapi/specParser.js' +import { resolveOpenApiSpec, SpecParser, OPENAPI_RESOURCE_TABLE_ALIASES } from '@stripe/openapi' import { processStripeEvent } from './process-event.js' import { processWebhookInput, createInputQueue, startWebhookServer } from './src-webhook.js' import { listApiBackfill } from './src-list-api.js' @@ -122,12 +117,11 @@ const source = { const resolved = await resolveOpenApiSpec({ apiVersion: config.api_version ?? '2020-08-27', }) - const registry = buildResourceRegistry(resolved.spec, config.api_key) + const registry = buildResourceRegistry(resolved.spec, config.api_key, resolved.apiVersion) try { const parser = new SpecParser() const parsed = parser.parse(resolved.spec, { resourceAliases: OPENAPI_RESOURCE_TABLE_ALIASES, - allowedTables: [...RUNTIME_REQUIRED_TABLES], }) return catalogFromOpenApi(parsed.tables, registry) } catch { @@ -178,7 +172,7 @@ const source = { const resolved = await resolveOpenApiSpec({ apiVersion: config.api_version ?? '2020-08-27', }) - const registry = buildResourceRegistry(resolved.spec, config.api_key) + const registry = buildResourceRegistry(resolved.spec, config.api_key, resolved.apiVersion) const streamNames = new Set(catalog.streams.map((s) => s.stream.name)) // Event-driven mode: iterate over incoming webhook inputs diff --git a/packages/source-stripe/src/resourceRegistry.ts b/packages/source-stripe/src/resourceRegistry.ts index 626f1d63f..a09d88889 100644 --- a/packages/source-stripe/src/resourceRegistry.ts +++ b/packages/source-stripe/src/resourceRegistry.ts @@ -6,7 +6,8 @@ import { isV2Path, buildListFn, buildRetrieveFn, - RUNTIME_REQUIRED_TABLES as OPENAPI_RUNTIME_REQUIRED_TABLES, + resolveTableName, + OPENAPI_RESOURCE_TABLE_ALIASES, } from '@stripe/openapi' /** @@ -52,8 +53,6 @@ export const REVALIDATE_ENTITIES = [ ] as const export type RevalidateEntityName = (typeof REVALIDATE_ENTITIES)[number] -export const RUNTIME_REQUIRED_TABLES: ReadonlyArray = OPENAPI_RUNTIME_REQUIRED_TABLES - export const RESOURCE_TABLE_NAME_MAP: Record = Object.fromEntries( DEFAULT_SYNC_OBJECTS.map((t) => [t, t]) ) @@ -64,7 +63,8 @@ export const RESOURCE_TABLE_NAME_MAP: Record = Object.fromEntrie */ export function buildResourceRegistry( spec: OpenApiSpec, - apiKey: string + apiKey: string, + apiVersion?: string ): Record { const endpoints = discoverListEndpoints(spec) const nestedEndpoints = discoverNestedEndpoints(spec, endpoints) @@ -91,8 +91,8 @@ export function buildResourceRegistry( supportsLimit: endpoint.supportsLimit, sync: true, dependencies: [], - listFn: buildListFn(apiKey, endpoint.apiPath), - retrieveFn: buildRetrieveFn(apiKey, endpoint.apiPath), + listFn: buildListFn(apiKey, endpoint.apiPath, apiVersion), + retrieveFn: buildRetrieveFn(apiKey, endpoint.apiPath, apiVersion), nestedResources: children.length > 0 ? children : undefined, } registry[tableName] = config From 33f99797a1bda392ca387da920da36a8e7756108 Mon Sep 17 00:00:00 2001 From: Yostra Date: Wed, 25 Mar 2026 04:13:32 +0100 Subject: [PATCH 11/19] use normalize --- packages/source-stripe/src/index.test.ts | 70 +++++++++---------- packages/source-stripe/src/process-event.ts | 7 +- .../source-stripe/src/resourceRegistry.ts | 9 +-- 3 files changed, 43 insertions(+), 43 deletions(-) diff --git a/packages/source-stripe/src/index.test.ts b/packages/source-stripe/src/index.test.ts index ba3224316..acbb7ce62 100644 --- a/packages/source-stripe/src/index.test.ts +++ b/packages/source-stripe/src/index.test.ts @@ -116,8 +116,8 @@ describe('StripeSource', () => { describe('discover()', () => { it('returns a CatalogMessage with known streams', async () => { const registry: Record = { - customer: makeConfig({ order: 1, tableName: 'customers' }), - invoice: makeConfig({ order: 2, tableName: 'invoices' }), + customers: makeConfig({ order: 1, tableName: 'customers' }), + invoices: makeConfig({ order: 2, tableName: 'invoices' }), } vi.mocked(buildResourceRegistry).mockReturnValue(registry as any) @@ -130,7 +130,7 @@ describe('StripeSource', () => { it('excludes resources with sync: false', async () => { const registry: Record = { - customer: makeConfig({ order: 1, tableName: 'customers' }), + customers: makeConfig({ order: 1, tableName: 'customers' }), internal: makeConfig({ order: 2, tableName: 'internal', sync: false }), } @@ -169,7 +169,7 @@ describe('StripeSource', () => { }) const registry: Record = { - customer: makeConfig({ + customers: makeConfig({ order: 1, tableName: 'customers', listFn: listFn as ResourceConfig['listFn'], @@ -236,12 +236,12 @@ describe('StripeSource', () => { }) const registry: Record = { - customer: makeConfig({ + customers: makeConfig({ order: 1, tableName: 'customers', listFn: custListFn as ResourceConfig['listFn'], }), - invoice: makeConfig({ + invoices: makeConfig({ order: 2, tableName: 'invoices', listFn: invListFn as ResourceConfig['listFn'], @@ -298,7 +298,7 @@ describe('StripeSource', () => { }) const registry: Record = { - customer: makeConfig({ + customers: makeConfig({ order: 1, tableName: 'customers', listFn: listFn as ResourceConfig['listFn'], @@ -334,7 +334,7 @@ describe('StripeSource', () => { }) const registry: Record = { - customer: makeConfig({ + customers: makeConfig({ order: 1, tableName: 'customers', listFn: listFn as ResourceConfig['listFn'], @@ -372,7 +372,7 @@ describe('StripeSource', () => { describe('fromWebhookEvent() — live mode scenarios', () => { it('webhook mode emits one RecordMessage + one StateMessage per event', () => { const registry: Record = { - customer: makeConfig({ order: 1, tableName: 'customers' }), + customers: makeConfig({ order: 1, tableName: 'customers' }), } const event = makeEvent({ @@ -404,7 +404,7 @@ describe('StripeSource', () => { it('returns null for unsupported object type', () => { const registry: Record = { - customer: makeConfig({ order: 1, tableName: 'customers' }), + customers: makeConfig({ order: 1, tableName: 'customers' }), } const event = makeEvent({ @@ -417,7 +417,7 @@ describe('StripeSource', () => { it('returns null for objects without id (preview/draft)', () => { const registry: Record = { - invoice: makeConfig({ order: 1, tableName: 'invoices' }), + invoices: makeConfig({ order: 1, tableName: 'invoices' }), } const event = makeEvent({ @@ -431,7 +431,7 @@ describe('StripeSource', () => { it('passes through deleted flag from event data', () => { const registry: Record = { - customer: makeConfig({ order: 1, tableName: 'customers' }), + customers: makeConfig({ order: 1, tableName: 'customers' }), } const event = makeEvent({ @@ -451,7 +451,7 @@ describe('StripeSource', () => { it('returns null when event data.object has no object field', () => { const registry: Record = { - customer: makeConfig({ order: 1, tableName: 'customers' }), + customers: makeConfig({ order: 1, tableName: 'customers' }), } const event = makeEvent({ @@ -467,7 +467,7 @@ describe('StripeSource', () => { // The same Stripe.Event structure is received regardless of transport. // This test verifies fromWebhookEvent works for any Stripe.Event input. const registry: Record = { - invoice: makeConfig({ order: 1, tableName: 'invoices' }), + invoices: makeConfig({ order: 1, tableName: 'invoices' }), } const event = makeEvent({ @@ -491,7 +491,7 @@ describe('StripeSource', () => { const listFn = vi.fn().mockRejectedValueOnce(new Error('Rate limit exceeded')) const registry: Record = { - customer: makeConfig({ + customers: makeConfig({ order: 1, tableName: 'customers', listFn: listFn as ResourceConfig['listFn'], @@ -541,7 +541,7 @@ describe('StripeSource', () => { const listFn = vi.fn().mockRejectedValueOnce(new Error('Connection refused')) const registry: Record = { - customer: makeConfig({ + customers: makeConfig({ order: 1, tableName: 'customers', listFn: listFn as ResourceConfig['listFn'], @@ -568,12 +568,12 @@ describe('StripeSource', () => { }) const registry: Record = { - customer: makeConfig({ + customers: makeConfig({ order: 1, tableName: 'customers', listFn: failingListFn as ResourceConfig['listFn'], }), - invoice: makeConfig({ + invoices: makeConfig({ order: 2, tableName: 'invoices', listFn: successListFn as ResourceConfig['listFn'], @@ -621,7 +621,7 @@ describe('StripeSource', () => { // Shared registry for these tests const listFn = vi.fn() const registry: Record = { - customer: makeConfig({ + customers: makeConfig({ order: 1, tableName: 'customers', listFn: listFn as ResourceConfig['listFn'], @@ -810,7 +810,7 @@ describe('StripeSource', () => { describe('read(input) — enriched webhook processing', () => { it('delete event yields record with deleted: true', async () => { const registry: Record = { - customer: makeConfig({ order: 1, tableName: 'customers' }), + customers: makeConfig({ order: 1, tableName: 'customers' }), } vi.mocked(buildResourceRegistry).mockReturnValue(registry as any) @@ -840,7 +840,7 @@ describe('StripeSource', () => { it('delete event detected by event type (not just deleted flag)', async () => { const registry: Record = { - product: makeConfig({ order: 1, tableName: 'products' }), + products: makeConfig({ order: 1, tableName: 'products' }), } vi.mocked(buildResourceRegistry).mockReturnValue(registry as any) @@ -866,7 +866,7 @@ describe('StripeSource', () => { it('subscription event yields subscription_items from nested items.data', async () => { const registry: Record = { - subscription: makeConfig({ order: 1, tableName: 'subscriptions' }), + subscriptions: makeConfig({ order: 1, tableName: 'subscriptions' }), } vi.mocked(buildResourceRegistry).mockReturnValue(registry as any) @@ -992,7 +992,7 @@ describe('StripeSource', () => { }) const registry: Record = { - subscription: makeConfig({ + subscriptions: makeConfig({ order: 1, tableName: 'subscriptions', retrieveFn: retrieveFn as ResourceConfig['retrieveFn'], @@ -1027,7 +1027,7 @@ describe('StripeSource', () => { const retrieveFn = vi.fn() const registry: Record = { - subscription: makeConfig({ + subscriptions: makeConfig({ order: 1, tableName: 'subscriptions', retrieveFn: retrieveFn as ResourceConfig['retrieveFn'], @@ -1061,7 +1061,7 @@ describe('StripeSource', () => { it('preview objects (no id) produce no output', async () => { const registry: Record = { - invoice: makeConfig({ order: 1, tableName: 'invoices' }), + invoices: makeConfig({ order: 1, tableName: 'invoices' }), } vi.mocked(buildResourceRegistry).mockReturnValue(registry as any) @@ -1105,7 +1105,7 @@ describe('StripeSource', () => { it('throws when raw webhook input is provided without webhook_secret', async () => { const registry: Record = { - customer: makeConfig({ order: 1, tableName: 'customers' }), + customers: makeConfig({ order: 1, tableName: 'customers' }), } vi.mocked(buildResourceRegistry).mockReturnValue(registry as any) @@ -1124,12 +1124,12 @@ describe('StripeSource', () => { describe('read() — WebSocket streaming', () => { const registry: Record = { - customer: makeConfig({ + customers: makeConfig({ order: 1, tableName: 'customers', listFn: (() => Promise.resolve({ data: [], has_more: false })) as ResourceConfig['listFn'], }), - invoice: makeConfig({ + invoices: makeConfig({ order: 2, tableName: 'invoices', listFn: (() => Promise.resolve({ data: [], has_more: false })) as ResourceConfig['listFn'], @@ -1256,7 +1256,7 @@ describe('StripeSource', () => { }) const wsRegistry: Record = { - customer: makeConfig({ + customers: makeConfig({ order: 1, tableName: 'customers', listFn: listFn as ResourceConfig['listFn'], @@ -1418,7 +1418,7 @@ describe('StripeSource', () => { it('starts an HTTP server on webhook_port and processes POSTed webhooks', async () => { const listFn = vi.fn().mockResolvedValue({ data: [], has_more: false }) const registry: Record = { - customer: makeConfig({ order: 1, tableName: 'customers', listFn }), + customers: makeConfig({ order: 1, tableName: 'customers', listFn }), } vi.mocked(buildResourceRegistry).mockReturnValue(registry as any) const cat = catalog({ name: 'customers' }) @@ -1452,7 +1452,7 @@ describe('StripeSource', () => { it('skips backfill when all streams are already complete', async () => { const listFn = vi.fn() const registry: Record = { - customer: makeConfig({ + customers: makeConfig({ order: 1, tableName: 'customers', listFn: listFn as ResourceConfig['listFn'], @@ -1481,7 +1481,7 @@ describe('StripeSource', () => { it('stamps initial events_cursor after first backfill completes', async () => { const listFn = vi.fn() const registry: Record = { - customer: makeConfig({ + customers: makeConfig({ order: 1, tableName: 'customers', listFn: listFn as ResourceConfig['listFn'], @@ -1511,7 +1511,7 @@ describe('StripeSource', () => { it('does not run events polling when poll_events is false/absent', async () => { const listFn = vi.fn() const registry: Record = { - customer: makeConfig({ + customers: makeConfig({ order: 1, tableName: 'customers', listFn: listFn as ResourceConfig['listFn'], @@ -1542,12 +1542,12 @@ describe('StripeSource', () => { }) const registry: Record = { - customer: makeConfig({ + customers: makeConfig({ order: 1, tableName: 'customers', listFn: custListFn as ResourceConfig['listFn'], }), - invoice: makeConfig({ + invoices: makeConfig({ order: 2, tableName: 'invoices', listFn: (() => diff --git a/packages/source-stripe/src/process-event.ts b/packages/source-stripe/src/process-event.ts index ed3433147..4bf89eee5 100644 --- a/packages/source-stripe/src/process-event.ts +++ b/packages/source-stripe/src/process-event.ts @@ -47,8 +47,7 @@ export function fromWebhookEvent( | undefined if (!dataObject?.object) return null - // Find config by matching registry keys to the Stripe object type - const objectType = dataObject.object + const objectType = normalizeStripeObjectName(dataObject.object) const config = registry[objectType] if (!config) return null @@ -147,7 +146,7 @@ export async function* processStripeEvent( // 5. Revalidation — re-fetch from Stripe API if configured let data: Record = dataObject if ( - config.revalidate_objects?.includes(objectType) && + config.revalidate_objects?.some((r) => normalizeStripeObjectName(r) === objectType) && resourceConfig.isFinalState && !resourceConfig.isFinalState(dataObject) ) { @@ -158,7 +157,7 @@ export async function* processStripeEvent( yield toRecordMessage(resourceConfig.tableName, data) // 7. Yield subscription items if applicable - if (objectType === 'subscription' && (data as { items?: { data?: unknown[] } }).items?.data) { + if (objectType === 'subscriptions' && (data as { items?: { data?: unknown[] } }).items?.data) { for (const item of (data as { items: { data: Record[] } }).items.data) { yield toRecordMessage('subscription_items', item) } diff --git a/packages/source-stripe/src/resourceRegistry.ts b/packages/source-stripe/src/resourceRegistry.ts index a09d88889..767ae857f 100644 --- a/packages/source-stripe/src/resourceRegistry.ts +++ b/packages/source-stripe/src/resourceRegistry.ts @@ -96,11 +96,10 @@ export function buildResourceRegistry( nestedResources: children.length > 0 ? children : undefined, } registry[tableName] = config - registry[endpoint.resourceId] = config } for (const nested of nestedEndpoints) { - if (!nested.parentTableName || registry[nested.tableName] || registry[nested.resourceId]) { + if (!nested.parentTableName || registry[nested.tableName]) { continue } if (seenNested.has(nested.tableName)) { @@ -122,7 +121,6 @@ export function buildResourceRegistry( } registry[nested.tableName] = config - registry[nested.resourceId] = config } return registry @@ -137,7 +135,10 @@ export const STRIPE_OBJECT_TO_SYNC_OBJECT_ALIASES: Record = { } export function normalizeStripeObjectName(stripeObjectName: string): string { - return STRIPE_OBJECT_TO_SYNC_OBJECT_ALIASES[stripeObjectName] ?? stripeObjectName + return resolveTableName(stripeObjectName, { + ...OPENAPI_RESOURCE_TABLE_ALIASES, + ...STRIPE_OBJECT_TO_SYNC_OBJECT_ALIASES, + }) } export const PREFIX_RESOURCE_MAP: Record = { From be074086edba633a0b6d6453b246e85f523acb09 Mon Sep 17 00:00:00 2001 From: Yostra Date: Wed, 25 Mar 2026 04:47:39 +0100 Subject: [PATCH 12/19] rebase cleanup --- packages/openapi/jsonSchemaConverter.ts | 2 +- packages/openapi/listFnResolver.ts | 4 ++-- packages/openapi/package.json | 15 +++++++++++---- packages/openapi/runtimeMappings.ts | 2 +- packages/openapi/specFetchHelper.ts | 2 +- packages/openapi/specParser.ts | 4 ++-- packages/openapi/tsconfig.json | 5 +++-- packages/source-stripe/src/process-event.ts | 2 +- packages/source-stripe/tsconfig.json | 2 +- pnpm-lock.yaml | 4 ---- 10 files changed, 23 insertions(+), 19 deletions(-) diff --git a/packages/openapi/jsonSchemaConverter.ts b/packages/openapi/jsonSchemaConverter.ts index 31d9b3924..1409d9a87 100644 --- a/packages/openapi/jsonSchemaConverter.ts +++ b/packages/openapi/jsonSchemaConverter.ts @@ -1,4 +1,4 @@ -import type { ParsedResourceTable, ScalarType } from './types' +import type { ParsedResourceTable, ScalarType } from './types.js' const SCALAR_TYPE_TO_JSON_SCHEMA: Record = { text: { type: 'string' }, diff --git a/packages/openapi/listFnResolver.ts b/packages/openapi/listFnResolver.ts index d919ab76d..e81d58058 100644 --- a/packages/openapi/listFnResolver.ts +++ b/packages/openapi/listFnResolver.ts @@ -1,5 +1,5 @@ -import type { OpenApiSchemaObject, OpenApiSpec } from './types' -import { OPENAPI_RESOURCE_TABLE_ALIASES } from './runtimeMappings' +import type { OpenApiSchemaObject, OpenApiSpec } from './types.js' +import { OPENAPI_RESOURCE_TABLE_ALIASES } from './runtimeMappings.js' const SCHEMA_REF_PREFIX = '#/components/schemas/' diff --git a/packages/openapi/package.json b/packages/openapi/package.json index ef99149ec..fcd4055b7 100644 --- a/packages/openapi/package.json +++ b/packages/openapi/package.json @@ -3,16 +3,23 @@ "version": "0.1.0", "private": true, "type": "module", - "main": "./dist/index.cjs", "exports": { ".": { "types": "./index.ts", - "import": "./dist/index.js", - "require": "./dist/index.cjs" + "import": "./index.ts", + "default": "./index.ts" + } + }, + "publishConfig": { + "exports": { + ".": { + "types": "./dist/index.d.ts", + "import": "./dist/index.js" + } } }, "scripts": { - "build": "tsup index.ts --format esm,cjs --dts", + "build": "tsc", "test": "vitest --passWithNoTests" }, "files": [ diff --git a/packages/openapi/runtimeMappings.ts b/packages/openapi/runtimeMappings.ts index c5ee8d336..423986662 100644 --- a/packages/openapi/runtimeMappings.ts +++ b/packages/openapi/runtimeMappings.ts @@ -1,4 +1,4 @@ -import type { ParsedColumn } from './types' +import type { ParsedColumn } from './types.js' /** * Overrides for x-resourceId values whose table name cannot be inferred by the diff --git a/packages/openapi/specFetchHelper.ts b/packages/openapi/specFetchHelper.ts index d3aafbe25..2b0dcec4b 100644 --- a/packages/openapi/specFetchHelper.ts +++ b/packages/openapi/specFetchHelper.ts @@ -1,7 +1,7 @@ import os from 'node:os' import fs from 'node:fs/promises' import path from 'node:path' -import type { OpenApiSpec, ResolveSpecConfig, ResolvedOpenApiSpec } from './types' +import type { OpenApiSpec, ResolveSpecConfig, ResolvedOpenApiSpec } from './types.js' const DEFAULT_CACHE_DIR = path.join(os.tmpdir(), 'stripe-sync-openapi-cache') diff --git a/packages/openapi/specParser.ts b/packages/openapi/specParser.ts index 8d24cf890..e670b4218 100644 --- a/packages/openapi/specParser.ts +++ b/packages/openapi/specParser.ts @@ -6,8 +6,8 @@ import type { ParsedColumn, ParsedOpenApiSpec, ScalarType, -} from './types' -import { OPENAPI_COMPATIBILITY_COLUMNS, OPENAPI_RESOURCE_TABLE_ALIASES } from './runtimeMappings' +} from './types.js' +import { OPENAPI_COMPATIBILITY_COLUMNS, OPENAPI_RESOURCE_TABLE_ALIASES } from './runtimeMappings.js' const SCHEMA_REF_PREFIX = '#/components/schemas/' diff --git a/packages/openapi/tsconfig.json b/packages/openapi/tsconfig.json index 362cfc683..05a09c37d 100644 --- a/packages/openapi/tsconfig.json +++ b/packages/openapi/tsconfig.json @@ -1,9 +1,10 @@ { "compilerOptions": { "target": "esnext", - "module": "esnext", - "moduleResolution": "bundler", + "module": "nodenext", + "moduleResolution": "nodenext", "declaration": true, + "declarationMap": true, "outDir": "dist", "esModuleInterop": true, "skipLibCheck": true, diff --git a/packages/source-stripe/src/process-event.ts b/packages/source-stripe/src/process-event.ts index 4bf89eee5..9ce197bcb 100644 --- a/packages/source-stripe/src/process-event.ts +++ b/packages/source-stripe/src/process-event.ts @@ -150,7 +150,7 @@ export async function* processStripeEvent( resourceConfig.isFinalState && !resourceConfig.isFinalState(dataObject) ) { - data = (await resourceConfig.retrieveFn(dataObject.id)) as Record + data = (await resourceConfig.retrieveFn!(dataObject.id)) as Record } // 6. Yield main record diff --git a/packages/source-stripe/tsconfig.json b/packages/source-stripe/tsconfig.json index 335f0ae21..b26b20b9a 100644 --- a/packages/source-stripe/tsconfig.json +++ b/packages/source-stripe/tsconfig.json @@ -13,5 +13,5 @@ "rootDir": "src" }, "include": ["src/**/*"], - "exclude": ["src/server", "src/**/*.test.ts", "src/**/__tests__/**"] + "exclude": ["src/server", "src/openapi", "src/**/*.test.ts", "src/**/__tests__/**"] } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 36b49b672..44b917972 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -346,10 +346,6 @@ importers: version: 3.2.4(@types/node@25.5.0)(jiti@2.6.1)(lightningcss@1.32.0)(terser@5.46.1)(tsx@4.21.0)(yaml@2.8.1) packages/openapi: - dependencies: - stripe: - specifier: ^17.7.0 - version: 17.7.0 devDependencies: '@types/node': specifier: ^24.5.0 From 74bd9a252acc47d2f716567a1bd6db72d5b70c43 Mon Sep 17 00:00:00 2001 From: Yostra Date: Wed, 25 Mar 2026 05:24:04 +0100 Subject: [PATCH 13/19] rebase cleanup --- packages/openapi/listFnResolver.ts | 30 ++++++++++++++----- packages/source-stripe/src/index.ts | 14 +++++++-- .../source-stripe/src/resourceRegistry.ts | 7 +++-- 3 files changed, 38 insertions(+), 13 deletions(-) diff --git a/packages/openapi/listFnResolver.ts b/packages/openapi/listFnResolver.ts index e81d58058..b407e87ce 100644 --- a/packages/openapi/listFnResolver.ts +++ b/packages/openapi/listFnResolver.ts @@ -205,7 +205,7 @@ export function isV2Path(apiPath: string): boolean { // HTTP-based list / retrieve builders (no Stripe SDK dependency) // --------------------------------------------------------------------------- -const STRIPE_API_BASE = 'https://api.stripe.com' +const DEFAULT_STRIPE_API_BASE = 'https://api.stripe.com' function authHeaders(apiKey: string): Record { return { Authorization: `Bearer ${apiKey}` } @@ -215,7 +215,14 @@ function authHeaders(apiKey: string): Record { * Build a callable list function that hits the Stripe HTTP API directly. * Supports both v1 (has_more pagination) and v2 (next_page_url pagination). */ -export function buildListFn(apiKey: string, apiPath: string, apiVersion?: string): ListFn { +export function buildListFn( + apiKey: string, + apiPath: string, + apiVersion?: string, + baseUrl?: string +): ListFn { + const base = baseUrl ?? DEFAULT_STRIPE_API_BASE + if (isV2Path(apiPath)) { return async (params) => { const qs = new URLSearchParams() @@ -225,7 +232,7 @@ export function buildListFn(apiKey: string, apiPath: string, apiVersion?: string const headers = authHeaders(apiKey) if (apiVersion) headers['Stripe-Version'] = apiVersion - const response = await fetch(`${STRIPE_API_BASE}${apiPath}?${qs}`, { headers }) + const response = await fetch(`${base}${apiPath}?${qs}`, { headers }) const body = (await response.json()) as { data: unknown[] next_page_url?: string | null @@ -246,7 +253,7 @@ export function buildListFn(apiKey: string, apiPath: string, apiVersion?: string } } - const response = await fetch(`${STRIPE_API_BASE}${apiPath}?${qs}`, { + const response = await fetch(`${base}${apiPath}?${qs}`, { headers: authHeaders(apiKey), }) const body = (await response.json()) as { data: unknown[]; has_more: boolean } @@ -257,19 +264,26 @@ export function buildListFn(apiKey: string, apiPath: string, apiVersion?: string /** * Build a callable retrieve function that hits the Stripe HTTP API directly. */ -export function buildRetrieveFn(apiKey: string, apiPath: string, apiVersion?: string): RetrieveFn { +export function buildRetrieveFn( + apiKey: string, + apiPath: string, + apiVersion?: string, + baseUrl?: string +): RetrieveFn { + const base = baseUrl ?? DEFAULT_STRIPE_API_BASE + if (isV2Path(apiPath)) { return async (id) => { const headers = authHeaders(apiKey) if (apiVersion) headers['Stripe-Version'] = apiVersion - const response = await fetch(`${STRIPE_API_BASE}${apiPath}/${id}`, { headers }) + const response = await fetch(`${base}${apiPath}/${id}`, { headers }) return await response.json() } } return async (id) => { - const response = await fetch(`${STRIPE_API_BASE}${apiPath}/${id}`, { + const response = await fetch(`${base}${apiPath}/${id}`, { headers: authHeaders(apiKey), }) return await response.json() @@ -279,7 +293,7 @@ export function buildRetrieveFn(apiKey: string, apiPath: string, apiVersion?: st function extractPageToken(nextPageUrl: string | null | undefined): string | undefined { if (!nextPageUrl) return undefined try { - const url = new URL(nextPageUrl, STRIPE_API_BASE) + const url = new URL(nextPageUrl, DEFAULT_STRIPE_API_BASE) return url.searchParams.get('page') ?? undefined } catch { return undefined diff --git a/packages/source-stripe/src/index.ts b/packages/source-stripe/src/index.ts index b72a7c045..ebceb3a66 100644 --- a/packages/source-stripe/src/index.ts +++ b/packages/source-stripe/src/index.ts @@ -117,7 +117,12 @@ const source = { const resolved = await resolveOpenApiSpec({ apiVersion: config.api_version ?? '2020-08-27', }) - const registry = buildResourceRegistry(resolved.spec, config.api_key, resolved.apiVersion) + const registry = buildResourceRegistry( + resolved.spec, + config.api_key, + resolved.apiVersion, + config.base_url + ) try { const parser = new SpecParser() const parsed = parser.parse(resolved.spec, { @@ -172,7 +177,12 @@ const source = { const resolved = await resolveOpenApiSpec({ apiVersion: config.api_version ?? '2020-08-27', }) - const registry = buildResourceRegistry(resolved.spec, config.api_key, resolved.apiVersion) + const registry = buildResourceRegistry( + resolved.spec, + config.api_key, + resolved.apiVersion, + config.base_url + ) const streamNames = new Set(catalog.streams.map((s) => s.stream.name)) // Event-driven mode: iterate over incoming webhook inputs diff --git a/packages/source-stripe/src/resourceRegistry.ts b/packages/source-stripe/src/resourceRegistry.ts index 767ae857f..6f80ef9e8 100644 --- a/packages/source-stripe/src/resourceRegistry.ts +++ b/packages/source-stripe/src/resourceRegistry.ts @@ -64,7 +64,8 @@ export const RESOURCE_TABLE_NAME_MAP: Record = Object.fromEntrie export function buildResourceRegistry( spec: OpenApiSpec, apiKey: string, - apiVersion?: string + apiVersion?: string, + baseUrl?: string ): Record { const endpoints = discoverListEndpoints(spec) const nestedEndpoints = discoverNestedEndpoints(spec, endpoints) @@ -91,8 +92,8 @@ export function buildResourceRegistry( supportsLimit: endpoint.supportsLimit, sync: true, dependencies: [], - listFn: buildListFn(apiKey, endpoint.apiPath, apiVersion), - retrieveFn: buildRetrieveFn(apiKey, endpoint.apiPath, apiVersion), + listFn: buildListFn(apiKey, endpoint.apiPath, apiVersion, baseUrl), + retrieveFn: buildRetrieveFn(apiKey, endpoint.apiPath, apiVersion, baseUrl), nestedResources: children.length > 0 ? children : undefined, } registry[tableName] = config From 14c90d47a550e7fbb9cd90b1f6cf9b72b3bf2725 Mon Sep 17 00:00:00 2001 From: Yostra Date: Wed, 25 Mar 2026 05:26:43 +0100 Subject: [PATCH 14/19] nit --- packages/openapi/types.ts | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/packages/openapi/types.ts b/packages/openapi/types.ts index cbd697c45..b4ccbebf9 100644 --- a/packages/openapi/types.ts +++ b/packages/openapi/types.ts @@ -43,7 +43,16 @@ export type OpenApiOperationObject = { responses?: Record } -export type OpenApiPathItem = Record +export type OpenApiPathItem = { + get?: OpenApiOperationObject + put?: OpenApiOperationObject + post?: OpenApiOperationObject + delete?: OpenApiOperationObject + options?: OpenApiOperationObject + head?: OpenApiOperationObject + patch?: OpenApiOperationObject + trace?: OpenApiOperationObject +} export type OpenApiSpec = { openapi: string From 2c58fa24ed9fe502e60469cdf6279e7a429ce84a Mon Sep 17 00:00:00 2001 From: Yostra Date: Wed, 25 Mar 2026 05:33:12 +0100 Subject: [PATCH 15/19] plugin loading --- e2e/connector-loading.test.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/e2e/connector-loading.test.sh b/e2e/connector-loading.test.sh index 81e084a7c..d1a33f769 100755 --- a/e2e/connector-loading.test.sh +++ b/e2e/connector-loading.test.sh @@ -191,14 +191,14 @@ echo "" # --------------------------------------------------------------------------- echo "--- Step 8: unknown connector name → not found ---" UNKNOWN_PARAMS='{"source_name":"nonexistent-xyz","source_config":{},"destination_name":"nonexistent-xyz","destination_config":{},"streams":[{"name":"x"}]}' -STEP8_OUTPUT=$(npx sync-engine check \ +unknown_output=$(npx sync-engine check \ --x-sync-params "$UNKNOWN_PARAMS" \ 2>&1 || true) -if echo "$STEP8_OUTPUT" | grep -qi "not found"; then +if echo "$unknown_output" | grep -qi "not found"; then echo " PASS: unknown connector correctly reports 'not found'" else echo " FAIL: unknown connector did not report 'not found'" - echo " Output: $STEP8_OUTPUT" + echo " Output: $unknown_output" exit 1 fi echo "" From b91b5fc0d6171f9a37c9303cf4da2c61634c03a6 Mon Sep 17 00:00:00 2001 From: Tony Xiao Date: Wed, 25 Mar 2026 01:48:02 -0700 Subject: [PATCH 16/19] =?UTF-8?q?Rename=20@stripe/openapi=20=E2=86=92=20@s?= =?UTF-8?q?tripe/sync-openapi;=20remove=20dead=20exports?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 1. Package naming convention: every package in the monorepo uses the @stripe/sync-* scope (sync-protocol, sync-engine, sync-source-stripe, etc). Rename to @stripe/sync-openapi for consistency. 2. Remove dead exports from resourceRegistry.ts that were never imported anywhere: StripeObject, CoreSyncObject, SyncObjectName (all just `string` aliases with no narrowing), CORE_SYNC_OBJECTS (redundant re-export of DEFAULT_SYNC_OBJECTS), SYNC_OBJECTS, and RESOURCE_TABLE_NAME_MAP (identity map of table→table). 3. Add missing cleanup line in e2e/connector-loading.test.sh for the openapi tarball — without this the cleanup() trap leaves a stale stripe-sync-openapi-*.tgz in the repo root. Committed-By-Agent: claude --- e2e/connector-loading.test.sh | 5 +++-- packages/openapi/package.json | 2 +- packages/source-stripe/package.json | 2 +- packages/source-stripe/src/catalog.ts | 4 ++-- packages/source-stripe/src/index.ts | 6 +++++- packages/source-stripe/src/resourceRegistry.ts | 18 ++---------------- packages/source-stripe/src/types.ts | 2 +- pnpm-lock.yaml | 4 ++-- 8 files changed, 17 insertions(+), 26 deletions(-) diff --git a/e2e/connector-loading.test.sh b/e2e/connector-loading.test.sh index d1a33f769..da5c44816 100755 --- a/e2e/connector-loading.test.sh +++ b/e2e/connector-loading.test.sh @@ -24,6 +24,7 @@ TMPDIR_BASE=$(mktemp -d) cleanup() { rm -rf "$TMPDIR_BASE" rm -f "$REPO_ROOT"/stripe-sync-protocol-*.tgz + rm -f "$REPO_ROOT"/stripe-sync-openapi-*.tgz rm -f "$REPO_ROOT"/stripe-sync-engine-*.tgz rm -f "$REPO_ROOT"/stripe-sync-source-stripe-*.tgz rm -f "$REPO_ROOT"/stripe-sync-destination-postgres-*.tgz @@ -44,7 +45,7 @@ echo "" echo "--- Step 1: Packing packages ---" PROTOCOL_TGZ=$(cd "$REPO_ROOT" && pnpm --filter @stripe/sync-protocol pack 2>/dev/null | tail -1) -OPENAPI_TGZ=$(cd "$REPO_ROOT" && pnpm --filter @stripe/openapi pack 2>/dev/null | tail -1) +OPENAPI_TGZ=$(cd "$REPO_ROOT" && pnpm --filter @stripe/sync-openapi pack 2>/dev/null | tail -1) ENGINE_TGZ=$(cd "$REPO_ROOT" && pnpm --filter @stripe/sync-engine pack 2>/dev/null | tail -1) SOURCE_TGZ=$(cd "$REPO_ROOT" && pnpm --filter @stripe/sync-source-stripe pack 2>/dev/null | tail -1) DEST_TGZ=$(cd "$REPO_ROOT" && pnpm --filter @stripe/sync-destination-postgres pack 2>/dev/null | tail -1) @@ -85,7 +86,7 @@ cat > package.json <): CatalogMessage { diff --git a/packages/source-stripe/src/index.ts b/packages/source-stripe/src/index.ts index ebceb3a66..8f4dd3b0a 100644 --- a/packages/source-stripe/src/index.ts +++ b/packages/source-stripe/src/index.ts @@ -8,7 +8,11 @@ import Stripe from 'stripe' import { z } from 'zod' import { buildResourceRegistry } from './resourceRegistry.js' import { catalogFromRegistry, catalogFromOpenApi } from './catalog.js' -import { resolveOpenApiSpec, SpecParser, OPENAPI_RESOURCE_TABLE_ALIASES } from '@stripe/openapi' +import { + resolveOpenApiSpec, + SpecParser, + OPENAPI_RESOURCE_TABLE_ALIASES, +} from '@stripe/sync-openapi' import { processStripeEvent } from './process-event.js' import { processWebhookInput, createInputQueue, startWebhookServer } from './src-webhook.js' import { listApiBackfill } from './src-list-api.js' diff --git a/packages/source-stripe/src/resourceRegistry.ts b/packages/source-stripe/src/resourceRegistry.ts index 6f80ef9e8..84a33fb51 100644 --- a/packages/source-stripe/src/resourceRegistry.ts +++ b/packages/source-stripe/src/resourceRegistry.ts @@ -1,5 +1,5 @@ import type { ResourceConfig } from './types.js' -import type { OpenApiSpec, NestedEndpoint } from '@stripe/openapi' +import type { OpenApiSpec, NestedEndpoint } from '@stripe/sync-openapi' import { discoverListEndpoints, discoverNestedEndpoints, @@ -8,7 +8,7 @@ import { buildRetrieveFn, resolveTableName, OPENAPI_RESOURCE_TABLE_ALIASES, -} from '@stripe/openapi' +} from '@stripe/sync-openapi' /** * The default set of table names synced when no explicit selection is made. @@ -35,16 +35,6 @@ export const DEFAULT_SYNC_OBJECTS: readonly string[] = [ 'checkout_sessions', ] -export type StripeObject = string - -export const CORE_SYNC_OBJECTS = DEFAULT_SYNC_OBJECTS as readonly string[] - -export type CoreSyncObject = string - -export const SYNC_OBJECTS = ['all', 'customer_with_entitlements', ...DEFAULT_SYNC_OBJECTS] as const - -export type SyncObjectName = string - export const REVALIDATE_ENTITIES = [ ...DEFAULT_SYNC_OBJECTS, 'radar.early_fraud_warning', @@ -53,10 +43,6 @@ export const REVALIDATE_ENTITIES = [ ] as const export type RevalidateEntityName = (typeof REVALIDATE_ENTITIES)[number] -export const RESOURCE_TABLE_NAME_MAP: Record = Object.fromEntries( - DEFAULT_SYNC_OBJECTS.map((t) => [t, t]) -) - /** * Build a ResourceConfig for every listable resource discovered in the OpenAPI spec. * All resources get list + retrieve functions derived dynamically from the spec paths. diff --git a/packages/source-stripe/src/types.ts b/packages/source-stripe/src/types.ts index bb549066c..91b1a7815 100644 --- a/packages/source-stripe/src/types.ts +++ b/packages/source-stripe/src/types.ts @@ -1,5 +1,5 @@ import type Stripe from 'stripe' -import type { ListFn, RetrieveFn } from '@stripe/openapi' +import type { ListFn, RetrieveFn } from '@stripe/sync-openapi' import type { RevalidateEntityName } from './resourceRegistry.js' /** diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 44b917972..9e716fa96 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -352,7 +352,7 @@ importers: version: 24.10.1 vitest: specifier: ^3.2.4 - version: 3.2.4(@types/node@24.10.1)(tsx@4.21.0)(yaml@2.8.1) + version: 3.2.4(@types/node@24.10.1)(terser@5.46.1)(tsx@4.21.0)(yaml@2.8.1) packages/protocol: dependencies: @@ -372,7 +372,7 @@ importers: packages/source-stripe: dependencies: - '@stripe/openapi': + '@stripe/sync-openapi': specifier: workspace:* version: link:../openapi '@stripe/sync-protocol': From 231a42a93fd74f94f2d88dfd07d9b1af77bdf999 Mon Sep 17 00:00:00 2001 From: Tony Xiao Date: Wed, 25 Mar 2026 02:35:13 -0700 Subject: [PATCH 17/19] Fix openapi package exports: use bun/types/import pattern MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The exports map pointed all conditions at source ./index.ts, which breaks Node after build — index.ts imports ./specParser.js but only .ts source files exist. Node needs the built dist/ files. Use the same pattern as other packages: bun→source for dev, types/import→dist for Node. Drop the unused publishConfig.exports since this package is private. Fixes: bin.js spec exited with code 1: ERR_MODULE_NOT_FOUND Committed-By-Agent: claude --- packages/openapi/package.json | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/packages/openapi/package.json b/packages/openapi/package.json index aa2942e75..b0c0fd040 100644 --- a/packages/openapi/package.json +++ b/packages/openapi/package.json @@ -5,17 +5,9 @@ "type": "module", "exports": { ".": { - "types": "./index.ts", - "import": "./index.ts", - "default": "./index.ts" - } - }, - "publishConfig": { - "exports": { - ".": { - "types": "./dist/index.d.ts", - "import": "./dist/index.js" - } + "bun": "./index.ts", + "types": "./dist/index.d.ts", + "import": "./dist/index.js" } }, "scripts": { From eabfe91325bb3f136683b650109024cdfd96a7ac Mon Sep 17 00:00:00 2001 From: Tony Xiao Date: Wed, 25 Mar 2026 19:22:45 -0700 Subject: [PATCH 18/19] fix: regenerate pnpm-lock.yaml after rebase (broken lockfile) Committed-By-Agent: claude --- pnpm-lock.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 9e716fa96..fd53d0085 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -352,7 +352,7 @@ importers: version: 24.10.1 vitest: specifier: ^3.2.4 - version: 3.2.4(@types/node@24.10.1)(terser@5.46.1)(tsx@4.21.0)(yaml@2.8.1) + version: 3.2.4(@types/node@24.10.1)(jiti@2.6.1)(lightningcss@1.32.0)(terser@5.46.1)(tsx@4.21.0)(yaml@2.8.1) packages/protocol: dependencies: From 4f729da0f6ebacc97d37d7847c0d28e5c02798d5 Mon Sep 17 00:00:00 2001 From: Tony Xiao Date: Thu, 26 Mar 2026 00:02:00 -0700 Subject: [PATCH 19/19] fix: restore deleted openapi/types.ts and include openapi dir in tsconfig types.ts was deleted in an early commit but was still imported by runtimeMappings.ts, specParser.ts, and re-exported via index.ts. Also remove src/openapi from tsconfig exclude so browser.ts barrel (the package's ./browser subpath export) gets compiled to dist. Committed-By-Agent: claude --- packages/source-stripe/src/openapi/types.ts | 81 +++++++++++++++++++++ packages/source-stripe/tsconfig.json | 2 +- 2 files changed, 82 insertions(+), 1 deletion(-) create mode 100644 packages/source-stripe/src/openapi/types.ts diff --git a/packages/source-stripe/src/openapi/types.ts b/packages/source-stripe/src/openapi/types.ts new file mode 100644 index 000000000..386b9e268 --- /dev/null +++ b/packages/source-stripe/src/openapi/types.ts @@ -0,0 +1,81 @@ +export type OpenApiSchemaObject = { + type?: string + format?: string + nullable?: boolean + properties?: Record + items?: OpenApiSchemaOrReference + oneOf?: OpenApiSchemaOrReference[] + anyOf?: OpenApiSchemaOrReference[] + allOf?: OpenApiSchemaOrReference[] + enum?: unknown[] + additionalProperties?: boolean | OpenApiSchemaOrReference + 'x-resourceId'?: string + 'x-expandableFields'?: string[] + 'x-expansionResources'?: { + oneOf?: OpenApiSchemaOrReference[] + } +} + +export type OpenApiReferenceObject = { + $ref: string +} + +export type OpenApiSchemaOrReference = OpenApiSchemaObject | OpenApiReferenceObject + +export type OpenApiSpec = { + openapi: string + info?: { + version?: string + } + components?: { + schemas?: Record + } +} + +export type ScalarType = 'text' | 'boolean' | 'bigint' | 'numeric' | 'json' | 'timestamptz' + +export type ParsedColumn = { + name: string + type: ScalarType + nullable: boolean + expandableReference?: boolean +} + +export type ParsedResourceTable = { + tableName: string + resourceId: string + sourceSchemaName: string + columns: ParsedColumn[] +} + +export type ParsedOpenApiSpec = { + apiVersion: string + tables: ParsedResourceTable[] +} + +export type ParseSpecOptions = { + /** + * Map Stripe x-resourceId values to concrete Postgres table names. + * Entries are matched case-sensitively. + */ + resourceAliases?: Record + /** + * Restrict parsing to these table names. + * If omitted, all resolvable x-resourceId entries are parsed. + */ + allowedTables?: string[] +} + +export type ResolveSpecConfig = { + apiVersion: string + openApiSpecPath?: string + cacheDir?: string +} + +export type ResolvedOpenApiSpec = { + apiVersion: string + spec: OpenApiSpec + source: 'explicit_path' | 'cache' | 'github' + cachePath?: string + commitSha?: string +} diff --git a/packages/source-stripe/tsconfig.json b/packages/source-stripe/tsconfig.json index b26b20b9a..335f0ae21 100644 --- a/packages/source-stripe/tsconfig.json +++ b/packages/source-stripe/tsconfig.json @@ -13,5 +13,5 @@ "rootDir": "src" }, "include": ["src/**/*"], - "exclude": ["src/server", "src/openapi", "src/**/*.test.ts", "src/**/__tests__/**"] + "exclude": ["src/server", "src/**/*.test.ts", "src/**/__tests__/**"] }