fix: improve local environment detection using SUPABASE_URL instead of API keys (#580)

# Fix Local Environment Detection and Optimize Step Output Storage

This PR fixes the local environment detection in pgflow by using `SUPABASE_URL` instead of API keys, which resolves compatibility issues with recent Supabase CLI versions. The previous detection method relied on matching specific API key patterns, but this became unreliable as Supabase CLI evolved.

Key changes:
- Changed local environment detection to check for `SUPABASE_URL: 'http://kong:8000'` instead of matching API keys
- Updated all tests to use the new URL-based detection method
- Renamed release notes from 0.13.0 to 0.13.1 since 0.13.0 was yanked due to the detection issue
- Added step output storage optimization that makes Map chains run 2x faster by storing outputs in step_states.output instead of aggregating them on-demand

This release (0.13.1) includes both the fix for local environment detection and the performance improvements for Map chains that were originally intended for 0.13.0.

Author: jumski

.changeset/fix-local-detection-step-output.md

@@ -0,0 +1,9 @@
+---
+'@pgflow/edge-worker': patch
+'@pgflow/core': patch
+---
+
+Note: Version 0.13.0 was yanked due to broken local environment detection. This release (0.13.1) includes both the fix and the features from 0.13.0.
+
+- Fix local environment detection to use SUPABASE_URL instead of API keys
+- Add step output storage optimization for 2x faster Map chains (outputs now stored in step_states.output instead of aggregated on-demand)

pkgs/edge-worker/src/shared/localDetection.ts

@@ -1,31 +1,19 @@
-// Old HS256 JWT keys (Supabase CLI v1)
-export const KNOWN_LOCAL_ANON_KEY =
-  'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJzdXBhYmFzZS1kZW1vIiwicm9sZSI6ImFub24iLCJleHAiOjE5ODM4MTI5OTZ9.CRXP1A7WOeoJeXxjNni43kdQwgnWNReilDMblYTn_I0';
-
-export const KNOWN_LOCAL_SERVICE_ROLE_KEY =
-  'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJzdXBhYmFzZS1kZW1vIiwicm9sZSI6InNlcnZpY2Vfcm9sZSIsImV4cCI6MTk4MzgxMjk5Nn0.EGIM96RAZx35lJzdJsyH-qQwv8Hdp7fsn3W0YpN81IU';
-
-// New opaque keys (Supabase CLI v2+)
-// Source: https://github.com/supabase/cli/blob/develop/pkg/config/apikeys.go
-export const KNOWN_LOCAL_PUBLISHABLE_KEY =
-  'sb_publishable_ACJWlzQHlZjBrEguHvfOxg_3BJgxAaH';
-
-export const KNOWN_LOCAL_SECRET_KEY = 'sb_secret_N7UND0UgjKTVK-Uodkm0Hg_xSvEMPvz';
+/**
+ * Local Supabase CLI sets SUPABASE_URL to this Docker-internal API gateway.
+ */
+export const LOCAL_SUPABASE_HOST = 'kong:8000';
 
 /**
- * Checks if the provided environment indicates local Supabase.
- * Use when you have access to an env record (e.g., from PlatformAdapter).
- * Supports both old HS256 JWT keys and new opaque keys from Supabase CLI v2+.
+ * Checks if the provided environment indicates local Supabase development.
+ * Uses SUPABASE_URL to detect local environment (more reliable than key matching).
  */
 export function isLocalSupabaseEnv(env: Record<string, string | undefined>): boolean {
-  const anonKey = env['SUPABASE_ANON_KEY'];
-  const serviceRoleKey = env['SUPABASE_SERVICE_ROLE_KEY'];
-
-  const isLocalAnonKey =
-    anonKey === KNOWN_LOCAL_ANON_KEY || anonKey === KNOWN_LOCAL_PUBLISHABLE_KEY;
-  const isLocalServiceRoleKey =
-    serviceRoleKey === KNOWN_LOCAL_SERVICE_ROLE_KEY ||
-    serviceRoleKey === KNOWN_LOCAL_SECRET_KEY;
+  const supabaseUrl = env['SUPABASE_URL'];
+  if (!supabaseUrl) return false;
 
-  return isLocalAnonKey || isLocalServiceRoleKey;
+  try {
+    return new URL(supabaseUrl).host === LOCAL_SUPABASE_HOST;
+  } catch {
+    return false;
+  }
 }

pkgs/edge-worker/supabase/functions/auth_test/index.ts

@@ -2,23 +2,21 @@ import { EdgeWorker } from '@pgflow/edge-worker';
 import { configurePlatform } from '@pgflow/edge-worker/testing';
 import { sql } from '../utils.ts';
 
-// Production-like keys (NOT the known local demo keys)
-// These must match the values used in tests/e2e/authorization.test.ts
-export const PRODUCTION_ANON_KEY = 'test-production-anon-key-abc123';
+// Production-like service role key for auth validation (must match test's Bearer token)
 export const PRODUCTION_SERVICE_ROLE_KEY = 'test-production-service-role-key-xyz789';
 
 // Docker-internal URL for Supabase transaction pooler
 const DOCKER_POOLER_URL = 'postgresql://postgres.pooler-dev:postgres@pooler:6543/postgres';
 
 // Override environment BEFORE EdgeWorker.start() to enable auth validation
 // This simulates production mode where auth is NOT bypassed
-// Note: We must include EDGE_WORKER_DB_URL because with production keys,
+// Note: We must include EDGE_WORKER_DB_URL because with production SUPABASE_URL,
 // isLocalSupabaseEnv() returns false and the docker pooler fallback won't trigger
 configurePlatform({
   getEnv: () => ({
     ...Deno.env.toObject(),
-    SUPABASE_ANON_KEY: PRODUCTION_ANON_KEY,
-    SUPABASE_SERVICE_ROLE_KEY: PRODUCTION_SERVICE_ROLE_KEY,
+    SUPABASE_URL: 'https://test-project.supabase.co', // Production URL
+    SUPABASE_SERVICE_ROLE_KEY: PRODUCTION_SERVICE_ROLE_KEY, // For auth validation
     EDGE_WORKER_DB_URL: DOCKER_POOLER_URL,
   }),
 });

pkgs/edge-worker/tests/unit/platform/SupabasePlatformAdapter.test.ts

@@ -214,16 +214,14 @@ Deno.test({
 // ============================================================
 
 Deno.test({
-  name: 'isLocalEnvironment returns true for local Supabase keys',
+  name: 'isLocalEnvironment returns true for local Supabase URL (kong)',
   sanitizeResources: false,
   fn: () => {
-    const KNOWN_LOCAL_ANON_KEY = 'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJzdXBhYmFzZS1kZW1vIiwicm9sZSI6ImFub24iLCJleHAiOjE5ODM4MTI5OTZ9.CRXP1A7WOeoJeXxjNni43kdQwgnWNReilDMblYTn_I0';
-
     const deps = createMockDeps({
       getEnv: () => ({
-        SUPABASE_URL: 'http://test.supabase.co',
-        SUPABASE_ANON_KEY: KNOWN_LOCAL_ANON_KEY,
-        SUPABASE_SERVICE_ROLE_KEY: 'test-service-key',
+        SUPABASE_URL: 'http://kong:8000', // Local dev URL
+        SUPABASE_ANON_KEY: 'any-anon-key',
+        SUPABASE_SERVICE_ROLE_KEY: 'any-service-key',
         SB_EXECUTION_ID: 'test-exec-id',
         EDGE_WORKER_DB_URL: 'postgresql://postgres:postgres@localhost:54322/postgres',
       }),
@@ -236,14 +234,14 @@ Deno.test({
 });
 
 Deno.test({
-  name: 'isLocalEnvironment returns false for production keys',
+  name: 'isLocalEnvironment returns false for production URL',
   sanitizeResources: false,
   fn: () => {
     const deps = createMockDeps({
       getEnv: () => ({
-        SUPABASE_URL: 'http://test.supabase.co',
-        SUPABASE_ANON_KEY: 'production-anon-key',
-        SUPABASE_SERVICE_ROLE_KEY: 'production-service-key',
+        SUPABASE_URL: 'https://abc123.supabase.co', // Production URL
+        SUPABASE_ANON_KEY: 'any-anon-key',
+        SUPABASE_SERVICE_ROLE_KEY: 'any-service-key',
         SB_EXECUTION_ID: 'test-exec-id',
         EDGE_WORKER_DB_URL: 'postgresql://postgres:postgres@localhost:54322/postgres',
       }),

pkgs/edge-worker/tests/unit/platform/connectionPriority.test.ts

@@ -1,8 +1,4 @@
 import { assertEquals, assertThrows } from '@std/assert';
-import {
-  KNOWN_LOCAL_ANON_KEY,
-  KNOWN_LOCAL_SERVICE_ROLE_KEY,
-} from '../../../src/shared/localDetection.ts';
 import {
   resolveConnectionString,
   assertConnectionAvailable,
@@ -11,31 +7,28 @@ import {
 } from '../../../src/platform/resolveConnection.ts';
 import postgres from 'postgres';
 
+// Local URL for test envs (triggers isLocalSupabaseEnv = true)
+const LOCAL_SUPABASE_URL = 'http://kong:8000';
+
 // ============================================================
 // Local environment tests
 // ============================================================
 
 Deno.test('connection priority - local env uses Docker pooler URL by default', () => {
-  const env = { SUPABASE_ANON_KEY: KNOWN_LOCAL_ANON_KEY };
-  const result = resolveConnectionString(env);
-  assertEquals(result, DOCKER_TRANSACTION_POOLER_URL);
-});
-
-Deno.test('connection priority - local env with service role key uses Docker pooler URL', () => {
-  const env = { SUPABASE_SERVICE_ROLE_KEY: KNOWN_LOCAL_SERVICE_ROLE_KEY };
+  const env = { SUPABASE_URL: LOCAL_SUPABASE_URL };
   const result = resolveConnectionString(env);
   assertEquals(result, DOCKER_TRANSACTION_POOLER_URL);
 });
 
 Deno.test('connection priority - local env respects config.connectionString override', () => {
-  const env = { SUPABASE_ANON_KEY: KNOWN_LOCAL_ANON_KEY };
+  const env = { SUPABASE_URL: LOCAL_SUPABASE_URL };
   const options = { connectionString: 'postgresql://custom:5432/db' };
   const result = resolveConnectionString(env, options);
   assertEquals(result, 'postgresql://custom:5432/db');
 });
 
 Deno.test('connection priority - local env respects config.sql override', () => {
-  const env = { SUPABASE_ANON_KEY: KNOWN_LOCAL_ANON_KEY };
+  const env = { SUPABASE_URL: LOCAL_SUPABASE_URL };
   const options = { hasSql: true };
   const result = resolveConnectionString(env, options);
   // When sql is provided, we don't use the local pooler URL
@@ -45,7 +38,7 @@ Deno.test('connection priority - local env respects config.sql override', () =>
 
 Deno.test('connection priority - local env with EDGE_WORKER_DB_URL uses it instead of docker pooler', () => {
   const env = {
-    SUPABASE_ANON_KEY: KNOWN_LOCAL_ANON_KEY,
+    SUPABASE_URL: LOCAL_SUPABASE_URL,
     EDGE_WORKER_DB_URL: 'postgresql://custom-local:5432/db',
   };
   const result = resolveConnectionString(env);
@@ -58,7 +51,7 @@ Deno.test('connection priority - local env with EDGE_WORKER_DB_URL uses it inste
 
 Deno.test('connection priority - production uses EDGE_WORKER_DB_URL', () => {
   const env = {
-    SUPABASE_ANON_KEY: 'prod-anon-key',
+    SUPABASE_URL: 'https://abc123.supabase.co',
     EDGE_WORKER_DB_URL: 'postgresql://prod:5432/db',
   };
   const result = resolveConnectionString(env);
@@ -67,7 +60,7 @@ Deno.test('connection priority - production uses EDGE_WORKER_DB_URL', () => {
 
 Deno.test('connection priority - production config.connectionString overrides env var', () => {
   const env = {
-    SUPABASE_ANON_KEY: 'prod-anon-key',
+    SUPABASE_URL: 'https://abc123.supabase.co',
     EDGE_WORKER_DB_URL: 'postgresql://prod:5432/db',
   };
   const options = { connectionString: 'postgresql://override:5432/db' };
@@ -76,7 +69,7 @@ Deno.test('connection priority - production config.connectionString overrides en
 });
 
 Deno.test('connection priority - production returns undefined when nothing configured', () => {
-  const env = { SUPABASE_ANON_KEY: 'prod-anon-key' };
+  const env = { SUPABASE_URL: 'https://abc123.supabase.co' };
   const result = resolveConnectionString(env);
   assertEquals(result, undefined);
 });
@@ -86,7 +79,7 @@ Deno.test('connection priority - production returns undefined when nothing confi
 // ============================================================
 
 Deno.test('connection validation - throws when no connection available on production', () => {
-  const env = { SUPABASE_ANON_KEY: 'prod-anon-key' };
+  const env = { SUPABASE_URL: 'https://abc123.supabase.co' };
   const connectionString = resolveConnectionString(env);
 
   assertThrows(
@@ -108,7 +101,7 @@ Deno.test('connection validation - does not throw when sql is provided', () => {
 });
 
 Deno.test('connection validation - error message lists all options', () => {
-  const env = { SUPABASE_ANON_KEY: 'prod-anon-key' };
+  const env = { SUPABASE_URL: 'https://abc123.supabase.co' };
   const connectionString = resolveConnectionString(env);
 
   try {
@@ -128,7 +121,7 @@ Deno.test('connection validation - error message lists all options', () => {
 Deno.test('connection priority - preview branch fallback pattern works', () => {
   // Simulates: connectionString: Deno.env.get('EDGE_WORKER_DB_URL') || Deno.env.get('SUPABASE_DB_URL')
   const env = {
-    SUPABASE_ANON_KEY: 'prod-anon-key',
+    SUPABASE_URL: 'https://abc123.supabase.co',
     // EDGE_WORKER_DB_URL not set (preview branch)
   };
 
@@ -146,7 +139,7 @@ Deno.test('connection priority - preview branch fallback pattern works', () => {
 
 Deno.test('resolveSqlConnection - priority 1: returns provided sql directly', () => {
   const mockSql = postgres('postgresql://mock:5432/db');
-  const env = { SUPABASE_ANON_KEY: 'prod-anon-key' };
+  const env = { SUPABASE_URL: 'https://abc123.supabase.co' };
 
   const result = resolveSqlConnection(env, { sql: mockSql });
 
@@ -155,7 +148,7 @@ Deno.test('resolveSqlConnection - priority 1: returns provided sql directly', ()
 });
 
 Deno.test('resolveSqlConnection - priority 2: creates sql from connectionString', () => {
-  const env = { SUPABASE_ANON_KEY: 'prod-anon-key' };
+  const env = { SUPABASE_URL: 'https://abc123.supabase.co' };
   const options = { connectionString: 'postgresql://custom:5432/db' };
 
   const result = resolveSqlConnection(env, options);
@@ -166,7 +159,7 @@ Deno.test('resolveSqlConnection - priority 2: creates sql from connectionString'
 
 Deno.test('resolveSqlConnection - priority 3: creates sql from EDGE_WORKER_DB_URL', () => {
   const env = {
-    SUPABASE_ANON_KEY: 'prod-anon-key',
+    SUPABASE_URL: 'https://abc123.supabase.co',
     EDGE_WORKER_DB_URL: 'postgresql://env-var:5432/db',
   };
 
@@ -177,7 +170,7 @@ Deno.test('resolveSqlConnection - priority 3: creates sql from EDGE_WORKER_DB_UR
 });
 
 Deno.test('resolveSqlConnection - priority 4: creates sql from Docker URL in local env', () => {
-  const env = { SUPABASE_ANON_KEY: KNOWN_LOCAL_ANON_KEY };
+  const env = { SUPABASE_URL: LOCAL_SUPABASE_URL };
 
   const result = resolveSqlConnection(env);
 
@@ -186,7 +179,7 @@ Deno.test('resolveSqlConnection - priority 4: creates sql from Docker URL in loc
 });
 
 Deno.test('resolveSqlConnection - throws when nothing configured in production', () => {
-  const env = { SUPABASE_ANON_KEY: 'prod-anon-key' };
+  const env = { SUPABASE_URL: 'https://abc123.supabase.co' };
 
   assertThrows(
     () => resolveSqlConnection(env),
@@ -197,7 +190,7 @@ Deno.test('resolveSqlConnection - throws when nothing configured in production',
 
 Deno.test('resolveSqlConnection - connectionString takes priority over EDGE_WORKER_DB_URL', () => {
   const env = {
-    SUPABASE_ANON_KEY: 'prod-anon-key',
+    SUPABASE_URL: 'https://abc123.supabase.co',
     EDGE_WORKER_DB_URL: 'postgresql://env-var:5432/db',
   };
   const options = { connectionString: 'postgresql://explicit:5432/db' };
@@ -212,7 +205,7 @@ Deno.test('resolveSqlConnection - connectionString takes priority over EDGE_WORK
 Deno.test('resolveSqlConnection - sql takes priority over everything', () => {
   const mockSql = postgres('postgresql://mock:5432/db');
   const env = {
-    SUPABASE_ANON_KEY: KNOWN_LOCAL_ANON_KEY,
+    SUPABASE_URL: LOCAL_SUPABASE_URL,
     EDGE_WORKER_DB_URL: 'postgresql://env-var:5432/db',
   };
   const options = {