feat(ci): flag overrides that exist only in CI builds

POSTHOG_WIZARD_CI_FLAG_OVERRIDES (a JSON object of flag key → value) merges
over the flags PostHog returns, so CI routes deterministically — a run that
tests the orchestrator arm says so instead of depending on a live flag.
Published builds inline NODE_ENV and tsdown strips the entire override path
from the bundle; the smoke test asserts the env var's name is physically
absent from production output and present in CI output, on every build.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

Author: gewenyu99

scripts/smoke-test.sh

@@ -19,7 +19,27 @@ node --input-type=module -e "import '$DIST_BIN'" 2>&1 | head -5 | grep -q 'PostH
   exit 1
 }
 
-# ── 2. --ci rejected in production builds ────────────────────────────────────
+# ── 2. CI flag overrides physically absent from production builds ───────────
+# The override path (src/utils/ci-flag-overrides.ts) is dead code in published
+# builds and tsdown strips it; its env var name appearing in dist/*.js means
+# dead-code elimination regressed and a prod surface leaked. Sourcemaps keep
+# the original source, so only .js output counts.
+OVERRIDE_MARKER='POSTHOG_WIZARD_CI_FLAG_OVERRIDES'
+if [ "${WIZARD_BUILD_NODE_ENV:-production}" = "ci" ]; then
+  # CI builds must keep the path — its absence means the override silently
+  # stopped working and CI is back to testing live flags.
+  if ! grep -q "$OVERRIDE_MARKER" ./dist/*.js; then
+    echo 'Smoke test failed: CI build is missing the CI flag-override path' >&2
+    exit 1
+  fi
+else
+  if grep -q "$OVERRIDE_MARKER" ./dist/*.js; then
+    echo 'Smoke test failed: CI flag-override code leaked into a production build' >&2
+    exit 1
+  fi
+fi
+
+# ── 3. --ci rejected in production builds ────────────────────────────────────
 # build:ci sets WIZARD_BUILD_NODE_ENV=ci → --ci stays enabled → skip the check.
 if [ "${WIZARD_BUILD_NODE_ENV:-production}" = "ci" ]; then
   exit 0

src/env.ts

@@ -41,6 +41,8 @@ export const IS_PRODUCTION_BUILD = process.env.NODE_ENV === 'production';
 type RuntimeEnvKey =
   // Wizard CLI configuration (yargs POSTHOG_WIZARD_ prefix)
   | 'POSTHOG_WIZARD_BENCHMARK_CONFIG'
+  // CI-build-only flag overrides (see utils/ci-flag-overrides.ts)
+  | 'POSTHOG_WIZARD_CI_FLAG_OVERRIDES'
   | 'POSTHOG_WIZARD_BENCHMARK_FILE'
   | 'POSTHOG_WIZARD_LOG_DIR'
   | 'POSTHOG_WIZARD_DEBUG'

src/utils/__tests__/ci-flag-overrides.test.ts

@@ -0,0 +1,63 @@
+import { applyCiFlagOverrides } from '@utils/ci-flag-overrides';
+
+jest.mock('@utils/debug', () => ({
+  logToFile: jest.fn(),
+  debug: jest.fn(),
+}));
+
+const ENV_KEY = 'POSTHOG_WIZARD_CI_FLAG_OVERRIDES';
+
+describe('applyCiFlagOverrides', () => {
+  afterEach(() => {
+    delete process.env[ENV_KEY];
+  });
+
+  // Jest runs with NODE_ENV=test, so IS_PRODUCTION_BUILD is false and the
+  // override path is live — the same shape a `build:ci` bundle has.
+  describe('in CI builds', () => {
+    it('returns the flags untouched when no override is set', () => {
+      const flags = { 'wizard-orchestrator': 'false' };
+      expect(applyCiFlagOverrides(flags)).toEqual(flags);
+    });
+
+    it('merges overrides over the fetched flags, stringifying values', () => {
+      process.env[ENV_KEY] = JSON.stringify({
+        'wizard-orchestrator': true,
+        'wizard-next-v2': 'legacy',
+      });
+      expect(
+        applyCiFlagOverrides({
+          'wizard-orchestrator': 'false',
+          'wizard-react-router': 'true',
+        }),
+      ).toEqual({
+        'wizard-orchestrator': 'true',
+        'wizard-next-v2': 'legacy',
+        'wizard-react-router': 'true',
+      });
+    });
+
+    it('fails loudly on malformed JSON instead of testing live flags', () => {
+      process.env[ENV_KEY] = 'wizard-orchestrator=true';
+      expect(() => applyCiFlagOverrides({})).toThrow(/not valid JSON/);
+    });
+  });
+
+  describe('in production builds', () => {
+    it('is inert: overrides are ignored even when the env var is set', () => {
+      const prevNodeEnv = process.env.NODE_ENV;
+      process.env.NODE_ENV = 'production';
+      process.env[ENV_KEY] = JSON.stringify({ 'wizard-orchestrator': true });
+      let result: Record<string, string> | undefined;
+      jest.isolateModules(() => {
+        // eslint-disable-next-line @typescript-eslint/no-var-requires
+        const prod = require('@utils/ci-flag-overrides') as {
+          applyCiFlagOverrides: typeof applyCiFlagOverrides;
+        };
+        result = prod.applyCiFlagOverrides({ 'wizard-orchestrator': 'false' });
+      });
+      process.env.NODE_ENV = prevNodeEnv;
+      expect(result).toEqual({ 'wizard-orchestrator': 'false' });
+    });
+  });
+});

src/utils/analytics.ts

@@ -8,6 +8,7 @@ import type { WizardSession } from '@lib/wizard-session';
 import type { ApiUser } from '@lib/api';
 import { v4 as uuidv4 } from 'uuid';
 import { debug, logToFile } from './debug';
+import { applyCiFlagOverrides } from './ci-flag-overrides';
 
 /**
  * Extract a standard property bag from the current session.
@@ -187,9 +188,9 @@ export class Analytics {
         if (value === undefined) continue;
         out[key] = typeof value === 'boolean' ? String(value) : String(value);
       }
-      this.activeFlags = out;
-      logToFile('[flags] evaluated', out);
-      return out;
+      this.activeFlags = applyCiFlagOverrides(out);
+      logToFile('[flags] evaluated', this.activeFlags);
+      return this.activeFlags;
     } catch (error) {
       debug('Failed to get all feature flags:', error);
       return {};

src/utils/ci-flag-overrides.ts

@@ -0,0 +1,46 @@
+/**
+ * CI-only feature-flag overrides.
+ *
+ * CI must route deterministically: a run that tests the orchestrator arm says
+ * so explicitly instead of depending on a live feature flag someone can edit
+ * mid-week. `POSTHOG_WIZARD_CI_FLAG_OVERRIDES` is a JSON object of flag key →
+ * value, merged over whatever PostHog returned.
+ *
+ * The override path exists only in CI builds (`pnpm build:ci`). Published
+ * builds inline NODE_ENV as the literal "production", the guard below
+ * collapses, and tsdown strips the rest from the bundle — and the smoke test
+ * asserts the env var's name is physically absent from production output, so
+ * this can never quietly become a production surface.
+ */
+import { runtimeEnv } from '@env';
+import { logToFile } from './debug';
+
+export function applyCiFlagOverrides(
+  flags: Record<string, string>,
+): Record<string, string> {
+  // Compared inline (not via env.ts's IS_PRODUCTION_BUILD) so tsdown replaces
+  // it with a literal right here and the bundler can prove the rest of this
+  // function unreachable in production builds. The smoke test enforces that.
+  if (process.env.NODE_ENV === 'production') return flags;
+
+  const raw = runtimeEnv('POSTHOG_WIZARD_CI_FLAG_OVERRIDES');
+  if (!raw) return flags;
+
+  let overrides: Record<string, unknown>;
+  try {
+    overrides = JSON.parse(raw) as Record<string, unknown>;
+  } catch {
+    // A malformed override is a CI misconfiguration. Fail the run loudly
+    // rather than silently testing whatever the live flags happen to say.
+    throw new Error(
+      'POSTHOG_WIZARD_CI_FLAG_OVERRIDES is not valid JSON (expected {"flag-key": value, ...}).',
+    );
+  }
+
+  const merged = { ...flags };
+  for (const [key, value] of Object.entries(overrides)) {
+    merged[key] = String(value);
+  }
+  logToFile('[flags] CI overrides applied', overrides);
+  return merged;
+}