feat(ci): flag overrides that exist only in CI builds

POSTHOG_WIZARD_CI_FLAG_OVERRIDES (a JSON object of flag key → value) merges
over the flags PostHog returns, so CI routes deterministically — a run that
tests the orchestrator arm says so instead of depending on a live flag.
Published builds inline NODE_ENV and tsdown strips the entire override path
from the bundle; the smoke test asserts the env var's name is physically
absent from production output and present in CI output, on every build.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

Author: gewenyu99

scripts/smoke-test.sh

@@ -19,7 +19,38 @@ node --input-type=module -e "import '$DIST_BIN'" 2>&1 | head -5 | grep -q 'PostH
   exit 1
 }
 
-# ── 2. --ci rejected in production builds ────────────────────────────────────
+# ── 2. CI flag overrides physically absent from production builds ───────────
+# The override path (src/utils/ci-flag-overrides.ts) is dead code in published
+# builds and tsdown strips it; its env var name appearing in dist/*.js means
+# dead-code elimination regressed and a prod surface leaked. Sourcemaps keep
+# the original source, so only .js output counts.
+OVERRIDE_MARKER='WIZARD_CI_FLAG_OVERRIDES'
+if [ "${WIZARD_BUILD_NODE_ENV:-production}" = "ci" ]; then
+  # CI builds must keep the path — its absence means the override silently
+  # stopped working and CI is back to testing live flags.
+  if ! grep -q "$OVERRIDE_MARKER" ./dist/*.js; then
+    echo 'Smoke test failed: CI build is missing the CI flag-override path' >&2
+    exit 1
+  fi
+  # And a real invocation must accept the env var. yargs claims every
+  # POSTHOG_WIZARD_-prefixed env var as a CLI option and strict-rejects
+  # unknown ones during command parse (--version/--help short-circuit and
+  # prove nothing). The run exits fast on the missing api key — all this
+  # asserts is that yargs did not reject the environment.
+  ci_probe=$(WIZARD_CI_FLAG_OVERRIDES='{"wizard-orchestrator":true}' node "$DIST_BIN" --ci --install-dir /tmp/wizard-smoke-probe 2>&1) || true
+  if echo "$ci_probe" | grep -q 'Unknown argument'; then
+    echo 'Smoke test failed: CI binary rejects WIZARD_CI_FLAG_OVERRIDES in the environment' >&2
+    echo "$ci_probe" | head -3 >&2
+    exit 1
+  fi
+else
+  if grep -q "$OVERRIDE_MARKER" ./dist/*.js; then
+    echo 'Smoke test failed: CI flag-override code leaked into a production build' >&2
+    exit 1
+  fi
+fi
+
+# ── 3. --ci rejected in production builds ────────────────────────────────────
 # build:ci sets WIZARD_BUILD_NODE_ENV=ci → --ci stays enabled → skip the check.
 if [ "${WIZARD_BUILD_NODE_ENV:-production}" = "ci" ]; then
   exit 0

src/env.ts

@@ -39,6 +39,10 @@ export const IS_PRODUCTION_BUILD = process.env.NODE_ENV === 'production';
  * Add new keys here when a new runtime dependency is needed.
  */
 type RuntimeEnvKey =
+  // CI-build-only flag overrides (see utils/ci-flag-overrides.ts).
+  // Deliberately NOT POSTHOG_WIZARD_-prefixed: yargs .env('POSTHOG_WIZARD')
+  // would claim it as an unknown CLI option and strict-reject the run.
+  | 'WIZARD_CI_FLAG_OVERRIDES'
   // Wizard CLI configuration (yargs POSTHOG_WIZARD_ prefix)
   | 'POSTHOG_WIZARD_BENCHMARK_CONFIG'
   | 'POSTHOG_WIZARD_BENCHMARK_FILE'

src/utils/__tests__/ci-flag-overrides.test.ts

@@ -0,0 +1,63 @@
+import { applyCiFlagOverrides } from '@utils/ci-flag-overrides';
+
+jest.mock('@utils/debug', () => ({
+  logToFile: jest.fn(),
+  debug: jest.fn(),
+}));
+
+const ENV_KEY = 'WIZARD_CI_FLAG_OVERRIDES';
+
+describe('applyCiFlagOverrides', () => {
+  afterEach(() => {
+    delete process.env[ENV_KEY];
+  });
+
+  // Jest runs with NODE_ENV=test, so IS_PRODUCTION_BUILD is false and the
+  // override path is live — the same shape a `build:ci` bundle has.
+  describe('in CI builds', () => {
+    it('returns the flags untouched when no override is set', () => {
+      const flags = { 'wizard-orchestrator': 'false' };
+      expect(applyCiFlagOverrides(flags)).toEqual(flags);
+    });
+
+    it('merges overrides over the fetched flags, stringifying values', () => {
+      process.env[ENV_KEY] = JSON.stringify({
+        'wizard-orchestrator': true,
+        'wizard-next-v2': 'legacy',
+      });
+      expect(
+        applyCiFlagOverrides({
+          'wizard-orchestrator': 'false',
+          'wizard-react-router': 'true',
+        }),
+      ).toEqual({
+        'wizard-orchestrator': 'true',
+        'wizard-next-v2': 'legacy',
+        'wizard-react-router': 'true',
+      });
+    });
+
+    it('fails loudly on malformed JSON instead of testing live flags', () => {
+      process.env[ENV_KEY] = 'wizard-orchestrator=true';
+      expect(() => applyCiFlagOverrides({})).toThrow(/not valid JSON/);
+    });
+  });
+
+  describe('in production builds', () => {
+    it('is inert: overrides are ignored even when the env var is set', () => {
+      const prevNodeEnv = process.env.NODE_ENV;
+      process.env.NODE_ENV = 'production';
+      process.env[ENV_KEY] = JSON.stringify({ 'wizard-orchestrator': true });
+      let result: Record<string, string> | undefined;
+      jest.isolateModules(() => {
+        // eslint-disable-next-line @typescript-eslint/no-var-requires
+        const prod = require('@utils/ci-flag-overrides') as {
+          applyCiFlagOverrides: typeof applyCiFlagOverrides;
+        };
+        result = prod.applyCiFlagOverrides({ 'wizard-orchestrator': 'false' });
+      });
+      process.env.NODE_ENV = prevNodeEnv;
+      expect(result).toEqual({ 'wizard-orchestrator': 'false' });
+    });
+  });
+});

src/utils/analytics.ts

@@ -9,6 +9,7 @@ import type { ApiUser } from '@lib/api';
 import { v4 as uuidv4 } from 'uuid';
 import { IS_PRODUCTION_BUILD } from '@env';
 import { debug, logToFile } from './debug';
+import { applyCiFlagOverrides } from './ci-flag-overrides';
 
 /**
  * Extract a standard property bag from the current session.
@@ -211,6 +212,7 @@ export class Analytics {
     if (this.activeFlags !== null) {
       return this.activeFlags;
     }
+    const out: Record<string, string> = {};
     try {
       const distinctId = this.distinctId ?? this.anonymousId;
       logToFile('[flags] evaluating as', {
@@ -222,18 +224,23 @@ export class Analytics {
         personProperties: this.flagPersonProperties(),
       });
       const flags = result.featureFlags ?? {};
-      const out: Record<string, string> = {};
       for (const [key, value] of Object.entries(flags)) {
         if (value === undefined) continue;
         out[key] = typeof value === 'boolean' ? String(value) : String(value);
       }
-      this.activeFlags = out;
-      logToFile('[flags] evaluated', out);
-      return out;
     } catch (error) {
       debug('Failed to get all feature flags:', error);
-      return {};
+      this.captureException(
+        error instanceof Error ? error : new Error(String(error)),
+        { step: 'get_all_flags' },
+      );
     }
+    // Outside the fetch guard on purpose: a malformed CI override must fail
+    // the run loudly, and a valid one applies even when the fetch failed —
+    // CI routing stays deterministic either way.
+    this.activeFlags = applyCiFlagOverrides(out);
+    logToFile('[flags] evaluated', this.activeFlags);
+    return this.activeFlags;
   }
 
   async shutdown(status: 'success' | 'error' | 'cancelled') {

src/utils/ci-flag-overrides.ts

@@ -0,0 +1,46 @@
+/**
+ * CI-only feature-flag overrides.
+ *
+ * CI must route deterministically: a run that tests the orchestrator arm says
+ * so explicitly instead of depending on a live feature flag someone can edit
+ * mid-week. `WIZARD_CI_FLAG_OVERRIDES` is a JSON object of flag key →
+ * value, merged over whatever PostHog returned.
+ *
+ * The override path exists only in CI builds (`pnpm build:ci`). Published
+ * builds inline NODE_ENV as the literal "production", the guard below
+ * collapses, and tsdown strips the rest from the bundle — and the smoke test
+ * asserts the env var's name is physically absent from production output, so
+ * this can never quietly become a production surface.
+ */
+import { runtimeEnv } from '@env';
+import { logToFile } from './debug';
+
+export function applyCiFlagOverrides(
+  flags: Record<string, string>,
+): Record<string, string> {
+  // Compared inline (not via env.ts's IS_PRODUCTION_BUILD) so tsdown replaces
+  // it with a literal right here and the bundler can prove the rest of this
+  // function unreachable in production builds. The smoke test enforces that.
+  if (process.env.NODE_ENV === 'production') return flags;
+
+  const raw = runtimeEnv('WIZARD_CI_FLAG_OVERRIDES');
+  if (!raw) return flags;
+
+  let overrides: Record<string, unknown>;
+  try {
+    overrides = JSON.parse(raw) as Record<string, unknown>;
+  } catch {
+    // A malformed override is a CI misconfiguration. Fail the run loudly
+    // rather than silently testing whatever the live flags happen to say.
+    throw new Error(
+      'WIZARD_CI_FLAG_OVERRIDES is not valid JSON (expected {"flag-key": value, ...}).',
+    );
+  }
+
+  const merged = { ...flags };
+  for (const [key, value] of Object.entries(overrides)) {
+    merged[key] = String(value);
+  }
+  logToFile('[flags] CI overrides applied', overrides);
+  return merged;
+}