docs: add homepage hero slogan A/B test with PostHog tracking. #3161
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,5 +1,20 @@ | ||
| import { cookies, headers } from "next/headers"; | ||
| import { Home } from "@/components/home"; | ||
| import { | ||
| HERO_SLOGAN_VARIANT_HEADER, | ||
| HERO_SLOGAN_VARIANT_KEY, | ||
| getHeroSloganVariantFromCookieValue, | ||
| } from "@/lib/hero-slogan-variant"; | ||
|
|
||
| export default async function HomePage() { | ||
| const [cookieStore, headerStore] = await Promise.all([ | ||
| cookies(), | ||
| headers(), | ||
| ]); | ||
| const heroSloganVariant = getHeroSloganVariantFromCookieValue( | ||
| headerStore.get(HERO_SLOGAN_VARIANT_HEADER) ?? | ||
| cookieStore.get(HERO_SLOGAN_VARIANT_KEY)?.value, | ||
| ); | ||
|
|
||
| return <Home heroSloganVariant={heroSloganVariant} />; | ||
|
Check failure on line 19 in app/(home)/page.tsx
|
||
| } | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,167 @@ | ||
| "use client"; | ||
|
|
||
| import { useEffect, useRef } from "react"; | ||
| import { usePostHog } from "posthog-js/react"; | ||
| import { Heading } from "@/components/ui/heading"; | ||
| import { TextHighlight } from "@/components/ui"; | ||
| import { cn } from "@/lib/utils"; | ||
| import { usePostHogClientCapture } from "@/src/usePostHogClientCapture"; | ||
| import { | ||
| HERO_SLOGAN_VARIANT_KEY, | ||
| type HeroSloganVariant, | ||
| } from "@/lib/hero-slogan-variant"; | ||
|
|
||
| function readStoredVariant(): HeroSloganVariant | null { | ||
| try { | ||
| const stored = localStorage.getItem(HERO_SLOGAN_VARIANT_KEY); | ||
| if (stored === "ai-engineering" || stored === "agent-evals") { | ||
| return stored; | ||
| } | ||
| } catch { | ||
| // localStorage unavailable (e.g. private browsing restrictions) | ||
| } | ||
| return null; | ||
| } | ||
|
|
||
| function persistVariant(variant: HeroSloganVariant) { | ||
| try { | ||
| localStorage.setItem(HERO_SLOGAN_VARIANT_KEY, variant); | ||
| } catch { | ||
| // ignore write failures | ||
| } | ||
| } | ||
|
|
||
| const headingClassName = cn( | ||
| "flex-col items-center gap-0.5 sm:gap-1 md:gap-1.5 text-center font-medium leading-[105%] max-md:max-w-[500px]", | ||
| "[leading-trim:both] [text-edge:cap]", | ||
| ); | ||
|
|
||
| const desktopWordSpacing = "max-[499px]:pr-1.75 min-[500px]:pr-2"; | ||
|
|
||
| function AiEngineeringSlogan() { | ||
| return ( | ||
| <> | ||
| <TextHighlight | ||
| highlightClassName="mix-blend-multiply" | ||
| className="whitespace-nowrap" | ||
| > | ||
| Open Source<span className="inline max-[499px]:hidden"> </span> | ||
| </TextHighlight> | ||
| <span className="flex flex-wrap justify-center min-[500px]:inline"> | ||
| <TextHighlight | ||
| highlightClassName="mix-blend-multiply" | ||
| className={desktopWordSpacing} | ||
| > | ||
| AI | ||
| </TextHighlight> | ||
| <TextHighlight | ||
| highlightClassName="mix-blend-multiply" | ||
| className="min-[500px]:pr-2" | ||
| > | ||
| Engineering | ||
| </TextHighlight> | ||
| </span> | ||
| <TextHighlight highlightClassName="mix-blend-multiply"> | ||
| Platform | ||
| </TextHighlight> | ||
| </> | ||
| ); | ||
| } | ||
|
|
||
| function AgentEvalsSlogan() { | ||
| return ( | ||
| <> | ||
| <TextHighlight | ||
| highlightClassName="mix-blend-multiply" | ||
| className="whitespace-nowrap" | ||
| > | ||
| Open Source | ||
| </TextHighlight> | ||
| <span className="flex flex-wrap justify-center min-[500px]:inline"> | ||
| <TextHighlight | ||
| highlightClassName="mix-blend-multiply" | ||
| className={desktopWordSpacing} | ||
| > | ||
| Agent | ||
| </TextHighlight> | ||
| <TextHighlight | ||
| highlightClassName="mix-blend-multiply" | ||
| className={desktopWordSpacing} | ||
| > | ||
| Evals | ||
| </TextHighlight> | ||
| <TextHighlight | ||
| highlightClassName="mix-blend-multiply" | ||
| className={desktopWordSpacing} | ||
| > | ||
| and | ||
| </TextHighlight> | ||
| <TextHighlight highlightClassName="mix-blend-multiply"> | ||
| Tracing | ||
| </TextHighlight> | ||
| </span> | ||
| </> | ||
| ); | ||
| } | ||
|
|
||
| function trackHeroSloganExposure( | ||
| posthog: ReturnType<typeof usePostHog>, | ||
| capture: ReturnType<typeof usePostHogClientCapture>, | ||
| variant: HeroSloganVariant, | ||
| isNewAssignment: boolean, | ||
| ) { | ||
| posthog.register({ hero_slogan_variant: variant }); | ||
| capture("hero_slogan_exposure", { | ||
| variant, | ||
| is_new_assignment: isNewAssignment, | ||
| }); | ||
| } | ||
|
|
||
| export function HeroSlogan({ | ||
| initialVariant, | ||
| }: { | ||
| initialVariant: HeroSloganVariant; | ||
| }) { | ||
| const posthog = usePostHog(); | ||
| const capture = usePostHogClientCapture(); | ||
| const trackedRef = useRef(false); | ||
|
|
||
| useEffect(() => { | ||
| const hadLocalStorage = readStoredVariant() !== null; | ||
| persistVariant(initialVariant); | ||
|
|
||
| if (trackedRef.current) { | ||
| return; | ||
| } | ||
|
|
||
|
Contributor
There was a problem hiding this comment.
Prompt To Fix With AIThis is a comment left during a code review.
Path: components/home/HeroSlogan.tsx
Line: 125
Comment:
**`posthog.register` may fire before PostHog has finished initialising**
`usePostHog()` returns the PostHog instance synchronously, but the underlying SDK's async bootstrap (feature-flag fetch, session init) may not have completed when this effect runs immediately after mount. Calling `posthog.register()` at this point can silently succeed but the super-property may not propagate to events captured in the same tick. A safer approach is to gate the registration on `posthog.isFeatureEnabled` or listen to the `onFeatureFlags` callback, ensuring the SDK is ready before registering super-properties.
How can I resolve this? If you propose a fix, please make it concise. |
||
| const runTracking = () => { | ||
| if (trackedRef.current) { | ||
| return; | ||
| } | ||
| trackedRef.current = true; | ||
| trackHeroSloganExposure( | ||
| posthog, | ||
| capture, | ||
| initialVariant, | ||
| !hadLocalStorage, | ||
| ); | ||
| }; | ||
|
Check failure on line 148 in components/home/HeroSlogan.tsx
|
||
|
Comment on lines
+129
to
+148
There was a problem hiding this comment. 🔴 🟡 Extended reasoning...What the bug isThe PR moves variant assignment from the client to the server: But const hadLocalStorage = readStoredVariant() !== null;
persistVariant(initialVariant);
// ...
trackHeroSloganExposure(posthog, capture, initialVariant, !hadLocalStorage);
Step-by-step proof — false positive (returning user, localStorage cleared)
Step-by-step proof — false negative (returning user, cookies cleared)
Why existing code doesn't prevent itThere is no signal threaded from middleware → page → component about whether the cookie was a hit or a miss. The component only sees the final ImpactThis biases FixCompute |
||
|
|
||
| if ((posthog as { __loaded?: boolean }).__loaded) { | ||
| runTracking(); | ||
| return; | ||
| } | ||
|
|
||
| posthog.onSessionId(runTracking); | ||
| }, [capture, initialVariant, posthog]); | ||
|
|
||
| return ( | ||
| <Heading as="h1" size="big" className={headingClassName}> | ||
| {initialVariant === "ai-engineering" ? ( | ||
| <AiEngineeringSlogan /> | ||
| ) : ( | ||
| <AgentEvalsSlogan /> | ||
| )} | ||
| </Heading> | ||
| ); | ||
| } | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,22 @@ | ||
| export const HERO_SLOGAN_VARIANT_HEADER = "x-hero-slogan-variant"; | ||
|
|
||
| export const HERO_SLOGAN_VARIANT_KEY = "langfuse-hero-slogan-variant"; | ||
|
|
||
| export type HeroSloganVariant = "ai-engineering" | "agent-evals"; | ||
|
|
||
| export function isHeroSloganVariant(value: string): value is HeroSloganVariant { | ||
| return value === "ai-engineering" || value === "agent-evals"; | ||
| } | ||
|
|
||
| export function pickHeroSloganVariant(): HeroSloganVariant { | ||
| return Math.random() < 0.5 ? "ai-engineering" : "agent-evals"; | ||
| } | ||
|
|
||
| export function getHeroSloganVariantFromCookieValue( | ||
| value: string | undefined, | ||
| ): HeroSloganVariant { | ||
| if (value && isHeroSloganVariant(value)) { | ||
| return value; | ||
| } | ||
| return pickHeroSloganVariant(); | ||
| } |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,38 @@ | ||
| import { NextResponse } from "next/server"; | ||
| import type { NextRequest } from "next/server"; | ||
| import { | ||
| HERO_SLOGAN_VARIANT_KEY, | ||
| isHeroSloganVariant, | ||
| pickHeroSloganVariant, | ||
| } from "@/lib/hero-slogan-variant"; | ||
|
|
||
| const HERO_SLOGAN_VARIANT_HEADER = "x-hero-slogan-variant"; | ||
|
Check warning on line 9 in middleware.ts
|
||
|
There was a problem hiding this comment. 🟡 Extended reasoning...What the bug is
Why this is not a live bug todayThe two string literals are byte-identical ( Why it is still worth fixingThis is the textbook "two strings that must stay in sync" footgun, with nothing — no test, no shared type — linking the two literals. If one is ever edited (e.g. namespacing to Step-by-step proof of the failure mode if literals driftSuppose someone later edits
FixTrivial — extend the existing import block at the top of import {
HERO_SLOGAN_VARIANT_HEADER,
HERO_SLOGAN_VARIANT_KEY,
isHeroSloganVariant,
pickHeroSloganVariant,
} from "@/lib/hero-slogan-variant";and delete |
||
|
|
||
| export function middleware(request: NextRequest) { | ||
| const existing = request.cookies.get(HERO_SLOGAN_VARIANT_KEY)?.value; | ||
| const variant = | ||
| existing && isHeroSloganVariant(existing) | ||
| ? existing | ||
| : pickHeroSloganVariant(); | ||
|
|
||
| const requestHeaders = new Headers(request.headers); | ||
| requestHeaders.set(HERO_SLOGAN_VARIANT_HEADER, variant); | ||
|
|
||
| const response = NextResponse.next({ | ||
| request: { headers: requestHeaders }, | ||
| }); | ||
|
|
||
| if (!existing || !isHeroSloganVariant(existing)) { | ||
| response.cookies.set(HERO_SLOGAN_VARIANT_KEY, variant, { | ||
| maxAge: 60 * 60 * 24 * 365, | ||
| path: "/", | ||
| sameSite: "lax", | ||
| }); | ||
| } | ||
|
|
||
| return response; | ||
| } | ||
|
|
||
| export const config = { | ||
| matcher: "/", | ||
| }; | ||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
🔴
pnpm build:static(package.json:16, used for the Cloudflare static-export build path with maintainedlib/_headers+scripts/generate-cloudflare-redirects.jsinfrastructure) breaks after this PR: underSTATIC_EXPORT=true→output: "export"(next.config.mjs:43), Next.js does not supportmiddleware.ts, and callingcookies()/headers()inapp/(home)/page.tsxunconditionally opts the route into dynamic rendering — which the export build rejects ("headerswas called in a route that is statically generated"). The active Vercel/CI path uses plainnext buildand is unaffected, but the static path is silently broken. Fix by either gating the dynamic code onprocess.env.STATIC_EXPORT !== "true"(with a deterministic default variant for static builds, plus an analogous matcher/no-op inmiddleware.ts), or by removingbuild:static+ the supporting Cloudflare scripts if that target is retired.Extended reasoning...
What the bug is
The repo ships two build paths:
pnpm build— plainnext build, used by Vercel and CI.pnpm build:static—cross-env STATIC_EXPORT=true ... next build && pnpm run postbuild:static(package.json:16), which next.config.mjs:43-48 wires tooutput: "export". The postbuild step runsscripts/generate-cloudflare-redirects.jsandcp lib/_headers out/— these are Cloudflare Pages convention artifacts (lib/_headersliterally contains "Static exports on Cloudflare should not be indexed").next-sitemap.config.jsandnext.config.mjsboth also branch onSTATIC_EXPORT === "true"(sitemap excludes,images.unoptimized), so this is deliberately maintained infrastructure, not dead code.This PR introduces two features that are explicitly incompatible with
output: "export":A new top-level
middleware.ts. Next.js docs explicitly list Middleware as unsupported in static export — there is no Node/Edge runtime to execute it, and the build refuses to bundle it.app/(home)/page.tsxnow callscookies()andheaders()unconditionally:These are dynamic Server APIs; calling them in a Server Component opts the route into dynamic rendering, which
output: "export"rejects at build time with "headerswas called in a route that is statically generated".Why existing code doesn't prevent it
There is no
STATIC_EXPORTgate in either the page or the middleware.middleware.tsexports a matcher of"/"unconditionally, and the page imports/awaitscookies()/headers()on every render path. Before this PR no file in the repo imported fromnext/headersand there was no middleware —grep-ing the tree confirms the static build was working.Step-by-step proof
pnpm build:static. The script setsSTATIC_EXPORT=trueand invokesnext build.process.env.STATIC_EXPORT === "true"and merges{ output: "export", trailingSlash: true, distDir: "out" }into the config.middleware.tsat the project root and emits an error: middleware is not supported withoutput: "export".app/(home)/page.tsxis a Server Component for a route that the export build attempts to statically render. When it hitsawait headers()/await cookies(), Next.js opts the route into dynamic rendering. Withoutput: "export"set, the build aborts with the static-rendering-required error.build:staticexits non-zero beforepostbuild:staticever runs, so the Cloudflareout/directory is never produced.Impact
CI (
pnpm build) and the Vercel production deploy use the regular build path and are unaffected — this is why no verifier flagged this as merge-blocking on the active deploy. But the static export path is a real, maintained alternate target (Cloudflare Pages noindex mirror), and this PR silently breaks it. Anyone who runspnpm build:staticlocally or in a separate deploy job will get a build failure that has nothing to do with their change. There is also a secondary semantic concern: even if it built, the prerendered HTML would freeze a single variant chosen at build time and never rotate per user — defeating the A/B test for the static deployment.How to fix
Either:
app/(home)/page.tsx, checkprocess.env.STATIC_EXPORT === "true"and short-circuit to a deterministic default variant (e.g.<Home heroSloganVariant="ai-engineering" />) without callingcookies()/headers(). Inmiddleware.ts, either gate the matcher to be empty underSTATIC_EXPORTor accept that middleware will be dropped (Next.js will still error unless the file is excluded from the build — easiest is to move the middleware behind a build-time check or rename it conditionally via a prebuild script).build:static+postbuild:staticfrom package.json, dropSTATIC_EXPORTbranches fromnext.config.mjsandnext-sitemap.config.js, removescripts/generate-cloudflare-redirects.jsandlib/_headers. Either way the codebase becomes coherent.