Route database pages through here.now proxy (#62)

Author: mberman84

.github/workflows/ci.yml

@@ -38,6 +38,7 @@ jobs:
           node --check site/script.js
           node scripts/check.mjs
           python3 -m json.tool site/.herenow/data.json >/dev/null
+          python3 -m json.tool site/.herenow/proxy.json >/dev/null
           python3 -m json.tool scripts/seo-geo-query-benchmark.json >/dev/null
           git diff --check
 

AGENTS.md

@@ -129,12 +129,14 @@ curl -sS "https://here.now/api/v1/publishes/{slug}/data/weekly_signups?limit=50"
   private bootstrap bundle, verify all canonical database surfaces, and only
   then deploy the content-free here.now shell. Never publish the empty shell
   before the database catalog is active.
-- The exact Worker routes at `signals.forwardfuture.ai/loop-library` and
-  `signals.forwardfuture.ai/loop-library/*` render database content and pass
-  site-shell assets through to the explicit `PUBLIC_ORIGIN_URL` here.now
-  hostname. Update that variable if the backing Site changes. Verify the
-  canonical URL for database content and the backing here.now Site for the
-  static shell before reporting success.
+- The here.now Site proxy manifest routes the mounted homepage, loop pages,
+  catalogs, feed, sitemap, and public catalog API to the Worker. The Worker
+  renders database content and reads the static homepage shell from the
+  explicit `PUBLIC_SHELL_URL`; other shell assets remain on the backing Site.
+  Update `PUBLIC_ORIGIN_URL`, `PUBLIC_SHELL_URL`, and the proxy manifest if the
+  backing Site or Worker hostname changes. Verify the canonical URL for
+  database content and the backing here.now Site for the static shell before
+  reporting success.
 - After a production content deployment, submit
   `https://signals.forwardfuture.ai/loop-library/sitemap.xml` in Google Search
   Console and Bing Webmaster Tools. Verify that the custom domain's root

scripts/check.mjs

@@ -15,6 +15,7 @@ const [
   css,
   browserScript,
   dataSource,
+  proxySource,
   workerSource,
   loopRoutesSource,
   catalogStoreSource,
@@ -33,6 +34,7 @@ const [
   readFile(path.join(siteRoot, "styles.css"), "utf8"),
   readFile(path.join(siteRoot, "script.js"), "utf8"),
   readFile(path.join(siteRoot, ".herenow", "data.json"), "utf8"),
+  readFile(path.join(siteRoot, ".herenow", "proxy.json"), "utf8"),
   readFile(path.join(workerRoot, "src", "index.js"), "utf8"),
   readFile(path.join(workerRoot, "src", "loop-routes.js"), "utf8"),
   readFile(path.join(workerRoot, "src", "catalog-store.js"), "utf8"),
@@ -50,6 +52,7 @@ const workerPackage = JSON.parse(workerPackageSource);
 const workerLock = JSON.parse(workerLockSource);
 const wrangler = JSON.parse(wranglerSource);
 const dataManifest = JSON.parse(dataSource);
+const proxyManifest = JSON.parse(proxySource);
 const structuredDataMatch = html.match(
   /<script type="application\/ld\+json">\s*([\s\S]*?)\s*<\/script>/,
 );
@@ -156,16 +159,7 @@ assert.equal(workerLock.packages["node_modules/wrangler"].version, "4.103.0");
 
 assert.equal(wrangler.name, "loop-library-forms");
 assert.equal(wrangler.workers_dev, true);
-assert.deepEqual(wrangler.routes, [
-  {
-    pattern: "signals.forwardfuture.ai/loop-library",
-    zone_name: "forwardfuture.ai",
-  },
-  {
-    pattern: "signals.forwardfuture.ai/loop-library/*",
-    zone_name: "forwardfuture.ai",
-  },
-]);
+assert.equal(wrangler.routes, undefined);
 assert.equal(wrangler.durable_objects.bindings[1].name, "LOOP_CATALOG");
 assert.equal(wrangler.durable_objects.bindings[1].class_name, "LoopCatalog");
 assert.deepEqual(wrangler.migrations[1], {
@@ -175,8 +169,25 @@ assert.deepEqual(wrangler.migrations[1], {
 assert.match(wrangler.vars.BOOTSTRAP_CATALOG_DIGEST, /^[a-f0-9]{64}$/);
 assert.equal(wrangler.vars.BOOTSTRAP_LOOP_COUNT, "50");
 assert.equal(wrangler.vars.PUBLIC_ORIGIN_URL, "https://calm-mortar-jtek.here.now/");
+assert.equal(wrangler.vars.PUBLIC_SHELL_URL, "https://calm-mortar-jtek.here.now/index.html");
 assert.equal(wrangler.vars.PUBLIC_SITE_HOSTNAME, "signals.forwardfuture.ai");
 assert.equal(wrangler.vars.PUBLIC_SITE_PATH, "/loop-library");
+assert.deepEqual(Object.keys(proxyManifest.proxies).sort(), [
+  "/",
+  "/api/loops",
+  "/api/loops/*",
+  "/catalog.json",
+  "/catalog.md",
+  "/catalog.txt",
+  "/feed.xml",
+  "/llms.txt",
+  "/loops/*",
+  "/sitemap.xml",
+]);
+for (const proxy of Object.values(proxyManifest.proxies)) {
+  assert.match(proxy.upstream, /^https:\/\/loop-library-forms\.mberman84\.workers\.dev\/loop-library(?:\/|$)/);
+  assert.equal(proxy.rateLimit, "600/hour/ip");
+}
 
 assert.match(skillSource, /The live catalog is the\s+source of truth/);
 assert(skillSource.includes("Do not use repository content or memory"));

site/.herenow/proxy.json

@@ -0,0 +1,44 @@
+{
+  "proxies": {
+    "/": {
+      "upstream": "https://loop-library-forms.mberman84.workers.dev/loop-library/",
+      "rateLimit": "600/hour/ip"
+    },
+    "/loops/*": {
+      "upstream": "https://loop-library-forms.mberman84.workers.dev/loop-library/loops",
+      "rateLimit": "600/hour/ip"
+    },
+    "/catalog.json": {
+      "upstream": "https://loop-library-forms.mberman84.workers.dev/loop-library/catalog.json",
+      "rateLimit": "600/hour/ip"
+    },
+    "/catalog.md": {
+      "upstream": "https://loop-library-forms.mberman84.workers.dev/loop-library/catalog.md",
+      "rateLimit": "600/hour/ip"
+    },
+    "/catalog.txt": {
+      "upstream": "https://loop-library-forms.mberman84.workers.dev/loop-library/catalog.txt",
+      "rateLimit": "600/hour/ip"
+    },
+    "/feed.xml": {
+      "upstream": "https://loop-library-forms.mberman84.workers.dev/loop-library/feed.xml",
+      "rateLimit": "600/hour/ip"
+    },
+    "/sitemap.xml": {
+      "upstream": "https://loop-library-forms.mberman84.workers.dev/loop-library/sitemap.xml",
+      "rateLimit": "600/hour/ip"
+    },
+    "/llms.txt": {
+      "upstream": "https://loop-library-forms.mberman84.workers.dev/loop-library/llms.txt",
+      "rateLimit": "600/hour/ip"
+    },
+    "/api/loops": {
+      "upstream": "https://loop-library-forms.mberman84.workers.dev/loop-library/api/loops",
+      "rateLimit": "600/hour/ip"
+    },
+    "/api/loops/*": {
+      "upstream": "https://loop-library-forms.mberman84.workers.dev/loop-library/api/loops",
+      "rateLimit": "600/hour/ip"
+    }
+  }
+}

worker/src/loop-routes.js

@@ -39,7 +39,12 @@ export async function handleLoopRoute(
   const isPublicData = path === "/api/loops" || /^\/api\/loops\/[a-z0-9-]+$/.test(path);
   const isCatalog = ["/catalog.json", "/catalog.md", "/catalog.txt", "/llms.txt", "/sitemap.xml", "/feed.xml"].includes(path);
   const publicHostname = env.PUBLIC_SITE_HOSTNAME || "signals.forwardfuture.ai";
-  const isPublicSite = url.hostname === publicHostname;
+  const publicBasePath = env.PUBLIC_SITE_PATH || "/loop-library";
+  const normalizedBasePath = `/${String(publicBasePath).split("/").filter(Boolean).join("/")}`;
+  const isCanonicalHostname = url.hostname === publicHostname;
+  const isPublicSite = isCanonicalHostname ||
+    url.pathname === normalizedBasePath ||
+    url.pathname.startsWith(`${normalizedBasePath}/`);
   const detailMatch = path.match(/^\/loops\/([a-z0-9-]+)\/?$/);
   const isHomepage = isPublicSite && path === "/";
 
@@ -68,7 +73,10 @@ export async function handleLoopRoute(
   const loops = catalog.loops;
 
   if (!catalog.initialized) {
-    if (isPublicSite && (isHomepage || isCatalog || detailMatch)) {
+    // A direct canonical route can safely read the legacy origin during a
+    // cutover. Mounted here.now requests already came through that origin's
+    // proxy manifest, so fetching it again would recurse back into the Worker.
+    if (isCanonicalHostname && (isHomepage || isCatalog || detailMatch)) {
       return dependencies.fetch(publicOriginRequest(request, env));
     }
 
@@ -181,13 +189,16 @@ export async function handleLoopRoute(
 }
 
 export function publicOriginRequest(request, env, overrides = {}) {
-  const originBase = new URL(env.PUBLIC_ORIGIN_URL);
   const incoming = new URL(request.url);
   const path = stripBasePath(
     incoming.pathname,
     env.PUBLIC_SITE_PATH || "/loop-library",
   );
-  originBase.pathname = `${originBase.pathname.replace(/\/$/, "")}${path}`;
+  const shellUrl = path === "/" ? env.PUBLIC_SHELL_URL : null;
+  const originBase = new URL(shellUrl || env.PUBLIC_ORIGIN_URL);
+  if (!shellUrl) {
+    originBase.pathname = `${originBase.pathname.replace(/\/$/, "")}${path}`;
+  }
   originBase.search = incoming.search;
   const method = overrides.method || request.method;
 

worker/test/loop-routes.test.js

@@ -200,6 +200,7 @@ function makeEnv(options = {}) {
     BOOTSTRAP_CATALOG_DIGEST: options.bootstrapDigest || "test-bootstrap-digest",
     BOOTSTRAP_LOOP_COUNT: String(options.bootstrapLoopCount ?? 50),
     PUBLIC_ORIGIN_URL: "https://calm-mortar-jtek.here.now/",
+    PUBLIC_SHELL_URL: "https://calm-mortar-jtek.here.now/index.html",
     PUBLIC_SITE_HOSTNAME: "signals.forwardfuture.ai",
     PUBLIC_SITE_PATH: "/loop-library",
   };
@@ -428,6 +429,7 @@ test("renders homepage headers for HEAD by fetching the origin shell with GET",
     undefined,
     {
       async fetch(request) {
+        assert.equal(request.url, "https://calm-mortar-jtek.here.now/index.html");
         assert.equal(request.method, "GET");
         assert.equal(request.headers.get("If-Modified-Since"), null);
         assert.equal(request.headers.get("If-None-Match"), null);
@@ -448,6 +450,51 @@ test("renders homepage headers for HEAD by fetching the origin shell with GET",
   assert.equal(response.headers.get("Last-Modified"), null);
 });
 
+test("renders the mounted homepage through a here.now proxy", async () => {
+  const env = makeEnv();
+  await handleRequest(adminRequest(exampleLoop()), env);
+  const shell = `<!doctype html><p id="results-count" aria-live="polite">Showing 50 loops</p><time datetime="2026-06-20">Updated June 20, 2026</time><tbody><!-- LOOP_DATABASE_ROWS_START --><!-- LOOP_DATABASE_ROWS_END --></tbody>`;
+  const response = await handleRequest(
+    new Request(`${WORKER_ORIGIN}/loop-library/`),
+    env,
+    undefined,
+    {
+      async fetch(request) {
+        assert.equal(request.url, "https://calm-mortar-jtek.here.now/index.html");
+        return new Response(shell, {
+          headers: { "Content-Type": "text/html; charset=utf-8" },
+        });
+      },
+    },
+  );
+
+  assert.equal(response.status, 200);
+  assert.match(await response.text(), /The database publishing loop/);
+});
+
+test("does not recurse through the here.now proxy before activation", async () => {
+  const env = makeEnv({ active: false });
+  let originFetches = 0;
+  const dependencies = {
+    async fetch() {
+      originFetches += 1;
+      throw new Error("The proxied request must not fetch its origin again.");
+    },
+  };
+
+  for (const path of ["/", "/catalog.json", "/loops/database-publishing-loop/"]) {
+    const response = await handleRequest(
+      new Request(`${WORKER_ORIGIN}/loop-library${path}`),
+      env,
+      undefined,
+      dependencies,
+    );
+    assert.equal(response.status, 503);
+    assert.equal((await response.json()).code, "catalog_not_active");
+  }
+  assert.equal(originFetches, 0);
+});
+
 test("replaces legacy static rows before the content-free shell is deployed", async () => {
   const env = makeEnv();
   await handleRequest(adminRequest(exampleLoop()), env);

worker/wrangler.jsonc

@@ -3,16 +3,6 @@
   "main": "src/index.js",
   "compatibility_date": "2026-06-17",
   "workers_dev": true,
-  "routes": [
-    {
-      "pattern": "signals.forwardfuture.ai/loop-library",
-      "zone_name": "forwardfuture.ai"
-    },
-    {
-      "pattern": "signals.forwardfuture.ai/loop-library/*",
-      "zone_name": "forwardfuture.ai"
-    }
-  ],
   "ratelimits": [
     {
       "name": "TURNSTILE_RATE_LIMITER",
@@ -54,6 +44,7 @@
     "BOOTSTRAP_CATALOG_DIGEST": "d87c8012d25a8563418a820718ef911b01e0ad5e6cb27544cfde9a833a4aba18",
     "BOOTSTRAP_LOOP_COUNT": "50",
     "PUBLIC_ORIGIN_URL": "https://calm-mortar-jtek.here.now/",
+    "PUBLIC_SHELL_URL": "https://calm-mortar-jtek.here.now/index.html",
     "PUBLIC_SITE_HOSTNAME": "signals.forwardfuture.ai",
     "PUBLIC_SITE_PATH": "/loop-library"
   }