chore(api): remove the data-layer scrape path (firecrawl#4029)

First step of moving routed data sources to a standalone service: drop
the data-layer pilot path (eligibility, capabilities cache, engine,
success-credits billing branch, and the blocklist bypass + terms gate
in blocklistMiddleware).

The professionalProfileCompanyDataBeta flag, the dashboard terms
acceptance, and organization_data_source_access are untouched - the
replacement path will reuse them.

Co-authored-by: Claude Fable 5 <noreply@anthropic.com>

Author: abimaelmartell

apps/api/src/__tests__/snips/v2/audio-routing.test.ts

@@ -6,8 +6,6 @@
 
 describe("Audio format engine routing (buildFallbackList)", () => {
   let buildFallbackList: typeof import("../../../scraper/scrapeURL/engines/index.js").buildFallbackList;
-  let clearDataLayerCapabilitiesForTest: typeof import("../../../lib/data-layer.js").clearDataLayerCapabilitiesForTest;
-  let setDataLayerCapabilitiesForTest: typeof import("../../../lib/data-layer.js").setDataLayerCapabilitiesForTest;
 
   const originalFireEngineUrl = process.env.FIRE_ENGINE_BETA_URL;
   const originalIndexUrl = process.env.INDEX_DATABASE_URL;
@@ -22,14 +20,6 @@ describe("Audio format engine routing (buildFallbackList)", () => {
     ({ buildFallbackList } = await import(
       "../../../scraper/scrapeURL/engines/index.js"
     ));
-    ({
-      clearDataLayerCapabilitiesForTest,
-      setDataLayerCapabilitiesForTest,
-    } = await import("../../../lib/data-layer.js"));
-  });
-
-  afterEach(() => {
-    clearDataLayerCapabilitiesForTest();
   });
 
   afterAll(() => {
@@ -120,20 +110,4 @@ describe("Audio format engine routing (buildFallbackList)", () => {
     expect(engines).toContain("fire-engine;chrome-cdp");
   });
 
-  it("does not route agent index-only requests through the data layer", async () => {
-    setDataLayerCapabilitiesForTest({
-      domains: ["profiles.example"],
-    });
-
-    const meta = buildStubMeta([]);
-    meta.url = "https://profiles.example/person/example-person";
-    meta.options.formats = [{ type: "markdown" }];
-    meta.internalOptions.agentIndexOnly = true;
-    meta.internalOptions.teamFlags = { enrichBeta: true };
-
-    const fallback = await buildFallbackList(meta);
-    const engines = fallback.map(f => f.engine);
-
-    expect(engines).toEqual(["index", "index;documents"]);
-  });
 });