fix(lockdown): gate remaining outbound paths on lockdown mode (firecrawl#3414)

* fix(lockdown): gate remaining outbound paths on lockdown mode

Lockdown promises no outbound requests to the target URL. Three paths
still fired: the checkRobotsOnScrape team-flag pre-fetch of robots.txt,
the fetchAudio transformer POSTing the URL to the avgrab service, and
sendDocumentToSearchIndex forwarding the URL + content to the search
service. Each now short-circuits when options.lockdown is true, with
unit tests locking in the invariant.

* test(lockdown): move unit tests under __tests__/, add audio-gate snip

Aligns with the repo's dominant pattern (__tests__ folder per feature dir
rather than colocated). Adds a snip that requests the audio format under
lockdown on a cached URL — succeeds only if the audio gate fires, since
firecrawl.dev would otherwise raise AudioUnsupportedUrlError.

Author: abimaelmartell

apps/api/src/__tests__/snips/v2/scrape-lockdown.test.ts

@@ -69,4 +69,36 @@ describeIf(TEST_PRODUCTION)("V2 Scrape Lockdown Mode", () => {
     },
     scrapeTimeout,
   );
+
+  test(
+    "should serve cache and skip audio fetch even when audio format is requested",
+    async () => {
+      const id = crypto.randomUUID();
+      const url = "https://firecrawl.dev/?testAudioGate=" + id;
+
+      const seed = await scrape({ url }, identity);
+      expect(seed).toBeDefined();
+      expect(seed.metadata.cacheState).toBe("miss");
+
+      await new Promise(resolve => setTimeout(resolve, 20000));
+
+      // Without the lockdown audio gate this would either throw
+      // AudioUnsupportedUrlError (firecrawl.dev is not an audio source) or
+      // POST to AVGRAB_SERVICE_URL with the target URL. Success here implies
+      // the gate short-circuited before any outbound call.
+      const data = await scrape(
+        {
+          url,
+          lockdown: true,
+          formats: ["markdown", "audio"],
+        },
+        identity,
+      );
+
+      expect(data).toBeDefined();
+      expect(data.metadata.cacheState).toBe("hit");
+      expect(data.audio).toBeUndefined();
+    },
+    scrapeTimeout * 2 + 20000,
+  );
 });

apps/api/src/scraper/scrapeURL/__tests__/shouldCheckRobots.test.ts

@@ -0,0 +1,37 @@
+import { shouldCheckRobots } from "../shouldCheckRobots";
+
+describe("shouldCheckRobots", () => {
+  it("returns false when lockdown is true, even if the team flag is set", () => {
+    expect(
+      shouldCheckRobots(
+        { lockdown: true },
+        { teamFlags: { checkRobotsOnScrape: true } as any },
+      ),
+    ).toBe(false);
+  });
+
+  it("returns false when the team flag is off, regardless of lockdown", () => {
+    expect(
+      shouldCheckRobots(
+        { lockdown: false },
+        { teamFlags: { checkRobotsOnScrape: false } as any },
+      ),
+    ).toBe(false);
+    expect(
+      shouldCheckRobots({ lockdown: true }, { teamFlags: {} as any }),
+    ).toBe(false);
+  });
+
+  it("returns false when teamFlags is missing", () => {
+    expect(shouldCheckRobots({ lockdown: false }, {})).toBe(false);
+  });
+
+  it("returns true only when the team flag is on and lockdown is off", () => {
+    expect(
+      shouldCheckRobots(
+        { lockdown: false },
+        { teamFlags: { checkRobotsOnScrape: true } as any },
+      ),
+    ).toBe(true);
+  });
+});

apps/api/src/scraper/scrapeURL/index.ts

@@ -57,6 +57,7 @@ import { ScrapeRetryTracker } from "./retryTracker";
 import { executeTransformers } from "./transformers";
 import { LLMRefusalError } from "./transformers/llmExtract";
 import { urlSpecificParams } from "./lib/urlSpecificParams";
+import { shouldCheckRobots } from "./shouldCheckRobots";
 import { loadMock, MockState } from "./lib/mock";
 import { CostTracking } from "../../lib/cost-tracking";
 import { getEngineForUrl } from "../WebScraper/utils/engine-forcing";
@@ -1052,7 +1053,7 @@ export async function scrapeURL(
       });
     }
 
-    if (internalOptions.teamFlags?.checkRobotsOnScrape) {
+    if (shouldCheckRobots(options, internalOptions)) {
       await withSpan("scrape.robots_check", async robotsSpan => {
         const urlToCheck = meta.rewrittenUrl || meta.url;
         meta.logger.info("Checking robots.txt", { url: urlToCheck });

apps/api/src/scraper/scrapeURL/shouldCheckRobots.ts

@@ -0,0 +1,15 @@
+import type { TeamFlags } from "../../controllers/v1/types";
+
+// The invariant "lockdown never fetches robots.txt" is load-bearing for the
+// lockdown guarantee (robots.txt is a request to the target domain). Keep this
+// in its own file so it can be unit-tested without dragging in scrapeURL's
+// ESM-heavy module graph.
+export function shouldCheckRobots(
+  options: { lockdown?: boolean },
+  internalOptions: { teamFlags?: TeamFlags },
+): boolean {
+  if (options.lockdown) {
+    return false;
+  }
+  return !!internalOptions.teamFlags?.checkRobotsOnScrape;
+}

apps/api/src/scraper/scrapeURL/transformers/__tests__/audio.test.ts

@@ -0,0 +1,50 @@
+import { fetchAudio } from "../audio";
+
+describe("fetchAudio lockdown guard", () => {
+  const originalFetch = global.fetch;
+
+  afterEach(() => {
+    global.fetch = originalFetch;
+    jest.clearAllMocks();
+  });
+
+  it("does not issue any fetch when lockdown is true, even if audio format is requested", async () => {
+    const fetchSpy = jest.fn();
+    global.fetch = fetchSpy as any;
+
+    const meta: any = {
+      url: "https://example.com/audio",
+      options: {
+        lockdown: true,
+        formats: [{ type: "audio" }],
+      },
+      logger: { warn: jest.fn(), info: jest.fn(), debug: jest.fn() },
+    };
+    const document: any = { markdown: "cached" };
+
+    const result = await fetchAudio(meta, document);
+
+    expect(fetchSpy).not.toHaveBeenCalled();
+    expect(result).toBe(document);
+    expect(result.audio).toBeUndefined();
+  });
+
+  it("returns early when audio format is not requested regardless of lockdown", async () => {
+    const fetchSpy = jest.fn();
+    global.fetch = fetchSpy as any;
+
+    const meta: any = {
+      url: "https://example.com/audio",
+      options: {
+        lockdown: false,
+        formats: [{ type: "markdown" }],
+      },
+      logger: { warn: jest.fn(), info: jest.fn(), debug: jest.fn() },
+    };
+    const document: any = { markdown: "cached" };
+
+    await fetchAudio(meta, document);
+
+    expect(fetchSpy).not.toHaveBeenCalled();
+  });
+});

apps/api/src/scraper/scrapeURL/transformers/__tests__/sendToSearchIndex.test.ts

@@ -0,0 +1,86 @@
+import { sendDocumentToSearchIndex } from "../sendToSearchIndex";
+import { indexDocumentIfEnabled } from "../../../../lib/search-index-client";
+import { config } from "../../../../config";
+
+jest.mock("../../../../lib/search-index-client", () => ({
+  indexDocumentIfEnabled: jest.fn(),
+}));
+
+const mockedIndex = indexDocumentIfEnabled as jest.MockedFunction<
+  typeof indexDocumentIfEnabled
+>;
+
+describe("sendDocumentToSearchIndex lockdown guard", () => {
+  const originalEnabled = config.ENABLE_SEARCH_INDEX;
+  const originalServiceUrl = config.SEARCH_SERVICE_URL;
+  const originalSampleRate = config.SEARCH_INDEX_SAMPLE_RATE;
+
+  beforeEach(() => {
+    // Force the transformer past the "enabled" and "sampling" early-returns so
+    // only the lockdown / should-index checks can stop it.
+    (config as any).ENABLE_SEARCH_INDEX = true;
+    (config as any).SEARCH_SERVICE_URL = "https://search.internal";
+    (config as any).SEARCH_INDEX_SAMPLE_RATE = 1;
+    mockedIndex.mockClear();
+  });
+
+  afterAll(() => {
+    (config as any).ENABLE_SEARCH_INDEX = originalEnabled;
+    (config as any).SEARCH_SERVICE_URL = originalServiceUrl;
+    (config as any).SEARCH_INDEX_SAMPLE_RATE = originalSampleRate;
+  });
+
+  const baseDocument: any = {
+    markdown: "a".repeat(500),
+    rawHtml: "<html><body>x</body></html>",
+    metadata: {
+      statusCode: 200,
+      indexId: "idx-123",
+    },
+  };
+
+  const baseMeta = (overrides: Partial<Record<string, unknown>> = {}) =>
+    ({
+      url: "https://example.com/page",
+      options: {
+        lockdown: false,
+        headers: undefined,
+        mobile: false,
+      },
+      internalOptions: {
+        isParse: false,
+        zeroDataRetention: false,
+      },
+      logger: {
+        warn: jest.fn(),
+        info: jest.fn(),
+        debug: jest.fn(),
+        error: jest.fn(),
+      },
+      ...overrides,
+    }) as any;
+
+  it("does not forward the URL to the search service when lockdown is true", async () => {
+    const meta = baseMeta({
+      options: { lockdown: true, headers: undefined, mobile: false },
+    });
+
+    await sendDocumentToSearchIndex(meta, { ...baseDocument });
+
+    // Fire-and-forget promise: give it a tick to settle.
+    await new Promise(r => setImmediate(r));
+
+    expect(mockedIndex).not.toHaveBeenCalled();
+  });
+
+  it("still forwards normal (non-lockdown) documents to the search service", async () => {
+    const meta = baseMeta();
+
+    await sendDocumentToSearchIndex(meta, { ...baseDocument });
+
+    await new Promise(r => setImmediate(r));
+
+    expect(mockedIndex).toHaveBeenCalledTimes(1);
+    expect(mockedIndex.mock.calls[0][0].url).toBe("https://example.com/page");
+  });
+});

apps/api/src/scraper/scrapeURL/transformers/audio.ts

@@ -42,6 +42,12 @@ export async function fetchAudio(
     return document;
   }
 
+  // Lockdown forbids any outbound request touching the target URL. The avgrab
+  // service fetches the source on our behalf, so we must skip it here.
+  if (meta.options.lockdown) {
+    return document;
+  }
+
   if (!config.AVGRAB_SERVICE_URL) {
     meta.logger.warn("AVGRAB_SERVICE_URL is not configured");
     document.warning =

apps/api/src/scraper/scrapeURL/transformers/sendToSearchIndex.ts

@@ -27,6 +27,11 @@ function shouldIndexForSearch(meta: Meta, document: Document): boolean {
     return false;
   }
 
+  // Lockdown must not forward the target URL to any external service.
+  if (meta.options.lockdown) {
+    return false;
+  }
+
   const statusCode = document.metadata.statusCode;
   if (statusCode < 200 || statusCode >= 300) {
     return false;