fix(api): enforce forced threat protection on v0 endpoints and team config (firecrawl#3965)

Author: mogery

apps/api/src/__tests__/snips/v0/threat-protection.test.ts

@@ -0,0 +1,72 @@
+import request from "supertest";
+import {
+  describeIf,
+  idmux,
+  Identity,
+  TEST_API_URL,
+  TEST_PRODUCTION,
+} from "../lib";
+import { scrapeRaw } from "./lib";
+
+// =========================================
+// v0 + forced threat protection
+//
+// The deprecated v0 endpoints never resolve a threat protection policy, so
+// teams whose flag is "forced" must be rejected on the content-fetching v0
+// endpoints (scrape, crawl, search) with 403. The rejection happens right
+// after auth — before any credit check or scraping — so no provider or
+// scrape target is needed. crawl-status/crawl-cancel intentionally stay
+// open so existing jobs can drain.
+// =========================================
+
+async function crawlRaw(body: unknown, identity: Identity) {
+  return await request(TEST_API_URL)
+    .post("/v0/crawl")
+    .set("Authorization", `Bearer ${identity.apiKey}`)
+    .set("Content-Type", "application/json")
+    .send(body as object);
+}
+
+async function searchRaw(body: unknown, identity: Identity) {
+  return await request(TEST_API_URL)
+    .post("/v0/search")
+    .set("Authorization", `Bearer ${identity.apiKey}`)
+    .set("Content-Type", "application/json")
+    .send(body as object);
+}
+
+// Requires idmux-provisioned team flags, which only exist in the production
+// test configuration.
+describeIf(TEST_PRODUCTION)("V0 threat protection (forced flag)", () => {
+  let identity: Identity;
+
+  beforeAll(async () => {
+    identity = await idmux({
+      name: "v0-threat-protection/forced",
+      flags: {
+        threatProtection: "forced",
+      },
+    });
+  }, 10000);
+
+  it.concurrent("rejects v0 scrape with 403", async () => {
+    const res = await scrapeRaw({ url: "https://firecrawl.dev" }, identity);
+    expect(res.statusCode).toBe(403);
+    expect(res.body.error).toContain("Threat protection");
+    expect(res.body.error).toContain("v0");
+  });
+
+  it.concurrent("rejects v0 crawl with 403", async () => {
+    const res = await crawlRaw({ url: "https://firecrawl.dev" }, identity);
+    expect(res.statusCode).toBe(403);
+    expect(res.body.error).toContain("Threat protection");
+    expect(res.body.error).toContain("v0");
+  });
+
+  it.concurrent("rejects v0 search with 403", async () => {
+    const res = await searchRaw({ query: "firecrawl" }, identity);
+    expect(res.statusCode).toBe(403);
+    expect(res.body.error).toContain("Threat protection");
+    expect(res.body.error).toContain("v0");
+  });
+});

apps/api/src/__tests__/snips/v2/team-threat-protection.test.ts

@@ -164,4 +164,35 @@ describeIf(TEST_PRODUCTION)("Team threat protection config API", () => {
       }
     });
   });
+
+  describe("with the forced team flag", () => {
+    let identity: Identity;
+
+    beforeAll(async () => {
+      identity = await idmux({
+        name: "team-threat-protection/forced",
+        flags: {
+          threatProtection: "forced",
+        },
+      });
+    });
+
+    it('PUT rejects mode "off" with 403', async () => {
+      const res = await putConfigRaw({ mode: "off" }, identity);
+      expect(res.statusCode).toBe(403);
+      expect(res.body.success).toBe(false);
+      expect(res.body.error).toContain("enforced");
+      expect(res.body.error).toContain('"off"');
+    });
+
+    it('PUT accepts mode "normal" with 200 (forced teams may tighten config)', async () => {
+      const res = await putConfigRaw({ mode: "normal" }, identity);
+      expect(res.statusCode).toBe(200);
+      expect(res.body.success).toBe(true);
+      expect(res.body.data).toMatchObject({
+        mode: "normal",
+        configured: true,
+      });
+    });
+  });
 });

apps/api/src/controllers/v0/crawl.ts

@@ -39,6 +39,10 @@ import {
 } from "../../services/worker/nuq-router";
 import { logRequest } from "../../services/logging/log_job";
 import { getScrapeZDR } from "../../lib/zdr-helpers";
+import {
+  isThreatProtectionForced,
+  THREAT_PROTECTION_V0_UNSUPPORTED_MESSAGE,
+} from "../../lib/threat-protection/request";
 import { applyAgentAuthDiscoveryHeader } from "../../lib/agent-auth-discovery";
 
 export async function crawlController(req: Request, res: Response) {
@@ -58,6 +62,12 @@ export async function crawlController(req: Request, res: Response) {
       });
     }
 
+    if (isThreatProtectionForced(chunk?.flags)) {
+      return res.status(403).json({
+        error: THREAT_PROTECTION_V0_UNSUPPORTED_MESSAGE,
+      });
+    }
+
     const id = uuidv7();
 
     await logRequest({

apps/api/src/controllers/v0/scrape.ts

@@ -26,6 +26,10 @@ import { scrapeQueue } from "../../services/worker/nuq-router";
 import { getErrorContactMessage } from "../../lib/deployment";
 import { logRequest } from "../../services/logging/log_job";
 import { getScrapeZDR } from "../../lib/zdr-helpers";
+import {
+  isThreatProtectionForced,
+  THREAT_PROTECTION_V0_UNSUPPORTED_MESSAGE,
+} from "../../lib/threat-protection/request";
 import { applyAgentAuthDiscoveryHeader } from "../../lib/agent-auth-discovery";
 
 async function scrapeHelper(
@@ -200,6 +204,12 @@ export async function scrapeController(req: Request, res: Response) {
       });
     }
 
+    if (isThreatProtectionForced(chunk?.flags)) {
+      return res.status(403).json({
+        error: THREAT_PROTECTION_V0_UNSUPPORTED_MESSAGE,
+      });
+    }
+
     const jobId = uuidv7();
 
     await logRequest({

apps/api/src/controllers/v0/search.ts

@@ -23,6 +23,10 @@ import { ScrapeJobTimeoutError } from "../../lib/error";
 import { scrapeQueue } from "../../services/worker/nuq-router";
 import { defaultOrigin } from "../../lib/default-values";
 import { getSearchZDR } from "../../lib/zdr-helpers";
+import {
+  isThreatProtectionForced,
+  THREAT_PROTECTION_V0_UNSUPPORTED_MESSAGE,
+} from "../../lib/threat-protection/request";
 import { applyAgentAuthDiscoveryHeader } from "../../lib/agent-auth-discovery";
 
 async function searchHelper(
@@ -192,6 +196,12 @@ export async function searchController(req: Request, res: Response) {
       });
     }
 
+    if (isThreatProtectionForced(chunk?.flags)) {
+      return res.status(403).json({
+        error: THREAT_PROTECTION_V0_UNSUPPORTED_MESSAGE,
+      });
+    }
+
     const jobId = uuidv7();
 
     await logRequest({

apps/api/src/controllers/v2/team-threat-protection.ts

@@ -4,6 +4,7 @@ import { logger as _logger } from "../../lib/logger";
 import { RequestWithAuth } from "./types";
 import { getThreatProtection } from "../../lib/zdr-helpers";
 import { threatProtectionConfigSchema } from "../../lib/threat-protection/config";
+import { THREAT_PROTECTION_CANNOT_TURN_OFF_MESSAGE } from "../../lib/threat-protection/request";
 import {
   getOrgIdForTeam,
   getOrgThreatProtectionConfig,
@@ -103,6 +104,19 @@ export async function putTeamThreatProtectionController(
 
   const input = threatProtectionConfigSchema.parse(req.body);
 
+  // "forced" guarantees enforcement: the org config may be tightened, but its
+  // mode may never be turned off through the API.
+  if (
+    getThreatProtection(req.acuc?.flags) === "forced" &&
+    input.mode === "off"
+  ) {
+    res.status(403).json({
+      success: false,
+      error: THREAT_PROTECTION_CANNOT_TURN_OFF_MESSAGE,
+    });
+    return;
+  }
+
   const orgId = await resolveOrgId(req, res);
   if (!orgId) return;
 

apps/api/src/lib/threat-protection/request.ts

@@ -25,6 +25,25 @@ export const THREAT_PROTECTION_OVERRIDES_DISABLED_MESSAGE =
 export const THREAT_PROTECTION_CANNOT_DISABLE_MESSAGE =
   'Threat protection is enforced for your team and cannot be disabled per-request (threatProtection.mode may not be "off"). Remove the mode field or contact your organization administrator.';
 
+export const THREAT_PROTECTION_CANNOT_TURN_OFF_MESSAGE =
+  'Threat protection is enforced for your team and its mode cannot be set to "off". Contact your organization administrator.';
+
+export const THREAT_PROTECTION_V0_UNSUPPORTED_MESSAGE =
+  "Threat protection is enforced for your team and is not supported on the deprecated v0 API. Please update your code to use the v1 or v2 API.";
+
+/**
+ * The deprecated v0 endpoints never resolve a threat protection policy, so
+ * teams whose flag is "forced" must not be able to fetch content through
+ * them. Content-fetching v0 controllers (scrape, crawl, search) call this
+ * right after auth and reject with 403 when it returns true; crawl-status and
+ * crawl-cancel intentionally do not, so existing jobs can drain.
+ */
+export function isThreatProtectionForced(
+  flags: TeamFlags | undefined,
+): boolean {
+  return getThreatProtection(flags) === "forced";
+}
+
 interface ResolvedThreatProtection {
   /** Set when the request must be rejected with a 403. */
   error?: string;