feat(js-sdk,python-sdk): add threatProtection request option (firecrawl#3966)

Author: mogery

apps/js-sdk/firecrawl/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mendable/firecrawl-js",
-  "version": "4.29.2",
+  "version": "4.30.0",
   "description": "JavaScript SDK for Firecrawl API",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",

apps/js-sdk/firecrawl/src/__tests__/unit/v2/threat-protection.unit.test.ts

@@ -0,0 +1,123 @@
+import { describe, test, expect, jest } from "@jest/globals";
+import { scrape } from "../../../v2/methods/scrape";
+import { startBatchScrape } from "../../../v2/methods/batch";
+import { startCrawl } from "../../../v2/methods/crawl";
+import { search } from "../../../v2/methods/search";
+import { map } from "../../../v2/methods/map";
+import { startExtract } from "../../../v2/methods/extract";
+import { startAgent } from "../../../v2/methods/agent";
+import type { ThreatProtectionOptions } from "../../../v2/types";
+
+const threatProtection: ThreatProtectionOptions = {
+  mode: "normal",
+  riskScoreThreshold: 80,
+  blacklist: ["*.blocked.example.com"],
+  whitelist: ["allowed.example.com"],
+  blockedTlds: ["zip"],
+  failurePolicy: "open",
+};
+
+function makeHttp(data: Record<string, unknown>) {
+  const post = jest.fn(async () => ({ status: 200, data }));
+  return {
+    post,
+    prepareHeaders: jest.fn(() => undefined),
+  } as any;
+}
+
+describe("v2 threatProtection request serialization", () => {
+  test("scrape sends threatProtection at top level", async () => {
+    const http = makeHttp({ success: true, data: {} });
+    await scrape(http, "https://example.com", { threatProtection });
+    expect(http.post).toHaveBeenCalledWith(
+      "/v2/scrape",
+      expect.objectContaining({ url: "https://example.com", threatProtection }),
+      {},
+    );
+  });
+
+  test("batch scrape sends threatProtection at top level", async () => {
+    const http = makeHttp({ success: true, id: "job", url: "u" });
+    await startBatchScrape(http, ["https://example.com"], {
+      options: { threatProtection },
+    });
+    expect(http.post).toHaveBeenCalledWith(
+      "/v2/batch/scrape",
+      expect.objectContaining({
+        urls: ["https://example.com"],
+        threatProtection,
+      }),
+      expect.anything(),
+    );
+  });
+
+  test("crawl sends threatProtection under scrapeOptions", async () => {
+    const http = makeHttp({ success: true, id: "job", url: "u" });
+    await startCrawl(http, {
+      url: "https://example.com",
+      scrapeOptions: { threatProtection },
+    });
+    expect(http.post).toHaveBeenCalledWith(
+      "/v2/crawl",
+      expect.objectContaining({
+        scrapeOptions: expect.objectContaining({ threatProtection }),
+      }),
+    );
+  });
+
+  test("search sends threatProtection at top level and under scrapeOptions", async () => {
+    const http = makeHttp({ success: true, data: {} });
+    await search(http, {
+      query: "firecrawl",
+      threatProtection,
+      scrapeOptions: { threatProtection },
+    });
+    expect(http.post).toHaveBeenCalledWith(
+      "/v2/search",
+      expect.objectContaining({
+        threatProtection,
+        scrapeOptions: expect.objectContaining({ threatProtection }),
+      }),
+      {},
+    );
+  });
+
+  test("map sends threatProtection at top level", async () => {
+    const http = makeHttp({ success: true, links: [] });
+    await map(http, "https://example.com", { threatProtection });
+    expect(http.post).toHaveBeenCalledWith(
+      "/v2/map",
+      expect.objectContaining({ threatProtection }),
+      {},
+    );
+  });
+
+  test("extract sends threatProtection at top level", async () => {
+    const http = makeHttp({ success: true, id: "job" });
+    await startExtract(http, {
+      urls: ["https://example.com"],
+      prompt: "extract",
+      threatProtection,
+    });
+    expect(http.post).toHaveBeenCalledWith(
+      "/v2/extract",
+      expect.objectContaining({ threatProtection }),
+    );
+  });
+
+  test("agent sends threatProtection at top level", async () => {
+    const http = makeHttp({ success: true, id: "job", status: "processing" });
+    await startAgent(http, { prompt: "find pricing", threatProtection });
+    expect(http.post).toHaveBeenCalledWith(
+      "/v2/agent",
+      expect.objectContaining({ threatProtection }),
+    );
+  });
+
+  test("threatProtection is omitted when not provided", async () => {
+    const http = makeHttp({ success: true, data: {} });
+    await scrape(http, "https://example.com", { onlyMainContent: true });
+    const body = http.post.mock.calls[0][1] as Record<string, unknown>;
+    expect(body).not.toHaveProperty("threatProtection");
+  });
+});

apps/js-sdk/firecrawl/src/v2/methods/agent.ts

@@ -1,4 +1,4 @@
-import { type AgentResponse, type AgentStatusResponse, type AgentWebhookConfig } from "../types";
+import { type AgentResponse, type AgentStatusResponse, type AgentWebhookConfig, type ThreatProtectionOptions } from "../types";
 import { HttpClient } from "../utils/httpClient";
 import { normalizeAxiosError, throwForBadResponse } from "../utils/errorHandler";
 import { isZodSchema, zodSchemaToJsonSchema } from "../../utils/zodSchemaToJson";
@@ -14,6 +14,7 @@ function prepareAgentPayload(args: {
   strictConstrainToURLs?: boolean;
   model?: "spark-1-pro" | "spark-1-mini";
   webhook?: string | AgentWebhookConfig;
+  threatProtection?: ThreatProtectionOptions;
 }): Record<string, unknown> {
   const body: Record<string, unknown> = {};
   if (args.urls) body.urls = args.urls;
@@ -27,6 +28,8 @@ function prepareAgentPayload(args: {
   if (args.strictConstrainToURLs !== null && args.strictConstrainToURLs !== undefined) body.strictConstrainToURLs = args.strictConstrainToURLs;
   if (args.model !== null && args.model !== undefined) body.model = args.model;
   if (args.webhook != null) body.webhook = args.webhook;
+  if (args.threatProtection != null)
+    body.threatProtection = args.threatProtection;
   return body;
 }
 

apps/js-sdk/firecrawl/src/v2/methods/extract.ts

@@ -1,4 +1,4 @@
-import { type ExtractResponse, type ScrapeOptions, type AgentOptions, type WebhookConfig } from "../types";
+import { type ExtractResponse, type ScrapeOptions, type AgentOptions, type ThreatProtectionOptions, type WebhookConfig } from "../types";
 import { HttpClient } from "../utils/httpClient";
 import { ensureValidScrapeOptions } from "../utils/validation";
 import { normalizeAxiosError, throwForBadResponse } from "../utils/errorHandler";
@@ -19,6 +19,7 @@ function prepareExtractPayload(args: {
   origin?: string;
   agent?: AgentOptions;
   webhook?: string | WebhookConfig | null;
+  threatProtection?: ThreatProtectionOptions;
 }): Record<string, unknown> {
   const body: Record<string, unknown> = {};
   if (args.urls) body.urls = args.urls;
@@ -35,6 +36,8 @@ function prepareExtractPayload(args: {
   if (args.origin) body.origin = args.origin;
   if (args.agent) body.agent = args.agent;
   if (args.webhook != null) body.webhook = args.webhook;
+  if (args.threatProtection != null)
+    body.threatProtection = args.threatProtection;
   if (args.scrapeOptions) {
     ensureValidScrapeOptions(args.scrapeOptions);
     body.scrapeOptions = args.scrapeOptions;

apps/js-sdk/firecrawl/src/v2/methods/map.ts

@@ -24,6 +24,8 @@ function prepareMapPayload(
       payload.integration = options.integration.trim();
     if (options.origin) payload.origin = options.origin;
     if (options.location != null) payload.location = options.location;
+    if (options.threatProtection != null)
+      payload.threatProtection = options.threatProtection;
   }
   return payload;
 }

apps/js-sdk/firecrawl/src/v2/methods/search.ts

@@ -41,6 +41,8 @@ function prepareSearchPayload(req: SearchRequest): Record<string, unknown> {
     payload.integration = req.integration.trim();
   if (req.origin) payload.origin = req.origin;
   if (req.enterprise) payload.enterprise = req.enterprise;
+  if (req.threatProtection != null)
+    payload.threatProtection = req.threatProtection;
   if (req.scrapeOptions) {
     ensureValidScrapeOptions(req.scrapeOptions as ScrapeOptions);
     payload.scrapeOptions = req.scrapeOptions;

apps/js-sdk/firecrawl/src/v2/types.ts

@@ -208,6 +208,7 @@ export interface ScrapeOptions {
   storeInCache?: boolean;
   lockdown?: boolean;
   redactPII?: boolean | RedactPIIOptions;
+  threatProtection?: ThreatProtectionOptions;
   profile?: {
     name: string;
     saveChanges?: boolean;
@@ -241,6 +242,27 @@ export interface RedactPIIOptions {
   replaceStyle?: "tag" | "mask" | "remove";
 }
 
+/**
+ * Enterprise: per-request field-level override of your team's threat
+ * protection policy. Requires threat protection to be enabled for your team
+ * and request overrides to be allowed in the team configuration. Only the
+ * fields you provide replace the team policy's values.
+ */
+export interface ThreatProtectionOptions {
+  /** "off" disables scanning for this request; "normal" applies the policy. */
+  mode?: "off" | "normal";
+  /** Block verdicts at or above this risk score (integer 0-100). */
+  riskScoreThreshold?: number;
+  /** Exact domains or globs like "*.example.com" to always block (max 1000). */
+  blacklist?: string[];
+  /** Exact domains or globs to always allow; wins over everything (max 1000). */
+  whitelist?: string[];
+  /** Lowercase TLDs without the leading dot, e.g. "zip" (max 1000). */
+  blockedTlds?: string[];
+  /** Behavior when scanning is unavailable: "closed" blocks, "open" allows. */
+  failurePolicy?: "open" | "closed";
+}
+
 export type ParseFileData =
   | Blob
   | File
@@ -267,6 +289,7 @@ export type ParseOptions = Omit<
   | "storeInCache"
   | "lockdown"
   | "proxy"
+  | "threatProtection"
 > & {
   formats?: ParseFormatOption[];
   proxy?: "basic" | "auto";
@@ -681,6 +704,7 @@ export interface SearchRequest {
    * your team.
    */
   enterprise?: Array<"default" | "anon" | "zdr">;
+  threatProtection?: ThreatProtectionOptions;
   integration?: string;
   origin?: string;
 }
@@ -769,6 +793,7 @@ export interface MapOptions {
   integration?: string;
   origin?: string;
   location?: LocationConfig;
+  threatProtection?: ThreatProtectionOptions;
 }
 
 export type FeedbackRating = "good" | "partial" | "bad";

apps/python-sdk/firecrawl/__init__.py

@@ -17,7 +17,7 @@
     V1ChangeTrackingOptions,
 )
 
-__version__ = "4.31.0"
+__version__ = "4.32.0"
 
 # Define the logger for the Firecrawl project
 logger: logging.Logger = logging.getLogger("firecrawl")

apps/python-sdk/firecrawl/__tests__/unit/v2/methods/aio/test_aio_threat_protection_request_preparation.py

@@ -0,0 +1,87 @@
+from firecrawl.v2.types import (
+    CrawlRequest,
+    MapOptions,
+    ScrapeOptions,
+    SearchRequest,
+    ThreatProtectionOptions,
+)
+from firecrawl.v2.methods.aio.agent import _prepare_agent_request
+from firecrawl.v2.methods.aio.batch import _prepare as _prepare_batch_request
+from firecrawl.v2.methods.aio.crawl import _prepare_crawl_request
+from firecrawl.v2.methods.aio.extract import _prepare_extract_request
+from firecrawl.v2.methods.aio.map import _prepare_map_request
+from firecrawl.v2.methods.aio.search import _prepare_search_request
+
+
+EXPECTED_WIRE_PAYLOAD = {
+    "mode": "normal",
+    "riskScoreThreshold": 80,
+    "blacklist": ["*.blocked.example.com"],
+    "whitelist": ["allowed.example.com"],
+    "blockedTlds": ["zip"],
+    "failurePolicy": "open",
+}
+
+
+def _threat_protection() -> ThreatProtectionOptions:
+    return ThreatProtectionOptions(
+        mode="normal",
+        risk_score_threshold=80,
+        blacklist=["*.blocked.example.com"],
+        whitelist=["allowed.example.com"],
+        blocked_tlds=["zip"],
+        failure_policy="open",
+    )
+
+
+class TestAioThreatProtectionRequestPreparation:
+    """The aio request preparers must serialize threat_protection too."""
+
+    def test_batch_scrape_top_level(self):
+        data = _prepare_batch_request(
+            ["https://example.com"],
+            options=ScrapeOptions(threat_protection=_threat_protection()),
+        )
+        assert data["threatProtection"] == EXPECTED_WIRE_PAYLOAD
+        assert "threat_protection" not in data
+
+    def test_crawl_scrape_options(self):
+        request = CrawlRequest(
+            url="https://example.com",
+            scrape_options=ScrapeOptions(threat_protection=_threat_protection()),
+        )
+        data = _prepare_crawl_request(request)
+        assert data["scrapeOptions"]["threatProtection"] == EXPECTED_WIRE_PAYLOAD
+
+    def test_search_top_level_and_scrape_options(self):
+        request = SearchRequest(
+            query="firecrawl",
+            threat_protection=_threat_protection(),
+            scrape_options=ScrapeOptions(threat_protection=_threat_protection()),
+        )
+        data = _prepare_search_request(request)
+        assert data["threatProtection"] == EXPECTED_WIRE_PAYLOAD
+        assert data["scrapeOptions"]["threatProtection"] == EXPECTED_WIRE_PAYLOAD
+        assert "threat_protection" not in data
+
+    def test_map_top_level(self):
+        options = MapOptions(threat_protection=_threat_protection())
+        data = _prepare_map_request("https://example.com", options)
+        assert data["threatProtection"] == EXPECTED_WIRE_PAYLOAD
+        assert "threat_protection" not in data
+
+    def test_extract_top_level(self):
+        data = _prepare_extract_request(
+            ["https://example.com"],
+            prompt="extract",
+            threat_protection=_threat_protection(),
+        )
+        assert data["threatProtection"] == EXPECTED_WIRE_PAYLOAD
+
+    def test_agent_top_level(self):
+        data = _prepare_agent_request(
+            None,
+            prompt="find pricing",
+            threat_protection=_threat_protection(),
+        )
+        assert data["threatProtection"] == EXPECTED_WIRE_PAYLOAD