Skip to content

Commit fea74fa

Browse files
abimaelmartellabimaelmartell
authored
feat(sdk): add redactPII option (firecrawl#3695)
* Add redactPII to SDK options * Remove unreleased PII wording from SDK PR * Fix Elixir batch redactPII option * Use test site for redactPII scrape snippet
1 parent 146828c commit fea74fa

34 files changed

Lines changed: 238 additions & 170 deletions

File tree

‎apps/api/src/__tests__/snips/v2/scrape-pii.test.ts‎

Lines changed: 15 additions & 117 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,9 @@
1-
import { ALLOW_TEST_SUITE_WEBSITE, describeIf, TEST_PRODUCTION } from "../lib";
2-
import { Identity, idmux, scrape, scrapeRaw, scrapeTimeout } from "./lib";
1+
import {
2+
ALLOW_TEST_SUITE_WEBSITE,
3+
describeIf,
4+
TEST_SUITE_WEBSITE,
5+
} from "../lib";
6+
import { Identity, idmux, scrapeRaw, scrapeTimeout } from "./lib";
37

48
let identity: Identity;
59

@@ -13,11 +17,11 @@ beforeAll(async () => {
1317

1418
describeIf(ALLOW_TEST_SUITE_WEBSITE)("V2 Scrape redactPII (schema)", () => {
1519
it.concurrent(
16-
"accepts redactPII: true when `pii` is not in formats",
20+
"accepts redactPII: true with markdown output",
1721
async () => {
1822
const res = await scrapeRaw(
1923
{
20-
url: "https://firecrawl.dev",
24+
url: TEST_SUITE_WEBSITE,
2125
formats: ["markdown"],
2226
redactPII: true,
2327
},
@@ -36,8 +40,8 @@ describeIf(ALLOW_TEST_SUITE_WEBSITE)("V2 Scrape redactPII (schema)", () => {
3640
async () => {
3741
const res = await scrapeRaw(
3842
{
39-
url: "https://firecrawl.dev",
40-
formats: ["markdown", "pii"],
43+
url: TEST_SUITE_WEBSITE,
44+
formats: ["markdown"],
4145
// typed as boolean, but we want to confirm the API rejects strings.
4246
// eslint-disable-next-line @typescript-eslint/no-explicit-any
4347
redactPII: "yes" as any,
@@ -56,8 +60,8 @@ describeIf(ALLOW_TEST_SUITE_WEBSITE)("V2 Scrape redactPII (schema)", () => {
5660
async () => {
5761
const res = await scrapeRaw(
5862
{
59-
url: "https://firecrawl.dev",
60-
formats: ["markdown", "pii"],
63+
url: TEST_SUITE_WEBSITE,
64+
formats: ["markdown"],
6165
// eslint-disable-next-line @typescript-eslint/no-explicit-any
6266
redactPII: {
6367
mode: "accurate",
@@ -68,12 +72,9 @@ describeIf(ALLOW_TEST_SUITE_WEBSITE)("V2 Scrape redactPII (schema)", () => {
6872
identity,
6973
);
7074

71-
// The page may not have PII (so spans can be empty), but the
72-
// request itself must validate and the response must include
73-
// the `pii` block.
7475
expect(res.statusCode).toBe(200);
7576
expect(res.body.success).toBe(true);
76-
expect(res.body.data.pii).toBeDefined();
77+
expect(res.body.data.pii).toBeUndefined();
7778
},
7879
scrapeTimeout,
7980
);
@@ -83,8 +84,8 @@ describeIf(ALLOW_TEST_SUITE_WEBSITE)("V2 Scrape redactPII (schema)", () => {
8384
async () => {
8485
const res = await scrapeRaw(
8586
{
86-
url: "https://firecrawl.dev",
87-
formats: ["markdown", "pii"],
87+
url: TEST_SUITE_WEBSITE,
88+
formats: ["markdown"],
8889
// "model" is the fire-privacy internal mode, not the
8990
// public surface — must be rejected.
9091
// eslint-disable-next-line @typescript-eslint/no-explicit-any
@@ -99,106 +100,3 @@ describeIf(ALLOW_TEST_SUITE_WEBSITE)("V2 Scrape redactPII (schema)", () => {
99100
scrapeTimeout,
100101
);
101102
});
102-
103-
describeIf(TEST_PRODUCTION)("V2 Scrape redactPII (e2e)", () => {
104-
it(
105-
"returns pii block with spans and replaces document.markdown with redacted content",
106-
async () => {
107-
// A page with named entities and an email so heuristic recognizers fire
108-
// even when fire-privacy is in heuristics-only mode.
109-
const data = await scrape(
110-
{
111-
url: "https://en.wikipedia.org/wiki/Alan_Turing",
112-
formats: ["markdown", "pii"],
113-
redactPII: true,
114-
},
115-
identity,
116-
);
117-
118-
// Scrape always succeeds regardless of fire-privacy outcome.
119-
expect(data.pii).toBeDefined();
120-
expect(["ok", "skipped", "failed"]).toContain(data.pii!.status);
121-
122-
if (data.pii!.status === "ok") {
123-
expect(typeof data.pii!.redactedMarkdown).toBe("string");
124-
// document.markdown is the redacted version, not the raw one.
125-
expect(data.markdown).toBe(data.pii!.redactedMarkdown);
126-
127-
expect(data.pii!.spans.length).toBeGreaterThan(0);
128-
expect(data.pii!.spans[0]).toEqual(
129-
expect.objectContaining({
130-
start: expect.any(Number),
131-
end: expect.any(Number),
132-
kind: expect.any(String),
133-
}),
134-
);
135-
// counts should sum to the number of spans with a mapped entity.
136-
const mappedSpanCount = data.pii!.spans.filter(s => s.entity).length;
137-
const totalCount = Object.values(data.pii!.counts ?? {}).reduce(
138-
(a, b) => a + (b ?? 0),
139-
0,
140-
);
141-
expect(totalCount).toBe(mappedSpanCount);
142-
} else if (data.pii!.status === "failed") {
143-
expect(data.pii!.redactedMarkdown).toBeNull();
144-
expect(data.pii!.reason).toBeDefined();
145-
// Fail closed: no raw markdown leaks through when redaction failed.
146-
expect(data.markdown).toBeUndefined();
147-
}
148-
},
149-
scrapeTimeout,
150-
);
151-
152-
it(
153-
"fails closed when fire-privacy is unreachable — pii.status is failed, markdown is empty",
154-
async () => {
155-
const data = await scrape(
156-
{
157-
url: "https://firecrawl.dev",
158-
formats: ["markdown", "pii"],
159-
redactPII: true,
160-
},
161-
identity,
162-
);
163-
164-
expect(data.pii).toBeDefined();
165-
// We don't pin to a specific outcome — could be ok if fire-privacy
166-
// is reachable in this environment, or one of skipped/failed otherwise.
167-
// The contract is: scrape still succeeds (no error response).
168-
expect(["ok", "skipped", "failed"]).toContain(data.pii!.status);
169-
170-
if (data.pii!.status === "ok") {
171-
expect(data.markdown).toBe(data.pii!.redactedMarkdown);
172-
} else {
173-
expect(data.pii!.reason).toBeDefined();
174-
// Markdown is empty on failed / skipped-with-null-redacted; on
175-
// upstream_skipped / empty_input it passes through (equal to
176-
// redactedMarkdown). Verify the invariant: markdown matches
177-
// redactedMarkdown (or is fail-closed empty).
178-
if (data.pii!.redactedMarkdown === null) {
179-
expect(data.markdown).toBe("");
180-
} else {
181-
expect(data.markdown).toBe(data.pii!.redactedMarkdown);
182-
}
183-
}
184-
},
185-
scrapeTimeout,
186-
);
187-
188-
it(
189-
"omits pii block when redactPII is false even if `pii` is in formats",
190-
async () => {
191-
const data = await scrape(
192-
{
193-
url: "https://firecrawl.dev",
194-
formats: ["markdown", "pii"],
195-
},
196-
identity,
197-
);
198-
199-
expect(typeof data.markdown).toBe("string");
200-
expect(data.pii).toBeUndefined();
201-
},
202-
scrapeTimeout,
203-
);
204-
});

‎apps/api/src/controllers/v2/__tests__/redactPII-schema.test.ts‎

Lines changed: 13 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ describe("v2 scrapeRequestSchema — redactPII", () => {
55

66
// ---- boolean form -------------------------------------------------------
77

8-
it("accepts redactPII: true without requiring `pii` in formats", () => {
8+
it("accepts redactPII: true with markdown output", () => {
99
const result = scrapeRequestSchema.safeParse({
1010
url: baseUrl,
1111
formats: ["markdown"],
@@ -21,10 +21,10 @@ describe("v2 scrapeRequestSchema — redactPII", () => {
2121
}
2222
});
2323

24-
it("accepts redactPII: false with `pii` in formats", () => {
24+
it("normalizes redactPII: false to unset", () => {
2525
const result = scrapeRequestSchema.safeParse({
2626
url: baseUrl,
27-
formats: ["markdown", "pii"],
27+
formats: ["markdown"],
2828
redactPII: false,
2929
});
3030
expect(result.success).toBe(true);
@@ -64,7 +64,7 @@ describe("v2 scrapeRequestSchema — redactPII", () => {
6464
it("accepts an explicit mode in the object form", () => {
6565
const result = scrapeRequestSchema.safeParse({
6666
url: baseUrl,
67-
formats: ["markdown", "pii"],
67+
formats: ["markdown"],
6868
redactPII: { mode: "aggressive" },
6969
});
7070
expect(result.success).toBe(true);
@@ -80,7 +80,7 @@ describe("v2 scrapeRequestSchema — redactPII", () => {
8080
for (const mode of ["accurate", "aggressive", "fast"] as const) {
8181
const result = scrapeRequestSchema.safeParse({
8282
url: baseUrl,
83-
formats: ["markdown", "pii"],
83+
formats: ["markdown"],
8484
redactPII: { mode },
8585
});
8686
expect(result.success).toBe(true);
@@ -93,7 +93,7 @@ describe("v2 scrapeRequestSchema — redactPII", () => {
9393
it("rejects an unknown mode", () => {
9494
const result = scrapeRequestSchema.safeParse({
9595
url: baseUrl,
96-
formats: ["markdown", "pii"],
96+
formats: ["markdown"],
9797
redactPII: { mode: "model" }, // internal mode name, not exposed
9898
});
9999
expect(result.success).toBe(false);
@@ -102,7 +102,7 @@ describe("v2 scrapeRequestSchema — redactPII", () => {
102102
it("accepts an entities allowlist", () => {
103103
const result = scrapeRequestSchema.safeParse({
104104
url: baseUrl,
105-
formats: ["markdown", "pii"],
105+
formats: ["markdown"],
106106
redactPII: { entities: ["EMAIL", "PHONE"] },
107107
});
108108
expect(result.success).toBe(true);
@@ -114,7 +114,7 @@ describe("v2 scrapeRequestSchema — redactPII", () => {
114114
it("rejects an unknown entity", () => {
115115
const result = scrapeRequestSchema.safeParse({
116116
url: baseUrl,
117-
formats: ["markdown", "pii"],
117+
formats: ["markdown"],
118118
redactPII: { entities: ["EMAIL", "NICKNAME"] },
119119
});
120120
expect(result.success).toBe(false);
@@ -124,7 +124,7 @@ describe("v2 scrapeRequestSchema — redactPII", () => {
124124
for (const replaceStyle of ["tag", "mask", "remove"] as const) {
125125
const result = scrapeRequestSchema.safeParse({
126126
url: baseUrl,
127-
formats: ["markdown", "pii"],
127+
formats: ["markdown"],
128128
redactPII: { replaceStyle },
129129
});
130130
expect(result.success).toBe(true);
@@ -137,13 +137,13 @@ describe("v2 scrapeRequestSchema — redactPII", () => {
137137
it("rejects unknown fields in the object form (strict)", () => {
138138
const result = scrapeRequestSchema.safeParse({
139139
url: baseUrl,
140-
formats: ["markdown", "pii"],
140+
formats: ["markdown"],
141141
redactPII: { mode: "accurate", typo: "yes" },
142142
});
143143
expect(result.success).toBe(false);
144144
});
145145

146-
it("accepts the object form without requiring `pii` in formats", () => {
146+
it("accepts the object form with markdown output", () => {
147147
const result = scrapeRequestSchema.safeParse({
148148
url: baseUrl,
149149
formats: ["markdown"],
@@ -163,7 +163,7 @@ describe("v2 scrapeRequestSchema — redactPII", () => {
163163
it("rejects redactPII as a string", () => {
164164
const result = scrapeRequestSchema.safeParse({
165165
url: baseUrl,
166-
formats: ["markdown", "pii"],
166+
formats: ["markdown"],
167167
redactPII: "yes",
168168
});
169169
expect(result.success).toBe(false);
@@ -172,7 +172,7 @@ describe("v2 scrapeRequestSchema — redactPII", () => {
172172
it("rejects redactPII as a number", () => {
173173
const result = scrapeRequestSchema.safeParse({
174174
url: baseUrl,
175-
formats: ["markdown", "pii"],
175+
formats: ["markdown"],
176176
redactPII: 1,
177177
});
178178
expect(result.success).toBe(false);
@@ -215,22 +215,4 @@ describe("v2 scrapeRequestSchema — redactPII", () => {
215215
expect(result.data.onlyMainContent).toBe(false);
216216
}
217217
});
218-
219-
// ---- formats schema -----------------------------------------------------
220-
221-
it("accepts `pii` as a string format", () => {
222-
const result = scrapeRequestSchema.safeParse({
223-
url: baseUrl,
224-
formats: ["pii"],
225-
});
226-
expect(result.success).toBe(true);
227-
});
228-
229-
it("accepts `pii` as an object format", () => {
230-
const result = scrapeRequestSchema.safeParse({
231-
url: baseUrl,
232-
formats: [{ type: "pii" }],
233-
});
234-
expect(result.success).toBe(true);
235-
});
236218
});

‎apps/api/src/scraper/scrapeURL/transformers/index.ts‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -498,15 +498,15 @@ function coerceFieldsToFormats(meta: Meta, document: Document): Document {
498498
);
499499
}
500500

501-
// pii is a diagnostic/report format. Redaction itself is controlled by
502-
// redactPII; include `pii` only when callers want spans/counts in the response.
501+
// Redaction itself is controlled by redactPII. Keep internal redaction
502+
// details only when explicitly requested.
503503
const hasPii = hasFormatOfType(meta.options.formats, "pii");
504504
const wantPii = !!(hasPii && meta.options.redactPII);
505505
if (!wantPii && document.pii !== undefined) {
506506
delete document.pii;
507507
} else if (wantPii && document.pii === undefined) {
508508
meta.logger.warn(
509-
"Request had format: pii with redactPII: true, but there was no pii field in the result.",
509+
"Redaction details were requested, but there was no pii field in the result.",
510510
);
511511
}
512512

‎apps/api/src/scraper/scrapeURL/transformers/redactPII.test.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ describe("performRedactPII", () => {
1212
({
1313
url: "https://example.com",
1414
options: {
15-
formats: [{ type: "markdown" }, { type: "pii" }],
15+
formats: [{ type: "markdown" }],
1616
redactPII: {
1717
mode: "accurate",
1818
replaceStyle: "tag",
@@ -54,7 +54,7 @@ describe("performRedactPII", () => {
5454
expect(document.pii?.redactedMarkdown).toBe("Hello <PERSON>");
5555
});
5656

57-
it("runs when redactPII is enabled without the pii format", async () => {
57+
it("runs when redactPII is enabled with markdown output", async () => {
5858
mockedRedactText.mockResolvedValue({
5959
status: "ok",
6060
redactedMarkdown: "Hello <PERSON>",

‎apps/dot-net-sdk/Firecrawl.Tests/ModelsTests.cs‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,8 @@ public void ScrapeOptions_SerializesCorrectly()
1616
Formats = new List<object> { "markdown", "html" },
1717
OnlyMainContent = true,
1818
Timeout = 30000,
19-
Mobile = false
19+
Mobile = false,
20+
RedactPII = true
2021
};
2122

2223
var json = JsonSerializer.Serialize(options, JsonOptions);
@@ -26,6 +27,7 @@ public void ScrapeOptions_SerializesCorrectly()
2627
Assert.Contains("\"onlyMainContent\":true", json);
2728
Assert.Contains("\"timeout\":30000", json);
2829
Assert.Contains("\"mobile\":false", json);
30+
Assert.Contains("\"redactPII\":true", json);
2931
}
3032

3133
[Fact]

‎apps/dot-net-sdk/Firecrawl.Tests/ParseTests.cs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -56,13 +56,15 @@ public void ParseOptions_Serializes_SupportedFields()
5656
OnlyMainContent = true,
5757
Timeout = 30000,
5858
Proxy = "auto",
59+
RedactPII = true,
5960
};
6061

6162
var json = JsonSerializer.Serialize(options, FirecrawlHttpClient.JsonOptions);
6263
Assert.Contains("\"formats\"", json);
6364
Assert.Contains("\"onlyMainContent\":true", json);
6465
Assert.Contains("\"timeout\":30000", json);
6566
Assert.Contains("\"proxy\":\"auto\"", json);
67+
Assert.Contains("\"redactPII\":true", json);
6668
}
6769

6870
[Fact]

‎apps/dot-net-sdk/Firecrawl/Firecrawl.csproj‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@
88

99
<!-- NuGet package metadata -->
1010
<PackageId>firecrawl-sdk</PackageId>
11-
<Version>1.6.0</Version>
11+
<Version>1.7.0</Version>
1212
<Authors>Firecrawl</Authors>
1313
<Company>Firecrawl</Company>
1414
<Description>.NET SDK for the Firecrawl API - web scraping, crawling, and data extraction</Description>

0 commit comments

Comments
 (0)