From cec5a26916c3820cb283bc7586ce3534e40b7289 Mon Sep 17 00:00:00 2001 From: mogery Date: Tue, 16 Jun 2026 09:24:24 -0700 Subject: [PATCH 1/3] fix(api/keyless): restore keyless_credit_usage audit logging after reserve refactor #3796 moved keyless credit accounting from chargeKeylessCredits (Redis counter + keyless_credit_usage insert) to reserveKeylessCredits/adjustKeylessCredits, which only touch Redis. The worker's chargeKeylessCredits call is gated on !keylessReserved, so for the now-default reserved path the audit insert never ran and keyless_credit_usage stopped getting rows. Extract the audit insert into logKeylessCreditUsage() and call it from each controller's reconciliation point with the actual billed credits. The worker fallback still logs via chargeKeylessCredits. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../src/__tests__/snips/v2/keyless.test.ts | 60 +++++++++++++++++++ apps/api/src/controllers/v1/scrape.ts | 7 ++- apps/api/src/controllers/v1/search.ts | 4 ++ apps/api/src/controllers/v2/parse.ts | 7 ++- apps/api/src/controllers/v2/scrape-browser.ts | 2 + apps/api/src/controllers/v2/scrape.ts | 7 ++- apps/api/src/controllers/v2/search.ts | 4 ++ apps/api/src/lib/keyless.ts | 45 +++++++++----- 8 files changed, 118 insertions(+), 18 deletions(-) diff --git a/apps/api/src/__tests__/snips/v2/keyless.test.ts b/apps/api/src/__tests__/snips/v2/keyless.test.ts index d79c91507b..fa8fb776e6 100644 --- a/apps/api/src/__tests__/snips/v2/keyless.test.ts +++ b/apps/api/src/__tests__/snips/v2/keyless.test.ts @@ -8,6 +8,9 @@ import { scrapeTimeout, } from "../lib"; import { redisRateLimitClient } from "../../../services/rate-limiter"; +import { db } from "../../../db/connection"; +import * as schema from "../../../db/schema"; +import { and, desc, eq, gt } from "drizzle-orm"; import request from "supertest"; // The keyless tier is disabled unless both limits are configured. The harness @@ -359,6 +362,63 @@ describeIf(KEYLESS_ENABLED)("Keyless free tier", () => { scrapeTimeout, ); + it( + "writes a keyless_credit_usage audit row for a successful keyless scrape (200)", + async () => { + // The audit insert is best-effort and fire-and-forget in the controller, + // so capture the latest id first, then poll for a newer row afterwards. + const beforeMaxId = ( + await db + .select({ id: schema.keyless_credit_usage.id }) + .from(schema.keyless_credit_usage) + .orderBy(desc(schema.keyless_credit_usage.id)) + .limit(1) + )[0]?.id; + + const response = await request(TEST_API_URL) + .post("/v2/scrape") + .set("Content-Type", "application/json") + .send({ + url: TEST_SUITE_WEBSITE, + origin: "mcp", + formats: ["markdown"], + }); + + expect(response.statusCode).toBe(200); + expect(response.body.success).toBe(true); + const creditsUsed = response.body.data.metadata.creditsUsed; + expect(creditsUsed).toBeGreaterThan(0); + + const ip = await currentKeylessIp(); + + let row: typeof schema.keyless_credit_usage.$inferSelect | undefined; + for (let i = 0; i < 20; i++) { + const rows = await db + .select() + .from(schema.keyless_credit_usage) + .where( + and( + eq(schema.keyless_credit_usage.ip, ip), + beforeMaxId !== undefined + ? gt(schema.keyless_credit_usage.id, beforeMaxId) + : undefined, + ), + ) + .orderBy(desc(schema.keyless_credit_usage.id)) + .limit(1); + if (rows.length > 0) { + row = rows[0]; + break; + } + await new Promise(resolve => setTimeout(resolve, 250)); + } + + expect(row).toBeDefined(); + expect(row!.credits_used).toBe(creditsUsed); + }, + scrapeTimeout, + ); + it( "rejects projected keyless search with scrape options before search (429)", async () => { diff --git a/apps/api/src/controllers/v1/scrape.ts b/apps/api/src/controllers/v1/scrape.ts index 3a1452c59c..93c22669f1 100644 --- a/apps/api/src/controllers/v1/scrape.ts +++ b/apps/api/src/controllers/v1/scrape.ts @@ -26,6 +26,7 @@ import { getScrapeZDR } from "../../lib/zdr-helpers"; import { KEYLESS_CREDITS_MESSAGE, adjustKeylessCredits, + logKeylessCreditUsage, reserveKeylessCredits, } from "../../lib/keyless"; import { projectScrapeCredits } from "../../lib/keyless-credit-projection"; @@ -306,10 +307,14 @@ export async function scrapeController( if (reservedKeylessCredits > 0 && !reconciledKeylessCredits) { reconciledKeylessCredits = true; + const actualKeylessCredits = doc?.metadata?.creditsUsed ?? 0; adjustKeylessCredits( req.auth.team_id, - (doc?.metadata?.creditsUsed ?? 0) - reservedKeylessCredits, + actualKeylessCredits - reservedKeylessCredits, ).catch(() => {}); + logKeylessCreditUsage(req.auth.team_id, actualKeylessCredits).catch( + () => {}, + ); } const totalRequestTime = new Date().getTime() - middlewareStartTime; diff --git a/apps/api/src/controllers/v1/search.ts b/apps/api/src/controllers/v1/search.ts index 0b4f37a199..394ed6a73f 100644 --- a/apps/api/src/controllers/v1/search.ts +++ b/apps/api/src/controllers/v1/search.ts @@ -30,6 +30,7 @@ import { getSearchForcedKind } from "../../lib/zdr-helpers"; import { KEYLESS_CREDITS_MESSAGE, adjustKeylessCredits, + logKeylessCreditUsage, reserveKeylessCredits, } from "../../lib/keyless"; import { projectSearchTotalCredits } from "../../lib/keyless-credit-projection"; @@ -254,6 +255,9 @@ export async function searchController( req.auth.team_id, result.totalCredits - reservedKeylessCredits, ).catch(() => {}); + logKeylessCreditUsage(req.auth.team_id, result.totalCredits).catch( + () => {}, + ); } const endTime = new Date().getTime(); diff --git a/apps/api/src/controllers/v2/parse.ts b/apps/api/src/controllers/v2/parse.ts index a39da32978..85fc0117fa 100644 --- a/apps/api/src/controllers/v2/parse.ts +++ b/apps/api/src/controllers/v2/parse.ts @@ -29,6 +29,7 @@ import { getScrapeZDR } from "../../lib/zdr-helpers"; import { KEYLESS_CREDITS_MESSAGE, adjustKeylessCredits, + logKeylessCreditUsage, reserveKeylessCredits, } from "../../lib/keyless"; import { projectScrapeCredits } from "../../lib/keyless-credit-projection"; @@ -656,10 +657,14 @@ export async function parseController( if (reservedKeylessCredits > 0 && !reconciledKeylessCredits) { reconciledKeylessCredits = true; + const actualKeylessCredits = doc?.metadata?.creditsUsed ?? 0; adjustKeylessCredits( req.auth.team_id, - (doc?.metadata?.creditsUsed ?? 0) - reservedKeylessCredits, + actualKeylessCredits - reservedKeylessCredits, ).catch(() => {}); + logKeylessCreditUsage(req.auth.team_id, actualKeylessCredits).catch( + () => {}, + ); } const totalRequestTime = new Date().getTime() - middlewareStartTime; diff --git a/apps/api/src/controllers/v2/scrape-browser.ts b/apps/api/src/controllers/v2/scrape-browser.ts index 6bb0676465..c54cbc110c 100644 --- a/apps/api/src/controllers/v2/scrape-browser.ts +++ b/apps/api/src/controllers/v2/scrape-browser.ts @@ -49,6 +49,7 @@ import { KEYLESS_CREDITS_MESSAGE, adjustKeylessCredits, keylessTeamUuid, + logKeylessCreditUsage, reserveKeylessCredits, } from "../../lib/keyless"; import { enqueueBrowserSessionActivity } from "../../lib/browser-session-activity"; @@ -482,6 +483,7 @@ export async function scrapeStopInteractiveBrowserController( adjustKeylessCredits(req.auth.team_id, creditsBilled - reservedCredits).catch( () => {}, ); + logKeylessCreditUsage(req.auth.team_id, creditsBilled).catch(() => {}); logger.info("Browser session destroyed", { sessionDurationMs: durationMs, diff --git a/apps/api/src/controllers/v2/scrape.ts b/apps/api/src/controllers/v2/scrape.ts index b99bbf2b0f..d779a90056 100644 --- a/apps/api/src/controllers/v2/scrape.ts +++ b/apps/api/src/controllers/v2/scrape.ts @@ -27,6 +27,7 @@ import { getScrapeZDR } from "../../lib/zdr-helpers"; import { KEYLESS_CREDITS_MESSAGE, adjustKeylessCredits, + logKeylessCreditUsage, reserveKeylessCredits, } from "../../lib/keyless"; import { projectScrapeCredits } from "../../lib/keyless-credit-projection"; @@ -447,10 +448,14 @@ export async function scrapeController( if (reservedKeylessCredits > 0 && !reconciledKeylessCredits) { reconciledKeylessCredits = true; + const actualKeylessCredits = doc?.metadata?.creditsUsed ?? 0; adjustKeylessCredits( req.auth.team_id, - (doc?.metadata?.creditsUsed ?? 0) - reservedKeylessCredits, + actualKeylessCredits - reservedKeylessCredits, ).catch(() => {}); + logKeylessCreditUsage(req.auth.team_id, actualKeylessCredits).catch( + () => {}, + ); } const totalRequestTime = new Date().getTime() - middlewareStartTime; diff --git a/apps/api/src/controllers/v2/search.ts b/apps/api/src/controllers/v2/search.ts index b661eef42d..05331a2691 100644 --- a/apps/api/src/controllers/v2/search.ts +++ b/apps/api/src/controllers/v2/search.ts @@ -10,6 +10,7 @@ import { billTeam } from "../../services/billing/credit_billing"; import { KEYLESS_CREDITS_MESSAGE, adjustKeylessCredits, + logKeylessCreditUsage, reserveKeylessCredits, } from "../../lib/keyless"; import { v7 as uuidv7 } from "uuid"; @@ -212,6 +213,9 @@ export async function searchController( req.auth.team_id, result.totalCredits - reservedKeylessCredits, ).catch(() => {}); + logKeylessCreditUsage(req.auth.team_id, result.totalCredits).catch( + () => {}, + ); } const endTime = new Date().getTime(); diff --git a/apps/api/src/lib/keyless.ts b/apps/api/src/lib/keyless.ts index 9b63063c7e..665ef558ff 100644 --- a/apps/api/src/lib/keyless.ts +++ b/apps/api/src/lib/keyless.ts @@ -249,9 +249,36 @@ export async function checkKeylessEligibility( } } +/** + * Append a row to `keyless_credit_usage` recording the actual credits a completed + * keyless request consumed (per-IP keyless team UUID + raw IP), for abuse + * monitoring. No-op for non-keyless teams, non-positive credits, or when DB auth + * is off. Best-effort — never throws. + */ +export async function logKeylessCreditUsage( + teamId: string, + credits: number, +): Promise { + const ip = keylessIpFromTeamId(teamId); + if (!ip || !Number.isFinite(credits) || credits <= 0) return; + const teamUuid = keylessTeamUuid(teamId); + if (config.USE_DB_AUTHENTICATION !== true || !teamUuid) return; + try { + await db.insert(schema.keyless_credit_usage).values({ + team_id: teamUuid, + ip, + credits_used: Math.ceil(credits), + }); + } catch { + // Logging is best-effort. + } +} + /** * Add the actual credits a completed request consumed to the IP's daily credit - * counter. No-op for non-keyless teams. Best-effort; never throws. + * counter. No-op for non-keyless teams. Best-effort; never throws. Used by the + * worker for the non-reserved path; the controllers reserve up front and call + * `logKeylessCreditUsage` directly at reconciliation. */ export async function chargeKeylessCredits( teamId: string, @@ -271,18 +298,6 @@ export async function chargeKeylessCredits( // extra free credits today. } - // Log the usage to keyless_credit_usage (per-IP keyless team UUID + raw IP) - // for abuse monitoring. Best-effort — never block the request. - const teamUuid = keylessTeamUuid(teamId); - if (config.USE_DB_AUTHENTICATION === true && teamUuid) { - try { - await db.insert(schema.keyless_credit_usage).values({ - team_id: teamUuid, - ip, - credits_used: inc, - }); - } catch { - // Logging is best-effort. - } - } + // Log the usage to keyless_credit_usage for abuse monitoring. Best-effort. + await logKeylessCreditUsage(teamId, credits); } From d47a4f304694120a69c1eea2a048085f5ba17888 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gerg=C5=91=20M=C3=B3ricz?= Date: Tue, 16 Jun 2026 09:46:17 -0700 Subject: [PATCH 2/3] [codex] Add v2 research proxy and SDKs (#3794) --- apps/api/src/__tests__/snips/v2/lib.ts | 14 + .../src/__tests__/snips/v2/research.test.ts | 139 +++++ apps/api/src/controllers/auth.ts | 11 +- apps/api/src/controllers/v2/research-proxy.ts | 489 ++++++++++++++++-- apps/api/src/db/schema/public.ts | 36 ++ apps/api/src/routes/v2.ts | 18 +- apps/api/src/services/logging/log_job.ts | 82 ++- apps/api/src/services/rate-limiter.ts | 2 +- apps/dot-net-sdk/Firecrawl/FirecrawlClient.cs | 140 +++++ .../Firecrawl/Models/ResearchModels.cs | 141 +++++ apps/elixir-sdk/lib/firecrawl.ex | 143 +++++ apps/go-sdk/firecrawl.go | 136 +++++ apps/go-sdk/models.go | 114 ++++ .../com/firecrawl/client/FirecrawlClient.java | 103 ++++ .../com/firecrawl/models/ResearchModels.java | 119 +++++ .../src/__tests__/unit/v2/research.test.ts | 16 +- .../firecrawl/src/v2/methods/research.ts | 2 +- apps/js-sdk/firecrawl/src/v2/types.ts | 34 +- apps/php-sdk/src/Client/FirecrawlClient.php | 95 ++++ .../src/Client/FirecrawlHttpClient.php | 2 +- apps/python-sdk/firecrawl/v2/client.py | 16 + apps/python-sdk/firecrawl/v2/client_async.py | 16 + .../firecrawl/v2/methods/aio/research.py | 117 +++++ .../firecrawl/v2/methods/research.py | 116 +++++ apps/ruby-sdk/lib/firecrawl/client.rb | 69 ++- apps/ruby-sdk/test/firecrawl/client_test.rb | 10 +- apps/rust-sdk/src/lib.rs | 2 + apps/rust-sdk/src/research.rs | 321 ++++++++++++ 28 files changed, 2407 insertions(+), 96 deletions(-) create mode 100644 apps/api/src/__tests__/snips/v2/research.test.ts create mode 100644 apps/dot-net-sdk/Firecrawl/Models/ResearchModels.cs create mode 100644 apps/java-sdk/src/main/java/com/firecrawl/models/ResearchModels.java create mode 100644 apps/python-sdk/firecrawl/v2/methods/aio/research.py create mode 100644 apps/python-sdk/firecrawl/v2/methods/research.py create mode 100644 apps/rust-sdk/src/research.rs diff --git a/apps/api/src/__tests__/snips/v2/lib.ts b/apps/api/src/__tests__/snips/v2/lib.ts index 0ed0e701e7..4092620e2d 100644 --- a/apps/api/src/__tests__/snips/v2/lib.ts +++ b/apps/api/src/__tests__/snips/v2/lib.ts @@ -638,6 +638,20 @@ export async function searchWithFailure( return raw.body; } +export async function researchRaw( + path: string, + query: Record | undefined, + identity?: Identity, +) { + const req = request(TEST_API_URL) + .get(path) + .set("Content-Type", "application/json"); + if (identity) { + req.set("Authorization", `Bearer ${identity.apiKey}`); + } + return query ? req.query(query) : req; +} + export async function searchRawFull( body: SearchRequestInput, identity: Identity, diff --git a/apps/api/src/__tests__/snips/v2/research.test.ts b/apps/api/src/__tests__/snips/v2/research.test.ts new file mode 100644 index 0000000000..d6c585b336 --- /dev/null +++ b/apps/api/src/__tests__/snips/v2/research.test.ts @@ -0,0 +1,139 @@ +import { config } from "../../../config"; +import { describeIf, TEST_PRODUCTION } from "../lib"; +import { creditUsage, idmux, researchRaw } from "./lib"; + +const HAS_RESEARCH = !!config.RESEARCH_PROXY_URL; +const KEYLESS_ENABLED = + process.env.KEYLESS_REQUESTS_PER_DAY !== undefined && + process.env.KEYLESS_CREDITS_PER_DAY !== undefined; + +const sleep = (ms: number) => new Promise(resolve => setTimeout(resolve, ms)); +const sleepForBilling = () => sleep(40000); + +describeIf(HAS_RESEARCH)("Research API", () => { + it("serves paper search from the canonical mount", async () => { + const identity = await idmux({ + name: "research/canonical paper search", + credits: 100, + }); + + const res = await researchRaw( + "/v2/search/research/papers", + { query: "retrieval augmented generation", k: 2 }, + identity, + ); + + expect(res.statusCode).toBe(200); + expect(res.body.success).toBe(true); + expect(Array.isArray(res.body.results)).toBe(true); + expect(res.body.results.length).toBeGreaterThan(0); + expect(res.body.results[0].paperId).toBeDefined(); + expect(res.body.results[0].paper_id).toBeUndefined(); + }, 120000); + + it("keeps the legacy research mount working", async () => { + const identity = await idmux({ + name: "research/legacy paper search", + credits: 100, + }); + + const res = await researchRaw( + "/v2/research/papers", + { query: "diffusion models", k: 1 }, + identity, + ); + + expect(res.statusCode).toBe(200); + expect(res.body.success).toBe(true); + expect(Array.isArray(res.body.results)).toBe(true); + expect(res.body.results[0].paperId).toBeDefined(); + expect(res.body.results[0].paper_id).toBe(res.body.results[0].paperId); + }, 120000); + + it("rejects invalid endpoint-specific query params", async () => { + const identity = await idmux({ + name: "research/invalid params", + credits: 100, + }); + + const res = await researchRaw( + "/v2/search/research/papers", + { query: "rag", magic: "true" } as any, + identity, + ); + + expect(res.statusCode).toBe(400); + expect(res.body.success).toBe(false); + }); + + it("rejects paper inspect k without read query", async () => { + const identity = await idmux({ + name: "research/inspect rejects k", + credits: 100, + }); + + const res = await researchRaw( + "/v2/search/research/papers/1706.03762", + { k: 1 }, + identity, + ); + + expect(res.statusCode).toBe(400); + expect(res.body.success).toBe(false); + }); + + describeIf(KEYLESS_ENABLED)("keyless research", () => { + it("permits keyless access on the canonical research index", async () => { + const res = await researchRaw("/v2/search/research/papers", { + query: "transformers", + k: 1, + }); + + expect(res.statusCode).not.toBe(401); + }, 120000); + }); + + describeIf(TEST_PRODUCTION)("research billing", () => { + it("bills read-paper as one scrape-like credit", async () => { + const identity = await idmux({ + name: "research/bills read paper", + credits: 100, + }); + const before = (await creditUsage(identity)).remainingCredits; + + const res = await researchRaw( + "/v2/search/research/papers/1706.03762", + { query: "attention", k: 1 }, + identity, + ); + expect(res.statusCode).toBe(200); + expect(res.body.success).toBe(true); + + await sleepForBilling(); + const after = (await creditUsage(identity)).remainingCredits; + expect(before - after).toBe(1); + }, 180000); + + it("bills search-like endpoints by returned result count", async () => { + const identity = await idmux({ + name: "research/bills search papers", + credits: 100, + }); + const before = (await creditUsage(identity)).remainingCredits; + + const res = await researchRaw( + "/v2/search/research/papers", + { query: "graph neural networks", k: 11 }, + identity, + ); + expect(res.statusCode).toBe(200); + expect(res.body.success).toBe(true); + expect(res.body.results.length).toBeGreaterThan(0); + const expectedCredits = Math.ceil(res.body.results.length / 10) * 2; + + await sleepForBilling(); + const after = (await creditUsage(identity)).remainingCredits; + expect(before - after).toBe(expectedCredits); + }, 180000); + }); +}); diff --git a/apps/api/src/controllers/auth.ts b/apps/api/src/controllers/auth.ts index 81eaeb9150..d7a50c6c01 100644 --- a/apps/api/src/controllers/auth.ts +++ b/apps/api/src/controllers/auth.ts @@ -561,9 +561,11 @@ async function handleKeylessAuth( const modeLabel = mode === RateLimiterMode.Search ? "search" - : mode === RateLimiterMode.BrowserExecute - ? "interact" - : "scrape"; + : mode === RateLimiterMode.Research + ? "research" + : mode === RateLimiterMode.BrowserExecute + ? "interact" + : "scrape"; let result: Awaited>; try { @@ -831,7 +833,8 @@ async function supaAuthenticateUser( mode === RateLimiterMode.Crawl || mode === RateLimiterMode.CrawlStatus || mode === RateLimiterMode.Extract || - mode === RateLimiterMode.Search) + mode === RateLimiterMode.Search || + mode === RateLimiterMode.Research) ) { return { success: true, diff --git a/apps/api/src/controllers/v2/research-proxy.ts b/apps/api/src/controllers/v2/research-proxy.ts index 2ea86ea5bd..c8f6a78a84 100644 --- a/apps/api/src/controllers/v2/research-proxy.ts +++ b/apps/api/src/controllers/v2/research-proxy.ts @@ -1,12 +1,28 @@ -import { NextFunction, Response } from "express"; +import express, { Request, Response } from "express"; import { Agent, fetch } from "undici"; +import { z } from "zod"; +import { v7 as uuidv7 } from "uuid"; import { config } from "../../config"; -import { logger } from "../../lib/logger"; -import { RequestWithAuth, RequestWithMaybeACUC } from "../v1/types"; +import { logger as rootLogger } from "../../lib/logger"; +import { chargeKeylessCredits } from "../../lib/keyless"; +import { billTeam } from "../../services/billing/credit_billing"; +import { getSearchForcedKind } from "../../lib/zdr-helpers"; +import { + logRequest, + logResearchEndpoint, +} from "../../services/logging/log_job"; +import type { + ResearchRequestKind, + ResearchTableName, +} from "../../services/logging/log_job"; +import type { RequestWithAuth } from "../v1/types"; +import { wrap } from "../../routes/shared"; const TIMEOUT_MS = 120_000; +const SEARCH_CREDITS_PER_TEN_RESULTS = 2; +const ZDR_SEARCH_CREDITS_PER_TEN_RESULTS = 10; -const FORWARDED_REQUEST_HEADERS = ["content-type", "accept", "x-request-id"]; +const FORWARDED_REQUEST_HEADERS = ["accept", "x-request-id"]; const FORWARDED_RESPONSE_HEADERS = ["content-type", "x-request-id"]; const dispatcher = new Agent({ @@ -15,36 +31,190 @@ const dispatcher = new Agent({ bodyTimeout: TIMEOUT_MS, }); -export function researchFlagMiddleware( - req: RequestWithMaybeACUC, - res: Response, - next: NextFunction, +const multiString = z + .union([z.string(), z.array(z.string())]) + .optional() + .transform(value => { + if (!value) return undefined; + return Array.isArray(value) ? value : [value]; + }); + +const kSchema = (max: number) => + z.coerce.number().int().positive().max(max).optional(); + +const dateSchema = z + .string() + .regex(/^\d{4}-\d{2}-\d{2}$/) + .optional(); + +const commonQuery = { + origin: z.string().optional(), + integration: z.string().optional(), +}; + +const searchPapersSchema = z.strictObject({ + query: z.string().min(1), + k: kSchema(500), + authors: multiString, + categories: multiString, + from: dateSchema, + to: dateSchema, + ...commonQuery, +}); + +const paperSchema = z + .strictObject({ + query: z.string().min(1).optional(), + k: kSchema(50), + ...commonQuery, + }) + .refine(value => value.query !== undefined || value.k === undefined, { + message: "k is only valid when query is present", + path: ["k"], + }); + +const similarPapersSchema = z.strictObject({ + intent: z.string().min(1), + mode: z.enum(["similar", "citers", "references"]).optional(), + k: kSchema(500), + rerank: z + .enum(["true", "false"]) + .optional() + .transform(value => (value === undefined ? undefined : value === "true")), + anchor: multiString, + ...commonQuery, +}); + +const githubSearchSchema = z.strictObject({ + query: z.string().min(1), + k: kSchema(100), + ...commonQuery, +}); + +type ResearchEndpointConfig = { + kind: ResearchRequestKind; + table: ResearchTableName; + action: string; + targetHint: ( + params: Record, + req: RequestWithAuth, + ) => string; + upstreamPath: ( + params: Record, + req: RequestWithAuth, + ) => string; + billAs: "scrape" | "search"; +}; + +type ResearchController = (req: Request, res: Response) => Promise; +type ResearchQueryParams = Record & { + origin?: string; + integration?: string; +}; + +const LEGACY_SNAKE_CASE_ALIASES: Record = { + paperId: "paper_id", + primaryId: "primary_id", + createdDate: "created_date", + updateDate: "update_date", + articleRank: "article_rank", + seedOverlap: "seed_overlap", + poolSize: "pool_size", + resultType: "result_type", + pageType: "page_type", + segmentCount: "segment_count", + readmeUrl: "readme_url", + contentMd: "content_md", +}; + +function addLegacySnakeCaseAliases(value: T): T { + if (Array.isArray(value)) { + return value.map(addLegacySnakeCaseAliases) as T; + } + + if (!value || typeof value !== "object") { + return value; + } + + const source = value as Record; + const object: Record = {}; + + for (const [key, item] of Object.entries(source)) { + object[key] = addLegacySnakeCaseAliases(item); + } + + for (const [camelKey, snakeKey] of Object.entries( + LEGACY_SNAKE_CASE_ALIASES, + )) { + if (Object.prototype.hasOwnProperty.call(source, camelKey)) { + object[snakeKey] = object[camelKey]; + } + } + + return object as T; +} + +function appendQuery( + url: URL, + params: Record, + allowed: string[], ) { - if (!req.acuc?.flags?.researchBeta) { - return res.status(403).json({ success: false, error: "Forbidden" }); + for (const key of allowed) { + const value = params[key]; + if (Array.isArray(value)) { + for (const item of value) { + if (item !== undefined && item !== null) { + url.searchParams.append(key, String(item)); + } + } + } else if (value !== undefined && value !== null) { + url.searchParams.append(key, String(value)); + } } - next(); } -export async function researchProxyController( +function resultCount(body: any): number { + return Array.isArray(body?.results) ? body.results.length : 0; +} + +function creditsFor( + config: ResearchEndpointConfig, + body: any, req: RequestWithAuth, +) { + if (config.billAs === "scrape") return 1; + const forcedKind = getSearchForcedKind(req.acuc?.flags); + const perTen = + forcedKind === "zdr" + ? ZDR_SEARCH_CREDITS_PER_TEN_RESULTS + : SEARCH_CREDITS_PER_TEN_RESULTS; + return Math.ceil(resultCount(body) / 10) * perTen; +} + +function researchError( res: Response, -): Promise { + status: number, + error: string, + details?: unknown, +) { + return res.status(status).json({ + success: false, + error, + ...(details === undefined ? {} : { details }), + }); +} + +async function fetchResearchUpstream( + req: RequestWithAuth, + path: string, + params: Record, + queryKeys: string[], +) { const base = config.RESEARCH_PROXY_URL; - if (!base) { - res.status(404).end(); - return; - } + if (!base) return null; - const fullPath = req.baseUrl + req.path || "/"; - const url = new URL(base.replace(/\/+$/, "") + fullPath); - for (const [k, v] of Object.entries(req.query)) { - if (Array.isArray(v)) { - v.forEach(vv => url.searchParams.append(k, String(vv))); - } else if (v !== undefined && v !== null) { - url.searchParams.append(k, String(v)); - } - } + const url = new URL(base.replace(/\/+$/, "") + path); + appendQuery(url, params, queryKeys); const headers: Record = {}; for (const h of FORWARDED_REQUEST_HEADERS) { @@ -53,38 +223,253 @@ export async function researchProxyController( } headers["firecrawl-team-id"] = req.auth.team_id; - const init: Parameters[1] = { - method: req.method, + return fetch(url, { + method: "GET", headers, signal: AbortSignal.timeout(TIMEOUT_MS), dispatcher, - }; + }); +} - if (req.method !== "GET" && req.method !== "HEAD") { - if ( - req.body && - typeof req.body === "object" && - Object.keys(req.body).length > 0 - ) { - init.body = JSON.stringify(req.body); - headers["content-type"] = "application/json"; - } - } +function createResearchController( + schema: z.ZodTypeAny, + queryKeys: string[], + endpoint: ResearchEndpointConfig, + options: { legacy?: boolean } = {}, +): ResearchController { + return async (req, res: Response) => { + const authedReq = req as RequestWithAuth; + const started = Date.now(); + const jobId = uuidv7(); + const logger = rootLogger.child({ + module: "api/v2/research", + method: endpoint.action, + jobId, + teamId: authedReq.auth.team_id, + }); - try { - const upstream = await fetch(url, init); - res.status(upstream.status); - for (const h of FORWARDED_RESPONSE_HEADERS) { - const v = upstream.headers.get(h); - if (v) res.setHeader(h, v); + const parsed = schema.safeParse(req.query); + if (!parsed.success) { + logger.warn("Invalid research query", { error: parsed.error.issues }); + return researchError( + res, + 400, + "Invalid query parameters", + parsed.error.issues, + ); } - res.send(Buffer.from(await upstream.arrayBuffer())); - } catch (err: unknown) { - if (err instanceof DOMException && err.name === "TimeoutError") { - res.status(504).end(); - return; + + const params = parsed.data as ResearchQueryParams; + const targetHint = endpoint.targetHint(params, authedReq); + await logRequest({ + id: jobId, + kind: endpoint.kind, + api_version: "v2", + team_id: authedReq.auth.team_id, + origin: params.origin ?? "api", + integration: params.integration ?? null, + target_hint: targetHint, + zeroDataRetention: false, + api_key_id: authedReq.acuc?.api_key_id ?? null, + }); + + let statusCode = 500; + let responseBody: any = null; + let error: string | undefined; + let credits = 0; + + try { + const upstream = await fetchResearchUpstream( + authedReq, + endpoint.upstreamPath(params, authedReq), + params, + queryKeys, + ); + if (!upstream) { + statusCode = 404; + error = "Research service is not configured"; + return res.status(404).end(); + } + + statusCode = upstream.status; + for (const h of FORWARDED_RESPONSE_HEADERS) { + const value = upstream.headers.get(h); + if (value) res.setHeader(h, value); + } + + const text = await upstream.text(); + try { + responseBody = text ? JSON.parse(text) : null; + } catch { + responseBody = text; + } + + if (upstream.ok) { + credits = creditsFor(endpoint, responseBody, authedReq); + if (credits > 0) { + billTeam( + authedReq.auth.team_id, + authedReq.acuc?.sub_id ?? undefined, + credits, + authedReq.acuc?.api_key_id ?? null, + { + endpoint: endpoint.billAs === "scrape" ? "scrape" : "search", + jobId, + }, + ).catch(billingError => { + logger.error("Failed to bill research request", { + error: billingError, + credits, + }); + }); + chargeKeylessCredits(authedReq.auth.team_id, credits).catch(() => {}); + } + } else { + error = + typeof responseBody === "object" && responseBody !== null + ? (responseBody.detail ?? responseBody.error ?? responseBody.title) + : undefined; + } + + if (responseBody === null || typeof responseBody === "string") { + return res.status(statusCode).send(responseBody ?? ""); + } + const response = + options.legacy && upstream.ok + ? addLegacySnakeCaseAliases(responseBody) + : responseBody; + return res.status(statusCode).json(response); + } catch (err: unknown) { + if (err instanceof DOMException && err.name === "TimeoutError") { + statusCode = 504; + error = "Research service timed out"; + return res.status(504).end(); + } + statusCode = 502; + error = "Research proxy error"; + logger.error("Research proxy error", { error: err }); + return res.status(502).end(); + } finally { + const timeTaken = (Date.now() - started) / 1000; + logResearchEndpoint({ + table: endpoint.table, + id: jobId, + request_id: jobId, + team_id: authedReq.auth.team_id, + target: targetHint, + options: params, + response: responseBody, + num_results: resultCount(responseBody), + time_taken: timeTaken, + credits_cost: statusCode >= 200 && statusCode < 300 ? credits : 0, + is_successful: statusCode >= 200 && statusCode < 300, + error, + zeroDataRetention: false, + }).catch(logError => { + logger.warn("Research endpoint log failed", { error: logError }); + }); } - logger.error("Research proxy error", { error: err }); - res.status(502).end(); + }; +} + +export function createResearchRouter(options: { legacy?: boolean } = {}) { + const router = express.Router(); + + if (options.legacy) { + router.use((req, _res, next) => { + rootLogger.warn("Legacy research endpoint used", { + module: "api/v2/research", + teamId: (req as RequestWithAuth).auth?.team_id, + method: req.method, + path: req.originalUrl, + requestId: req.headers["x-request-id"], + }); + next(); + }); } + + router.get( + "/papers", + wrap( + createResearchController( + searchPapersSchema, + ["query", "k", "authors", "categories", "from", "to"], + { + kind: "research_paper_search", + table: "research_paper_searches", + action: "searchPapers", + targetHint: params => String(params.query), + upstreamPath: () => "/v2/research/papers", + billAs: "search", + }, + options, + ), + ), + ); + + router.get( + "/papers/:id/similar", + wrap( + createResearchController( + similarPapersSchema, + ["intent", "mode", "k", "rerank", "anchor"], + { + kind: "research_related_papers", + table: "research_related_papers", + action: "similarPapers", + targetHint: (params, req) => + `${req.params.id}: ${String(params.intent)}`, + upstreamPath: (_params, req) => + `/v2/research/papers/${encodeURIComponent(req.params.id)}/similar`, + billAs: "search", + }, + options, + ), + ), + ); + + router.get( + "/papers/:id", + wrap(async (req: Request, res: Response) => { + const authedReq = req as RequestWithAuth; + const parsed = paperSchema.safeParse(req.query); + const isRead = parsed.success && parsed.data.query !== undefined; + const controller = createResearchController( + paperSchema, + ["query", "k"], + { + kind: isRead ? "research_paper_read" : "research_paper_inspect", + table: isRead ? "research_paper_reads" : "research_paper_inspects", + action: isRead ? "readPaper" : "inspectPaper", + targetHint: (_params, request) => request.params.id, + upstreamPath: (_params, request) => + `/v2/research/papers/${encodeURIComponent(request.params.id)}`, + billAs: "scrape", + }, + options, + ); + return controller(authedReq, res); + }), + ); + + router.get( + "/github", + wrap( + createResearchController( + githubSearchSchema, + ["query", "k"], + { + kind: "research_github_search", + table: "research_github_searches", + action: "searchGithub", + targetHint: params => String(params.query), + upstreamPath: () => "/v2/research/github", + billAs: "search", + }, + options, + ), + ), + ); + + return router; } diff --git a/apps/api/src/db/schema/public.ts b/apps/api/src/db/schema/public.ts index f73752a359..f03241d653 100644 --- a/apps/api/src/db/schema/public.ts +++ b/apps/api/src/db/schema/public.ts @@ -162,6 +162,42 @@ export const deep_researches = pgTable("deep_researches", { options: jsonb("options"), }); +const researchEndpointTable = (name: string) => + pgTable(name, { + id: uuid("id").notNull(), + request_id: uuid("request_id").notNull(), + target: text("target").notNull(), + team_id: uuid("team_id").notNull(), + options: jsonb("options"), + response: jsonb("response"), + num_results: integer("num_results").notNull(), + time_taken: num("time_taken").notNull(), + credits_cost: integer("credits_cost").notNull(), + is_successful: boolean("is_successful").notNull(), + error: text("error"), + created_at: ts("created_at").notNull().defaultNow(), + }); + +export const research_paper_searches = researchEndpointTable( + "research_paper_searches", +); + +export const research_paper_inspects = researchEndpointTable( + "research_paper_inspects", +); + +export const research_paper_reads = researchEndpointTable( + "research_paper_reads", +); + +export const research_related_papers = researchEndpointTable( + "research_related_papers", +); + +export const research_github_searches = researchEndpointTable( + "research_github_searches", +); + export const deterministic_json_scripts = pgTable( "deterministic_json_scripts", { diff --git a/apps/api/src/routes/v2.ts b/apps/api/src/routes/v2.ts index 9df0a6cbf8..b1cd15866b 100644 --- a/apps/api/src/routes/v2.ts +++ b/apps/api/src/routes/v2.ts @@ -62,10 +62,7 @@ import { } from "../controllers/v2/browser"; import { activityController } from "../controllers/v1/activity"; import { supportProxyController } from "../controllers/v2/support-proxy"; -import { - researchFlagMiddleware, - researchProxyController, -} from "../controllers/v2/research-proxy"; +import { createResearchRouter } from "../controllers/v2/research-proxy"; import { scrapeInteractController, scrapeStopInteractiveBrowserController, @@ -593,11 +590,16 @@ v2Router.post( ); if (config.RESEARCH_PROXY_URL) { - v2Router.all( - "/research/*", + v2Router.use( + "/search/research", + authMiddleware(RateLimiterMode.Research, { allowKeyless: true }), + createResearchRouter(), + ); + + v2Router.use( + "/research", authMiddleware(RateLimiterMode.Research), - researchFlagMiddleware, - wrap(researchProxyController), + createResearchRouter({ legacy: true }), ); } diff --git a/apps/api/src/services/logging/log_job.ts b/apps/api/src/services/logging/log_job.ts index d9a3eab416..20852deeff 100644 --- a/apps/api/src/services/logging/log_job.ts +++ b/apps/api/src/services/logging/log_job.ts @@ -45,6 +45,11 @@ const tableMap: Record = { crawls: schema.crawls, batch_scrapes: schema.batch_scrapes, searches: schema.searches, + research_paper_searches: schema.research_paper_searches, + research_paper_inspects: schema.research_paper_inspects, + research_paper_reads: schema.research_paper_reads, + research_related_papers: schema.research_related_papers, + research_github_searches: schema.research_github_searches, extracts: schema.extracts, maps: schema.maps, llmstxts: schema.llmstxts, @@ -166,7 +171,12 @@ type LoggedRequest = { | "parse" | "agent" | "browser" - | "interact"; + | "interact" + | "research_paper_search" + | "research_paper_inspect" + | "research_paper_read" + | "research_related_papers" + | "research_github_search"; api_version: string; team_id: string; origin?: string; @@ -490,6 +500,76 @@ export async function logSearch(search: LoggedSearch, force: boolean = false) { } } +export type ResearchRequestKind = + | "research_paper_search" + | "research_paper_inspect" + | "research_paper_read" + | "research_related_papers" + | "research_github_search"; + +export type ResearchTableName = + | "research_paper_searches" + | "research_paper_inspects" + | "research_paper_reads" + | "research_related_papers" + | "research_github_searches"; + +type LoggedResearchEndpoint = { + table: ResearchTableName; + id: string; + request_id: string; + target: string; + team_id: string; + options: any; + response: any; + num_results: number; + time_taken: number; + credits_cost: number; + is_successful: boolean; + error?: string; + zeroDataRetention: boolean; +}; + +export async function logResearchEndpoint( + research: LoggedResearchEndpoint, + force: boolean = false, +) { + const logger = _logger.child({ + module: "log_job", + method: "logResearchEndpoint", + researchId: research.id, + requestId: research.request_id, + teamId: research.team_id, + zeroDataRetention: research.zeroDataRetention, + }); + + await robustInsert( + research.table, + { + id: research.id, + request_id: research.request_id, + target: research.zeroDataRetention + ? "" + : (sanitizeString(research.target) ?? ""), + team_id: + keylessTeamUuid(research.team_id) ?? + (research.team_id === "preview" || + research.team_id?.startsWith("preview_") + ? previewTeamId + : research.team_id), + options: research.zeroDataRetention ? null : research.options, + response: research.zeroDataRetention ? null : research.response, + num_results: research.num_results, + time_taken: research.time_taken, + credits_cost: research.credits_cost, + is_successful: research.is_successful, + error: research.zeroDataRetention ? null : (research.error ?? null), + }, + force, + logger, + ); +} + export type LoggedExtract = { id: string; request_id: string; diff --git a/apps/api/src/services/rate-limiter.ts b/apps/api/src/services/rate-limiter.ts index a711638668..c8ea8cc04a 100644 --- a/apps/api/src/services/rate-limiter.ts +++ b/apps/api/src/services/rate-limiter.ts @@ -32,7 +32,7 @@ const fallbackRateLimits: AuthCreditUsageChunk["rate_limits"] = { account: 1000, supportAsk: 3, supportDocsSearch: 3, - research: 20, + research: 100, }; export function getRateLimiter( diff --git a/apps/dot-net-sdk/Firecrawl/FirecrawlClient.cs b/apps/dot-net-sdk/Firecrawl/FirecrawlClient.cs index fde84b672f..1b5b4b0051 100644 --- a/apps/dot-net-sdk/Firecrawl/FirecrawlClient.cs +++ b/apps/dot-net-sdk/Firecrawl/FirecrawlClient.cs @@ -461,6 +461,113 @@ public async Task SearchAsync( return response.Data ?? throw new FirecrawlException("Search response contained no data"); } + /// + /// Searches research papers. + /// + public async Task SearchPapersAsync( + string query, + SearchPapersOptions? options = null, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(query); + + return await _http.GetAsync( + "/v2/search/research/papers" + BuildResearchQuery(new Dictionary + { + ["query"] = query, + ["origin"] = SdkOrigin, + ["k"] = options?.K, + ["authors"] = options?.Authors, + ["categories"] = options?.Categories, + ["from"] = options?.From, + ["to"] = options?.To + }), + cancellationToken); + } + + /// + /// Fetches paper metadata. + /// + public async Task InspectPaperAsync( + string paperId, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(paperId); + + return await _http.GetAsync( + $"/v2/search/research/papers/{Uri.EscapeDataString(paperId)}", + cancellationToken); + } + + /// + /// Fetches paper metadata and query-guided passages. + /// + public async Task ReadPaperAsync( + string paperId, + string query, + ReadPaperOptions? options = null, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(paperId); + ArgumentNullException.ThrowIfNull(query); + + return await _http.GetAsync( + $"/v2/search/research/papers/{Uri.EscapeDataString(paperId)}" + + BuildResearchQuery(new Dictionary + { + ["query"] = query, + ["origin"] = SdkOrigin, + ["k"] = options?.K + }), + cancellationToken); + } + + /// + /// Finds papers related to a paper. + /// + public async Task RelatedPapersAsync( + string paperId, + string intent, + RelatedPapersOptions? options = null, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(paperId); + ArgumentNullException.ThrowIfNull(intent); + + return await _http.GetAsync( + $"/v2/search/research/papers/{Uri.EscapeDataString(paperId)}/similar" + + BuildResearchQuery(new Dictionary + { + ["intent"] = intent, + ["origin"] = SdkOrigin, + ["mode"] = options?.Mode, + ["k"] = options?.K, + ["rerank"] = options?.Rerank, + ["anchor"] = options?.Anchor + }), + cancellationToken); + } + + /// + /// Searches GitHub research content. + /// + public async Task SearchGitHubAsync( + string query, + SearchGitHubOptions? options = null, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(query); + + return await _http.GetAsync( + "/v2/search/research/github" + BuildResearchQuery(new Dictionary + { + ["query"] = query, + ["origin"] = SdkOrigin, + ["k"] = options?.K + }), + cancellationToken); + } + // ================================================================ // USAGE & METRICS // ================================================================ @@ -657,6 +764,39 @@ private static string BuildMonitorCheckQuery(int? limit = null, int? skip = null return query.Count == 0 ? string.Empty : "?" + string.Join("&", query); } + private static string BuildResearchQuery(Dictionary parameters) + { + var query = new List(); + foreach (var (key, value) in parameters) + { + if (value == null) + continue; + + if (value is System.Collections.IEnumerable values && value is not string) + { + foreach (var item in values) + { + if (item != null) + AddQueryValue(query, key, item); + } + continue; + } + + AddQueryValue(query, key, value); + } + + return query.Count == 0 ? string.Empty : "?" + string.Join("&", query); + } + + private static void AddQueryValue(List query, string key, object value) + { + var stringValue = value is bool boolValue + ? boolValue.ToString().ToLowerInvariant() + : value.ToString() ?? string.Empty; + + query.Add($"{Uri.EscapeDataString(key)}={Uri.EscapeDataString(stringValue)}"); + } + private static string? ResolveApiKey(string? apiKey) { if (!string.IsNullOrWhiteSpace(apiKey)) diff --git a/apps/dot-net-sdk/Firecrawl/Models/ResearchModels.cs b/apps/dot-net-sdk/Firecrawl/Models/ResearchModels.cs new file mode 100644 index 0000000000..be794f5c23 --- /dev/null +++ b/apps/dot-net-sdk/Firecrawl/Models/ResearchModels.cs @@ -0,0 +1,141 @@ +using System.Text.Json.Serialization; + +namespace Firecrawl.Models; + +public class PaperResult +{ + [JsonPropertyName("paperId")] + public string? PaperId { get; set; } + + [JsonPropertyName("primaryId")] + public string? PrimaryId { get; set; } + + public Dictionary? Ids { get; set; } + public string? Title { get; set; } + public string? Abstract { get; set; } + public double? Score { get; set; } + public int? Year { get; set; } + public List? Authors { get; set; } + public string? Venue { get; set; } + public string? Url { get; set; } + public Dictionary? Signals { get; set; } +} + +public class PaperMetadata +{ + [JsonPropertyName("paperId")] + public string? PaperId { get; set; } + + public Dictionary? Ids { get; set; } + public string? Title { get; set; } + public string? Abstract { get; set; } + public string? Authors { get; set; } + public List? Categories { get; set; } + + [JsonPropertyName("createdDate")] + public string? CreatedDate { get; set; } + + [JsonPropertyName("updateDate")] + public string? UpdateDate { get; set; } +} + +public class Passage +{ + public string? Text { get; set; } + public string? Section { get; set; } + public int? Page { get; set; } + public double? Score { get; set; } + public Dictionary? Metadata { get; set; } +} + +public class SearchPapersResponse +{ + public bool Success { get; set; } + public List? Results { get; set; } +} + +public class PaperMetadataResponse +{ + public bool Success { get; set; } + public PaperMetadata? Paper { get; set; } +} + +public class ReadPaperResponse +{ + public bool Success { get; set; } + public PaperMetadata? Paper { get; set; } + + [JsonPropertyName("paperId")] + public string? PaperId { get; set; } + + public string? Query { get; set; } + public List? Passages { get; set; } +} + +public class SimilarPapersResponse +{ + public bool Success { get; set; } + public List? Results { get; set; } + + [JsonPropertyName("poolSize")] + public int? PoolSize { get; set; } + + public bool Truncated { get; set; } + public string? Note { get; set; } +} + +public class GitHubSearchItem +{ + public string? ResultType { get; set; } + + public string? Repo { get; set; } + public string? Url { get; set; } + + public string? PageType { get; set; } + + public int? Number { get; set; } + + public int? SegmentCount { get; set; } + + public string? ReadmeUrl { get; set; } + + public string? Title { get; set; } + public string? Snippet { get; set; } + + public string? ContentMd { get; set; } + + public Dictionary? Scores { get; set; } +} + +public class GitHubSearchResponse +{ + public bool Success { get; set; } + public List? Results { get; set; } +} + +public class SearchPapersOptions +{ + public int? K { get; set; } + public List? Authors { get; set; } + public List? Categories { get; set; } + public string? From { get; set; } + public string? To { get; set; } +} + +public class ReadPaperOptions +{ + public int? K { get; set; } +} + +public class RelatedPapersOptions +{ + public string? Mode { get; set; } + public int? K { get; set; } + public bool? Rerank { get; set; } + public List? Anchor { get; set; } +} + +public class SearchGitHubOptions +{ + public int? K { get; set; } +} diff --git a/apps/elixir-sdk/lib/firecrawl.ex b/apps/elixir-sdk/lib/firecrawl.ex index dc1ea6008c..f81a95ecf3 100644 --- a/apps/elixir-sdk/lib/firecrawl.ex +++ b/apps/elixir-sdk/lib/firecrawl.ex @@ -1196,6 +1196,149 @@ defmodule Firecrawl do Req.post!(client(opts), url: "/search", json: to_body(params, @search_and_scrape_key_mapping)) end + @research_search_papers_schema NimbleOptions.new!([ + authors: [type: {:list, :string}], + categories: [type: {:list, :string}], + from: [type: :string], + k: [type: :integer], + query: [type: :string, required: true], + to: [type: :string] + ]) + + @research_search_papers_key_mapping %{authors: "authors", categories: "categories", from: "from", k: "k", query: "query", to: "to"} + + @doc """ + Search research papers. + + `GET /search/research/papers` + """ + @spec search_papers(keyword(), keyword()) :: response() + def search_papers(params \\ [], opts \\ []) do + with {:ok, params} <- NimbleOptions.validate(params, @research_search_papers_schema) do + Req.get(client(opts), url: "/search/research/papers", params: [{"origin", @sdk_origin} | to_query(params, @research_search_papers_key_mapping)]) + end + end + + @doc """ + Bang variant of `search_papers`. Raises on error. + """ + @spec search_papers!(keyword(), keyword()) :: Req.Response.t() + def search_papers!(params \\ [], opts \\ []) do + params = NimbleOptions.validate!(params, @research_search_papers_schema) + Req.get!(client(opts), url: "/search/research/papers", params: [{"origin", @sdk_origin} | to_query(params, @research_search_papers_key_mapping)]) + end + + @doc """ + Inspect paper metadata. + + `GET /search/research/papers/{id}` + """ + @spec inspect_paper(String.t(), keyword()) :: response() + def inspect_paper(id, opts \\ []) do + Req.get(client(opts), + url: "/search/research/papers/#{URI.encode_www_form(id)}", + params: [{"origin", @sdk_origin}] + ) + end + + @doc """ + Bang variant of `inspect_paper`. Raises on error. + """ + @spec inspect_paper!(String.t(), keyword()) :: Req.Response.t() + def inspect_paper!(id, opts \\ []) do + Req.get!(client(opts), + url: "/search/research/papers/#{URI.encode_www_form(id)}", + params: [{"origin", @sdk_origin}] + ) + end + + @research_read_paper_schema NimbleOptions.new!([ + k: [type: :integer], + query: [type: :string, required: true] + ]) + + @research_read_paper_key_mapping %{k: "k", query: "query"} + + @doc """ + Read a paper with query-guided passages. + + `GET /search/research/papers/{id}` + """ + @spec read_paper(String.t(), keyword(), keyword()) :: response() + def read_paper(id, params \\ [], opts \\ []) do + with {:ok, params} <- NimbleOptions.validate(params, @research_read_paper_schema) do + Req.get(client(opts), url: "/search/research/papers/#{URI.encode_www_form(id)}", params: [{"origin", @sdk_origin} | to_query(params, @research_read_paper_key_mapping)]) + end + end + + @doc """ + Bang variant of `read_paper`. Raises on error. + """ + @spec read_paper!(String.t(), keyword(), keyword()) :: Req.Response.t() + def read_paper!(id, params \\ [], opts \\ []) do + params = NimbleOptions.validate!(params, @research_read_paper_schema) + Req.get!(client(opts), url: "/search/research/papers/#{URI.encode_www_form(id)}", params: [{"origin", @sdk_origin} | to_query(params, @research_read_paper_key_mapping)]) + end + + @research_related_papers_schema NimbleOptions.new!([ + anchor: [type: {:list, :string}], + intent: [type: :string, required: true], + k: [type: :integer], + mode: [type: {:in, [:similar, :citers, :references, "similar", "citers", "references"]}], + rerank: [type: :boolean] + ]) + + @research_related_papers_key_mapping %{anchor: "anchor", intent: "intent", k: "k", mode: "mode", rerank: "rerank"} + + @doc """ + Find papers related to a paper. + + `GET /search/research/papers/{id}/similar` + """ + @spec related_papers(String.t(), keyword(), keyword()) :: response() + def related_papers(id, params \\ [], opts \\ []) do + with {:ok, params} <- NimbleOptions.validate(params, @research_related_papers_schema) do + Req.get(client(opts), url: "/search/research/papers/#{URI.encode_www_form(id)}/similar", params: [{"origin", @sdk_origin} | to_query(params, @research_related_papers_key_mapping)]) + end + end + + @doc """ + Bang variant of `related_papers`. Raises on error. + """ + @spec related_papers!(String.t(), keyword(), keyword()) :: Req.Response.t() + def related_papers!(id, params \\ [], opts \\ []) do + params = NimbleOptions.validate!(params, @research_related_papers_schema) + Req.get!(client(opts), url: "/search/research/papers/#{URI.encode_www_form(id)}/similar", params: [{"origin", @sdk_origin} | to_query(params, @research_related_papers_key_mapping)]) + end + + @research_search_github_schema NimbleOptions.new!([ + k: [type: :integer], + query: [type: :string, required: true] + ]) + + @research_search_github_key_mapping %{k: "k", query: "query"} + + @doc """ + Search GitHub research content. + + `GET /search/research/github` + """ + @spec search_github(keyword(), keyword()) :: response() + def search_github(params \\ [], opts \\ []) do + with {:ok, params} <- NimbleOptions.validate(params, @research_search_github_schema) do + Req.get(client(opts), url: "/search/research/github", params: [{"origin", @sdk_origin} | to_query(params, @research_search_github_key_mapping)]) + end + end + + @doc """ + Bang variant of `search_github`. Raises on error. + """ + @spec search_github!(keyword(), keyword()) :: Req.Response.t() + def search_github!(params \\ [], opts \\ []) do + params = NimbleOptions.validate!(params, @research_search_github_schema) + Req.get!(client(opts), url: "/search/research/github", params: [{"origin", @sdk_origin} | to_query(params, @research_search_github_key_mapping)]) + end + @start_agent_schema NimbleOptions.new!([ max_credits: [type: {:or, [:integer, :float]}, doc: "Maximum credits to spend on this agent task. Defaults to 2500 if not set. Values above 2,500 are always billed as paid requests."], diff --git a/apps/go-sdk/firecrawl.go b/apps/go-sdk/firecrawl.go index 84495c3e08..409bb2e9a9 100644 --- a/apps/go-sdk/firecrawl.go +++ b/apps/go-sdk/firecrawl.go @@ -557,6 +557,142 @@ func (c *Client) Search(ctx context.Context, query string, opts *SearchOptions) return data, nil } +// SearchPapers searches research papers. +func (c *Client) SearchPapers(ctx context.Context, query string, opts *SearchPapersOptions) (*SearchPapersResponse, error) { + if query == "" { + return nil, &FirecrawlError{Message: "query is required"} + } + values := url.Values{} + values.Set("query", query) + values.Set("origin", "go-sdk@"+Version) + if opts != nil { + if opts.K != nil { + values.Set("k", fmt.Sprint(*opts.K)) + } + for _, author := range opts.Authors { + values.Add("authors", author) + } + for _, category := range opts.Categories { + values.Add("categories", category) + } + if opts.From != "" { + values.Set("from", opts.From) + } + if opts.To != "" { + values.Set("to", opts.To) + } + } + + raw, err := c.http.get(ctx, "/v2/search/research/papers?"+values.Encode()) + if err != nil { + return nil, err + } + var resp SearchPapersResponse + if err := json.Unmarshal(raw, &resp); err != nil { + return nil, &FirecrawlError{Message: fmt.Sprintf("failed to decode response: %v", err)} + } + return &resp, nil +} + +// InspectPaper fetches paper metadata. +func (c *Client) InspectPaper(ctx context.Context, paperID string) (*PaperMetadataResponse, error) { + if paperID == "" { + return nil, &FirecrawlError{Message: "paper ID is required"} + } + raw, err := c.http.get(ctx, "/v2/search/research/papers/"+url.PathEscape(paperID)) + if err != nil { + return nil, err + } + var resp PaperMetadataResponse + if err := json.Unmarshal(raw, &resp); err != nil { + return nil, &FirecrawlError{Message: fmt.Sprintf("failed to decode response: %v", err)} + } + return &resp, nil +} + +// ReadPaper fetches paper metadata and relevant passages. +func (c *Client) ReadPaper(ctx context.Context, paperID, query string, opts *ReadPaperOptions) (*ReadPaperResponse, error) { + if paperID == "" { + return nil, &FirecrawlError{Message: "paper ID is required"} + } + if query == "" { + return nil, &FirecrawlError{Message: "query is required"} + } + values := url.Values{} + values.Set("query", query) + values.Set("origin", "go-sdk@"+Version) + if opts != nil && opts.K != nil { + values.Set("k", fmt.Sprint(*opts.K)) + } + raw, err := c.http.get(ctx, "/v2/search/research/papers/"+url.PathEscape(paperID)+"?"+values.Encode()) + if err != nil { + return nil, err + } + var resp ReadPaperResponse + if err := json.Unmarshal(raw, &resp); err != nil { + return nil, &FirecrawlError{Message: fmt.Sprintf("failed to decode response: %v", err)} + } + return &resp, nil +} + +// RelatedPapers finds papers related to a paper. +func (c *Client) RelatedPapers(ctx context.Context, paperID, intent string, opts *RelatedPapersOptions) (*SimilarPapersResponse, error) { + if paperID == "" { + return nil, &FirecrawlError{Message: "paper ID is required"} + } + if intent == "" { + return nil, &FirecrawlError{Message: "intent is required"} + } + values := url.Values{} + values.Set("intent", intent) + values.Set("origin", "go-sdk@"+Version) + if opts != nil { + if opts.Mode != "" { + values.Set("mode", opts.Mode) + } + if opts.K != nil { + values.Set("k", fmt.Sprint(*opts.K)) + } + if opts.Rerank != nil { + values.Set("rerank", fmt.Sprint(*opts.Rerank)) + } + for _, anchor := range opts.Anchor { + values.Add("anchor", anchor) + } + } + raw, err := c.http.get(ctx, "/v2/search/research/papers/"+url.PathEscape(paperID)+"/similar?"+values.Encode()) + if err != nil { + return nil, err + } + var resp SimilarPapersResponse + if err := json.Unmarshal(raw, &resp); err != nil { + return nil, &FirecrawlError{Message: fmt.Sprintf("failed to decode response: %v", err)} + } + return &resp, nil +} + +// SearchGitHub searches GitHub research content. +func (c *Client) SearchGitHub(ctx context.Context, query string, opts *SearchGitHubOptions) (*GitHubSearchResponse, error) { + if query == "" { + return nil, &FirecrawlError{Message: "query is required"} + } + values := url.Values{} + values.Set("query", query) + values.Set("origin", "go-sdk@"+Version) + if opts != nil && opts.K != nil { + values.Set("k", fmt.Sprint(*opts.K)) + } + raw, err := c.http.get(ctx, "/v2/search/research/github?"+values.Encode()) + if err != nil { + return nil, err + } + var resp GitHubSearchResponse + if err := json.Unmarshal(raw, &resp); err != nil { + return nil, &FirecrawlError{Message: fmt.Sprintf("failed to decode response: %v", err)} + } + return &resp, nil +} + // ================================================================ // AGENT // ================================================================ diff --git a/apps/go-sdk/models.go b/apps/go-sdk/models.go index 858bfcce98..51d3a15414 100644 --- a/apps/go-sdk/models.go +++ b/apps/go-sdk/models.go @@ -24,6 +24,120 @@ type Document struct { Branding map[string]interface{} `json:"branding,omitempty"` } +// PaperResult represents a ranked research paper result. +type PaperResult struct { + PaperID string `json:"paperId"` + PrimaryID string `json:"primaryId,omitempty"` + IDs map[string]interface{} `json:"ids,omitempty"` + Title string `json:"title,omitempty"` + Abstract string `json:"abstract,omitempty"` + Score *float64 `json:"score,omitempty"` + Year *int `json:"year,omitempty"` + Authors []string `json:"authors,omitempty"` + Venue string `json:"venue,omitempty"` + URL string `json:"url,omitempty"` + Signals map[string]interface{} `json:"signals,omitempty"` +} + +// PaperMetadata represents paper metadata returned by inspect/read endpoints. +type PaperMetadata struct { + PaperID string `json:"paperId,omitempty"` + IDs map[string]interface{} `json:"ids,omitempty"` + Title string `json:"title,omitempty"` + Abstract string `json:"abstract,omitempty"` + Authors string `json:"authors,omitempty"` + Categories []string `json:"categories,omitempty"` + CreatedDate string `json:"createdDate,omitempty"` + UpdateDate string `json:"updateDate,omitempty"` +} + +// Passage is a relevant paper passage. +type Passage struct { + Text string `json:"text,omitempty"` + Section string `json:"section,omitempty"` + Page *int `json:"page,omitempty"` + Score *float64 `json:"score,omitempty"` + Metadata map[string]interface{} `json:"metadata,omitempty"` +} + +// SearchPapersResponse is returned by SearchPapers. +type SearchPapersResponse struct { + Success bool `json:"success"` + Results []PaperResult `json:"results"` +} + +// PaperMetadataResponse is returned by InspectPaper. +type PaperMetadataResponse struct { + Success bool `json:"success"` + Paper PaperMetadata `json:"paper"` +} + +// ReadPaperResponse is returned by ReadPaper. +type ReadPaperResponse struct { + Success bool `json:"success"` + Paper PaperMetadata `json:"paper"` + PaperID string `json:"paperId,omitempty"` + Query string `json:"query,omitempty"` + Passages []Passage `json:"passages,omitempty"` +} + +// SimilarPapersResponse is returned by RelatedPapers. +type SimilarPapersResponse struct { + Success bool `json:"success"` + Results []PaperResult `json:"results"` + PoolSize *int `json:"poolSize,omitempty"` + Truncated bool `json:"truncated"` + Note *string `json:"note,omitempty"` +} + +// GitHubSearchItem represents a GitHub research search result. +type GitHubSearchItem struct { + ResultType string `json:"resultType,omitempty"` + Repo string `json:"repo,omitempty"` + URL string `json:"url,omitempty"` + PageType string `json:"pageType,omitempty"` + Number *int `json:"number,omitempty"` + SegmentCount *int `json:"segmentCount,omitempty"` + ReadmeURL string `json:"readmeUrl,omitempty"` + Title string `json:"title,omitempty"` + Snippet string `json:"snippet,omitempty"` + ContentMD string `json:"contentMd,omitempty"` + Scores map[string]interface{} `json:"scores,omitempty"` +} + +// GitHubSearchResponse is returned by SearchGitHub. +type GitHubSearchResponse struct { + Success bool `json:"success"` + Results []GitHubSearchItem `json:"results"` +} + +// Research options for search-like endpoints. +type SearchPapersOptions struct { + K *int `json:"k,omitempty"` + Authors []string `json:"authors,omitempty"` + Categories []string `json:"categories,omitempty"` + From string `json:"from,omitempty"` + To string `json:"to,omitempty"` +} + +// ReadPaperOptions configures ReadPaper. +type ReadPaperOptions struct { + K *int `json:"k,omitempty"` +} + +// RelatedPapersOptions configures RelatedPapers. +type RelatedPapersOptions struct { + Mode string `json:"mode,omitempty"` + K *int `json:"k,omitempty"` + Rerank *bool `json:"rerank,omitempty"` + Anchor []string `json:"anchor,omitempty"` +} + +// SearchGitHubOptions configures SearchGitHub. +type SearchGitHubOptions struct { + K *int `json:"k,omitempty"` +} + // CrawlResponse is returned when starting an async crawl. type CrawlResponse struct { ID string `json:"id"` diff --git a/apps/java-sdk/src/main/java/com/firecrawl/client/FirecrawlClient.java b/apps/java-sdk/src/main/java/com/firecrawl/client/FirecrawlClient.java index 71688c500c..a2b43e53b3 100644 --- a/apps/java-sdk/src/main/java/com/firecrawl/client/FirecrawlClient.java +++ b/apps/java-sdk/src/main/java/com/firecrawl/client/FirecrawlClient.java @@ -556,6 +556,65 @@ public SearchData search(String query, SearchOptions options) { return extractData(http.post("/v2/search", body, Map.class), SearchData.class); } + public ResearchModels.SearchPapersResponse searchPapers(String query) { + return searchPapers(query, null); + } + + public ResearchModels.SearchPapersResponse searchPapers(String query, ResearchModels.SearchPapersOptions options) { + Objects.requireNonNull(query, "Query is required"); + Map params = new LinkedHashMap<>(); + params.put("query", query); + params.put("origin", SDK_ORIGIN); + if (options != null) mergeOptions(params, options); + return http.get("/v2/search/research/papers" + researchQuery(params), ResearchModels.SearchPapersResponse.class); + } + + public ResearchModels.PaperMetadataResponse inspectPaper(String paperId) { + Objects.requireNonNull(paperId, "Paper ID is required"); + return http.get("/v2/search/research/papers/" + urlEncode(paperId), ResearchModels.PaperMetadataResponse.class); + } + + public ResearchModels.ReadPaperResponse readPaper(String paperId, String query) { + return readPaper(paperId, query, null); + } + + public ResearchModels.ReadPaperResponse readPaper(String paperId, String query, ResearchModels.ReadPaperOptions options) { + Objects.requireNonNull(paperId, "Paper ID is required"); + Objects.requireNonNull(query, "Query is required"); + Map params = new LinkedHashMap<>(); + params.put("query", query); + params.put("origin", SDK_ORIGIN); + if (options != null) mergeOptions(params, options); + return http.get("/v2/search/research/papers/" + urlEncode(paperId) + researchQuery(params), ResearchModels.ReadPaperResponse.class); + } + + public ResearchModels.SimilarPapersResponse relatedPapers(String paperId, String intent) { + return relatedPapers(paperId, intent, null); + } + + public ResearchModels.SimilarPapersResponse relatedPapers(String paperId, String intent, ResearchModels.RelatedPapersOptions options) { + Objects.requireNonNull(paperId, "Paper ID is required"); + Objects.requireNonNull(intent, "Intent is required"); + Map params = new LinkedHashMap<>(); + params.put("intent", intent); + params.put("origin", SDK_ORIGIN); + if (options != null) mergeOptions(params, options); + return http.get("/v2/search/research/papers/" + urlEncode(paperId) + "/similar" + researchQuery(params), ResearchModels.SimilarPapersResponse.class); + } + + public ResearchModels.GitHubSearchResponse searchGitHub(String query) { + return searchGitHub(query, null); + } + + public ResearchModels.GitHubSearchResponse searchGitHub(String query, ResearchModels.SearchGitHubOptions options) { + Objects.requireNonNull(query, "Query is required"); + Map params = new LinkedHashMap<>(); + params.put("query", query); + params.put("origin", SDK_ORIGIN); + if (options != null) mergeOptions(params, options); + return http.get("/v2/search/research/github" + researchQuery(params), ResearchModels.GitHubSearchResponse.class); + } + // ================================================================ // AGENT // ================================================================ @@ -913,6 +972,26 @@ public CompletableFuture searchAsync(String query, SearchOptions opt return CompletableFuture.supplyAsync(() -> search(query, options), asyncExecutor); } + public CompletableFuture searchPapersAsync(String query, ResearchModels.SearchPapersOptions options) { + return CompletableFuture.supplyAsync(() -> searchPapers(query, options), asyncExecutor); + } + + public CompletableFuture inspectPaperAsync(String paperId) { + return CompletableFuture.supplyAsync(() -> inspectPaper(paperId), asyncExecutor); + } + + public CompletableFuture readPaperAsync(String paperId, String query, ResearchModels.ReadPaperOptions options) { + return CompletableFuture.supplyAsync(() -> readPaper(paperId, query, options), asyncExecutor); + } + + public CompletableFuture relatedPapersAsync(String paperId, String intent, ResearchModels.RelatedPapersOptions options) { + return CompletableFuture.supplyAsync(() -> relatedPapers(paperId, intent, options), asyncExecutor); + } + + public CompletableFuture searchGitHubAsync(String query, ResearchModels.SearchGitHubOptions options) { + return CompletableFuture.supplyAsync(() -> searchGitHub(query, options), asyncExecutor); + } + /** * Asynchronously runs a map operation. * @@ -1264,6 +1343,30 @@ private String monitorCheckQuery(Integer limit, Integer skip, String status) { return parts.isEmpty() ? "" : "?" + String.join("&", parts); } + private String researchQuery(Map params) { + List parts = new ArrayList<>(); + for (Map.Entry entry : params.entrySet()) { + Object value = entry.getValue(); + if (value == null) continue; + if (value instanceof Collection) { + for (Object item : (Collection) value) { + if (item != null) parts.add(urlEncode(entry.getKey()) + "=" + urlEncode(stringValue(item))); + } + } else { + parts.add(urlEncode(entry.getKey()) + "=" + urlEncode(stringValue(value))); + } + } + return parts.isEmpty() ? "" : "?" + String.join("&", parts); + } + + private String stringValue(Object value) { + return value instanceof Boolean ? value.toString().toLowerCase(Locale.ROOT) : value.toString(); + } + + private String urlEncode(String value) { + return java.net.URLEncoder.encode(value, java.nio.charset.StandardCharsets.UTF_8); + } + /** * Merges a typed options object into a request body map, using Jackson serialization. */ diff --git a/apps/java-sdk/src/main/java/com/firecrawl/models/ResearchModels.java b/apps/java-sdk/src/main/java/com/firecrawl/models/ResearchModels.java new file mode 100644 index 0000000000..6243119c1c --- /dev/null +++ b/apps/java-sdk/src/main/java/com/firecrawl/models/ResearchModels.java @@ -0,0 +1,119 @@ +package com.firecrawl.models; + +import com.fasterxml.jackson.annotation.JsonProperty; +import java.util.List; +import java.util.Map; + +public final class ResearchModels { + private ResearchModels() {} + + public static class PaperResult { + @JsonProperty("paperId") + public String paperId; + @JsonProperty("primaryId") + public String primaryId; + public Map ids; + public String title; + @JsonProperty("abstract") + public String abstractText; + public Double score; + public Integer year; + public List authors; + public String venue; + public String url; + public Map signals; + } + + public static class PaperMetadata { + @JsonProperty("paperId") + public String paperId; + public Map ids; + public String title; + @JsonProperty("abstract") + public String abstractText; + public String authors; + public List categories; + @JsonProperty("createdDate") + public String createdDate; + @JsonProperty("updateDate") + public String updateDate; + } + + public static class Passage { + public String text; + public String section; + public Integer page; + public Double score; + public Map metadata; + } + + public static class SearchPapersResponse { + public boolean success; + public List results; + } + + public static class PaperMetadataResponse { + public boolean success; + public PaperMetadata paper; + } + + public static class ReadPaperResponse { + public boolean success; + public PaperMetadata paper; + @JsonProperty("paperId") + public String paperId; + public String query; + public List passages; + } + + public static class SimilarPapersResponse { + public boolean success; + public List results; + @JsonProperty("poolSize") + public Integer poolSize; + public boolean truncated; + public String note; + } + + public static class GitHubSearchItem { + public String resultType; + public String repo; + public String url; + public String pageType; + public Integer number; + public Integer segmentCount; + public String readmeUrl; + public String title; + public String snippet; + public String contentMd; + public Map scores; + } + + public static class GitHubSearchResponse { + public boolean success; + public List results; + } + + public static class SearchPapersOptions { + public Integer k; + public List authors; + public List categories; + public String from; + public String to; + } + + public static class ReadPaperOptions { + public Integer k; + } + + public static class RelatedPapersOptions { + public String mode; + public Integer k; + public Boolean rerank; + public List anchor; + } + + public static class SearchGitHubOptions { + public Integer k; + } +} diff --git a/apps/js-sdk/firecrawl/src/__tests__/unit/v2/research.test.ts b/apps/js-sdk/firecrawl/src/__tests__/unit/v2/research.test.ts index 93d76558f0..9e1995d38f 100644 --- a/apps/js-sdk/firecrawl/src/__tests__/unit/v2/research.test.ts +++ b/apps/js-sdk/firecrawl/src/__tests__/unit/v2/research.test.ts @@ -39,7 +39,7 @@ describe("research.searchPapers", () => { to: "2024-12-31", }); const url = calls[0]; - expect(url.startsWith("/v2/research/papers?")).toBe(true); + expect(url.startsWith("/v2/search/research/papers?")).toBe(true); const qs = new URLSearchParams(url.split("?")[1]); expect(qs.get("query")).toBe("diffusion models"); expect(qs.get("k")).toBe("10"); @@ -67,7 +67,7 @@ describe("research.searchPapers", () => { }); test("returns the response body verbatim", async () => { - const payload = { results: [{ paper_id: "1", title: "t", abstract: "a", score: 0.1 }] }; + const payload = { results: [{ paperId: "1", title: "t", abstract: "a", score: 0.1 }] }; const { client } = makeClient(() => ({ status: 200, data: payload })); await expect(client.searchPapers("q")).resolves.toEqual(payload); }); @@ -77,17 +77,17 @@ describe("research.getPaper", () => { test("detail mode encodes the id and sends no query params", async () => { const { client, calls } = makeClient(() => ({ status: 200, data: { paper: {} } })); await client.getPaper("arxiv:2105.05233"); - expect(calls[0]).toBe("/v2/research/papers/arxiv%3A2105.05233"); + expect(calls[0]).toBe("/v2/search/research/papers/arxiv%3A2105.05233"); }); test("read mode adds query and k", async () => { const { client, calls } = makeClient(() => ({ status: 200, - data: { paper: {}, paper_id: "1", query: "q", passages: [] }, + data: { paper: {}, paperId: "1", query: "q", passages: [] }, })); await client.getPaper("123", { query: "noise schedule", k: 4 }); const [path, query] = calls[0].split("?"); - expect(path).toBe("/v2/research/papers/123"); + expect(path).toBe("/v2/search/research/papers/123"); const qs = new URLSearchParams(query); expect(qs.get("query")).toBe("noise schedule"); expect(qs.get("k")).toBe("4"); @@ -112,7 +112,7 @@ describe("research.similarPapers", () => { test("builds path and query with repeated anchors and rerank", async () => { const { client, calls } = makeClient(() => ({ status: 200, - data: { results: [], pool_size: 0, truncated: false }, + data: { results: [], poolSize: 0, truncated: false }, })); await client.similarPapers("2105.05233", { intent: "diffusion image synthesis", @@ -122,7 +122,7 @@ describe("research.similarPapers", () => { anchor: ["arxiv:2006.11239", "1503.03585"], }); const [path, query] = calls[0].split("?"); - expect(path).toBe("/v2/research/papers/2105.05233/similar"); + expect(path).toBe("/v2/search/research/papers/2105.05233/similar"); const qs = new URLSearchParams(query); expect(qs.get("intent")).toBe("diffusion image synthesis"); expect(qs.get("mode")).toBe("citers"); @@ -137,7 +137,7 @@ describe("research.searchGithub", () => { const { client, calls } = makeClient(() => ({ status: 200, data: { results: [] } })); await client.searchGithub("milvus hybrid search", { k: 10 }); const qs = new URLSearchParams(calls[0].split("?")[1]); - expect(calls[0].startsWith("/v2/research/github?")).toBe(true); + expect(calls[0].startsWith("/v2/search/research/github?")).toBe(true); expect(qs.get("query")).toBe("milvus hybrid search"); expect(qs.get("k")).toBe("10"); }); diff --git a/apps/js-sdk/firecrawl/src/v2/methods/research.ts b/apps/js-sdk/firecrawl/src/v2/methods/research.ts index c84bf70ce2..c764c44003 100644 --- a/apps/js-sdk/firecrawl/src/v2/methods/research.ts +++ b/apps/js-sdk/firecrawl/src/v2/methods/research.ts @@ -13,7 +13,7 @@ import { SdkError } from "../types"; import { HttpClient } from "../utils/httpClient"; import { throwForBadResponse } from "../utils/errorHandler"; -const BASE = "/v2/research"; +const BASE = "/v2/search/research"; /** Append a value (or repeated array values) to a URLSearchParams instance. */ function appendParam( diff --git a/apps/js-sdk/firecrawl/src/v2/types.ts b/apps/js-sdk/firecrawl/src/v2/types.ts index edc7a8378f..d617335ddc 100644 --- a/apps/js-sdk/firecrawl/src/v2/types.ts +++ b/apps/js-sdk/firecrawl/src/v2/types.ts @@ -1175,16 +1175,18 @@ export interface PaperSignals { structural: number; /** Semantic score from the intent abstract search (0 if absent). */ semantic: number; - /** Citation-graph PageRank of the candidate. */ - pagerank: number; + /** Citation-graph article-rank score of the candidate. */ + articleRank: number; /** Number of distinct seeds connected to this candidate. */ - seed_overlap: number; + seedOverlap: number; } -/** A ranked paper. `paper_id` is canonical; arXiv lives in `ids`. */ +/** A ranked paper. `paperId` is canonical; arXiv lives in `ids`. */ export interface PaperResult { /** Canonical paper id — the Milvus INT64 primary key as a decimal string. */ - paper_id: string; + paperId: string; + /** Preferred cite/fetch identifier such as `arxiv:`, `pmid:`, or `doi:`. */ + primaryId: string; ids?: IdMap; title: string; abstract: string; @@ -1195,7 +1197,7 @@ export interface PaperResult { } export interface PaperMetadata { - paper_id: string; + paperId: string; ids?: IdMap; title: string; abstract: string; @@ -1204,9 +1206,9 @@ export interface PaperMetadata { /** arXiv categories. Omitted if unknown. */ categories?: string[]; /** Original creation date string (format varies). Omitted if unknown. */ - created_date?: string; + createdDate?: string; /** Last-updated date string. Omitted if unknown. */ - update_date?: string; + updateDate?: string; } export interface Passage { @@ -1217,17 +1219,20 @@ export interface Passage { } export interface SearchPapersResponse { + success: boolean; results: PaperResult[]; } export interface PaperMetadataResponse { + success: boolean; paper: PaperMetadata; } export interface ReadPaperResponse { + success: boolean; paper: PaperMetadata; /** Resolved canonical paper id (empty string if not found via id-key). */ - paper_id: string; + paperId: string; /** Echo of the read query. */ query: string; /** Top matching in-body passages. */ @@ -1235,10 +1240,11 @@ export interface ReadPaperResponse { } export interface SimilarPapersResponse { + success: boolean; /** Ranked related papers; each carries `signals`. */ results: PaperResult[]; /** Number of resolved candidates considered before truncation to `k`. */ - pool_size: number; + poolSize: number; /** True if more resolved candidates existed than were returned. */ truncated: boolean; /** Human-readable note when no results are produced. */ @@ -1251,11 +1257,12 @@ export interface GitHubScoreBreakdown { semantic?: number; lexical?: number; fusion?: number; + rerank?: number; } export interface GitHubSearchItem { - resultType: "github_history" | "repo_readme"; - /** `owner/name`. */ + resultType: "github_history" | "repo_readme" | "web"; + /** `owner/name`; empty for web results whose URL is not a repo page. */ repo: string; url: string; /** History page type (e.g. `issue`, `pull`). Omitted for readmes. */ @@ -1266,6 +1273,8 @@ export interface GitHubSearchItem { segmentCount?: number; /** Readme URL (readme results). Omitted otherwise. */ readmeUrl?: string; + /** SERP page title. Only set on web results. */ + title?: string; /** Short matched excerpt. */ snippet: string; /** Full matched content in markdown. Omitted unless available. */ @@ -1274,6 +1283,7 @@ export interface GitHubSearchItem { } export interface GitHubSearchResponse { + success: boolean; results: GitHubSearchItem[]; } diff --git a/apps/php-sdk/src/Client/FirecrawlClient.php b/apps/php-sdk/src/Client/FirecrawlClient.php index 9758ca9fe8..8660c0c91f 100644 --- a/apps/php-sdk/src/Client/FirecrawlClient.php +++ b/apps/php-sdk/src/Client/FirecrawlClient.php @@ -117,6 +117,78 @@ public function scrape(string $url, ?ScrapeOptions $options = null): Document return Document::fromArray($response['data'] ?? $response); } + /** + * Search research papers. + * + * @param array $options + * @return array + */ + public function searchPapers(string $query, array $options = []): array + { + return $this->http->get('/v2/search/research/papers' . $this->queryArray(array_merge( + ['query' => $query, 'origin' => 'php-sdk@' . Version::SDK_VERSION], + $options, + ))); + } + + /** + * Inspect paper metadata. + * + * @return array + */ + public function inspectPaper(string $paperId): array + { + return $this->http->get('/v2/search/research/papers/' . rawurlencode($paperId)); + } + + /** + * Read a paper with query-guided passages. + * + * @param array $options + * @return array + */ + public function readPaper(string $paperId, string $query, array $options = []): array + { + return $this->http->get( + '/v2/search/research/papers/' . rawurlencode($paperId) + . $this->queryArray(array_merge( + ['query' => $query, 'origin' => 'php-sdk@' . Version::SDK_VERSION], + $options, + )), + ); + } + + /** + * Find papers related to a paper. + * + * @param array $options + * @return array + */ + public function relatedPapers(string $paperId, string $intent, array $options = []): array + { + return $this->http->get( + '/v2/search/research/papers/' . rawurlencode($paperId) . '/similar' + . $this->queryArray(array_merge( + ['intent' => $intent, 'origin' => 'php-sdk@' . Version::SDK_VERSION], + $options, + )), + ); + } + + /** + * Search GitHub research content. + * + * @param array $options + * @return array + */ + public function searchGithub(string $query, array $options = []): array + { + return $this->http->get('/v2/search/research/github' . $this->queryArray(array_merge( + ['query' => $query, 'origin' => 'php-sdk@' . Version::SDK_VERSION], + $options, + ))); + } + /** * Interact with the scrape-bound browser session for a scrape job. */ @@ -660,6 +732,29 @@ private function query(array $params): string return $params === [] ? '' : '?' . http_build_query($params); } + /** + * @param array $params + */ + private function queryArray(array $params): string + { + $pairs = []; + foreach ($params as $key => $value) { + if ($value === null || $value === '') { + continue; + } + $values = is_array($value) ? $value : [$value]; + foreach ($values as $item) { + if ($item === null || $item === '') { + continue; + } + $stringValue = is_bool($item) ? ($item ? 'true' : 'false') : (string) $item; + $pairs[] = rawurlencode((string) $key) . '=' . rawurlencode($stringValue); + } + } + + return $pairs === [] ? '' : '?' . implode('&', $pairs); + } + private function pollCrawl( ?string $jobId, int $pollIntervalSec, diff --git a/apps/php-sdk/src/Client/FirecrawlHttpClient.php b/apps/php-sdk/src/Client/FirecrawlHttpClient.php index 44e7c8d205..0b2284d7dc 100644 --- a/apps/php-sdk/src/Client/FirecrawlHttpClient.php +++ b/apps/php-sdk/src/Client/FirecrawlHttpClient.php @@ -141,7 +141,7 @@ private function request( ]; // Omit the Authorization header entirely when no key is set so that // scrape/search/interact can use the keyless free tier. - if ($this->apiKey !== null && $this->apiKey !== '') { + if ($this->apiKey !== '') { $defaultHeaders['Authorization'] = 'Bearer ' . $this->apiKey; } diff --git a/apps/python-sdk/firecrawl/v2/client.py b/apps/python-sdk/firecrawl/v2/client.py index b34ce4a38e..3b6e3594e4 100644 --- a/apps/python-sdk/firecrawl/v2/client.py +++ b/apps/python-sdk/firecrawl/v2/client.py @@ -65,6 +65,7 @@ from .methods import agent as agent_module from .methods import browser as browser_module from .methods import monitor as monitor_module +from .methods import research as research_module from .watcher import Watcher # Kwargs that map to ScrapeOptions fields. Used by async crawl normalization @@ -215,6 +216,21 @@ def scrape( ) if any(v is not None for v in [formats, headers, include_tags, exclude_tags, only_main_content, timeout, wait_for, mobile, parsers, actions, location, skip_tls_verification, remove_base64_images, fast_mode, use_mock, block_ads, proxy, max_age, store_in_cache, lockdown, profile, integration]) else None return scrape_module.scrape(self.http_client, url, options) + def search_papers(self, query: str, **kwargs): + return research_module.search_papers(self.http_client, query, **kwargs) + + def inspect_paper(self, paper_id: str): + return research_module.inspect_paper(self.http_client, paper_id) + + def read_paper(self, paper_id: str, query: str, **kwargs): + return research_module.read_paper(self.http_client, paper_id, query, **kwargs) + + def related_papers(self, paper_id: str, intent: str, **kwargs): + return research_module.related_papers(self.http_client, paper_id, intent, **kwargs) + + def search_github(self, query: str, **kwargs): + return research_module.search_github(self.http_client, query, **kwargs) + def interact( self, job_id: str, diff --git a/apps/python-sdk/firecrawl/v2/client_async.py b/apps/python-sdk/firecrawl/v2/client_async.py index 528b78f4f6..20e8184be7 100644 --- a/apps/python-sdk/firecrawl/v2/client_async.py +++ b/apps/python-sdk/firecrawl/v2/client_async.py @@ -60,6 +60,7 @@ from .methods.aio import agent as async_agent # type: ignore[attr-defined] from .methods.aio import browser as async_browser # type: ignore[attr-defined] from .methods.aio import monitor as async_monitor # type: ignore[attr-defined] +from .methods.aio import research as async_research # type: ignore[attr-defined] from .client import _SCRAPE_OPTION_KEYS from .watcher_async import AsyncWatcher @@ -105,6 +106,21 @@ async def scrape( options = ScrapeOptions(**{k: v for k, v in kwargs.items() if v is not None}) if kwargs else None return await async_scrape.scrape(self.async_http_client, url, options) + async def search_papers(self, query: str, **kwargs): + return await async_research.search_papers(self.async_http_client, query, **kwargs) + + async def inspect_paper(self, paper_id: str): + return await async_research.inspect_paper(self.async_http_client, paper_id) + + async def read_paper(self, paper_id: str, query: str, **kwargs): + return await async_research.read_paper(self.async_http_client, paper_id, query, **kwargs) + + async def related_papers(self, paper_id: str, intent: str, **kwargs): + return await async_research.related_papers(self.async_http_client, paper_id, intent, **kwargs) + + async def search_github(self, query: str, **kwargs): + return await async_research.search_github(self.async_http_client, query, **kwargs) + async def interact( self, job_id: str, diff --git a/apps/python-sdk/firecrawl/v2/methods/aio/research.py b/apps/python-sdk/firecrawl/v2/methods/aio/research.py new file mode 100644 index 0000000000..07897c9baf --- /dev/null +++ b/apps/python-sdk/firecrawl/v2/methods/aio/research.py @@ -0,0 +1,117 @@ +""" +Async research functionality for Firecrawl v2 API. +""" + +from typing import Any, Dict, List, Optional +from urllib.parse import quote + +from ...utils import handle_response_error +from ...utils.http_client_async import AsyncHttpClient +from ...utils.get_version import get_version + + +BASE = "/v2/search/research" +ORIGIN = f"python-sdk@{get_version()}" + + +def _query(params: Dict[str, Any]) -> str: + pairs: List[str] = [] + for key, value in params.items(): + if value is None: + continue + values = value if isinstance(value, list) else [value] + for item in values: + if item is not None: + pairs.append(f"{quote(str(key), safe='')}={quote(str(item), safe='')}") + return ("?" + "&".join(pairs)) if pairs else "" + + +async def _get(client: AsyncHttpClient, path: str) -> Dict[str, Any]: + response = await client.get(path) + if response.status_code != 200: + handle_response_error(response, "research") + return response.json() + + +async def search_papers( + client: AsyncHttpClient, + query: str, + *, + k: Optional[int] = None, + authors: Optional[List[str]] = None, + categories: Optional[List[str]] = None, + from_date: Optional[str] = None, + to_date: Optional[str] = None, +) -> Dict[str, Any]: + return await _get( + client, + BASE + + "/papers" + + _query( + { + "query": query, + "k": k, + "authors": authors, + "categories": categories, + "from": from_date, + "to": to_date, + "origin": ORIGIN, + } + ), + ) + + +async def inspect_paper(client: AsyncHttpClient, paper_id: str) -> Dict[str, Any]: + return await _get(client, f"{BASE}/papers/{quote(paper_id, safe='')}") + + +async def read_paper( + client: AsyncHttpClient, + paper_id: str, + query: str, + *, + k: Optional[int] = None, +) -> Dict[str, Any]: + return await _get( + client, + f"{BASE}/papers/{quote(paper_id, safe='')}" + + _query({"query": query, "k": k, "origin": ORIGIN}), + ) + + +async def related_papers( + client: AsyncHttpClient, + paper_id: str, + intent: str, + *, + mode: Optional[str] = None, + k: Optional[int] = None, + rerank: Optional[bool] = None, + anchor: Optional[List[str]] = None, +) -> Dict[str, Any]: + return await _get( + client, + f"{BASE}/papers/{quote(paper_id, safe='')}/similar" + + _query( + { + "intent": intent, + "mode": mode, + "k": k, + "rerank": None if rerank is None else str(rerank).lower(), + "anchor": anchor, + "origin": ORIGIN, + } + ), + ) + + +async def search_github( + client: AsyncHttpClient, + query: str, + *, + k: Optional[int] = None, +) -> Dict[str, Any]: + return await _get( + client, + BASE + "/github" + _query({"query": query, "k": k, "origin": ORIGIN}), + ) diff --git a/apps/python-sdk/firecrawl/v2/methods/research.py b/apps/python-sdk/firecrawl/v2/methods/research.py new file mode 100644 index 0000000000..3255396acb --- /dev/null +++ b/apps/python-sdk/firecrawl/v2/methods/research.py @@ -0,0 +1,116 @@ +""" +Research functionality for Firecrawl v2 API. +""" + +from typing import Any, Dict, List, Optional +from urllib.parse import quote + +from ..utils import HttpClient, handle_response_error +from ..utils.get_version import get_version + + +BASE = "/v2/search/research" +ORIGIN = f"python-sdk@{get_version()}" + + +def _query(params: Dict[str, Any]) -> str: + pairs: List[str] = [] + for key, value in params.items(): + if value is None: + continue + values = value if isinstance(value, list) else [value] + for item in values: + if item is not None: + pairs.append(f"{quote(str(key), safe='')}={quote(str(item), safe='')}") + return ("?" + "&".join(pairs)) if pairs else "" + + +def _get(client: HttpClient, path: str) -> Dict[str, Any]: + response = client.get(path) + if response.status_code != 200: + handle_response_error(response, "research") + return response.json() + + +def search_papers( + client: HttpClient, + query: str, + *, + k: Optional[int] = None, + authors: Optional[List[str]] = None, + categories: Optional[List[str]] = None, + from_date: Optional[str] = None, + to_date: Optional[str] = None, +) -> Dict[str, Any]: + return _get( + client, + BASE + + "/papers" + + _query( + { + "query": query, + "k": k, + "authors": authors, + "categories": categories, + "from": from_date, + "to": to_date, + "origin": ORIGIN, + } + ), + ) + + +def inspect_paper(client: HttpClient, paper_id: str) -> Dict[str, Any]: + return _get(client, f"{BASE}/papers/{quote(paper_id, safe='')}") + + +def read_paper( + client: HttpClient, + paper_id: str, + query: str, + *, + k: Optional[int] = None, +) -> Dict[str, Any]: + return _get( + client, + f"{BASE}/papers/{quote(paper_id, safe='')}" + + _query({"query": query, "k": k, "origin": ORIGIN}), + ) + + +def related_papers( + client: HttpClient, + paper_id: str, + intent: str, + *, + mode: Optional[str] = None, + k: Optional[int] = None, + rerank: Optional[bool] = None, + anchor: Optional[List[str]] = None, +) -> Dict[str, Any]: + return _get( + client, + f"{BASE}/papers/{quote(paper_id, safe='')}/similar" + + _query( + { + "intent": intent, + "mode": mode, + "k": k, + "rerank": None if rerank is None else str(rerank).lower(), + "anchor": anchor, + "origin": ORIGIN, + } + ), + ) + + +def search_github( + client: HttpClient, + query: str, + *, + k: Optional[int] = None, +) -> Dict[str, Any]: + return _get( + client, + BASE + "/github" + _query({"query": query, "k": k, "origin": ORIGIN}), + ) diff --git a/apps/ruby-sdk/lib/firecrawl/client.rb b/apps/ruby-sdk/lib/firecrawl/client.rb index 3bb199d664..d7829677be 100644 --- a/apps/ruby-sdk/lib/firecrawl/client.rb +++ b/apps/ruby-sdk/lib/firecrawl/client.rb @@ -85,6 +85,57 @@ def scrape(url, options = nil) Models::Document.new(data) end + # Search research papers. + # + # @param query [String] research query + # @param options [Hash] optional query parameters + # @return [Hash] + def search_papers(query, options = {}) + @http.get("/v2/search/research/papers#{query(options.merge("query" => query, "origin" => "ruby-sdk@#{Firecrawl::VERSION}"))}") + end + + # Inspect paper metadata. + # + # @param paper_id [String] paper identifier + # @return [Hash] + def inspect_paper(paper_id) + raise ArgumentError, "Paper ID is required" if paper_id.nil? + @http.get("/v2/search/research/papers/#{URI.encode_www_form_component(paper_id)}") + end + + # Read a paper with query-guided passages. + # + # @param paper_id [String] paper identifier + # @param query_text [String] passage query + # @param options [Hash] optional query parameters + # @return [Hash] + def read_paper(paper_id, query_text, options = {}) + raise ArgumentError, "Paper ID is required" if paper_id.nil? + path = "/v2/search/research/papers/#{URI.encode_www_form_component(paper_id)}" + @http.get("#{path}#{query(options.merge("query" => query_text, "origin" => "ruby-sdk@#{Firecrawl::VERSION}"))}") + end + + # Find papers related to a paper. + # + # @param paper_id [String] paper identifier + # @param intent [String] relatedness intent + # @param options [Hash] optional query parameters + # @return [Hash] + def related_papers(paper_id, intent, options = {}) + raise ArgumentError, "Paper ID is required" if paper_id.nil? + path = "/v2/search/research/papers/#{URI.encode_www_form_component(paper_id)}/similar" + @http.get("#{path}#{query(options.merge("intent" => intent, "origin" => "ruby-sdk@#{Firecrawl::VERSION}"))}") + end + + # Search GitHub research content. + # + # @param query_text [String] GitHub query + # @param options [Hash] optional query parameters + # @return [Hash] + def search_github(query_text, options = {}) + @http.get("/v2/search/research/github#{query(options.merge("query" => query_text, "origin" => "ruby-sdk@#{Firecrawl::VERSION}"))}") + end + # Interacts with the scrape-bound browser session for a scrape job. # # @param job_id [String] the scrape job ID @@ -465,9 +516,21 @@ def get_credit_usage private - def query(**params) - compact = params.compact - compact.empty? ? "" : "?#{URI.encode_www_form(compact)}" + def query(params = nil, **kwargs) + params = (params || {}).merge(kwargs) + pairs = [] + params.each do |key, value| + next if value.nil? || value == "" + + values = value.is_a?(Array) ? value : [value] + values.each do |item| + next if item.nil? || item == "" + + string_value = item == true ? "true" : item == false ? "false" : item.to_s + pairs << [key.to_s, string_value] + end + end + pairs.empty? ? "" : "?#{URI.encode_www_form(pairs)}" end def poll_crawl(job_id, poll_interval, timeout) diff --git a/apps/ruby-sdk/test/firecrawl/client_test.rb b/apps/ruby-sdk/test/firecrawl/client_test.rb index 8771887cd8..9581408db2 100644 --- a/apps/ruby-sdk/test/firecrawl/client_test.rb +++ b/apps/ruby-sdk/test/firecrawl/client_test.rb @@ -15,14 +15,14 @@ def setup # CLIENT INITIALIZATION # ================================================================ - def test_raises_when_no_api_key + def test_allows_no_api_key_for_keyless_endpoints ENV.delete("FIRECRAWL_API_KEY") - assert_raises(Firecrawl::FirecrawlError) { Firecrawl::Client.new } + assert_instance_of Firecrawl::Client, Firecrawl::Client.new end - def test_raises_when_whitespace_only_api_key + def test_allows_whitespace_only_api_key_for_keyless_endpoints ENV.delete("FIRECRAWL_API_KEY") - assert_raises(Firecrawl::FirecrawlError) { Firecrawl::Client.new(api_key: " ") } + assert_instance_of Firecrawl::Client, Firecrawl::Client.new(api_key: " ") end def test_raises_when_api_url_not_http @@ -69,7 +69,7 @@ def test_custom_api_url_from_env def test_scrape_basic stub_request(:post, "#{BASE_URL}/v2/scrape") .with( - body: { url: "https://example.com" }.to_json, + body: { url: "https://example.com", origin: "ruby-sdk@#{Firecrawl::VERSION}" }.to_json, headers: { "Authorization" => "Bearer #{API_KEY}", "Content-Type" => "application/json" } ) .to_return( diff --git a/apps/rust-sdk/src/lib.rs b/apps/rust-sdk/src/lib.rs index 4639f0cdb1..d71a0bd976 100644 --- a/apps/rust-sdk/src/lib.rs +++ b/apps/rust-sdk/src/lib.rs @@ -27,6 +27,7 @@ mod crawl; mod map; mod monitor; mod parse; +mod research; mod scrape; mod search; mod types; @@ -39,6 +40,7 @@ pub use error::FirecrawlError; pub use map::*; pub use monitor::*; pub use parse::*; +pub use research::*; pub use scrape::*; pub use search::*; pub use types::*; diff --git a/apps/rust-sdk/src/research.rs b/apps/rust-sdk/src/research.rs new file mode 100644 index 0000000000..232c523ebe --- /dev/null +++ b/apps/rust-sdk/src/research.rs @@ -0,0 +1,321 @@ +//! Research endpoints for Firecrawl API v2. + +use serde::{Deserialize, Serialize}; +use serde_json::Value; +use std::collections::HashMap; + +use crate::client::Client; +use crate::FirecrawlError; + +#[serde_with::skip_serializing_none] +#[derive(Deserialize, Serialize, Debug, Default, Clone)] +#[serde(rename_all = "camelCase")] +pub struct PaperResult { + pub paper_id: Option, + pub primary_id: Option, + pub ids: Option>, + pub title: Option, + #[serde(rename = "abstract")] + pub abstract_text: Option, + pub score: Option, + pub year: Option, + pub authors: Option>, + pub venue: Option, + pub url: Option, + pub signals: Option>, +} + +#[serde_with::skip_serializing_none] +#[derive(Deserialize, Serialize, Debug, Default, Clone)] +#[serde(rename_all = "camelCase")] +pub struct PaperMetadata { + pub paper_id: Option, + pub ids: Option>, + pub title: Option, + #[serde(rename = "abstract")] + pub abstract_text: Option, + pub authors: Option, + pub categories: Option>, + pub created_date: Option, + pub update_date: Option, +} + +#[serde_with::skip_serializing_none] +#[derive(Deserialize, Serialize, Debug, Default, Clone)] +pub struct Passage { + pub text: Option, + pub section: Option, + pub page: Option, + pub score: Option, + pub metadata: Option>, +} + +#[derive(Deserialize, Serialize, Debug, Default, Clone)] +pub struct SearchPapersResponse { + pub success: bool, + pub results: Vec, +} + +#[derive(Deserialize, Serialize, Debug, Default, Clone)] +pub struct PaperMetadataResponse { + pub success: bool, + pub paper: PaperMetadata, +} + +#[derive(Deserialize, Serialize, Debug, Default, Clone)] +#[serde(rename_all = "camelCase")] +pub struct ReadPaperResponse { + pub success: bool, + pub paper: PaperMetadata, + pub paper_id: Option, + pub query: Option, + pub passages: Option>, +} + +#[derive(Deserialize, Serialize, Debug, Default, Clone)] +#[serde(rename_all = "camelCase")] +pub struct SimilarPapersResponse { + pub success: bool, + pub results: Vec, + pub pool_size: Option, + pub truncated: bool, + pub note: Option, +} + +#[serde_with::skip_serializing_none] +#[derive(Deserialize, Serialize, Debug, Default, Clone)] +#[serde(rename_all = "camelCase")] +pub struct GitHubSearchItem { + pub result_type: Option, + pub repo: Option, + pub url: Option, + pub page_type: Option, + pub number: Option, + pub segment_count: Option, + pub readme_url: Option, + pub title: Option, + pub snippet: Option, + pub content_md: Option, + pub scores: Option>, +} + +#[derive(Deserialize, Serialize, Debug, Default, Clone)] +pub struct GitHubSearchResponse { + pub success: bool, + pub results: Vec, +} + +#[derive(Debug, Default, Clone)] +pub struct SearchPapersOptions { + pub k: Option, + pub authors: Option>, + pub categories: Option>, + pub from: Option, + pub to: Option, +} + +#[derive(Debug, Default, Clone)] +pub struct ReadPaperOptions { + pub k: Option, +} + +#[derive(Debug, Default, Clone)] +pub struct RelatedPapersOptions { + pub mode: Option, + pub k: Option, + pub rerank: Option, + pub anchor: Option>, +} + +#[derive(Debug, Default, Clone)] +pub struct SearchGitHubOptions { + pub k: Option, +} + +fn push_query(query: &mut Vec<(String, String)>, key: &str, value: impl ToString) { + query.push((key.to_string(), value.to_string())); +} + +fn path_escape(value: &str) -> String { + const HEX: &[u8; 16] = b"0123456789ABCDEF"; + let mut out = String::with_capacity(value.len()); + for byte in value.bytes() { + if byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_' | b'.' | b'~') { + out.push(byte as char); + } else { + out.push('%'); + out.push(HEX[(byte >> 4) as usize] as char); + out.push(HEX[(byte & 0x0f) as usize] as char); + } + } + out +} + +impl Client { + pub async fn search_papers( + &self, + query_text: impl AsRef, + options: impl Into>, + ) -> Result { + let mut query = vec![ + ("query".to_string(), query_text.as_ref().to_string()), + ( + "origin".to_string(), + format!("rust-sdk@{}", env!("CARGO_PKG_VERSION")), + ), + ]; + if let Some(options) = options.into() { + if let Some(k) = options.k { + push_query(&mut query, "k", k); + } + for author in options.authors.unwrap_or_default() { + push_query(&mut query, "authors", author); + } + for category in options.categories.unwrap_or_default() { + push_query(&mut query, "categories", category); + } + if let Some(from) = options.from { + push_query(&mut query, "from", from); + } + if let Some(to) = options.to { + push_query(&mut query, "to", to); + } + } + + let response = self + .client + .get(self.url("/search/research/papers")) + .headers(self.prepare_headers(None)) + .query(&query) + .send() + .await + .map_err(|e| FirecrawlError::HttpError("search papers".to_string(), e))?; + + self.handle_response(response, "search papers").await + } + + pub async fn inspect_paper( + &self, + paper_id: impl AsRef, + ) -> Result { + let response = self + .client + .get(self.url(&format!( + "/search/research/papers/{}", + path_escape(paper_id.as_ref()) + ))) + .headers(self.prepare_headers(None)) + .send() + .await + .map_err(|e| FirecrawlError::HttpError("inspect paper".to_string(), e))?; + + self.handle_response(response, "inspect paper").await + } + + pub async fn read_paper( + &self, + paper_id: impl AsRef, + query_text: impl AsRef, + options: impl Into>, + ) -> Result { + let mut query = vec![ + ("query".to_string(), query_text.as_ref().to_string()), + ( + "origin".to_string(), + format!("rust-sdk@{}", env!("CARGO_PKG_VERSION")), + ), + ]; + if let Some(options) = options.into() { + if let Some(k) = options.k { + push_query(&mut query, "k", k); + } + } + + let response = self + .client + .get(self.url(&format!( + "/search/research/papers/{}", + path_escape(paper_id.as_ref()) + ))) + .headers(self.prepare_headers(None)) + .query(&query) + .send() + .await + .map_err(|e| FirecrawlError::HttpError("read paper".to_string(), e))?; + + self.handle_response(response, "read paper").await + } + + pub async fn related_papers( + &self, + paper_id: impl AsRef, + intent: impl AsRef, + options: impl Into>, + ) -> Result { + let mut query = vec![ + ("intent".to_string(), intent.as_ref().to_string()), + ( + "origin".to_string(), + format!("rust-sdk@{}", env!("CARGO_PKG_VERSION")), + ), + ]; + if let Some(options) = options.into() { + if let Some(mode) = options.mode { + push_query(&mut query, "mode", mode); + } + if let Some(k) = options.k { + push_query(&mut query, "k", k); + } + if let Some(rerank) = options.rerank { + push_query(&mut query, "rerank", rerank); + } + for anchor in options.anchor.unwrap_or_default() { + push_query(&mut query, "anchor", anchor); + } + } + + let response = self + .client + .get(self.url(&format!( + "/search/research/papers/{}/similar", + path_escape(paper_id.as_ref()) + ))) + .headers(self.prepare_headers(None)) + .query(&query) + .send() + .await + .map_err(|e| FirecrawlError::HttpError("related papers".to_string(), e))?; + + self.handle_response(response, "related papers").await + } + + pub async fn search_github( + &self, + query_text: impl AsRef, + options: impl Into>, + ) -> Result { + let mut query = vec![ + ("query".to_string(), query_text.as_ref().to_string()), + ( + "origin".to_string(), + format!("rust-sdk@{}", env!("CARGO_PKG_VERSION")), + ), + ]; + if let Some(options) = options.into() { + if let Some(k) = options.k { + push_query(&mut query, "k", k); + } + } + + let response = self + .client + .get(self.url("/search/research/github")) + .headers(self.prepare_headers(None)) + .query(&query) + .send() + .await + .map_err(|e| FirecrawlError::HttpError("search github".to_string(), e))?; + + self.handle_response(response, "search github").await + } +} From 37c68d3f818bbead35d4366e8bfc8d3d7c78bdff Mon Sep 17 00:00:00 2001 From: mogery Date: Tue, 16 Jun 2026 09:54:44 -0700 Subject: [PATCH 3/3] chore(sdks): bump versions to release v2 research support Follow-up to #3794 which added the research methods across all SDKs but did not bump versions, so the publish workflows no-op'd at the version gate. - js: 4.26.0 -> 4.27.0 - python: 4.28.3 -> 4.29.0 - rust: 2.8.1 -> 2.9.0 - go: 1.6.1 -> 1.7.0 - ruby: 1.8.1 -> 1.9.0 - php: 1.6.1 -> 1.7.0 - java: 1.9.1 -> 1.10.0 - dotnet: 1.7.1 -> 1.8.0 - elixir: 1.6.1 -> 1.7.0 Co-Authored-By: Claude Opus 4.8 (1M context) --- .pnpm-store/v11/.pnpm-needs-build-marker | 0 .pnpm-store/v11/index.db | Bin 0 -> 8192 bytes apps/dot-net-sdk/Firecrawl/Firecrawl.csproj | 2 +- apps/elixir-sdk/mix.exs | 2 +- apps/go-sdk/version.go | 2 +- apps/java-sdk/build.gradle.kts | 2 +- apps/js-sdk/firecrawl/package.json | 2 +- apps/php-sdk/src/Version.php | 2 +- apps/python-sdk/firecrawl/__init__.py | 2 +- apps/ruby-sdk/lib/firecrawl/version.rb | 2 +- apps/rust-sdk/Cargo.lock | 2 +- apps/rust-sdk/Cargo.toml | 2 +- 12 files changed, 10 insertions(+), 10 deletions(-) create mode 100644 .pnpm-store/v11/.pnpm-needs-build-marker create mode 100644 .pnpm-store/v11/index.db diff --git a/.pnpm-store/v11/.pnpm-needs-build-marker b/.pnpm-store/v11/.pnpm-needs-build-marker new file mode 100644 index 0000000000..e69de29bb2 diff --git a/.pnpm-store/v11/index.db b/.pnpm-store/v11/index.db new file mode 100644 index 0000000000000000000000000000000000000000..d583c087b2ea7237a82fd6abbd7a82897e4249b9 GIT binary patch literal 8192 zcmeIuzpBD86bA4#2p0s=&GDX11#$5OY&BppTCFMSqD0LV@%|C%po4=C;Y;~cx0O=t zfB*y_009U<00Izz00bZafgB3lKCO>xt!CY>pipIV>wEYDQ#G?7q-|A44BRz*ko}y78W!h}e%vF6aP~>|v kx0l=(WA#c7>G359KmY;|fB*y_009U<00Izz00dHje+G3k*8l(j literal 0 HcmV?d00001 diff --git a/apps/dot-net-sdk/Firecrawl/Firecrawl.csproj b/apps/dot-net-sdk/Firecrawl/Firecrawl.csproj index cda2565164..168f9fe809 100644 --- a/apps/dot-net-sdk/Firecrawl/Firecrawl.csproj +++ b/apps/dot-net-sdk/Firecrawl/Firecrawl.csproj @@ -8,7 +8,7 @@ firecrawl-sdk - 1.7.1 + 1.8.0 Firecrawl Firecrawl .NET SDK for the Firecrawl API - web scraping, crawling, and data extraction diff --git a/apps/elixir-sdk/mix.exs b/apps/elixir-sdk/mix.exs index 183f12340b..f4e59ee411 100644 --- a/apps/elixir-sdk/mix.exs +++ b/apps/elixir-sdk/mix.exs @@ -1,7 +1,7 @@ defmodule Firecrawl.MixProject do use Mix.Project - @version "1.6.1" + @version "1.7.0" @source_url "https://github.com/firecrawl/firecrawl/tree/main/apps/elixir-sdk" def project do diff --git a/apps/go-sdk/version.go b/apps/go-sdk/version.go index 296f5f6af5..7c0127dce3 100644 --- a/apps/go-sdk/version.go +++ b/apps/go-sdk/version.go @@ -9,4 +9,4 @@ package firecrawl // Bump this when preparing a new release. The publish-go-sdk GitHub workflow // reads this value and creates the corresponding monorepo-prefixed tag on // merge to main. -const Version = "1.6.1" +const Version = "1.7.0" diff --git a/apps/java-sdk/build.gradle.kts b/apps/java-sdk/build.gradle.kts index 1f2affb906..5f76edbfc8 100644 --- a/apps/java-sdk/build.gradle.kts +++ b/apps/java-sdk/build.gradle.kts @@ -4,7 +4,7 @@ plugins { } group = "com.firecrawl" -version = "1.9.1" +version = "1.10.0" java { sourceCompatibility = JavaVersion.VERSION_11 diff --git a/apps/js-sdk/firecrawl/package.json b/apps/js-sdk/firecrawl/package.json index e4721691ed..26eb9c64f2 100644 --- a/apps/js-sdk/firecrawl/package.json +++ b/apps/js-sdk/firecrawl/package.json @@ -1,6 +1,6 @@ { "name": "@mendable/firecrawl-js", - "version": "4.26.0", + "version": "4.27.0", "description": "JavaScript SDK for Firecrawl API", "main": "dist/index.js", "types": "dist/index.d.ts", diff --git a/apps/php-sdk/src/Version.php b/apps/php-sdk/src/Version.php index 5f15f46efe..9cdde7df4e 100644 --- a/apps/php-sdk/src/Version.php +++ b/apps/php-sdk/src/Version.php @@ -6,5 +6,5 @@ final class Version { - public const SDK_VERSION = '1.6.1'; + public const SDK_VERSION = '1.7.0'; } diff --git a/apps/python-sdk/firecrawl/__init__.py b/apps/python-sdk/firecrawl/__init__.py index aba9a1c64a..a0e20204e0 100644 --- a/apps/python-sdk/firecrawl/__init__.py +++ b/apps/python-sdk/firecrawl/__init__.py @@ -17,7 +17,7 @@ V1ChangeTrackingOptions, ) -__version__ = "4.28.3" +__version__ = "4.29.0" # Define the logger for the Firecrawl project logger: logging.Logger = logging.getLogger("firecrawl") diff --git a/apps/ruby-sdk/lib/firecrawl/version.rb b/apps/ruby-sdk/lib/firecrawl/version.rb index 52751598ce..23865bfe84 100644 --- a/apps/ruby-sdk/lib/firecrawl/version.rb +++ b/apps/ruby-sdk/lib/firecrawl/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module Firecrawl - VERSION = "1.8.1" + VERSION = "1.9.0" end diff --git a/apps/rust-sdk/Cargo.lock b/apps/rust-sdk/Cargo.lock index 091dc733cb..eb499e5eb0 100644 --- a/apps/rust-sdk/Cargo.lock +++ b/apps/rust-sdk/Cargo.lock @@ -250,7 +250,7 @@ checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" [[package]] name = "firecrawl" -version = "2.8.1" +version = "2.9.0" dependencies = [ "mockito", "reqwest", diff --git a/apps/rust-sdk/Cargo.toml b/apps/rust-sdk/Cargo.toml index a924c654b4..9f4eec34ea 100644 --- a/apps/rust-sdk/Cargo.toml +++ b/apps/rust-sdk/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "firecrawl" -version = "2.8.1" +version = "2.9.0" edition = "2021" license = "MIT" homepage = "https://www.firecrawl.dev/"