[pull] main from firecrawl:main

apps/api/src/__tests__/snips/v2/keyless.test.ts

@@ -8,6 +8,9 @@ import {
   scrapeTimeout,
 } from "../lib";
 import { redisRateLimitClient } from "../../../services/rate-limiter";
+import { db } from "../../../db/connection";
+import * as schema from "../../../db/schema";
+import { and, desc, eq, gt } from "drizzle-orm";
 import request from "supertest";
 
 // The keyless tier is disabled unless both limits are configured. The harness
@@ -359,6 +362,63 @@ describeIf(KEYLESS_ENABLED)("Keyless free tier", () => {
     scrapeTimeout,
   );
 
+  it(
+    "writes a keyless_credit_usage audit row for a successful keyless scrape (200)",
+    async () => {
+      // The audit insert is best-effort and fire-and-forget in the controller,
+      // so capture the latest id first, then poll for a newer row afterwards.
+      const beforeMaxId = (
+        await db
+          .select({ id: schema.keyless_credit_usage.id })
+          .from(schema.keyless_credit_usage)
+          .orderBy(desc(schema.keyless_credit_usage.id))
+          .limit(1)
+      )[0]?.id;
+
+      const response = await request(TEST_API_URL)
+        .post("/v2/scrape")
+        .set("Content-Type", "application/json")
+        .send({
+          url: TEST_SUITE_WEBSITE,
+          origin: "mcp",
+          formats: ["markdown"],
+        });
+
+      expect(response.statusCode).toBe(200);
+      expect(response.body.success).toBe(true);
+      const creditsUsed = response.body.data.metadata.creditsUsed;
+      expect(creditsUsed).toBeGreaterThan(0);
+
+      const ip = await currentKeylessIp();
+
+      let row: typeof schema.keyless_credit_usage.$inferSelect | undefined;
+      for (let i = 0; i < 20; i++) {
+        const rows = await db
+          .select()
+          .from(schema.keyless_credit_usage)
+          .where(
+            and(
+              eq(schema.keyless_credit_usage.ip, ip),
+              beforeMaxId !== undefined
+                ? gt(schema.keyless_credit_usage.id, beforeMaxId)
+                : undefined,
+            ),
+          )
+          .orderBy(desc(schema.keyless_credit_usage.id))
+          .limit(1);
+        if (rows.length > 0) {
+          row = rows[0];
+          break;
+        }
+        await new Promise(resolve => setTimeout(resolve, 250));
+      }
+
+      expect(row).toBeDefined();
+      expect(row!.credits_used).toBe(creditsUsed);
+    },
+    scrapeTimeout,
+  );
+
   it(
     "rejects projected keyless search with scrape options before search (429)",
     async () => {

apps/api/src/__tests__/snips/v2/lib.ts

@@ -638,6 +638,20 @@ export async function searchWithFailure(
   return raw.body;
 }
 
+export async function researchRaw(
+  path: string,
+  query: Record<string, string | number | boolean | string[]> | undefined,
+  identity?: Identity,
+) {
+  const req = request(TEST_API_URL)
+    .get(path)
+    .set("Content-Type", "application/json");
+  if (identity) {
+    req.set("Authorization", `Bearer ${identity.apiKey}`);
+  }
+  return query ? req.query(query) : req;
+}
+
 export async function searchRawFull(
   body: SearchRequestInput,
   identity: Identity,

apps/api/src/__tests__/snips/v2/research.test.ts

@@ -0,0 +1,139 @@
+import { config } from "../../../config";
+import { describeIf, TEST_PRODUCTION } from "../lib";
+import { creditUsage, idmux, researchRaw } from "./lib";
+
+const HAS_RESEARCH = !!config.RESEARCH_PROXY_URL;
+const KEYLESS_ENABLED =
+  process.env.KEYLESS_REQUESTS_PER_DAY !== undefined &&
+  process.env.KEYLESS_CREDITS_PER_DAY !== undefined;
+
+const sleep = (ms: number) => new Promise(resolve => setTimeout(resolve, ms));
+const sleepForBilling = () => sleep(40000);
+
+describeIf(HAS_RESEARCH)("Research API", () => {
+  it("serves paper search from the canonical mount", async () => {
+    const identity = await idmux({
+      name: "research/canonical paper search",
+      credits: 100,
+    });
+
+    const res = await researchRaw(
+      "/v2/search/research/papers",
+      { query: "retrieval augmented generation", k: 2 },
+      identity,
+    );
+
+    expect(res.statusCode).toBe(200);
+    expect(res.body.success).toBe(true);
+    expect(Array.isArray(res.body.results)).toBe(true);
+    expect(res.body.results.length).toBeGreaterThan(0);
+    expect(res.body.results[0].paperId).toBeDefined();
+    expect(res.body.results[0].paper_id).toBeUndefined();
+  }, 120000);
+
+  it("keeps the legacy research mount working", async () => {
+    const identity = await idmux({
+      name: "research/legacy paper search",
+      credits: 100,
+    });
+
+    const res = await researchRaw(
+      "/v2/research/papers",
+      { query: "diffusion models", k: 1 },
+      identity,
+    );
+
+    expect(res.statusCode).toBe(200);
+    expect(res.body.success).toBe(true);
+    expect(Array.isArray(res.body.results)).toBe(true);
+    expect(res.body.results[0].paperId).toBeDefined();
+    expect(res.body.results[0].paper_id).toBe(res.body.results[0].paperId);
+  }, 120000);
+
+  it("rejects invalid endpoint-specific query params", async () => {
+    const identity = await idmux({
+      name: "research/invalid params",
+      credits: 100,
+    });
+
+    const res = await researchRaw(
+      "/v2/search/research/papers",
+      { query: "rag", magic: "true" } as any,
+      identity,
+    );
+
+    expect(res.statusCode).toBe(400);
+    expect(res.body.success).toBe(false);
+  });
+
+  it("rejects paper inspect k without read query", async () => {
+    const identity = await idmux({
+      name: "research/inspect rejects k",
+      credits: 100,
+    });
+
+    const res = await researchRaw(
+      "/v2/search/research/papers/1706.03762",
+      { k: 1 },
+      identity,
+    );
+
+    expect(res.statusCode).toBe(400);
+    expect(res.body.success).toBe(false);
+  });
+
+  describeIf(KEYLESS_ENABLED)("keyless research", () => {
+    it("permits keyless access on the canonical research index", async () => {
+      const res = await researchRaw("/v2/search/research/papers", {
+        query: "transformers",
+        k: 1,
+      });
+
+      expect(res.statusCode).not.toBe(401);
+    }, 120000);
+  });
+
+  describeIf(TEST_PRODUCTION)("research billing", () => {
+    it("bills read-paper as one scrape-like credit", async () => {
+      const identity = await idmux({
+        name: "research/bills read paper",
+        credits: 100,
+      });
+      const before = (await creditUsage(identity)).remainingCredits;
+
+      const res = await researchRaw(
+        "/v2/search/research/papers/1706.03762",
+        { query: "attention", k: 1 },
+        identity,
+      );
+      expect(res.statusCode).toBe(200);
+      expect(res.body.success).toBe(true);
+
+      await sleepForBilling();
+      const after = (await creditUsage(identity)).remainingCredits;
+      expect(before - after).toBe(1);
+    }, 180000);
+
+    it("bills search-like endpoints by returned result count", async () => {
+      const identity = await idmux({
+        name: "research/bills search papers",
+        credits: 100,
+      });
+      const before = (await creditUsage(identity)).remainingCredits;
+
+      const res = await researchRaw(
+        "/v2/search/research/papers",
+        { query: "graph neural networks", k: 11 },
+        identity,
+      );
+      expect(res.statusCode).toBe(200);
+      expect(res.body.success).toBe(true);
+      expect(res.body.results.length).toBeGreaterThan(0);
+      const expectedCredits = Math.ceil(res.body.results.length / 10) * 2;
+
+      await sleepForBilling();
+      const after = (await creditUsage(identity)).remainingCredits;
+      expect(before - after).toBe(expectedCredits);
+    }, 180000);
+  });
+});

apps/api/src/controllers/auth.ts

@@ -561,9 +561,11 @@ async function handleKeylessAuth(
   const modeLabel =
     mode === RateLimiterMode.Search
       ? "search"
-      : mode === RateLimiterMode.BrowserExecute
-        ? "interact"
-        : "scrape";
+      : mode === RateLimiterMode.Research
+        ? "research"
+        : mode === RateLimiterMode.BrowserExecute
+          ? "interact"
+          : "scrape";
 
   let result: Awaited<ReturnType<typeof consumeKeylessRequest>>;
   try {
@@ -831,7 +833,8 @@ async function supaAuthenticateUser(
       mode === RateLimiterMode.Crawl ||
       mode === RateLimiterMode.CrawlStatus ||
       mode === RateLimiterMode.Extract ||
-      mode === RateLimiterMode.Search)
+      mode === RateLimiterMode.Search ||
+      mode === RateLimiterMode.Research)
   ) {
     return {
       success: true,

apps/api/src/controllers/v1/scrape.ts

@@ -26,6 +26,7 @@ import { getScrapeZDR } from "../../lib/zdr-helpers";
 import {
   KEYLESS_CREDITS_MESSAGE,
   adjustKeylessCredits,
+  logKeylessCreditUsage,
   reserveKeylessCredits,
 } from "../../lib/keyless";
 import { projectScrapeCredits } from "../../lib/keyless-credit-projection";
@@ -306,10 +307,14 @@ export async function scrapeController(
 
   if (reservedKeylessCredits > 0 && !reconciledKeylessCredits) {
     reconciledKeylessCredits = true;
+    const actualKeylessCredits = doc?.metadata?.creditsUsed ?? 0;
     adjustKeylessCredits(
       req.auth.team_id,
-      (doc?.metadata?.creditsUsed ?? 0) - reservedKeylessCredits,
+      actualKeylessCredits - reservedKeylessCredits,
     ).catch(() => {});
+    logKeylessCreditUsage(req.auth.team_id, actualKeylessCredits).catch(
+      () => {},
+    );
   }
 
   const totalRequestTime = new Date().getTime() - middlewareStartTime;

apps/api/src/controllers/v1/search.ts

@@ -30,6 +30,7 @@ import { getSearchForcedKind } from "../../lib/zdr-helpers";
 import {
   KEYLESS_CREDITS_MESSAGE,
   adjustKeylessCredits,
+  logKeylessCreditUsage,
   reserveKeylessCredits,
 } from "../../lib/keyless";
 import { projectSearchTotalCredits } from "../../lib/keyless-credit-projection";
@@ -254,6 +255,9 @@ export async function searchController(
         req.auth.team_id,
         result.totalCredits - reservedKeylessCredits,
       ).catch(() => {});
+      logKeylessCreditUsage(req.auth.team_id, result.totalCredits).catch(
+        () => {},
+      );
     }
 
     const endTime = new Date().getTime();

apps/api/src/controllers/v2/parse.ts

@@ -29,6 +29,7 @@ import { getScrapeZDR } from "../../lib/zdr-helpers";
 import {
   KEYLESS_CREDITS_MESSAGE,
   adjustKeylessCredits,
+  logKeylessCreditUsage,
   reserveKeylessCredits,
 } from "../../lib/keyless";
 import { projectScrapeCredits } from "../../lib/keyless-credit-projection";
@@ -656,10 +657,14 @@ export async function parseController(
 
       if (reservedKeylessCredits > 0 && !reconciledKeylessCredits) {
         reconciledKeylessCredits = true;
+        const actualKeylessCredits = doc?.metadata?.creditsUsed ?? 0;
         adjustKeylessCredits(
           req.auth.team_id,
-          (doc?.metadata?.creditsUsed ?? 0) - reservedKeylessCredits,
+          actualKeylessCredits - reservedKeylessCredits,
         ).catch(() => {});
+        logKeylessCreditUsage(req.auth.team_id, actualKeylessCredits).catch(
+          () => {},
+        );
       }
 
       const totalRequestTime = new Date().getTime() - middlewareStartTime;