[pull] main from firecrawl:main

apps/api/src/__tests__/snips/lib.ts

@@ -23,6 +23,7 @@ export const TEST_PRODUCTION = !TEST_SELF_HOST;
 
 // TODO: do we want to run AI tests when users run this command locally? It may lead to increased spending for them, depending on configuration
 export const HAS_AI = !!(config.OPENAI_API_KEY || config.OLLAMA_BASE_URL);
+export const HAS_FIREWORKS = !!process.env.FIREWORKS_API_KEY;
 export const HAS_FIRE_ENGINE = !!config.FIRE_ENGINE_BETA_URL;
 export const HAS_PLAYWRIGHT = !!config.PLAYWRIGHT_MICROSERVICE_URL;
 export const HAS_PROXY = !!config.PROXY_SERVER;

apps/api/src/__tests__/snips/v2/crawl.test.ts

@@ -437,7 +437,7 @@ describe("Crawl tests", () => {
         async () => {
           const res = await crawl(
             {
-              url: base,
+              url: new URL("/blog", base).href,
               prompt:
                 "Crawl everything including external links and subdomains",
               // Explicit options that should override the prompt

apps/api/src/__tests__/snips/v2/lib.ts

@@ -85,6 +85,96 @@ export async function scrapeWithFailure(
   return raw.body;
 }
 
+// =========================================
+// Monitor API
+// =========================================
+
+export type MonitorCreateInput = {
+  name: string;
+  schedule: { cron: string; timezone?: string };
+  webhook?: { url: string; headers?: Record<string, string> };
+  notification?: {
+    email?: {
+      enabled?: boolean;
+      recipients?: string[];
+      includeDiffs?: boolean;
+    };
+  };
+  targets: Array<
+    | {
+        type: "scrape";
+        urls: string[];
+        scrapeOptions?: Record<string, unknown>;
+      }
+    | {
+        type: "crawl";
+        url: string;
+        crawlOptions?: Record<string, unknown>;
+        scrapeOptions?: Record<string, unknown>;
+      }
+  >;
+  retentionDays?: number;
+};
+
+export async function monitorCreateRaw(
+  body: MonitorCreateInput,
+  identity: Identity,
+) {
+  return await request(TEST_API_URL)
+    .post("/v2/monitor")
+    .set("Authorization", `Bearer ${identity.apiKey}`)
+    .set("Content-Type", "application/json")
+    .send(body);
+}
+
+export async function monitorListRaw(identity: Identity) {
+  return await request(TEST_API_URL)
+    .get("/v2/monitor")
+    .set("Authorization", `Bearer ${identity.apiKey}`);
+}
+
+export async function monitorGetRaw(id: string, identity: Identity) {
+  return await request(TEST_API_URL)
+    .get(`/v2/monitor/${id}`)
+    .set("Authorization", `Bearer ${identity.apiKey}`);
+}
+
+export async function monitorPatchRaw(
+  id: string,
+  body: Partial<MonitorCreateInput> & { status?: "active" | "paused" },
+  identity: Identity,
+) {
+  return await request(TEST_API_URL)
+    .patch(`/v2/monitor/${id}`)
+    .set("Authorization", `Bearer ${identity.apiKey}`)
+    .set("Content-Type", "application/json")
+    .send(body);
+}
+
+export async function monitorDeleteRaw(id: string, identity: Identity) {
+  return await request(TEST_API_URL)
+    .delete(`/v2/monitor/${id}`)
+    .set("Authorization", `Bearer ${identity.apiKey}`);
+}
+
+export async function monitorRunRaw(id: string, identity: Identity) {
+  return await request(TEST_API_URL)
+    .post(`/v2/monitor/${id}/run`)
+    .set("Authorization", `Bearer ${identity.apiKey}`);
+}
+
+export async function monitorCheckRaw(
+  monitorId: string,
+  checkId: string,
+  identity: Identity,
+  query?: Record<string, string | number>,
+) {
+  const req = request(TEST_API_URL)
+    .get(`/v2/monitor/${monitorId}/checks/${checkId}`)
+    .set("Authorization", `Bearer ${identity.apiKey}`);
+  return query ? req.query(query) : req;
+}
+
 export async function parseRaw(
   body: {
     options?: Omit<ParseRequestInput, "file">;

apps/api/src/__tests__/snips/v2/monitor.test.ts

@@ -0,0 +1,142 @@
+import {
+  createTestIdUrl,
+  describeIf,
+  ALLOW_TEST_SUITE_WEBSITE,
+  TEST_SELF_HOST,
+} from "../lib";
+import {
+  idmux,
+  Identity,
+  monitorCheckRaw,
+  monitorCreateRaw,
+  monitorDeleteRaw,
+  monitorGetRaw,
+  monitorListRaw,
+  monitorPatchRaw,
+  monitorRunRaw,
+  scrapeTimeout,
+} from "./lib";
+
+describeIf(ALLOW_TEST_SUITE_WEBSITE && !TEST_SELF_HOST)("/v2/monitor", () => {
+  let identity: Identity;
+
+  beforeAll(async () => {
+    identity = await idmux({
+      name: "monitor",
+      concurrency: 20,
+      credits: 1000000,
+    });
+  }, 10000);
+
+  it("creates, lists, gets, pauses, and deletes a monitor", async () => {
+    const create = await monitorCreateRaw(
+      {
+        name: "snips monitor",
+        schedule: { cron: "*/30 * * * *", timezone: "UTC" },
+        targets: [
+          {
+            type: "scrape",
+            urls: [createTestIdUrl(), createTestIdUrl()],
+            scrapeOptions: { formats: ["markdown"] },
+          },
+        ],
+        notification: { email: { enabled: false } },
+      },
+      identity,
+    );
+
+    expect(create.statusCode).toBe(200);
+    expect(create.body.success).toBe(true);
+    expect(create.body.data.id).toEqual(expect.any(String));
+    expect(create.body.data.targets[0].id).toEqual(expect.any(String));
+
+    const id = create.body.data.id;
+    const list = await monitorListRaw(identity);
+    expect(list.statusCode).toBe(200);
+    expect(list.body.data.some((x: any) => x.id === id)).toBe(true);
+
+    const get = await monitorGetRaw(id, identity);
+    expect(get.statusCode).toBe(200);
+    expect(get.body.data.id).toBe(id);
+
+    const patch = await monitorPatchRaw(id, { status: "paused" }, identity);
+    expect(patch.statusCode).toBe(200);
+    expect(patch.body.data.status).toBe("paused");
+
+    const del = await monitorDeleteRaw(id, identity);
+    expect(del.statusCode).toBe(200);
+    expect(del.body.success).toBe(true);
+  });
+
+  it("rejects cron schedules under 15 minutes", async () => {
+    const response = await monitorCreateRaw(
+      {
+        name: "too frequent",
+        schedule: { cron: "*/5 * * * *", timezone: "UTC" },
+        targets: [
+          {
+            type: "scrape",
+            urls: [createTestIdUrl()],
+          },
+        ],
+      },
+      identity,
+    );
+
+    expect(response.statusCode).toBe(400);
+    expect(response.body.success).toBe(false);
+    expect(response.body.error).toContain("15 minutes");
+  });
+
+  it(
+    "runs a manual scrape monitor check",
+    async () => {
+      const create = await monitorCreateRaw(
+        {
+          name: "manual monitor",
+          schedule: { cron: "*/30 * * * *", timezone: "UTC" },
+          targets: [
+            {
+              type: "scrape",
+              urls: [createTestIdUrl(), createTestIdUrl()],
+              scrapeOptions: { formats: ["markdown"] },
+            },
+          ],
+        },
+        identity,
+      );
+      expect(create.statusCode).toBe(200);
+
+      const monitorId = create.body.data.id;
+      const run = await monitorRunRaw(monitorId, identity);
+      expect(run.statusCode).toBe(200);
+      const checkId = run.body.id;
+
+      let check: any;
+      for (let i = 0; i < 90; i++) {
+        const raw = await monitorCheckRaw(monitorId, checkId, identity);
+        expect(raw.statusCode).toBe(200);
+        check = raw.body.data;
+        if (["completed", "partial", "failed"].includes(check.status)) break;
+        await new Promise(resolve => setTimeout(resolve, 1000));
+      }
+
+      expect(["completed", "partial"]).toContain(check.status);
+      expect(check.summary.totalPages).toBeGreaterThanOrEqual(2);
+      expect(check.pages.length).toBeGreaterThanOrEqual(1);
+      expect(check.next).toBeUndefined();
+
+      const firstPage = await monitorCheckRaw(monitorId, checkId, identity, {
+        limit: 1,
+      });
+      expect(firstPage.statusCode).toBe(200);
+      expect(firstPage.body.next).toContain("skip=1");
+      expect(firstPage.body.next).toContain("limit=1");
+      expect(firstPage.body.data.next).toBe(firstPage.body.next);
+      expect(firstPage.body.data.pages).toHaveLength(1);
+
+      await monitorDeleteRaw(monitorId, identity);
+    },
+    2 * scrapeTimeout,
+  );
+});

apps/api/src/__tests__/snips/v2/scrape-query.test.ts

@@ -1,4 +1,4 @@
-import { concurrentIf, HAS_AI, TEST_PRODUCTION } from "../lib";
+import { concurrentIf, HAS_AI, HAS_FIREWORKS, TEST_PRODUCTION } from "../lib";
 import {
   scrape,
   scrapeRaw,
@@ -38,6 +38,25 @@ describe("Query format", () => {
     scrapeTimeout,
   );
 
+  concurrentIf(TEST_PRODUCTION || HAS_AI)(
+    "returns a non-empty answer for a valid question",
+    async () => {
+      const response = await scrape(
+        {
+          url: "https://firecrawl.dev",
+          formats: [{ type: "question", question: "What is Firecrawl?" }],
+        },
+        identity,
+      );
+
+      expect(response.answer).toBeDefined();
+      expect(typeof response.answer).toBe("string");
+      expect(response.answer!.length).toBeGreaterThan(0);
+      expect(response.markdown).toBeUndefined();
+    },
+    scrapeTimeout,
+  );
+
   concurrentIf(TEST_PRODUCTION || HAS_AI)(
     "returns both answer and markdown when formats include markdown and query",
     async () => {
@@ -61,7 +80,26 @@ describe("Query format", () => {
     scrapeTimeout,
   );
 
-  concurrentIf(TEST_PRODUCTION || HAS_AI)(
+  concurrentIf(TEST_PRODUCTION || HAS_FIREWORKS)(
+    "returns non-empty highlights for a valid highlights query",
+    async () => {
+      const response = await scrape(
+        {
+          url: "https://firecrawl.dev",
+          formats: [{ type: "highlights", query: "What is Firecrawl?" }],
+        },
+        identity,
+      );
+
+      expect(response.highlights).toBeDefined();
+      expect(typeof response.highlights).toBe("string");
+      expect(response.highlights!.length).toBeGreaterThan(0);
+      expect(response.answer).toBeUndefined();
+    },
+    scrapeTimeout,
+  );
+
+  concurrentIf(TEST_PRODUCTION || HAS_FIREWORKS)(
     "returns a direct quote answer when query mode is directQuote",
     async () => {
       const response = await scrape(
@@ -118,4 +156,38 @@ describe("Query format", () => {
     },
     scrapeTimeout,
   );
+
+  it(
+    "rejects question over 10000 characters",
+    async () => {
+      const response = await scrapeWithFailure(
+        {
+          url: "https://firecrawl.dev",
+          formats: [{ type: "question", question: "a".repeat(10001) }],
+        } as any,
+        identity,
+      );
+
+      expect(response.success).toBe(false);
+      expect(response.error).toBeDefined();
+    },
+    scrapeTimeout,
+  );
+
+  it(
+    "rejects highlights query over 10000 characters",
+    async () => {
+      const response = await scrapeWithFailure(
+        {
+          url: "https://firecrawl.dev",
+          formats: [{ type: "highlights", query: "a".repeat(10001) }],
+        } as any,
+        identity,
+      );
+
+      expect(response.success).toBe(false);
+      expect(response.error).toBeDefined();
+    },
+    scrapeTimeout,
+  );
 });