[pull] main from firecrawl:main

apps/api/src/__tests__/snips/v2/monitor.test.ts

@@ -139,4 +139,190 @@ describeIf(ALLOW_TEST_SUITE_WEBSITE && !TEST_SELF_HOST)("/v2/monitor", () => {
     },
     2 * scrapeTimeout,
   );
+
+  it(
+    "runs a JSON-mode monitor and surfaces a snapshot",
+    async () => {
+      const jsonSchema = {
+        type: "object",
+        additionalProperties: false,
+        required: ["title"],
+        properties: {
+          title: {
+            type: "string",
+            description: "The page title, verbatim.",
+          },
+        },
+      };
+
+      const create = await monitorCreateRaw(
+        {
+          name: "json-mode monitor",
+          schedule: { cron: "*/30 * * * *", timezone: "UTC" },
+          targets: [
+            {
+              type: "scrape",
+              urls: [createTestIdUrl()],
+              scrapeOptions: {
+                formats: [
+                  "markdown",
+                  {
+                    type: "json",
+                    prompt: "Extract the page title verbatim.",
+                    schema: jsonSchema,
+                  },
+                ],
+              },
+            },
+          ],
+        },
+        identity,
+      );
+      expect(create.statusCode).toBe(200);
+
+      const monitorId = create.body.data.id;
+      const run = await monitorRunRaw(monitorId, identity);
+      expect(run.statusCode).toBe(200);
+      const checkId = run.body.id;
+
+      let check: any;
+      for (let i = 0; i < 90; i++) {
+        const raw = await monitorCheckRaw(monitorId, checkId, identity);
+        expect(raw.statusCode).toBe(200);
+        check = raw.body.data;
+        if (["completed", "partial", "failed"].includes(check.status)) break;
+        await new Promise(resolve => setTimeout(resolve, 1000));
+      }
+      expect(["completed", "partial"]).toContain(check.status);
+      // First run: status is "new" (no previous scrape to diff against), no
+      // snapshot persisted to GCS. We can't assert snapshot.json here
+      // without a mutating fixture; a second run with a changed page would
+      // be needed. The contract assertion is: when JSON mode is requested,
+      // the monitor doesn't fall through the markdown path and crash.
+      expect(check.pages[0].status).toBe("new");
+
+      await monitorDeleteRaw(monitorId, identity);
+    },
+    2 * scrapeTimeout,
+  );
+
+  it(
+    "accepts changeTracking-json format on a monitor target",
+    async () => {
+      const jsonSchema = {
+        type: "object",
+        additionalProperties: false,
+        required: ["title"],
+        properties: {
+          title: { type: "string", description: "The page title." },
+        },
+      };
+
+      const create = await monitorCreateRaw(
+        {
+          name: "ct-json monitor",
+          schedule: { cron: "*/30 * * * *", timezone: "UTC" },
+          targets: [
+            {
+              type: "scrape",
+              urls: [createTestIdUrl()],
+              scrapeOptions: {
+                formats: [
+                  "markdown",
+                  {
+                    type: "changeTracking",
+                    modes: ["json"],
+                    prompt: "Extract the title.",
+                    schema: jsonSchema,
+                  },
+                ],
+              },
+            },
+          ],
+        },
+        identity,
+      );
+      expect(create.statusCode).toBe(200);
+
+      const monitorId = create.body.data.id;
+      const run = await monitorRunRaw(monitorId, identity);
+      expect(run.statusCode).toBe(200);
+      const checkId = run.body.id;
+
+      let check: any;
+      for (let i = 0; i < 90; i++) {
+        const raw = await monitorCheckRaw(monitorId, checkId, identity);
+        expect(raw.statusCode).toBe(200);
+        check = raw.body.data;
+        if (["completed", "partial", "failed"].includes(check.status)) break;
+        await new Promise(resolve => setTimeout(resolve, 1000));
+      }
+      expect(["completed", "partial"]).toContain(check.status);
+      expect(check.pages[0].status).toBe("new");
+
+      await monitorDeleteRaw(monitorId, identity);
+    },
+    2 * scrapeTimeout,
+  );
+
+  it(
+    "accepts mixed json + git-diff changeTracking modes",
+    async () => {
+      const jsonSchema = {
+        type: "object",
+        additionalProperties: false,
+        required: ["title"],
+        properties: {
+          title: { type: "string", description: "The page title." },
+        },
+      };
+
+      const create = await monitorCreateRaw(
+        {
+          name: "ct-mixed monitor",
+          schedule: { cron: "*/30 * * * *", timezone: "UTC" },
+          targets: [
+            {
+              type: "scrape",
+              urls: [createTestIdUrl()],
+              scrapeOptions: {
+                formats: [
+                  "markdown",
+                  {
+                    type: "changeTracking",
+                    modes: ["json", "git-diff"],
+                    prompt: "Extract the title.",
+                    schema: jsonSchema,
+                  },
+                ],
+              },
+            },
+          ],
+        },
+        identity,
+      );
+      expect(create.statusCode).toBe(200);
+
+      const monitorId = create.body.data.id;
+      const run = await monitorRunRaw(monitorId, identity);
+      expect(run.statusCode).toBe(200);
+      const checkId = run.body.id;
+
+      let check: any;
+      for (let i = 0; i < 90; i++) {
+        const raw = await monitorCheckRaw(monitorId, checkId, identity);
+        expect(raw.statusCode).toBe(200);
+        check = raw.body.data;
+        if (["completed", "partial", "failed"].includes(check.status)) break;
+        await new Promise(resolve => setTimeout(resolve, 1000));
+      }
+      expect(["completed", "partial"]).toContain(check.status);
+      // First run is always "new"; the assertion here is that mixed-mode
+      // configuration is accepted end-to-end without erroring.
+      expect(check.pages[0].status).toBe("new");
+
+      await monitorDeleteRaw(monitorId, identity);
+    },
+    2 * scrapeTimeout,
+  );
 });

apps/api/src/controllers/v2/monitor.ts

@@ -389,19 +389,38 @@ export async function getMonitorCheckController(
   ]);
 
   const pagesWithDiffs = await Promise.all(
-    pages.map(async page => ({
-      id: page.id,
-      targetId: page.target_id,
-      url: page.url,
-      status: page.status,
-      previousScrapeId: page.previous_scrape_id,
-      currentScrapeId: page.current_scrape_id,
-      statusCode: page.status_code,
-      error: page.error,
-      metadata: page.metadata,
-      diff: await getMonitorDiffArtifact(page.diff_gcs_key),
-      createdAt: page.created_at,
-    })),
+    pages.map(async page => {
+      const artifact = await getMonitorDiffArtifact(page.diff_gcs_key);
+      const base = {
+        id: page.id,
+        targetId: page.target_id,
+        url: page.url,
+        status: page.status,
+        previousScrapeId: page.previous_scrape_id,
+        currentScrapeId: page.current_scrape_id,
+        statusCode: page.status_code,
+        error: page.error,
+        metadata: page.metadata,
+        createdAt: page.created_at,
+      };
+      if (!artifact) {
+        return { ...base, diff: null };
+      }
+      if (artifact.kind === "json") {
+        return {
+          ...base,
+          diff: {
+            json: artifact.json,
+            ...(artifact.markdown ? { text: artifact.markdown.text } : {}),
+          },
+          snapshot: { json: artifact.snapshot },
+        };
+      }
+      return {
+        ...base,
+        diff: { text: artifact.text, json: artifact.json },
+      };
+    }),
   );
   const nextSkip = skip + pagesWithDiffs.length;
   const next = (() => {

apps/api/src/lib/gcs-monitoring.ts

@@ -1,15 +1,37 @@
 import { config } from "../config";
 import { storage } from "./gcs-jobs";
 
-type MonitorDiffArtifact = {
+type MonitorDiffArtifactBase = {
   url: string;
   previousScrapeId: string | null;
   currentScrapeId: string | null;
-  text: string;
-  json: unknown;
   generatedAt: string;
 };
 
+export type MonitorDiffArtifact =
+  | (MonitorDiffArtifactBase & {
+      kind: "markdown";
+      text: string;
+      json: unknown;
+    })
+  | (MonitorDiffArtifactBase & {
+      kind: "json";
+      /** Per-field {previous, current} diff. */
+      json: Record<string, { previous: unknown; current: unknown }>;
+      /** Full current JSON extraction (the snapshot at this run). */
+      snapshot: Record<string, unknown>;
+      /**
+       * Optional markdown diff sidecar. Populated only when the monitor's
+       * formats requested both `"json"` and `"git-diff"` change-tracking
+       * modes — in that case we run both diffs and report `changed` if
+       * either path saw a change.
+       */
+      markdown?: {
+        text: string;
+        json: unknown;
+      };
+    });
+
 const contentType = "application/json";
 
 export function monitorDiffGcsKey(params: {
@@ -21,16 +43,29 @@ export function monitorDiffGcsKey(params: {
   return `monitors/${params.teamId}/${params.monitorId}/${params.checkId}/${params.pageId}.diff.json`;
 }
 
+function artifactBytes(artifact: MonitorDiffArtifact): {
+  textBytes: number;
+  jsonBytes: number;
+} {
+  const jsonBytes = Buffer.byteLength(JSON.stringify(artifact.json ?? null));
+  let textBytes = 0;
+  if (artifact.kind === "markdown") {
+    textBytes = Buffer.byteLength(artifact.text);
+  } else if (artifact.kind === "json" && artifact.markdown) {
+    // Sidecar markdown diff (mixed-mode monitor) — count it so storage
+    // accounting stays honest.
+    textBytes = Buffer.byteLength(artifact.markdown.text);
+  }
+  return { textBytes, jsonBytes };
+}
+
 export async function saveMonitorDiffArtifact(
   key: string,
   artifact: MonitorDiffArtifact,
 ): Promise<{ textBytes: number; jsonBytes: number }> {
   const payload = JSON.stringify(artifact);
   if (!config.GCS_BUCKET_NAME) {
-    return {
-      textBytes: Buffer.byteLength(artifact.text),
-      jsonBytes: Buffer.byteLength(JSON.stringify(artifact.json ?? null)),
-    };
+    return artifactBytes(artifact);
   }
 
   const bucket = storage.bucket(config.GCS_BUCKET_NAME);
@@ -39,10 +74,7 @@ export async function saveMonitorDiffArtifact(
     resumable: false,
   });
 
-  return {
-    textBytes: Buffer.byteLength(artifact.text),
-    jsonBytes: Buffer.byteLength(JSON.stringify(artifact.json ?? null)),
-  };
+  return artifactBytes(artifact);
 }
 
 export async function getMonitorDiffArtifact(
@@ -53,7 +85,26 @@ export async function getMonitorDiffArtifact(
   const bucket = storage.bucket(config.GCS_BUCKET_NAME);
   try {
     const [contents] = await bucket.file(key).download();
-    return JSON.parse(contents.toString()) as MonitorDiffArtifact;
+    let parsed: unknown;
+    try {
+      parsed = JSON.parse(contents.toString());
+    } catch {
+      // Corrupt or truncated artifact — surface as "no diff" instead of
+      // letting JSON.parse throw and break the entire check response.
+      return null;
+    }
+    if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) {
+      // An unexpected payload shape (e.g. number, array, null) was written
+      // here; treat as missing rather than risk reading kind off a non-object.
+      return null;
+    }
+    const asPartial = parsed as Partial<MonitorDiffArtifact>;
+    // Backwards compat: historical artifacts predate the `kind` field and
+    // are always markdown.
+    if (!asPartial.kind) {
+      return { ...(asPartial as any), kind: "markdown" } as MonitorDiffArtifact;
+    }
+    return asPartial as MonitorDiffArtifact;
   } catch (error) {
     const maybeGcsError = error as { code?: number; statusCode?: number };
     if (maybeGcsError.code === 404 || maybeGcsError.statusCode === 404) {

apps/api/src/services/monitoring/cron.ts

@@ -1,4 +1,4 @@
-const MIN_MONITOR_INTERVAL_MS = 15 * 60 * 1000;
+const MIN_MONITOR_INTERVAL_MS = 5 * 60 * 1000;
 const SEARCH_LIMIT_MINUTES = 366 * 24 * 60;
 
 type CronField = Set<number>;
@@ -234,7 +234,7 @@ export function validateMonitorCron(
   const intervalMs = secondRunAt.getTime() - nextRunAt.getTime();
   if (intervalMs < MIN_MONITOR_INTERVAL_MS) {
     throw new Error(
-      "Monitor schedule must not run more often than every 15 minutes",
+      "Monitor schedule must not run more often than every 5 minutes",
     );
   }