feat: implement FirecrawlService.mapSite tests and sanitizeMapOptions function

Author: Rieranthony

apps/api/src/services/firecrawl.test.ts

@@ -0,0 +1,106 @@
+import { afterEach, beforeEach, describe, expect, it, mock } from "bun:test";
+import { FirecrawlService } from "./firecrawl";
+
+describe("FirecrawlService.mapSite", () => {
+	const originalFetch = globalThis.fetch;
+	const originalWarn = console.warn;
+
+	let capturedRequestBody: Record<string, unknown> | null = null;
+	let warnSpy = mock(() => {});
+
+	beforeEach(() => {
+		capturedRequestBody = null;
+		warnSpy = mock(() => {});
+		console.warn = warnSpy as typeof console.warn;
+
+		globalThis.fetch = (async (
+			_input: string | URL | Request,
+			init?: RequestInit
+		) => {
+			capturedRequestBody = JSON.parse(String(init?.body ?? "{}")) as Record<
+				string,
+				unknown
+			>;
+
+			return new Response(
+				JSON.stringify({
+					success: true,
+					links: ["https://example.com/docs"],
+				}),
+				{
+					status: 200,
+					headers: {
+						"Content-Type": "application/json",
+					},
+				}
+			);
+		}) as typeof fetch;
+	});
+
+	afterEach(() => {
+		globalThis.fetch = originalFetch;
+		console.warn = originalWarn;
+	});
+
+	it("drops unsupported map options like maxAge from request body", async () => {
+		const firecrawlService = new FirecrawlService("test-key");
+
+		const result = await firecrawlService.mapSite("https://example.com", {
+			limit: 100,
+			ignoreCache: false,
+			maxAge: 3_600_000,
+		} as unknown as Parameters<FirecrawlService["mapSite"]>[1]);
+
+		expect(result).toEqual({
+			success: true,
+			urls: ["https://example.com/docs"],
+		});
+		expect(capturedRequestBody).toEqual({
+			url: "https://example.com",
+			includeSubdomains: false,
+			limit: 100,
+			ignoreCache: false,
+		});
+		expect(capturedRequestBody?.maxAge).toBeUndefined();
+		expect(warnSpy).toHaveBeenCalledTimes(1);
+	});
+
+	it("maps sitemapOnly and ignoreCache to supported Firecrawl /map fields", async () => {
+		const firecrawlService = new FirecrawlService("test-key");
+
+		await firecrawlService.mapSite("https://example.com", {
+			search: "docs",
+			includeSubdomains: true,
+			limit: 42,
+			sitemapOnly: true,
+			ignoreSitemap: true,
+			ignoreCache: true,
+		});
+
+		expect(capturedRequestBody).toEqual({
+			url: "https://example.com",
+			search: "docs",
+			includeSubdomains: true,
+			limit: 42,
+			sitemap: "only",
+			ignoreCache: true,
+		});
+		expect(warnSpy).not.toHaveBeenCalled();
+	});
+
+	it("maps ignoreSitemap to sitemap=skip", async () => {
+		const firecrawlService = new FirecrawlService("test-key");
+
+		await firecrawlService.mapSite("https://example.com", {
+			ignoreSitemap: true,
+		});
+
+		expect(capturedRequestBody).toEqual({
+			url: "https://example.com",
+			includeSubdomains: false,
+			limit: 100,
+			sitemap: "skip",
+		});
+		expect(warnSpy).not.toHaveBeenCalled();
+	});
+});

apps/api/src/services/firecrawl.ts

@@ -256,14 +256,61 @@ export type MapOptions = {
 	limit?: number;
 	/** Firecrawl v2 map option for bypassing cache */
 	ignoreCache?: boolean;
-	/**
-	 * Deprecated compatibility field.
-	 * Firecrawl map endpoint does not support maxAge.
-	 * When set to 0 or less, it forces ignoreCache=true.
-	 */
-	maxAge?: number;
 };
 
+const FIRECRAWL_MAP_OPTION_KEYS = new Set<keyof MapOptions>([
+	"search",
+	"ignoreSitemap",
+	"sitemapOnly",
+	"includeSubdomains",
+	"limit",
+	"ignoreCache",
+]);
+
+type SanitizedMapOptions = {
+	search?: string;
+	ignoreSitemap?: boolean;
+	sitemapOnly?: boolean;
+	includeSubdomains: boolean;
+	limit: number;
+	ignoreCache?: boolean;
+};
+
+function sanitizeMapOptions(options: MapOptions): {
+	sanitized: SanitizedMapOptions;
+	unknownKeys: string[];
+} {
+	const unknownKeys = Object.keys(options as Record<string, unknown>).filter(
+		(key) => !FIRECRAWL_MAP_OPTION_KEYS.has(key as keyof MapOptions)
+	);
+
+	const sanitized: SanitizedMapOptions = {
+		includeSubdomains: options.includeSubdomains ?? false,
+		limit: options.limit ?? 100,
+	};
+
+	if (options.search) {
+		sanitized.search = options.search;
+	}
+
+	if (options.ignoreSitemap) {
+		sanitized.ignoreSitemap = true;
+	}
+
+	if (options.sitemapOnly) {
+		sanitized.sitemapOnly = true;
+	}
+
+	if (options.ignoreCache !== undefined) {
+		sanitized.ignoreCache = options.ignoreCache;
+	}
+
+	return {
+		sanitized,
+		unknownKeys,
+	};
+}
+
 function sleep(ms: number): Promise<void> {
 	return new Promise((resolve) => setTimeout(resolve, ms));
 }
@@ -1153,26 +1200,32 @@ export class FirecrawlService {
 		console.log("[firecrawl] mapSite called for:", url, "options:", options);
 
 		try {
+			const { sanitized: sanitizedOptions, unknownKeys } =
+				sanitizeMapOptions(options);
+			if (unknownKeys.length > 0) {
+				console.warn("[firecrawl] mapSite dropping unsupported options", {
+					unknownKeys,
+				});
+			}
+
 			const requestBody: Record<string, unknown> = {
 				url,
-				includeSubdomains: options.includeSubdomains ?? false,
-				limit: options.limit ?? 100,
+				includeSubdomains: sanitizedOptions.includeSubdomains,
+				limit: sanitizedOptions.limit,
 			};
 
-			if (options.search) {
-				requestBody.search = options.search;
+			if (sanitizedOptions.search) {
+				requestBody.search = sanitizedOptions.search;
 			}
 
-			if (options.sitemapOnly) {
+			if (sanitizedOptions.sitemapOnly) {
 				requestBody.sitemap = "only";
-			} else if (options.ignoreSitemap) {
+			} else if (sanitizedOptions.ignoreSitemap) {
 				requestBody.sitemap = "skip";
 			}
 
-			if (options.ignoreCache !== undefined) {
-				requestBody.ignoreCache = options.ignoreCache;
-			} else if (options.maxAge !== undefined && options.maxAge <= 0) {
-				requestBody.ignoreCache = true;
+			if (sanitizedOptions.ignoreCache !== undefined) {
+				requestBody.ignoreCache = sanitizedOptions.ignoreCache;
 			}
 
 			const response = await this.requestWithRetry("/map", {

apps/api/src/trpc/routers/ai-agent.test.ts

@@ -1,5 +1,8 @@
 import { describe, expect, it } from "bun:test";
-import { getModelSelectionError } from "./ai-agent";
+import {
+	getGenerateBasePromptFirecrawlOptions,
+	getModelSelectionError,
+} from "./ai-agent";
 
 describe("ai-agent router model validation", () => {
 	it("rejects unknown models", () => {
@@ -29,3 +32,20 @@ describe("ai-agent router model validation", () => {
 		expect(result).toBeNull();
 	});
 });
+
+describe("generateBasePrompt Firecrawl options", () => {
+	it("uses scrape maxAge and map ignoreCache without maxAge", () => {
+		const firecrawlOptions = getGenerateBasePromptFirecrawlOptions();
+
+		expect(firecrawlOptions.scrapeOptions).toEqual({
+			maxAge: 3_600_000,
+		});
+		expect(firecrawlOptions.mapOptions).toEqual({
+			limit: 100,
+			ignoreCache: false,
+		});
+		expect(
+			"maxAge" in (firecrawlOptions.mapOptions as Record<string, unknown>)
+		).toBe(false);
+	});
+});

apps/api/src/trpc/routers/ai-agent.ts

@@ -427,6 +427,25 @@ export function getModelSelectionError(params: {
 	return null;
 }
 
+const GENERATE_BASE_PROMPT_FIRECRAWL_SCRAPE_MAX_AGE_MS = 3_600_000;
+
+export function getGenerateBasePromptFirecrawlOptions(): {
+	scrapeOptions: NonNullable<
+		Parameters<typeof firecrawlService.extractBrandInfo>[1]
+	>;
+	mapOptions: Parameters<typeof firecrawlService.mapSite>[1];
+} {
+	return {
+		scrapeOptions: {
+			maxAge: GENERATE_BASE_PROMPT_FIRECRAWL_SCRAPE_MAX_AGE_MS,
+		},
+		mapOptions: {
+			limit: 100,
+			ignoreCache: false,
+		},
+	};
+}
+
 async function resolveAiAgentModelForRead(params: {
 	db: Parameters<typeof updateAiAgentModel>[0];
 	agent: Awaited<ReturnType<typeof getAiAgentForWebsite>>;
@@ -715,15 +734,18 @@ export const aiAgentRouter = createTRPCRouter({
 			if (input.sourceUrl) {
 				// Run brand extraction (which scrapes internally) and site mapping in parallel
 				// extractBrandInfo returns company name, description, logo, favicon, AND markdown content
-				// Use maxAge of 1 hour (in ms) to enable Firecrawl caching - avoids re-paying
+				// Use scrape maxAge of 1 hour (in ms) to enable Firecrawl caching - avoids re-paying
 				// for API calls when user refreshes the page during onboarding
-				const cacheOptions = { maxAge: 3_600_000 };
+				const firecrawlOptions = getGenerateBasePromptFirecrawlOptions();
 				[brandInfo, mapResult] = await Promise.all([
-					firecrawlService.extractBrandInfo(input.sourceUrl, cacheOptions),
-					firecrawlService.mapSite(input.sourceUrl, {
-						limit: 100,
-						ignoreCache: false,
-					}),
+					firecrawlService.extractBrandInfo(
+						input.sourceUrl,
+						firecrawlOptions.scrapeOptions
+					),
+					firecrawlService.mapSite(
+						input.sourceUrl,
+						firecrawlOptions.mapOptions
+					),
 				]);
 
 				// Log what Firecrawl returned for debugging