diff --git a/docs/superpowers/plans/2026-07-01-web-fetch-domain-filters.md b/docs/superpowers/plans/2026-07-01-web-fetch-domain-filters.md new file mode 100644 index 00000000..1e565351 --- /dev/null +++ b/docs/superpowers/plans/2026-07-01-web-fetch-domain-filters.md @@ -0,0 +1,74 @@ +# Web Fetch Domain Filters Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Expose Anthropic-compatible `allowed_domains` and `blocked_domains` parameters on the `web_fetch` MCP tool and route them to the existing engine host filters. + +**Architecture:** Keep enforcement in the existing URL safety and fetch engine path. Extend only the MCP schema and handler adapter so `allowed_domains` maps to `allowHosts` and `blocked_domains` maps to `denyHosts`. + +**Tech Stack:** TypeScript, Vitest, existing `@frontagent/mcp-web-fetch` package. + +--- + +### Task 1: Add Schema and Handler Coverage + +**Files:** +- Modify: `packages/mcp-web-fetch/src/tools.security.test.ts` + +- [x] **Step 1: Add schema assertions** + +Import `webFetchSchema` and `vi`, then add tests asserting the schema exposes `allowed_domains` and `blocked_domains` as optional string arrays. + +- [x] **Step 2: Add handler no-network assertions** + +Patch `globalThis.fetch` with a throwing mock for each domain-filter test, call `handleWebFetchTool('web_fetch', ...)`, assert the result fails with allow/block wording, and assert the fetch mock was not called. + +- [x] **Step 3: Run focused test and confirm failure before implementation** + +Run: `pnpm --dir packages/mcp-web-fetch test -- --run src/tools.security.test.ts` + +Expected before implementation: schema/domain-filter tests fail because schema fields are missing and handler does not pass filters to the engine. + +### Task 2: Map Tool Parameters to Engine Options + +**Files:** +- Modify: `packages/mcp-web-fetch/src/tools.ts` + +- [x] **Step 1: Extend `webFetchSchema`** + +Add optional `allowed_domains` and `blocked_domains` properties with `type: 'array'`, string `items`, and descriptions noting their mapping to `allowHosts` and `denyHosts`. + +- [x] **Step 2: Extend `handleWebFetchTool`** + +Pass `allowHosts: args.allowed_domains as string[] | undefined` and `denyHosts: args.blocked_domains as string[] | undefined` to `fetchUrl`. + +- [x] **Step 3: Run focused tests** + +Run: `pnpm --dir packages/mcp-web-fetch test -- --run src/tools.security.test.ts` + +Expected after implementation: all tests in `tools.security.test.ts` pass. + +### Task 3: Verify and Prepare PR + +**Files:** +- No additional planned code files. + +- [x] **Step 1: Run mcp-web-fetch package tests** + +Run: `pnpm --dir packages/mcp-web-fetch test` + +Expected: package test suite passes. + +- [x] **Step 2: Run required broader gate** + +Run: `pnpm quality:precommit` + +Expected: lint, typecheck, test, and workflow tests pass. + +- [x] **Step 3: Run GitNexus final diff analysis** + +Run GitNexus `detect_changes` for `/tmp/frontagent-issue-380` before committing and include the blast radius summary in the PR body. + +- [ ] **Step 4: Commit, push, and open PR** + +Commit with a focused message, push `feat/web-fetch-domain-filters-380`, and open a PR to `develop` with `Closes #380`. diff --git a/packages/mcp-web-fetch/src/tools.security.test.ts b/packages/mcp-web-fetch/src/tools.security.test.ts index 48f82822..867fed50 100644 --- a/packages/mcp-web-fetch/src/tools.security.test.ts +++ b/packages/mcp-web-fetch/src/tools.security.test.ts @@ -1,5 +1,5 @@ -import { describe, expect, it } from 'vitest'; -import { handleWebFetchTool } from './tools.js'; +import { describe, expect, it, vi } from 'vitest'; +import { handleWebFetchTool, webFetchSchema } from './tools.js'; describe('mcp-web-fetch SSRF guards (no network)', () => { it('rejects loopback IPv4 addresses', async () => { @@ -54,4 +54,81 @@ describe('mcp-web-fetch SSRF guards (no network)', () => { expect(result.success).toBe(false); expect(result.error).toMatch(/unknown/i); }); + + it('exposes allowed_domains and blocked_domains schema parameters', () => { + expect(webFetchSchema.inputSchema.properties.allowed_domains).toMatchObject({ + type: 'array', + items: { type: 'string' }, + minItems: 1, + }); + expect(webFetchSchema.inputSchema.properties.blocked_domains).toMatchObject({ + type: 'array', + items: { type: 'string' }, + minItems: 1, + }); + expect(webFetchSchema.inputSchema.required).toEqual(['url']); + }); + + it('maps allowed_domains to allowHosts and rejects before fetch', async () => { + const originalFetch = globalThis.fetch; + const fetchMock = vi.fn(() => { + throw new Error('network attempted'); + }); + globalThis.fetch = fetchMock as unknown as typeof fetch; + + try { + const result = await handleWebFetchTool('web_fetch', { + url: 'https://example.com/', + allowed_domains: ['docs.example.com'], + }); + + expect(result.success).toBe(false); + expect(result.error).toMatch(/not in allowlist|allow/i); + expect(fetchMock).not.toHaveBeenCalled(); + } finally { + globalThis.fetch = originalFetch; + } + }); + + it('maps blocked_domains to denyHosts and rejects before fetch', async () => { + const originalFetch = globalThis.fetch; + const fetchMock = vi.fn(() => { + throw new Error('network attempted'); + }); + globalThis.fetch = fetchMock as unknown as typeof fetch; + + try { + const result = await handleWebFetchTool('web_fetch', { + url: 'https://example.com/', + blocked_domains: ['example.com'], + }); + + expect(result.success).toBe(false); + expect(result.error).toMatch(/denylist|blocked|denied/i); + expect(fetchMock).not.toHaveBeenCalled(); + } finally { + globalThis.fetch = originalFetch; + } + }); + + it('rejects an empty allowed_domains list before fetch', async () => { + const originalFetch = globalThis.fetch; + const fetchMock = vi.fn(() => { + throw new Error('network attempted'); + }); + globalThis.fetch = fetchMock as unknown as typeof fetch; + + try { + const result = await handleWebFetchTool('web_fetch', { + url: 'https://example.com/', + allowed_domains: [], + }); + + expect(result.success).toBe(false); + expect(result.error).toMatch(/allowed_domains.*at least one domain/i); + expect(fetchMock).not.toHaveBeenCalled(); + } finally { + globalThis.fetch = originalFetch; + } + }); }); diff --git a/packages/mcp-web-fetch/src/tools.ts b/packages/mcp-web-fetch/src/tools.ts index cb0d91c4..d8a4be03 100644 --- a/packages/mcp-web-fetch/src/tools.ts +++ b/packages/mcp-web-fetch/src/tools.ts @@ -38,6 +38,18 @@ export const webFetchSchema = { maximum: 5000000, description: '响应体最大字节数，超出后截断，默认 2000000，硬上限 5000000', }, + allowed_domains: { + type: 'array', + items: { type: 'string' }, + minItems: 1, + description: '可选允许域名列表；映射到底层引擎 allowHosts，仅允许抓取这些主机名。', + }, + blocked_domains: { + type: 'array', + items: { type: 'string' }, + minItems: 1, + description: '可选阻止域名列表；映射到底层引擎 denyHosts，拒绝抓取这些主机名。', + }, }, required: ['url'], }, @@ -56,10 +68,28 @@ export async function handleWebFetchTool( try { switch (toolName) { case 'web_fetch': { + const allowHosts = args.allowed_domains as string[] | undefined; + const denyHosts = args.blocked_domains as string[] | undefined; + + if (Array.isArray(allowHosts) && allowHosts.length === 0) { + return { + success: false, + error: 'allowed_domains must contain at least one domain when provided', + }; + } + if (Array.isArray(denyHosts) && denyHosts.length === 0) { + return { + success: false, + error: 'blocked_domains must contain at least one domain when provided', + }; + } + const data = await fetchUrl(args.url as string, { format: args.format as 'text' | 'html' | undefined, timeoutMs: args.timeoutMs as number | undefined, maxBytes: args.maxBytes as number | undefined, + allowHosts, + denyHosts, }); return { success: true, data }; }