Merge pull request #54 from ai-action/feat/web_fetch

Author: remarkablemark

src/constants/tool.ts

@@ -6,3 +6,4 @@ export const LIST_DIR = 'list_dir';
 export const GREP_SEARCH = 'grep_search';
 export const VIEW_RANGE = 'view_range';
 export const WEB_SEARCH = 'web_search';
+export const WEB_FETCH = 'web_fetch';

src/utils/tools/definitions.test.ts

@@ -3,7 +3,7 @@ import { READ_TOOLS, TOOLS, WRITE_TOOLS } from './definitions';
 describe('definitions', () => {
   describe('TOOLS', () => {
     it('exports tool definitions', () => {
-      expect(TOOLS).toHaveLength(8);
+      expect(TOOLS).toHaveLength(9);
       expect(TOOLS.map((t) => t.function.name)).toContain('read_file');
       expect(TOOLS.map((t) => t.function.name)).toContain('write_file');
       expect(TOOLS.map((t) => t.function.name)).toContain('edit_file');
@@ -12,6 +12,7 @@ describe('definitions', () => {
       expect(TOOLS.map((t) => t.function.name)).toContain('grep_search');
       expect(TOOLS.map((t) => t.function.name)).toContain('view_range');
       expect(TOOLS.map((t) => t.function.name)).toContain('web_search');
+      expect(TOOLS.map((t) => t.function.name)).toContain('web_fetch');
     });
   });
 

src/utils/tools/definitions.ts

@@ -126,6 +126,15 @@ export const TOOLS = [
     },
     ['query'],
   ),
+
+  defineTool(
+    TOOL.WEB_FETCH,
+    'Fetch the readable content of a webpage at the given URL',
+    {
+      url: { type: 'string', description: 'The full URL of the page to fetch' },
+    },
+    ['url'],
+  ),
 ];
 
 // tools that can be used during plan mode
@@ -135,6 +144,7 @@ export const READ_TOOLS = new Set<string>([
   TOOL.GREP_SEARCH,
   TOOL.VIEW_RANGE,
   TOOL.WEB_SEARCH,
+  TOOL.WEB_FETCH,
 ]);
 
 // tools that require approval before execution (safe mode or plan approval)

src/utils/tools/dispatcher.test.ts

@@ -158,5 +158,18 @@ describe('dispatcher', () => {
       expect(result.content).toContain('Source');
       expect(result.error).toBeUndefined();
     });
+
+    it('executes web_fetch tool', async () => {
+      mockFetch.mockResolvedValueOnce({
+        ok: true,
+        text: vi.fn().mockResolvedValue('# Fetched Page\n\nContent here.'),
+      } as unknown as Response);
+
+      const result = await executeTool('web_fetch', {
+        url: 'https://example.com',
+      });
+      expect(result.content).toContain('Fetched Page');
+      expect(result.error).toBeUndefined();
+    });
   });
 });

src/utils/tools/dispatcher.ts

@@ -9,7 +9,7 @@ import {
   writeFile,
 } from './filesystem';
 import { runShell } from './shell';
-import { webSearch } from './web/search';
+import { webFetch, webSearch } from './web';
 
 interface ToolOptions {
   allowedTools?: ReadonlySet<string>;
@@ -63,6 +63,9 @@ export async function executeTool(
     case TOOL.WEB_SEARCH:
       return await webSearch(args.query as string);
 
+    case TOOL.WEB_FETCH:
+      return await webFetch(args.url as string);
+
     default:
       return { content: '', error: `Unknown tool: ${name as string}` };
   }

src/utils/tools/web/fetch-page.test.ts

@@ -0,0 +1,100 @@
+import { webFetch } from './fetch-page';
+
+const mockFetch = vi.fn<typeof fetch>();
+
+describe('fetch-page', () => {
+  beforeEach(() => {
+    vi.clearAllMocks();
+    vi.stubGlobal('fetch', mockFetch);
+  });
+
+  describe('webFetch', () => {
+    it('returns error for empty URL', async () => {
+      const result = await webFetch('   ');
+      expect(result.error).toBe('URL cannot be empty');
+    });
+
+    it('returns content from Jina Reader on success', async () => {
+      mockFetch.mockResolvedValueOnce(
+        createFetchResponse('# Page Title\n\nSome content.', 200),
+      );
+
+      const result = await webFetch('https://example.com');
+      expect(result.content).toBe('# Page Title\n\nSome content.');
+      expect(result.error).toBeUndefined();
+      const calledUrl = (mockFetch.mock.calls[0] as [string, ...unknown[]])[0];
+      expect(calledUrl).toBe('https://r.jina.ai/https://example.com');
+    });
+
+    it('falls back to direct fetch when Jina Reader fails', async () => {
+      mockFetch
+        .mockRejectedValueOnce(new Error('rate limited'))
+        .mockResolvedValueOnce(
+          createFetchResponse(
+            '<html><body><p>Direct content</p></body></html>',
+            200,
+          ),
+        );
+
+      const result = await webFetch('https://example.com');
+      expect(result.content).toContain('Note: Jina Reader unavailable');
+      expect(result.content).toContain('Direct content');
+      expect(result.error).toBeUndefined();
+    });
+
+    it('returns error when both Jina and fallback fail', async () => {
+      mockFetch
+        .mockRejectedValueOnce(new Error('network error'))
+        .mockRejectedValueOnce(new Error('also failed'));
+
+      const result = await webFetch('https://example.com');
+      expect(result.error).toContain('Failed to fetch page: also failed');
+      expect(result.content).toBe('');
+    });
+
+    it('returns error when both fail with non-Error rejection', async () => {
+      mockFetch
+        .mockRejectedValueOnce('jina string error')
+        .mockRejectedValueOnce('fallback string error');
+
+      const result = await webFetch('https://example.com');
+      expect(result.error).toContain(
+        'Failed to fetch page: fallback string error',
+      );
+    });
+
+    it('returns error when fallback responds with non-OK status', async () => {
+      mockFetch
+        .mockRejectedValueOnce(new Error('jina down'))
+        .mockResolvedValueOnce(createFetchResponse('bad gateway', 502));
+
+      const result = await webFetch('https://example.com');
+      expect(result.error).toContain('Failed to fetch page: HTTP 502');
+    });
+
+    it('strips HTML tags in fallback content', async () => {
+      mockFetch
+        .mockRejectedValueOnce(new Error('jina error'))
+        .mockResolvedValueOnce(
+          createFetchResponse(
+            '<html><head><title>Title</title></head><body><h1>Heading</h1><p>Paragraph text.</p></body></html>',
+            200,
+          ),
+        );
+
+      const result = await webFetch('https://example.com');
+      expect(result.content).toContain('Heading');
+      expect(result.content).toContain('Paragraph text.');
+      expect(result.content).not.toContain('<h1>');
+      expect(result.content).not.toContain('<p>');
+    });
+  });
+});
+
+function createFetchResponse(body: string, status: number): Response {
+  return {
+    ok: status >= 200 && status < 300,
+    status,
+    text: vi.fn().mockResolvedValue(body),
+  } as unknown as Response;
+}

src/utils/tools/web/fetch-page.ts

@@ -0,0 +1,37 @@
+import type { ToolResult } from '../../../types';
+import { fetchText } from './fetch';
+import { cleanText, stripTags } from './utils';
+
+const JINA_READER_BASE_URL = 'https://r.jina.ai/';
+
+/**
+ * Fetch readable page content via Jina Reader, with fallback to direct fetch + HTML stripping
+ */
+export async function webFetch(url: string): Promise<ToolResult> {
+  const trimmedUrl = url.trim();
+  if (!trimmedUrl) {
+    return { content: '', error: 'URL cannot be empty' };
+  }
+
+  try {
+    const content = await fetchText(`${JINA_READER_BASE_URL}${trimmedUrl}`, {
+      Accept: 'text/plain',
+    });
+    return { content };
+  } catch {
+    // Fallback: direct fetch + strip HTML
+  }
+
+  try {
+    const html = await fetchText(trimmedUrl, { Accept: 'text/html' });
+    const content = cleanText(stripTags(html));
+    return {
+      content: `Note: Jina Reader unavailable, falling back to raw fetch.\n\n${content}`,
+    };
+  } catch (error) {
+    return {
+      content: '',
+      error: `Failed to fetch page: ${error instanceof Error ? error.message : String(error)}`,
+    };
+  }
+}

src/utils/tools/web/fetch.test.ts

@@ -0,0 +1,62 @@
+import { fetchJSON, fetchText } from './fetch';
+
+const mockFetch = vi.fn<typeof fetch>();
+
+describe('web/fetch', () => {
+  beforeEach(() => {
+    vi.clearAllMocks();
+    vi.stubGlobal('fetch', mockFetch);
+  });
+
+  describe('fetchText', () => {
+    it('returns response text on success', async () => {
+      mockFetch.mockResolvedValueOnce({
+        ok: true,
+        status: 200,
+        text: vi.fn().mockResolvedValue('hello world'),
+      } as unknown as Response);
+
+      const result = await fetchText('https://example.com', {
+        Accept: 'text/plain',
+      });
+      expect(result).toBe('hello world');
+    });
+
+    it('throws on non-OK status', async () => {
+      mockFetch.mockResolvedValueOnce({
+        ok: false,
+        status: 404,
+        text: vi.fn(),
+      } as unknown as Response);
+
+      await expect(fetchText('https://example.com', {})).rejects.toThrow(
+        'HTTP 404',
+      );
+    });
+  });
+
+  describe('fetchJSON', () => {
+    it('returns parsed JSON on success', async () => {
+      mockFetch.mockResolvedValueOnce({
+        ok: true,
+        status: 200,
+        json: vi.fn().mockResolvedValue({ key: 'value' }),
+      } as unknown as Response);
+
+      const result = await fetchJSON<{ key: string }>('https://example.com');
+      expect(result).toEqual({ key: 'value' });
+    });
+
+    it('throws on non-OK status', async () => {
+      mockFetch.mockResolvedValueOnce({
+        ok: false,
+        status: 500,
+        json: vi.fn(),
+      } as unknown as Response);
+
+      await expect(fetchJSON('https://example.com')).rejects.toThrow(
+        'HTTP 500',
+      );
+    });
+  });
+});

src/utils/tools/web/fetch.ts

@@ -2,16 +2,22 @@ import { PACKAGE } from '../../../constants';
 
 const FETCH_TIMEOUT_MS = 10_000;
 
+const BASE_HEADERS = {
+  'user-agent': `${PACKAGE.NAME}/${PACKAGE.VERSION}`,
+};
+
+type Headers = Record<string, string>;
+
 /**
  * Fetch text from URL with timeout and headers
  */
 export async function fetchText(
   url: string,
-  headers: Record<string, string>,
+  headers: Headers,
 ): Promise<string> {
   const response = await fetch(url, {
     headers: {
-      'user-agent': `${PACKAGE.NAME}/${PACKAGE.VERSION}`,
+      ...BASE_HEADERS,
       ...headers,
     },
     signal: AbortSignal.timeout(FETCH_TIMEOUT_MS),
@@ -23,3 +29,26 @@ export async function fetchText(
 
   return response.text();
 }
+
+/**
+ * Fetch and parse JSON from URL with timeout and headers
+ */
+export async function fetchJSON<T>(
+  url: string,
+  headers: Headers = {},
+): Promise<T> {
+  const response = await fetch(url, {
+    headers: {
+      ...BASE_HEADERS,
+      Accept: 'application/json',
+      ...headers,
+    },
+    signal: AbortSignal.timeout(FETCH_TIMEOUT_MS),
+  });
+
+  if (!response.ok) {
+    throw new Error(`HTTP ${response.status.toString()}`);
+  }
+
+  return response.json() as Promise<T>;
+}

src/utils/tools/web/index.ts

@@ -1 +1,2 @@
+export { webFetch } from './fetch-page';
 export { webSearch } from './search';