diff --git a/adapters/_fixtures/llmstxt_example.txt b/adapters/_fixtures/llmstxt_example.txt new file mode 100644 index 0000000..209945c --- /dev/null +++ b/adapters/_fixtures/llmstxt_example.txt @@ -0,0 +1,18 @@ +# Example Agent Surface + +> A compact map of agent-readable documentation and APIs. + +Use the stable docs before the API reference when bootstrapping an integration. + +## Docs + +- [Quickstart](./docs/quickstart.md): Start here for setup. +- [API Reference](https://example.com/api): Endpoint documentation. + +## Support + +- [Status Page](https://status.example.com): Current service status. + +## Optional + +- [Release Notes](/releases): Lower-priority product updates. diff --git a/adapters/_test/run.mjs b/adapters/_test/run.mjs index d18b2f3..5a0b21a 100644 --- a/adapters/_test/run.mjs +++ b/adapters/_test/run.mjs @@ -88,6 +88,175 @@ test("openapi exports an action handler", async () => { assert.equal(typeof openapi.actions.openapi_request, "function"); }); +// --- llms.txt --- +test("llmstxt detects direct llms.txt URLs only", async () => { + const llmstxt = await import("../llmstxt.js"); + assert.ok(llmstxt.detect({ url: "https://example.com/llms.txt" })); + assert.ok(llmstxt.detect({ url: "https://example.com/llms-full.txt?raw=1" })); + assert.equal(llmstxt.detect({ url: "file:///tmp/llms.txt" }), null); + assert.equal(llmstxt.detect({ url: "ftp://example.com/llms.txt" }), null); + assert.equal(llmstxt.detect({ url: "https://example.com/docs" }), null); + assert.equal(llmstxt.detect({ url: "https://example.com/not-llms.txt.backup" }), null); +}); + +test("llmstxt extract parses sections and declared links into tools", async () => { + const llmstxt = await import("../llmstxt.js"); + const body = await fixture("llmstxt_example.txt"); + const originalFetch = globalThis.fetch; + globalThis.fetch = async (url, opts) => { + assert.equal(String(url), "https://example.com/llms.txt"); + assert.equal(opts.credentials, "omit"); + return new Response(body, { + status: 200, + headers: { "content-type": "text/plain; charset=utf-8" }, + }); + }; + try { + const data = await llmstxt.extract({ adapter: "llmstxt", sourceUrl: "https://example.com/llms.txt" }); + assert.equal(data.product.title, "Example Agent Surface"); + assert.equal(data.product.summary, "A compact map of agent-readable documentation and APIs."); + assert.equal(data.product.description, "Use the stable docs before the API reference when bootstrapping an integration."); + assert.equal(data.product.section_count, 3); + assert.equal(data.product.link_count, 4); + const names = data.tools.map((t) => t.name); + assert.deepEqual(names, ["list_sections", "get_section", "fetch_link"]); + const list = data.tools.find((t) => t.name === "list_sections"); + assert.equal(list.result.sections[0].name, "Docs"); + assert.equal(list.result.sections[0].links[0].url, "https://example.com/docs/quickstart.md"); + const optional = list.result.sections.find((section) => section.name === "Optional"); + assert.equal(optional.optional, true); + assert.equal(optional.links[0].optional, true); + assert.equal(optional.links[0].url, "https://example.com/releases"); + const fetchTool = data.tools.find((t) => t.name === "fetch_link"); + assert.equal(fetchTool.inputSchema.required[0], "url"); + assert.equal(fetchTool.action.kind, "llmstxt_fetch_link"); + } finally { + globalThis.fetch = originalFetch; + } +}); + +test("llmstxt action returns declared sections by name", async () => { + const llmstxt = await import("../llmstxt.js"); + const section = await llmstxt.actions.llmstxt_get_section({ + sections: [{ name: "Docs", links: [{ name: "Quickstart", url: "https://example.com/docs" }] }], + args: { section: "docs" }, + }); + assert.equal(section.name, "Docs"); + await assert.rejects( + llmstxt.actions.llmstxt_get_section({ sections: [], args: { section: "Docs" } }), + /not declared/ + ); +}); + +test("llmstxt action fetches only declared links", async () => { + const llmstxt = await import("../llmstxt.js"); + const links = [ + { name: "Quickstart", url: "https://example.com/docs/quickstart.md", description: "Start here." }, + ]; + const originalFetch = globalThis.fetch; + globalThis.fetch = async (url, opts) => { + assert.equal(String(url), "https://example.com/docs/quickstart.md"); + assert.equal(opts.credentials, "omit"); + return new Response("# Quickstart\n\nHello agents.", { + status: 200, + headers: { "content-type": "text/markdown" }, + }); + }; + try { + const value = await llmstxt.actions.llmstxt_fetch_link({ + links, + args: { url: "https://example.com/docs/quickstart.md" }, + }); + assert.equal(value.status, 200); + assert.equal(value.url, "https://example.com/docs/quickstart.md"); + assert.match(value.body, /Hello agents/); + await assert.rejects( + llmstxt.actions.llmstxt_fetch_link({ links, args: { url: "https://other.example.com/" } }), + /not declared/ + ); + } finally { + globalThis.fetch = originalFetch; + } +}); + +test("llmstxt rejects non-http declared links", async () => { + const llmstxt = await import("../llmstxt.js"); + let fetchCalls = 0; + const originalFetch = globalThis.fetch; + globalThis.fetch = async () => { + fetchCalls++; + return new Response( + "# Example\n\n## Docs\n\n- [Safe](https://example.com/docs)\n- [Mail](mailto:docs@example.com)\n- [Local](http://127.0.0.1/admin)\n- [Metadata](http://169.254.169.254/latest)", + { status: 200 } + ); + }; + try { + const data = await llmstxt.extract({ adapter: "llmstxt", sourceUrl: "https://example.com/llms.txt" }); + const fetchTool = data.tools.find((t) => t.name === "fetch_link"); + assert.deepEqual(fetchTool.inputSchema.properties.url.enum, ["https://example.com/docs"]); + await assert.rejects( + llmstxt.actions.llmstxt_fetch_link({ + links: [{ name: "Local", url: "file:///etc/passwd" }], + args: { url: "file:///etc/passwd" }, + }), + /unsupported URL/ + ); + assert.equal(fetchCalls, 1); + } finally { + globalThis.fetch = originalFetch; + } +}); + +test("llmstxt action truncates large declared-link responses", async () => { + const llmstxt = await import("../llmstxt.js"); + const originalFetch = globalThis.fetch; + globalThis.fetch = async () => new Response("x".repeat(20005), { status: 200 }); + try { + const value = await llmstxt.actions.llmstxt_fetch_link({ + links: [{ name: "Large", url: "https://example.com/large.md" }], + args: { url: "https://example.com/large.md" }, + }); + assert.equal(value.body.length, 20000); + assert.equal(value.truncated, true); + } finally { + globalThis.fetch = originalFetch; + } +}); + +test("llmstxt full context files can produce a bounded context tool without links", async () => { + const llmstxt = await import("../llmstxt.js"); + const originalFetch = globalThis.fetch; + globalThis.fetch = async () => + new Response("# Full Context\n\nThis file contains complete documentation for agents.", { + status: 200, + headers: { "content-type": "text/plain" }, + }); + try { + const data = await llmstxt.extract(llmstxt.detect({ url: "https://example.com/llms-full.txt" })); + assert.equal(data.product.title, "Full Context"); + assert.equal(data.product.link_count, 0); + assert.deepEqual(data.tools.map((t) => t.name), ["get_full_context"]); + assert.match(data.tools[0].result.body, /complete documentation/); + assert.equal(data.tools[0].result.truncated, false); + } finally { + globalThis.fetch = originalFetch; + } +}); + +test("llmstxt rejects malformed files without declared links", async () => { + const llmstxt = await import("../llmstxt.js"); + const originalFetch = globalThis.fetch; + globalThis.fetch = async () => new Response("# Empty\n\nNo links here.", { status: 200 }); + try { + await assert.rejects( + llmstxt.extract({ adapter: "llmstxt", sourceUrl: "https://example.com/llms.txt" }), + /no declared links/ + ); + } finally { + globalThis.fetch = originalFetch; + } +}); + // --- llm fallback --- test("llm.detect needs an API key", async () => { const llm = await import("../llm.js"); diff --git a/adapters/llmstxt.js b/adapters/llmstxt.js new file mode 100644 index 0000000..7db57ae --- /dev/null +++ b/adapters/llmstxt.js @@ -0,0 +1,347 @@ +// llmstxt.js — turn a site's declared llms.txt map into agent-readable tools. +// +// Detection is URL-only and side-effect-free: direct /llms.txt or +// /llms-full.txt URLs. Extraction fetches the text file and parses the common +// H1 + blockquote + H2 sections + markdown link-list shape. + +export const ID = "llmstxt"; + +const LLMS_PATH_RE = /\/llms(?:-full)?\.txt$/i; +const LLMS_FULL_PATH_RE = /\/llms-full\.txt$/i; +const MAX_FETCH_BODY = 20000; + +const CHROME_UA = + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 " + + "(KHTML, like Gecko) Chrome/132.0.0.0 Safari/537.36"; + +export function detect({ url }) { + try { + const parsed = new URL(url); + if (!isHttpUrl(parsed.toString())) return null; + if (!LLMS_PATH_RE.test(parsed.pathname)) return null; + return { adapter: ID, sourceUrl: parsed.toString(), full: LLMS_FULL_PATH_RE.test(parsed.pathname) }; + } catch { + return null; + } +} + +export async function extract(ctx) { + const res = await fetch(ctx.sourceUrl, { + headers: { + accept: "text/plain, text/markdown, text/*, */*", + "user-agent": CHROME_UA, + }, + credentials: "omit", + }); + if (!res.ok) throw new Error(`llmstxt fetch failed: ${res.status}`); + + const text = await res.text(); + const parsed = parseLlmsTxt(text, ctx.sourceUrl); + const links = parsed.sections.flatMap((section) => + section.links.map((link) => ({ ...link, section: section.name })) + ); + if (!links.length) { + if (ctx.full) return buildFullContextPayload(parsed, text, ctx.sourceUrl); + throw new Error("llmstxt: no declared links found"); + } + + const product = { + title: parsed.title, + name: parsed.title, + url: ctx.sourceUrl, + summary: parsed.summary, + description: parsed.description, + section_count: parsed.sections.length, + link_count: links.length, + }; + + return { + product, + variants: [], + tools: [ + { + name: "list_sections", + description: `List the declared llms.txt sections for ${parsed.title}.`, + inputSchema: { type: "object", properties: {}, required: [] }, + result: { + title: parsed.title, + summary: parsed.summary, + description: parsed.description, + sections: parsed.sections, + }, + }, + { + name: "get_section", + description: "Return one declared llms.txt section by name.", + inputSchema: { + type: "object", + properties: { + section: { + type: "string", + enum: parsed.sections.map((section) => section.name), + }, + }, + required: ["section"], + }, + action: { kind: "llmstxt_get_section", sections: parsed.sections }, + }, + { + name: "fetch_link", + description: "Fetch one URL declared by llms.txt.", + inputSchema: { + type: "object", + properties: { + url: { + type: "string", + enum: links.map((link) => link.url), + }, + }, + required: ["url"], + }, + action: { kind: "llmstxt_fetch_link", links }, + }, + ], + }; +} + +function parseLlmsTxt(text, sourceUrl) { + const lines = String(text || "").replace(/^\uFEFF/, "").split(/\r?\n/); + let title = ""; + const summary = []; + const details = []; + const sections = []; + let current = null; + let beforeFirstSection = true; + + for (const raw of lines) { + const line = raw.trim(); + if (!line) continue; + + const h1 = line.match(/^#\s+(.+)$/); + if (h1 && !title) { + title = cleanText(h1[1]); + continue; + } + + const h2 = line.match(/^##\s+(.+)$/); + if (h2) { + const name = cleanText(h2[1]); + current = { name, links: [] }; + if (isOptionalSectionName(name)) current.optional = true; + sections.push(current); + beforeFirstSection = false; + continue; + } + + if (beforeFirstSection && line.startsWith(">")) { + summary.push(cleanText(line.replace(/^>\s?/, ""))); + continue; + } + + const link = parseMarkdownListLink(line, sourceUrl); + if (link) { + if (!current) { + current = { name: "Overview", links: [] }; + sections.push(current); + } + current.links.push(current.optional ? { ...link, optional: true } : link); + beforeFirstSection = false; + continue; + } + + if (beforeFirstSection) { + details.push(cleanText(line)); + } + } + + let fallbackTitle = "llms.txt"; + try { + fallbackTitle = new URL(sourceUrl).hostname; + } catch {} + + return { + title: title || fallbackTitle, + summary: summary.join(" ").trim() || undefined, + description: details.join(" ").trim() || undefined, + sections: sections.filter((section) => section.links.length), + }; +} + +function parseMarkdownListLink(line, sourceUrl) { + const match = line.match(/^\s*[-*]\s+\[([^\]]+)]\(([^)]+)\)(?:\s*[:—-]\s*(.*))?\s*$/); + if (!match) return null; + const name = cleanText(match[1]); + const rawUrl = match[2].trim(); + const description = cleanText(match[3] || ""); + let url; + try { + url = new URL(rawUrl, sourceUrl).toString(); + } catch { + return null; + } + if (!isFetchableUrl(url)) return null; + return description ? { name, url, description } : { name, url }; +} + +function buildFullContextPayload(parsed, text, sourceUrl) { + const body = String(text || "").slice(0, MAX_FETCH_BODY); + return { + product: { + title: parsed.title, + name: parsed.title, + url: sourceUrl, + summary: parsed.summary, + description: parsed.description, + section_count: 0, + link_count: 0, + }, + variants: [], + tools: [ + { + name: "get_full_context", + description: `Return the bounded llms-full.txt context for ${parsed.title}.`, + inputSchema: { type: "object", properties: {}, required: [] }, + result: { + title: parsed.title, + summary: parsed.summary, + description: parsed.description, + url: sourceUrl, + body, + truncated: String(text || "").length > MAX_FETCH_BODY, + }, + }, + ], + }; +} + +function cleanText(value) { + return String(value || "").replace(/\s+/g, " ").trim(); +} + +function isOptionalSectionName(name) { + return name.trim().toLowerCase() === "optional"; +} + +function isHttpUrl(url) { + try { + const parsed = new URL(url); + return parsed.protocol === "http:" || parsed.protocol === "https:"; + } catch { + return false; + } +} + +function isFetchableUrl(url) { + try { + const parsed = new URL(url); + if (!isHttpUrl(parsed.toString())) return false; + return !isBlockedHost(parsed.hostname); + } catch { + return false; + } +} + +function isBlockedHost(hostname) { + const host = String(hostname || "").toLowerCase().replace(/^\[/, "").replace(/]$/, "").replace(/\.$/, ""); + if (!host) return true; + if ( + host === "localhost" || + host.endsWith(".localhost") || + host.endsWith(".internal") || + host === "metadata.google.internal" + ) { + return true; + } + + const mappedIpv4 = host.match(/^::ffff:(\d{1,3}(?:\.\d{1,3}){3})$/); + if (mappedIpv4) return isBlockedHost(mappedIpv4[1]); + + const ipv4 = host.match(/^(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})$/); + if (ipv4) { + const parts = ipv4.slice(1).map((part) => Number(part)); + if (parts.some((part) => part > 255)) return true; + const [a, b] = parts; + return ( + a === 0 || + a === 10 || + a === 127 || + (a === 169 && b === 254) || + (a === 172 && b >= 16 && b <= 31) || + (a === 192 && b === 168) || + (a === 100 && b >= 64 && b <= 127) || + a >= 224 + ); + } + + return ( + host === "::1" || + host.startsWith("fe80:") || + host.startsWith("fc") || + host.startsWith("fd") + ); +} + +async function readTextWithLimit(res, maxChars) { + const reader = res.body?.getReader?.(); + if (!reader) { + const text = await res.text(); + return { body: text.slice(0, maxChars), truncated: text.length > maxChars }; + } + + const decoder = new TextDecoder(); + let body = ""; + let truncated = false; + + while (true) { + const { done, value } = await reader.read(); + if (done) break; + if (!value) continue; + const chunk = decoder.decode(value, { stream: true }); + const remaining = maxChars - body.length; + if (remaining > 0) body += chunk.slice(0, remaining); + if (chunk.length > remaining) { + truncated = true; + await reader.cancel(); + break; + } + } + + if (!truncated) body += decoder.decode(); + return { body, truncated }; +} + +export const actions = { + llmstxt_get_section: async ({ sections, args }) => { + const wanted = String(args?.section || "").trim().toLowerCase(); + const section = (sections || []).find((item) => item.name.toLowerCase() === wanted); + if (!section) throw new Error("llmstxt section not declared"); + return section; + }, + + llmstxt_fetch_link: async ({ links, args }) => { + const wanted = String(args?.url || args?.name || "").trim(); + const link = (links || []).find( + (item) => item.url === wanted || item.name.toLowerCase() === wanted.toLowerCase() + ); + if (!link) throw new Error("llmstxt link not declared"); + if (!isFetchableUrl(link.url)) throw new Error("llmstxt unsupported URL"); + + const res = await fetch(link.url, { + headers: { + accept: "text/plain, text/markdown, text/html, application/json, */*", + "user-agent": CHROME_UA, + }, + credentials: "omit", + }); + if (res.url && !isFetchableUrl(res.url)) throw new Error("llmstxt unsupported URL"); + const { body, truncated } = await readTextWithLimit(res, MAX_FETCH_BODY); + return { + name: link.name, + url: link.url, + status: res.status, + content_type: res.headers.get("content-type") || undefined, + body, + truncated, + }; + }, +}; diff --git a/worker/src/engine.ts b/worker/src/engine.ts index 273db8b..8a75b15 100644 --- a/worker/src/engine.ts +++ b/worker/src/engine.ts @@ -10,6 +10,7 @@ import * as shopify from "../../adapters/shopify.js"; import * as jsonld from "../../adapters/jsonld.js"; import * as openapi from "../../adapters/openapi.js"; +import * as llmstxt from "../../adapters/llmstxt.js"; import * as llm from "../../adapters/llm.js"; import * as coingecko from "../../adapters/coingecko.js"; import * as defillama from "../../adapters/defillama.js"; @@ -30,8 +31,12 @@ export type EngineEnv = { // Minimal shape of what Workers' ExecutionContext gives us (waitUntil only). type WaitCtx = { waitUntil(p: Promise): void }; -// Crypto/data adapters share the same structure: detect → extract → action. +// Format and crypto/data adapters share the same structure: detect → extract → action. // They sit after openapi and before the HTML fetch in the chain. +export const FORMAT_ADAPTERS = [ + { name: "llmstxt", mod: llmstxt, ttl: 24 * 3600 }, +]; + export const CRYPTO_ADAPTERS = [ { name: "coingecko", mod: coingecko, ttl: 60 }, { name: "dexscreener", mod: dexscreener, ttl: 60 }, @@ -103,7 +108,7 @@ export type ResolveResult = | { ok: true; payload: ToolsPayload; from: "cache" | "live" | "llm"; cached_at?: number } | { ok: false; status: number; body: any }; -// The 5-tier cascade (cache → shopify → openapi → crypto → jsonld → llm). +// The cascade: cache → shopify → openapi → format → crypto → jsonld → llm. // Mirrors what /api/v1/tools used to do inline; callers decide how to gate + // serialize. `authUserId` lets the LLM fallback use the caller's connected // Anthropic/Claude-Max OAuth token (shifts inference cost to their account). @@ -149,6 +154,24 @@ export async function resolveTools( } } + for (const { name, mod, ttl } of FORMAT_ADAPTERS) { + const fctx = (mod as any).detect({ url }); + if (!fctx) continue; + try { + const data = await (mod as any).extract(fctx); + const payload: ToolsPayload = { + adapter: name, + tools: data.tools, + product: data.product, + variants: data.variants, + }; + ctx.waitUntil(writeCache(env, url, payload, ttl)); + return { ok: true, payload, from: "live" }; + } catch { + // fall through to crypto/jsonld + } + } + for (const { name, mod, ttl } of CRYPTO_ADAPTERS) { const cctx = (mod as any).detect({ url }); if (!cctx) continue; @@ -279,11 +302,32 @@ export async function executeTool( } } - for (const { name, mod } of CRYPTO_ADAPTERS) { - const cctx = (mod as any).detect({ url }); - if (!cctx) continue; + const oaCtx = openapi.detect({ url }); + if (oaCtx) { try { - const data = await (mod as any).extract(cctx); + const data = await openapi.extract(oaCtx); + const tool = data.tools.find((t: any) => t.name === toolName); + if (!tool) return { ok: false, status: 404, body: { error: `tool ${toolName} not found` } }; + const kind = tool.action?.kind; + const handler = (openapi.actions as any)[kind]; + if (!handler) return { ok: false, status: 500, body: { error: "no action handler" } }; + const resolveToken = async (host: string) => { + const target = `https://${host}`; + const r = await resolveTokenForUrl(env, opts.userId, target); + return r?.access_token || null; + }; + const value = await handler({ ...tool.action, args: args || {}, resolveToken }); + return { ok: true, value }; + } catch (err: any) { + return { ok: false, status: 500, body: { ok: false, error: String(err?.message || err) } }; + } + } + + for (const { name, mod } of FORMAT_ADAPTERS) { + const fctx = (mod as any).detect({ url }); + if (!fctx) continue; + try { + const data = await (mod as any).extract(fctx); const tool = data.tools.find((t: any) => t.name === toolName); if (!tool) return { ok: false, status: 404, body: { error: `tool ${toolName} not found` } }; if (tool.result !== undefined) return { ok: true, value: tool.result }; @@ -298,24 +342,22 @@ export async function executeTool( } } - const oaCtx = openapi.detect({ url }); - if (oaCtx) { + for (const { name, mod } of CRYPTO_ADAPTERS) { + const cctx = (mod as any).detect({ url }); + if (!cctx) continue; try { - const data = await openapi.extract(oaCtx); + const data = await (mod as any).extract(cctx); const tool = data.tools.find((t: any) => t.name === toolName); if (!tool) return { ok: false, status: 404, body: { error: `tool ${toolName} not found` } }; + if (tool.result !== undefined) return { ok: true, value: tool.result }; const kind = tool.action?.kind; - const handler = (openapi.actions as any)[kind]; - if (!handler) return { ok: false, status: 500, body: { error: "no action handler" } }; - const resolveToken = async (host: string) => { - const target = `https://${host}`; - const r = await resolveTokenForUrl(env, opts.userId, target); - return r?.access_token || null; - }; - const value = await handler({ ...tool.action, args: args || {}, resolveToken }); + if (!kind) return { ok: false, status: 400, body: { error: "static result tool — no action to execute" } }; + const handler = (mod as any).actions?.[kind]; + if (!handler) return { ok: false, status: 500, body: { error: `no action handler for ${kind}` } }; + const value = await handler({ ...tool.action, args: args || {} }); return { ok: true, value }; } catch (err: any) { - return { ok: false, status: 500, body: { ok: false, error: String(err?.message || err) } }; + return { ok: false, status: 500, body: { ok: false, error: String(err?.message || err), adapter: name } }; } } diff --git a/worker/test/engine_order.test.ts b/worker/test/engine_order.test.ts new file mode 100644 index 0000000..6f493c3 --- /dev/null +++ b/worker/test/engine_order.test.ts @@ -0,0 +1,52 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { executeTool, resolveTools } from "../src/engine"; +import { kvMock } from "./helpers"; + +const OVERLAP_URL = "https://example.com/api-docs/llms.txt"; +const OPENAPI_SPEC = { + openapi: "3.0.0", + info: { title: "Overlap API", version: "1.0.0" }, + servers: [{ url: "https://api.example.com" }], + paths: { + "/ping": { + get: { + operationId: "ping", + summary: "Ping the API", + responses: {}, + }, + }, + }, +}; + +describe("engine adapter order", () => { + const originalFetch = globalThis.fetch; + + afterEach(() => { + globalThis.fetch = originalFetch; + vi.restoreAllMocks(); + }); + + it("executes with the same adapter precedence used by resolveTools", async () => { + globalThis.fetch = vi.fn(async (url: any) => { + const href = String(url); + if (href === OVERLAP_URL) return new Response(JSON.stringify(OPENAPI_SPEC), { status: 200 }); + if (href === "https://api.example.com/ping") return new Response(JSON.stringify({ ok: true }), { status: 200 }); + return new Response("not found", { status: 404 }); + }) as any; + + const env = { CACHE: kvMock(), KEYS: kvMock() } as any; + const ctx = { waitUntil: (_promise: Promise) => {} }; + + const resolved = await resolveTools(env, ctx, OVERLAP_URL, { fresh: true }); + expect(resolved.ok).toBe(true); + if (!resolved.ok) return; + expect(resolved.payload.adapter).toBe("openapi"); + expect(resolved.payload.tools.map((tool: any) => tool.name)).toContain("ping"); + + const executed = await executeTool(env, { url: OVERLAP_URL, tool: "ping" }); + expect(executed.ok).toBe(true); + if (!executed.ok) return; + expect(executed.value.status).toBe(200); + expect(executed.value.data).toEqual({ ok: true }); + }); +}); diff --git a/worker/test/llmstxt_engine.test.ts b/worker/test/llmstxt_engine.test.ts new file mode 100644 index 0000000..5b87d5d --- /dev/null +++ b/worker/test/llmstxt_engine.test.ts @@ -0,0 +1,65 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { executeTool, resolveTools } from "../src/engine"; +import { kvMock } from "./helpers"; + +const LLMS_URL = "https://example.com/llms.txt"; +const QUICKSTART_URL = "https://example.com/docs/quickstart.md"; +const LLMS_BODY = `# Example Agent Surface + +> A compact map of agent-readable documentation and APIs. + +## Docs + +- [Quickstart](./docs/quickstart.md): Start here for setup. +`; + +describe("llmstxt engine integration", () => { + const originalFetch = globalThis.fetch; + + afterEach(() => { + globalThis.fetch = originalFetch; + vi.restoreAllMocks(); + }); + + it("resolves llms.txt tools and executes a declared fetch action", async () => { + globalThis.fetch = vi.fn(async (url: any) => { + const href = String(url); + if (href === LLMS_URL) { + return new Response(LLMS_BODY, { status: 200, headers: { "content-type": "text/plain" } }); + } + if (href === QUICKSTART_URL) { + return new Response("# Quickstart\n\nHello agents.", { + status: 200, + headers: { "content-type": "text/markdown" }, + }); + } + return new Response("not found", { status: 404 }); + }) as any; + + const env = { CACHE: kvMock(), KEYS: kvMock() } as any; + const waited: Promise[] = []; + const ctx = { waitUntil: (promise: Promise) => waited.push(promise) }; + + const resolved = await resolveTools(env, ctx, LLMS_URL, { fresh: true }); + expect(resolved.ok).toBe(true); + if (!resolved.ok) return; + expect(resolved.payload.adapter).toBe("llmstxt"); + expect(resolved.payload.product.title).toBe("Example Agent Surface"); + expect(resolved.payload.tools.map((tool: any) => tool.name)).toEqual([ + "list_sections", + "get_section", + "fetch_link", + ]); + await Promise.all(waited); + + const executed = await executeTool(env, { + url: LLMS_URL, + tool: "fetch_link", + args: { url: QUICKSTART_URL }, + }); + expect(executed.ok).toBe(true); + if (!executed.ok) return; + expect(executed.value.status).toBe(200); + expect(executed.value.body).toContain("Hello agents."); + }); +});