Fix agent readiness discovery endpoints (#768)

Author: kentcdodds

.github/workflows/deploy-site.yml

@@ -148,8 +148,10 @@ jobs:
           restore-keys: |
             playwright-${{ runner.os }}-node${{ env.NODE_VERSION }}-
 
-      - name: 🎬 Setup ffmpeg
-        uses: FedericoCarboni/setup-ffmpeg@v3
+      - name: 🎬 Install ffmpeg
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y ffmpeg
 
       - name: 🌐 Install Playwright Browsers
         if: steps.playwright-cache.outputs.cache-hit != 'true'

services/site/app/routes/[.]well-known/__tests__/agent-skills-index.test.ts

@@ -1,15 +1,60 @@
 // @vitest-environment node
+import { createHash } from 'node:crypto'
 import { expect, test } from 'vitest'
+import {
+	getContentSearchSkillDigest,
+	getContentSearchSkillMarkdown,
+} from '#app/utils/agent-skills.ts'
+import { loader as skillIndexLoader } from '../agent-skills.index[.]json.ts'
+import { loader as skillMarkdownLoader } from '../agent-skills.content-search.SKILL[.]md.ts'
+import { type Route as IndexRoute } from '../+types/agent-skills.index[.]json'
 
-import { loader } from '../agent-skills.index[.]json.ts'
+function createIndexLoaderArgs(request: Request): IndexRoute.LoaderArgs {
+	return {
+		context: {},
+		params: {},
+		request,
+		unstable_pattern: '/.well-known/agent-skills/index.json',
+	}
+}
 
-test('loader returns the placeholder agent skills discovery index', async () => {
-	const response = await loader()
+test('loader returns a populated agent skills discovery index', async () => {
+	const request = new Request(
+		'https://kentcdodds.com/.well-known/agent-skills/index.json',
+		{
+			headers: { host: 'kentcdodds.com' },
+		},
+	)
+	const response = await skillIndexLoader(createIndexLoaderArgs(request))
 	const payload = await response.json()
 
 	expect(response.headers.get('Content-Type')).toContain('application/json')
+	expect(response.headers.get('Cache-Control')).toBe('public, max-age=3600')
 	expect(payload).toEqual({
 		$schema: 'https://schemas.agentskills.io/discovery/0.2.0/schema.json',
-		skills: [],
+		skills: [
+			{
+				name: 'content-search',
+				type: 'skill-md',
+				description:
+					'Find relevant kentcdodds.com content and retrieve it from canonical, agent-friendly endpoints.',
+				url: 'https://kentcdodds.com/.well-known/agent-skills/content-search/SKILL.md',
+				digest: getContentSearchSkillDigest(),
+			},
+		],
 	})
 })
+
+test('skill markdown route serves the same artifact referenced by the index', async () => {
+	const response = await skillMarkdownLoader()
+	const body = await response.text()
+	const expectedContentDigest = `sha-256=:${createHash('sha256')
+		.update(getContentSearchSkillMarkdown(), 'utf8')
+		.digest('base64')}:`
+
+	expect(response.headers.get('Content-Type')).toContain('text/markdown')
+	expect(response.headers.get('Content-Digest')).toBe(expectedContentDigest)
+	expect(body).toBe(getContentSearchSkillMarkdown())
+	expect(body).toContain('https://kentcdodds.com/search?query=<terms>')
+	expect(body).toContain('Accept: text/markdown')
+})

services/site/app/routes/[.]well-known/agent-skills.content-search.SKILL[.]md.ts

@@ -0,0 +1,17 @@
+import {
+	getContentSearchSkillContentDigest,
+	getContentSearchSkillMarkdown,
+} from '#app/utils/agent-skills.ts'
+
+export function loader() {
+	const markdown = getContentSearchSkillMarkdown()
+
+	return new Response(markdown, {
+		headers: {
+			'Cache-Control': 'public, max-age=3600',
+			'Content-Digest': getContentSearchSkillContentDigest(),
+			'Content-Length': String(Buffer.byteLength(markdown)),
+			'Content-Type': 'text/markdown; charset=utf-8',
+		},
+	})
+}

services/site/app/routes/[.]well-known/agent-skills.index[.]json.ts

@@ -1,13 +1,12 @@
-const agentSkillsDiscoveryDocument = {
-	$schema: 'https://schemas.agentskills.io/discovery/0.2.0/schema.json',
-	skills: [],
-} as const
+import { getAgentSkillsDiscoveryDocument } from '#app/utils/agent-skills.ts'
+import { type Route } from './+types/agent-skills.index[.]json'
 
-export function loader() {
-	const body = JSON.stringify(agentSkillsDiscoveryDocument)
+export function loader({ request }: Route.LoaderArgs) {
+	const body = JSON.stringify(getAgentSkillsDiscoveryDocument(request))
 
 	return new Response(body, {
 		headers: {
+			'Cache-Control': 'public, max-age=3600',
 			'Content-Length': String(Buffer.byteLength(body)),
 			'Content-Type': 'application/json',
 		},

services/site/app/utils/agent-skills.ts

@@ -0,0 +1,76 @@
+import { createHash } from 'node:crypto'
+import { getDomainUrl } from './misc.ts'
+
+const agentSkillsDiscoverySchemaUrl =
+	'https://schemas.agentskills.io/discovery/0.2.0/schema.json'
+
+function createSha256Digest(contents: string) {
+	return `sha256:${createHash('sha256').update(contents, 'utf8').digest('hex')}`
+}
+
+function createContentDigestHeader(contents: string) {
+	return `sha-256=:${createHash('sha256').update(contents, 'utf8').digest('base64')}:`
+}
+
+const contentSearchSkillMarkdown = `---
+name: content-search
+description: Search kentcdodds.com content quickly and prefer canonical, agent-friendly sources.
+---
+
+# Search kentcdodds.com content
+
+Use this skill when you need articles, podcast episodes, or other published content from kentcdodds.com.
+
+## Preferred workflow
+
+1. Query \`https://kentcdodds.com/search?query=<terms>\` first and review the highest-signal results.
+2. Fetch the canonical page that best matches the request.
+3. If the request might involve broader site discovery or you need alternate candidates, fetch \`https://kentcdodds.com/sitemap.xml\`.
+4. When reading a page, prefer \`Accept: text/markdown\` so the site can return markdown instead of HTML when available.
+
+## Selection heuristics
+
+- Prefer canonical \`https://kentcdodds.com\` URLs over mirrors, embeds, or syndicated copies.
+- Prefer \`/blog/\` articles when the user wants detailed written guidance.
+- Use the search results page to shortlist candidates before opening full pages.
+- If several results match, prefer the newest relevant post unless the user asks for historical context.
+`
+
+const contentSearchSkill = {
+	name: 'content-search',
+	type: 'skill-md',
+	description:
+		'Find relevant kentcdodds.com content and retrieve it from canonical, agent-friendly endpoints.',
+	path: '/.well-known/agent-skills/content-search/SKILL.md',
+	markdown: contentSearchSkillMarkdown,
+	digest: createSha256Digest(contentSearchSkillMarkdown),
+	contentDigest: createContentDigestHeader(contentSearchSkillMarkdown),
+} as const
+
+const agentSkillDefinitions = [contentSearchSkill] as const
+export function getAgentSkillsDiscoveryDocument(request: Request) {
+	const origin = getDomainUrl(request)
+
+	return {
+		$schema: agentSkillsDiscoverySchemaUrl,
+		skills: agentSkillDefinitions.map((skill) => ({
+			name: skill.name,
+			type: skill.type,
+			description: skill.description,
+			url: `${origin}${skill.path}`,
+			digest: skill.digest,
+		})),
+	} as const
+}
+
+export function getContentSearchSkillMarkdown() {
+	return contentSearchSkill.markdown
+}
+
+export function getContentSearchSkillDigest() {
+	return contentSearchSkill.digest
+}
+
+export function getContentSearchSkillContentDigest() {
+	return contentSearchSkill.contentDigest
+}

services/site/public/robots.txt

@@ -1,5 +1,10 @@
+# Signals:
+# search - build a search index and show results
+# ai-input - use content as live input for AI answers
+# ai-train - train or fine-tune AI models
+
 User-agent: *
-Content-Signal: ai-train=yes, search=yes, ai-input=yes
+Content-Signal: ai-train=no, search=yes, ai-input=no
 Allow: /
 
 Sitemap: https://kentcdodds.com/sitemap.xml