Add tests for markdown serving based on user agents

packages/gitbook/tests/robots.test.ts

@@ -1,6 +1,56 @@
 import { describe, expect, it } from 'bun:test';
 import { getContentTestURL } from './utils';
 
+const TEST_PAGE_URL = 'https://gitbook.gitbook.io/test-gitbook-open/text-page';
+
+describe('markdown serving based on user agent', () => {
+    it('should serve markdown to GPTBot (ua-match AI agent)', async () => {
+        const response = await fetch(getContentTestURL(TEST_PAGE_URL), {
+            headers: {
+                'User-Agent': 'GPTBot/1.2',
+            },
+        });
+
+        expect(response.status).toBe(200);
+        expect(response.headers.get('content-type')).toContain('text/markdown');
+    });
+
+    it('should serve markdown to ClaudeBot (ua-match AI agent)', async () => {
+        const response = await fetch(getContentTestURL(TEST_PAGE_URL), {
+            headers: {
+                'User-Agent': 'ClaudeBot/1.0',
+            },
+        });
+
+        expect(response.status).toBe(200);
+        expect(response.headers.get('content-type')).toContain('text/markdown');
+    });
+
+    it('should NOT serve markdown to Slackbot (heuristic detection only)', async () => {
+        const response = await fetch(getContentTestURL(TEST_PAGE_URL), {
+            headers: {
+                'User-Agent':
+                    'Slackbot-LinkExpanding 1.0 (+https://api.slack.com/robots)',
+            },
+        });
+
+        expect(response.status).toBe(200);
+        expect(response.headers.get('content-type')).toContain('text/html');
+    });
+
+    it('should NOT serve markdown to Googlebot (traditional bot, not an AI agent)', async () => {
+        const response = await fetch(getContentTestURL(TEST_PAGE_URL), {
+            headers: {
+                'User-Agent':
+                    'Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)',
+            },
+        });
+
+        expect(response.status).toBe(200);
+        expect(response.headers.get('content-type')).toContain('text/html');
+    });
+});
+
 describe('robots.txt', () => {
     it('declares allow content signals for public sites', async () => {
         const response = await fetch(