Add "Content-Signal" to robots.txt (#4202)

Author: SamyPesse

.changeset/honest-parts-shine.md

@@ -0,0 +1,5 @@
+---
+"gitbook": minor
+---
+
+Add "Content-Signal" to "robots.txt"

packages/gitbook/src/routes/robots.ts

@@ -16,6 +16,7 @@ export async function serveRobotsTxt(context: GitBookSiteContext) {
     const lines = isIndexable
         ? [
               'User-agent: *',
+              'Content-Signal: ai-train=yes, search=yes, ai-input=yes',
               // Disallow only internal search
               'Disallow: /*?*q=*',
               'Disallow: /*?*ask=*',
@@ -26,7 +27,7 @@ export async function serveRobotsTxt(context: GitBookSiteContext) {
               'Allow: /',
               `Sitemap: ${sitemapUrl}`,
           ]
-        : ['User-agent: *', 'Disallow: /'];
+        : ['User-agent: *', 'Content-Signal: ai-train=no, search=no, ai-input=no', 'Disallow: /'];
 
-    return new Response(lines.join('\n'), { headers: { 'Content-Type': 'text/plain' } });
+    return new Response(`${lines.join('\n')}\n`, { headers: { 'Content-Type': 'text/plain' } });
 }

packages/gitbook/tests/robots.test.ts

@@ -0,0 +1,32 @@
+import { describe, expect, it } from 'bun:test';
+import { getContentTestURL } from './utils';
+
+describe('robots.txt', () => {
+    it('declares allow content signals for public sites', async () => {
+        const response = await fetch(
+            getContentTestURL(
+                'https://gitbook.gitbook.io/test-gitbook-open/robots.txt?x-gitbook-search-indexation=1'
+            )
+        );
+
+        expect(response.status).toBe(200);
+        expect(response.headers.get('content-type')).toContain('text/plain');
+        const content = await response.text();
+        expect(content).toContain('Allow: /\n');
+        expect(content).toContain('Content-Signal: ai-train=yes, search=yes, ai-input=yes');
+    });
+
+    it('declares deny content signals for share link sites', async () => {
+        const response = await fetch(
+            getContentTestURL(
+                'https://gitbook-open-e2e-sites.gitbook.io/api-multi-versions-share-links/8tNo6MeXg7CkFMzSSz81/robots.txt?x-gitbook-search-indexation=1'
+            )
+        );
+
+        expect(response.status).toBe(200);
+        expect(response.headers.get('content-type')).toContain('text/plain');
+        const content = await response.text();
+        expect(content).toContain('Disallow: /\n');
+        expect(content).toContain('Content-Signal: ai-train=no, search=no, ai-input=no');
+    });
+});