Allow AI agents to ask questions using ?ask= (#4205)

Author: SamyPesse

.changeset/nice-news-study.md

@@ -0,0 +1,5 @@
+---
+"gitbook": minor
+---
+
+Allow AI agents to ask questions and get the answer in markdown when fetching with `?ask=<question>`.

packages/gitbook/src/app/sites/static/[mode]/[siteURL]/[siteData]/~gitbook/markdown-ask/[question]/route.ts

@@ -0,0 +1,16 @@
+import { type RouteLayoutParams, getStaticSiteContext } from '@/app/utils';
+import { serveAskMarkdown } from '@/routes/markdownAsk';
+import type { NextRequest } from 'next/server';
+
+export const dynamic = 'force-static';
+
+export async function GET(
+    _request: NextRequest,
+    { params }: { params: Promise<RouteLayoutParams & { question: string }> }
+) {
+    const { question: encodedQuestion } = await params;
+    const { context } = await getStaticSiteContext(await params);
+    const question = decodeURIComponent(encodedQuestion);
+
+    return serveAskMarkdown(context, question);
+}

packages/gitbook/src/lib/links.ts

@@ -1,11 +1,7 @@
-import path from 'node:path';
 import { getPagePath } from '@/lib/pages';
 import { withLeadingSlash, withTrailingSlash } from '@/lib/paths';
 import type { RevisionPage, RevisionPageDocument, RevisionPageGroup } from '@gitbook/api';
-import type { Link, Root } from 'mdast';
-import { visit } from 'unist-util-visit';
 import warnOnce from 'warn-once';
-import { checkIsAnchor, checkIsExternalURL } from './urls';
 
 /**
  * Generic interface to generate links based on a given context.
@@ -228,6 +224,18 @@ export function linkerWithAbsoluteURLs(linker: GitBookLinker): GitBookLinker {
     };
 }
 
+/**
+ * Create a new linker that resolves pages to their markdown version.
+ */
+export function linkerWithMarkdownPages(linker: GitBookLinker): GitBookLinker {
+    return {
+        ...linker,
+        toPathForPage: (input) => {
+            return `${linker.toPathInSpace(input.page.path)}.md${input.anchor ? `#${input.anchor}` : ''}`;
+        },
+    };
+}
+
 function joinPaths(prefix: string, path: string): string {
     const prefixPath = prefix.endsWith('/') ? prefix : `${prefix}/`;
     const suffixPath = path.startsWith('/') ? path.slice(1) : path;
@@ -238,34 +246,3 @@ function joinPaths(prefix: string, path: string): string {
 function removeTrailingSlash(path: string): string {
     return path.endsWith('/') ? path.slice(0, -1) : path;
 }
-
-/**
- * Re-writes the URL of every relative <a> link so it is expressed from the site-root.
- */
-export function relativeToAbsoluteLinks(
-    linker: GitBookLinker,
-    tree: Root,
-    currentPagePath: string
-): Root {
-    const currentDir = path.posix.dirname(currentPagePath);
-
-    visit(tree, 'link', (node: Link) => {
-        const original = node.url;
-
-        // Skip anchors, mailto:, http(s):, protocol-like, or already-rooted paths
-        if (checkIsExternalURL(original) || checkIsAnchor(original) || original.startsWith('/')) {
-            return;
-        }
-
-        // Resolve against the current page’s directory and strip any leading “/” or "../"
-        // Sometimes the path can be "../" if we are on the default section
-        // but it means we are just at the root of the site.
-        const pathInPage = path.posix
-            .normalize(path.posix.join(currentDir, original))
-            .replace(/^[\/\.]+/, '');
-
-        node.url = linker.toAbsoluteURL(linker.toPathInSpace(pathInPage));
-    });
-
-    return tree;
-}

packages/gitbook/src/lib/markdownPage.ts

@@ -1,4 +1,5 @@
-import type { GitBookSiteContext } from '@/lib/context';
+import path from 'node:path';
+import type { GitBookAnyContext, GitBookSiteContext } from '@/lib/context';
 import { DataFetcherError, throwIfDataError } from '@/lib/data';
 import type { ResolvedPagePath } from '@/lib/pages';
 import { getIndexablePages } from '@/lib/sitemap';
@@ -10,15 +11,18 @@ import {
     RevisionPageType,
     type SiteSpace,
 } from '@gitbook/api';
-import type { Root } from 'mdast';
+import type { Link, Root } from 'mdast';
 import { fromMarkdown } from 'mdast-util-from-markdown';
 import { frontmatterFromMarkdown } from 'mdast-util-frontmatter';
 import { gfmFromMarkdown, gfmToMarkdown } from 'mdast-util-gfm';
 import { toMarkdown } from 'mdast-util-to-markdown';
 import { frontmatter } from 'micromark-extension-frontmatter';
 import { gfm } from 'micromark-extension-gfm';
 import { remove } from 'unist-util-remove';
-import { type GitBookLinker, relativeToAbsoluteLinks } from './links';
+import { visit } from 'unist-util-visit';
+import type { GitBookLinker } from './links';
+import { resolveContentRef, resolveStringContentRef } from './references';
+import { checkIsAnchor, checkIsExternalURL } from './urls';
 
 /**
  * Generate a markdown version of a page.
@@ -53,8 +57,7 @@ export async function getMarkdownForPage(
         throw error;
     }
 
-    const tree = fromPageMarkdown({
-        linker: context.linker,
+    const tree = await fromPageMarkdown(context, {
         markdown: rawMarkdown,
         pagePath: page.path,
     });
@@ -98,8 +101,7 @@ export async function getMarkdownForPageInSpace(
         })
     );
 
-    const tree = fromPageMarkdown({
-        linker,
+    const tree = await fromPageMarkdown(context, {
         markdown: rawMarkdown,
         pagePath: page.path,
     });
@@ -120,11 +122,13 @@ export async function getMarkdownForPageInSpace(
  * Parse markdown from a page, removing frontmatter and rewriting relative links to absolute links.
  * Returns the markdown AST that can be further processed or converted back to markdown using `toPageMarkdown`.
  */
-export function fromPageMarkdown(args: {
-    linker: GitBookLinker;
-    markdown: string;
-    pagePath: string;
-}): Root {
+export async function fromPageMarkdown(
+    context: GitBookAnyContext,
+    args: {
+        markdown: string;
+        pagePath: string;
+    }
+): Promise<Root> {
     const tree = fromMarkdown(args.markdown, {
         extensions: [frontmatter(['yaml']), gfm()],
         mdastExtensions: [frontmatterFromMarkdown(['yaml']), gfmFromMarkdown()],
@@ -133,7 +137,7 @@ export function fromPageMarkdown(args: {
     // Remove frontmatter
     remove(tree, 'yaml');
 
-    relativeToAbsoluteLinks(args.linker, tree, args.pagePath);
+    await rewriteMarkdownLinks(context, tree, args.pagePath);
 
     return tree;
 }
@@ -230,3 +234,55 @@ async function renderGroupPageMarkdown(args: {
         bullet: '-',
     });
 }
+
+/**
+ * Re-writes URLs in a markdown content:
+ * -
+ * - the URL of every relative <a> link so it is expressed from the site-root.
+ */
+async function rewriteMarkdownLinks(
+    context: GitBookAnyContext,
+    tree: Root,
+    currentPagePath: string
+): Promise<Root> {
+    const currentDir = path.posix.dirname(currentPagePath);
+
+    const pending: Array<Promise<void>> = [];
+
+    visit(tree, 'link', (node: Link) => {
+        const original = node.url;
+
+        // Skip anchors, mailto:, http(s):, protocol-like
+        if (checkIsExternalURL(original) || checkIsAnchor(original)) {
+            return;
+        }
+
+        const contentRef = resolveStringContentRef(original);
+
+        if (contentRef) {
+            pending.push(
+                (async () => {
+                    const resolved = await resolveContentRef(contentRef, context);
+                    if (resolved?.href) {
+                        node.url = resolved.href;
+                    }
+                })()
+            );
+        } else {
+            // Resolve against the current page’s directory and strip any leading “/” or "../"
+            // Sometimes the path can be "../" if we are on the default section
+            // but it means we are just at the root of the site.
+            const pathInPage = path.posix
+                .normalize(path.posix.join(currentDir, original))
+                .replace(/^[\/\.]+/, '');
+
+            node.url = context.linker.toAbsoluteURL(context.linker.toPathInSpace(pathInPage));
+        }
+    });
+
+    if (pending.length > 0) {
+        await Promise.all(pending);
+    }
+
+    return tree;
+}

packages/gitbook/src/lib/references.test.ts

@@ -0,0 +1,138 @@
+import { describe, expect, it } from 'bun:test';
+
+import { resolveStringContentRef } from './references';
+
+describe('resolveStringContentRef', () => {
+    it.each([
+        {
+            label: 'an external URL',
+            input: 'https://docs.gitbook.com/product-tour',
+            expected: {
+                kind: 'url',
+                url: 'https://docs.gitbook.com/product-tour',
+            },
+        },
+        {
+            label: 'a page ref in the current space',
+            input: '/pages/page_123',
+            expected: {
+                kind: 'page',
+                page: 'page_123',
+            },
+        },
+        {
+            label: 'a page ref in another space',
+            input: '/spaces/space-123/pages/page_123',
+            expected: {
+                kind: 'page',
+                space: 'space-123',
+                page: 'page_123',
+            },
+        },
+        {
+            label: 'an anchor in the current page',
+            input: '#heading-1',
+            expected: {
+                kind: 'anchor',
+                anchor: 'heading-1',
+            },
+        },
+        {
+            label: 'an anchor on a page in another space',
+            input: '/spaces/space-123/pages/page-123#heading_1',
+            expected: {
+                kind: 'anchor',
+                space: 'space-123',
+                page: 'page-123',
+                anchor: 'heading_1',
+            },
+        },
+        {
+            label: 'a file ref',
+            input: '/spaces/space-123/files/file_123',
+            expected: {
+                kind: 'file',
+                space: 'space-123',
+                file: 'file_123',
+            },
+        },
+        {
+            label: 'a space ref',
+            input: '/spaces/space-123',
+            expected: {
+                kind: 'space',
+                space: 'space-123',
+            },
+        },
+        {
+            label: 'a collection ref',
+            input: '/collections/collection-123',
+            expected: {
+                kind: 'collection',
+                collection: 'collection-123',
+            },
+        },
+        {
+            label: 'a user ref',
+            input: '/users/user_123',
+            expected: {
+                kind: 'user',
+                user: 'user_123',
+            },
+        },
+        {
+            label: 'a reusable content ref',
+            input: '/spaces/space-123/reusable-content/reusable_123',
+            expected: {
+                kind: 'reusable-content',
+                space: 'space-123',
+                reusableContent: 'reusable_123',
+            },
+        },
+        {
+            label: 'a tag ref',
+            input: '/spaces/space-123/tags/tag_123',
+            expected: {
+                kind: 'tag',
+                space: 'space-123',
+                tag: 'tag_123',
+            },
+        },
+        {
+            label: 'an OpenAPI ref',
+            input: '/openapi/spec-v1',
+            expected: {
+                kind: 'openapi',
+                spec: 'spec-v1',
+            },
+        },
+    ])('parses $label', ({ input, expected }) => {
+        // @ts-expect-error
+        expect(resolveStringContentRef(input)).toEqual(expected);
+    });
+
+    it.each([
+        {
+            label: 'a relative page path',
+            input: 'getting-started',
+        },
+        {
+            label: 'a page path with an extra segment',
+            input: '/pages/page-123/child',
+        },
+        {
+            label: 'a missing page identifier',
+            input: '/pages/',
+        },
+        {
+            label: 'a space path with a trailing slash',
+            input: '/spaces/space-123/',
+        },
+        {
+            label: 'an OpenAPI path with nested segments',
+            input: '/openapi/spec/v1',
+        },
+    ])('returns null for $label', ({ input }) => {
+        expect(resolveStringContentRef(input)).toBeNull();
+    });
+});