Block search indexation in deployment configuration (#4203)

Author: conico974

.github/composite/deploy-cloudflare/action.yaml

@@ -64,6 +64,7 @@ runs:
           run: bun run turbo build:cloudflare
           env:
             GITBOOK_RUNTIME: cloudflare
+            GITBOOK_BLOCK_SEARCH_INDEXATION: ${{ inputs.environment == 'preview' && 'true' || '' }}
           shell: bash
 
         - name: Upload the DO worker

.github/composite/deploy-vercel/action.yaml

@@ -70,6 +70,7 @@ runs:
             echo "resolved HEAD_SHA: $HEAD_SHA"
             echo "GITBOOK_HEAD_SHA=$HEAD_SHA" >> .vercel/.env.${{ inputs.environment }}.local
             echo "GITBOOK_RUNTIME=vercel" >> .vercel/.env.${{ inputs.environment }}.local
+            echo "GITBOOK_BLOCK_SEARCH_INDEXATION=true" >> .vercel/.env.${{ inputs.environment }}.local
             echo "--- .vercel/.env.${{ inputs.environment }}.local after inject ---"
             cat .vercel/.env.${{ inputs.environment }}.local
         - name: Build Project Artifacts

packages/gitbook/next.config.mjs

@@ -59,6 +59,7 @@ const nextConfig = {
         GITBOOK_IMAGE_RESIZE_MODE: process.env.GITBOOK_IMAGE_RESIZE_MODE,
         GITBOOK_FONTS_URL: process.env.GITBOOK_FONTS_URL,
         GITBOOK_RUNTIME: process.env.GITBOOK_RUNTIME,
+        GITBOOK_BLOCK_SEARCH_INDEXATION: process.env.GITBOOK_BLOCK_SEARCH_INDEXATION,
 
         // Next.js envs
         NEXT_SERVER_ACTIONS_ENCRYPTION_KEY: process.env.NEXT_SERVER_ACTIONS_ENCRYPTION_KEY,

packages/gitbook/src/components/SiteLayout/SiteLayout.tsx

@@ -193,6 +193,6 @@ export async function generateSiteLayoutMetadata(context: GitBookSiteContext): P
                     ? 'black'
                     : 'default',
         },
-        robots: (await isSiteIndexable(context)) ? 'index, follow' : 'noindex, nofollow',
+        robots: isSiteIndexable(context) ? 'index, follow' : 'noindex, nofollow',
     };
 }

packages/gitbook/src/components/SitePage/SitePage.tsx

@@ -228,7 +228,7 @@ export async function generateSitePageMetadata(props: SitePageProps): Promise<Me
             ],
         },
         robots:
-            (await isSiteIndexable(context)) && isPageIndexable(ancestors, page)
+            isSiteIndexable(context) && isPageIndexable(ancestors, page)
                 ? 'index, follow'
                 : 'noindex, nofollow',
     };

packages/gitbook/src/lib/context.ts

@@ -59,6 +59,12 @@ export type SiteURLData = Pick<
      * By knowing it's a fallback, we can redirect to the space base path instead of returning a 404.
      */
     isFallback?: boolean;
+
+    /**
+     * Whether search indexation is blocked for this deployment.
+     * Computed in middleware from GITBOOK_BLOCK_SEARCH_INDEXATION env var and x-gitbook-search-indexation header.
+     */
+    noIndexSearch?: boolean;
 };
 
 /**
@@ -150,6 +156,9 @@ export type GitBookSiteContext = GitBookSpaceContext & {
 
     /** Whether this request is a fallback rendering. */
     isFallback: boolean;
+
+    /** Whether search indexation is blocked for this deployment. */
+    noIndexSearch: boolean;
 };
 
 /**
@@ -229,6 +238,7 @@ export async function fetchSiteContextByURLLookup(
         revision: data.revision,
         contextId: data.contextId,
         isFallback: data.isFallback ?? false,
+        noIndexSearch: data.noIndexSearch ?? false,
     });
 }
 
@@ -248,6 +258,7 @@ export async function fetchSiteContextByIds(
         revision: string | undefined;
         contextId?: string;
         isFallback: boolean;
+        noIndexSearch: boolean;
     }
 ): Promise<GitBookSiteContext> {
     const { dataFetcher } = baseContext;
@@ -371,6 +382,7 @@ export async function fetchSiteContextByIds(
         scripts,
         contextId: ids.contextId,
         isFallback: ids.isFallback,
+        noIndexSearch: ids.noIndexSearch,
     };
 }
 

packages/gitbook/src/lib/seo.ts

@@ -1,6 +1,5 @@
 import type { GitBookSiteContext } from '@/lib/context';
 import { type RevisionPageDocument, type RevisionPageGroup, SiteVisibility } from '@gitbook/api';
-import { headers } from 'next/headers';
 
 /**
  * Return true if a page is indexable in search.
@@ -24,13 +23,8 @@ export function isPageIndexable(
 /**
  * Return true if a space should be indexed by search engines.
  */
-export async function isSiteIndexable(context: GitBookSiteContext) {
-    const headersList = await headers();
-
-    if (
-        process.env.GITBOOK_BLOCK_SEARCH_INDEXATION &&
-        !headersList.has('x-gitbook-search-indexation')
-    ) {
+export function isSiteIndexable(context: GitBookSiteContext) {
+    if (context.noIndexSearch) {
         return false;
     }
 

packages/gitbook/src/middleware.ts

@@ -369,6 +369,11 @@ async function serveSiteRoutes(requestURL: URL, request: NextRequest) {
             imagesContextId: imagesContextId,
             contextId: siteURLData.contextId,
             isFallback: requestURL.searchParams.get('fallback') === 'true' ? true : undefined,
+            noIndexSearch:
+                Boolean(process.env.GITBOOK_BLOCK_SEARCH_INDEXATION) &&
+                !requestURL.searchParams.has('x-gitbook-search-indexation')
+                    ? true
+                    : undefined,
         };
 
         const requestHeaders = new Headers(request.headers);

packages/gitbook/src/routes/robots.ts

@@ -8,7 +8,7 @@ export async function serveRobotsTxt(context: GitBookSiteContext) {
     const { linker } = context;
 
     const isRoot = checkIsRootSiteContext(context);
-    const isIndexable = await isSiteIndexable(context);
+    const isIndexable = isSiteIndexable(context);
 
     const sitemapPath = linker.toPathInSpace(isRoot ? '/sitemap.xml' : '/sitemap-pages.xml');
     const sitemapUrl = linker.toAbsoluteURL(sitemapPath);