Improve 404 recovery with related page suggestions

.github/workflows/assembler-preview.yml

@@ -101,6 +101,14 @@ jobs:
           yq -i ".environments.preview.path_prefix = \"${ASSEMBLER_PREVIEW_PATH_PREFIX}\"" config/assembler.yml
           dotnet run --project src/tooling/docs-builder -- assemble --skip-private-repositories --environment preview
 
+      # Temporary: remove this step after docs-infra serves the 404 page from each preview's path prefix.
+      - name: Publish temporary 404 experience preview
+        run: |
+          preview_root=".artifacts/assembly/${ASSEMBLER_PREVIEW_PATH_PREFIX}"
+          preview_path="${preview_root}/opentelemetry-collector"
+          mkdir -p "${preview_path}"
+          cp "${preview_root}/404/index.html" "${preview_path}/index.html"
+
       - uses: elastic/docs-builder/.github/actions/aws-auth@main
 
       - name: Upload assembled docs to S3

config/assembler.yml

@@ -34,6 +34,7 @@ environments:
       enabled: false
     feature_flags:
       SEARCH_OR_ASK_AI: true
+      RELATED_PAGES: true
   dev:
     uri: http://localhost:4000
     content_source: current
@@ -50,6 +51,7 @@ environments:
       enabled: false
     feature_flags:
       SEARCH_OR_ASK_AI: true
+      RELATED_PAGES: true
   air-gapped:
     uri: http://localhost:8080
     path_prefix: docs

src/Elastic.Documentation.Configuration/Builder/FeatureFlags.cs

@@ -32,6 +32,12 @@ public bool SearchOrAskAiEnabled
 		set => _featureFlags["search-or-ask-ai"] = value;
 	}
 
+	public bool RelatedPagesEnabled
+	{
+		get => IsEnabled("related-pages");
+		set => _featureFlags["related-pages"] = value;
+	}
+
 	public bool StagingElasticNavEnabled
 	{
 		get => IsEnabled("staging-elastic-nav");

src/Elastic.Documentation.Site/Assets/main.ts

@@ -55,6 +55,7 @@ import('./web-components/AskAi/AskAi')
 import('./web-components/VersionDropdown')
 import('./web-components/AppliesToPopover')
 import('./web-components/FullPageSearch/FullPageSearchComponent')
+import('./web-components/RelatedPages/RelatedPagesComponent')
 import('./web-components/Diagnostics/DiagnosticsComponent')
 import('./web-components/StorybookStory/StorybookStoryComponent')
 

src/Elastic.Documentation.Site/Assets/web-components/RelatedPages/RelatedPages.test.tsx

@@ -0,0 +1,113 @@
+import * as logging from '../../telemetry/logging'
+import { getRelatedPagesPath, RelatedPages } from './RelatedPages'
+import { EuiProvider } from '@elastic/eui'
+import { fireEvent, render, screen, waitFor } from '@testing-library/react'
+
+jest.mock('../../telemetry/logging', () => ({
+    logInfo: jest.fn(),
+}))
+
+const fetchMock = jest.fn()
+
+const renderRelatedPages = () =>
+    render(
+        <EuiProvider
+            colorMode="light"
+            globalStyles={false}
+            utilityClasses={false}
+        >
+            <RelatedPages path="/docs/old/index-lifecycle-management.html" />
+        </EuiProvider>
+    )
+
+describe('RelatedPages', () => {
+    beforeEach(() => {
+        jest.clearAllMocks()
+        Object.defineProperty(global, 'fetch', {
+            configurable: true,
+            writable: true,
+            value: fetchMock,
+        })
+    })
+
+    afterEach(() => {
+        jest.restoreAllMocks()
+    })
+
+    it.each([
+        [
+            '/elastic/docs-builder/docs/3655/opentelemetry-collector',
+            '/elastic/docs-builder/docs/3655',
+            '/opentelemetry-collector',
+        ],
+        ['/docs/missing-page', '/docs', '/missing-page'],
+        ['/docs-preview/missing-page', '/docs', '/docs-preview/missing-page'],
+        ['/missing-page', '', '/missing-page'],
+    ])(
+        'normalizes %s relative to the configured root path',
+        (pathname, rootPath, expected) => {
+            expect(getRelatedPagesPath(pathname, rootPath)).toBe(expected)
+        }
+    )
+
+    it('renders related pages returned by the API and tracks clicks', async () => {
+        fetchMock.mockResolvedValue({
+            ok: true,
+            json: async () => ({
+                query: 'index lifecycle management',
+                results: [
+                    {
+                        url: '/docs/manage-data/lifecycle/index-lifecycle-management',
+                        title: 'Manage the index lifecycle',
+                        description:
+                            'Automate how indices are managed over time.',
+                        parents: [
+                            { title: 'Manage data', url: '/docs/manage-data' },
+                        ],
+                    },
+                ],
+            }),
+        } as Response)
+
+        renderRelatedPages()
+
+        const link = await screen.findByRole('link', {
+            name: /manage the index lifecycle/i,
+        })
+        expect(screen.getByText('You might be looking for')).toBeInTheDocument()
+        expect(global.fetch).toHaveBeenCalledWith(
+            '/docs/_api/v1/related-pages?path=%2Fdocs%2Fold%2Findex-lifecycle-management.html',
+            expect.objectContaining({ signal: expect.any(AbortSignal) })
+        )
+
+        link.addEventListener('click', (event) => event.preventDefault())
+        fireEvent.click(link, { button: 1 })
+
+        expect(logging.logInfo).toHaveBeenCalledWith(
+            '404 related page clicked',
+            {
+                '404.related_pages.result_url':
+                    '/docs/manage-data/lifecycle/index-lifecycle-management',
+                '404.related_pages.result_position': 0,
+            }
+        )
+    })
+
+    it('renders no section when the API has no confident suggestions', async () => {
+        fetchMock.mockResolvedValue({
+            ok: true,
+            json: async () => ({ query: 'missing', results: [] }),
+        } as Response)
+
+        renderRelatedPages()
+
+        await waitFor(() =>
+            expect(
+                screen.queryByText('Finding related pages')
+            ).not.toBeInTheDocument()
+        )
+        expect(
+            screen.queryByText('You might be looking for')
+        ).not.toBeInTheDocument()
+    })
+})

src/Elastic.Documentation.Site/Assets/web-components/RelatedPages/RelatedPages.tsx

@@ -0,0 +1,180 @@
+import { config } from '../../config'
+import { logInfo } from '../../telemetry/logging'
+import {
+    EuiFlexGroup,
+    EuiFlexItem,
+    EuiIcon,
+    EuiLoadingSpinner,
+    EuiPanel,
+    EuiText,
+    useEuiTheme,
+} from '@elastic/eui'
+import { css } from '@emotion/react'
+import { useEffect, useState } from 'react'
+
+export interface RelatedPage {
+    url: string
+    title: string
+    description: string
+    parents: Array<{ title: string; url: string }>
+}
+
+interface RelatedPagesResponse {
+    query: string
+    results: RelatedPage[]
+}
+
+export const getRelatedPagesPath = (
+    pathname = window.location.pathname,
+    rootPath = config.rootPath
+) => {
+    const root = rootPath.replace(/\/$/, '')
+    if (!root || !pathname.startsWith(`${root}/`)) return pathname
+
+    return pathname.slice(root.length)
+}
+
+export const RelatedPages = ({
+    path = getRelatedPagesPath(),
+}: {
+    path?: string
+}) => {
+    const { euiTheme } = useEuiTheme()
+    const [results, setResults] = useState<RelatedPage[]>([])
+    const [isLoading, setIsLoading] = useState(true)
+
+    useEffect(() => {
+        const controller = new AbortController()
+
+        const load = async () => {
+            try {
+                const response = await fetch(
+                    `${config.apiBasePath}/v1/related-pages?path=${encodeURIComponent(path)}`,
+                    { signal: controller.signal }
+                )
+                if (!response.ok) return
+
+                const data = (await response.json()) as RelatedPagesResponse
+                setResults(data.results)
+                logInfo('404 related pages loaded', {
+                    '404.related_pages.results_count': data.results.length,
+                })
+            } catch (error) {
+                if (error instanceof Error && error.name === 'AbortError')
+                    return
+            } finally {
+                if (!controller.signal.aborted) setIsLoading(false)
+            }
+        }
+
+        if (config.airGapped) {
+            setIsLoading(false)
+            return () => controller.abort()
+        }
+
+        void load()
+        return () => controller.abort()
+    }, [path])
+
+    if (isLoading) {
+        return (
+            <div
+                role="status"
+                css={css`
+                    display: flex;
+                    justify-content: center;
+                    padding: ${euiTheme.size.l};
+                `}
+            >
+                <EuiLoadingSpinner size="l" />
+                <span className="sr-only">Finding related pages</span>
+            </div>
+        )
+    }
+
+    if (results.length === 0) return null
+
+    return (
+        <section aria-labelledby="related-pages-heading">
+            <EuiText>
+                <h2 id="related-pages-heading">You might be looking for</h2>
+            </EuiText>
+            <EuiFlexGroup
+                direction="column"
+                gutterSize="m"
+                css={css`
+                    margin-top: ${euiTheme.size.base};
+                `}
+            >
+                {results.map((result, index) => (
+                    <EuiFlexItem key={result.url}>
+                        <EuiPanel hasBorder paddingSize="m">
+                            <a
+                                href={result.url}
+                                onClick={() =>
+                                    logInfo('404 related page clicked', {
+                                        '404.related_pages.result_url':
+                                            result.url,
+                                        '404.related_pages.result_position':
+                                            index,
+                                    })
+                                }
+                                css={css`
+                                    display: block;
+                                    color: inherit;
+                                    text-decoration: none;
+
+                                    &:hover h3 {
+                                        color: ${euiTheme.colors.primary};
+                                        text-decoration: underline;
+                                    }
+                                `}
+                            >
+                                {result.parents.length > 0 && (
+                                    <div
+                                        css={css`
+                                            margin-bottom: ${euiTheme.size.xs};
+                                            color: ${euiTheme.colors
+                                                .subduedText};
+                                            font-size: ${euiTheme.font.scale
+                                                .xs * euiTheme.base}px;
+                                        `}
+                                    >
+                                        {result.parents
+                                            .map((parent) => parent.title)
+                                            .join(' › ')}
+                                    </div>
+                                )}
+                                <EuiFlexGroup
+                                    alignItems="center"
+                                    gutterSize="s"
+                                    responsive={false}
+                                >
+                                    <EuiFlexItem grow={false}>
+                                        <EuiIcon type="document" />
+                                    </EuiFlexItem>
+                                    <EuiFlexItem>
+                                        <EuiText>
+                                            <h3>{result.title}</h3>
+                                        </EuiText>
+                                    </EuiFlexItem>
+                                </EuiFlexGroup>
+                                {result.description && (
+                                    <EuiText size="s" color="subdued">
+                                        <p
+                                            css={css`
+                                                margin-top: ${euiTheme.size.xs};
+                                            `}
+                                        >
+                                            {result.description}
+                                        </p>
+                                    </EuiText>
+                                )}
+                            </a>
+                        </EuiPanel>
+                    </EuiFlexItem>
+                ))}
+            </EuiFlexGroup>
+        </section>
+    )
+}

src/Elastic.Documentation.Site/Assets/web-components/RelatedPages/RelatedPagesComponent.tsx

@@ -0,0 +1,22 @@
+import { sharedQueryClient } from '../shared/queryClient'
+import { RelatedPages } from './RelatedPages'
+import { EuiProvider } from '@elastic/eui'
+import r2wc from '@r2wc/react-to-web-component'
+import { QueryClientProvider } from '@tanstack/react-query'
+import { StrictMode } from 'react'
+
+const RelatedPagesWrapper = () => (
+    <StrictMode>
+        <EuiProvider
+            colorMode="light"
+            globalStyles={false}
+            utilityClasses={false}
+        >
+            <QueryClientProvider client={sharedQueryClient}>
+                <RelatedPages />
+            </QueryClientProvider>
+        </EuiProvider>
+    </StrictMode>
+)
+
+customElements.define('related-pages', r2wc(RelatedPagesWrapper))

src/Elastic.Documentation.Site/synthetics/journeys/404-page.journey.ts

@@ -0,0 +1,22 @@
+import { journey, step, monitor, expect } from '@elastic/synthetics'
+
+journey('404 recovery page', ({ page, params }) => {
+    monitor.use({
+        id: `elastic-co-docs-404-${params.environment}`,
+        schedule: params.environment === 'prod' ? 1 : 15,
+        tags: [`env:${params.environment}`],
+    })
+
+    const host = params.baseUrl
+    step('Open a missing documentation page', async () => {
+        const response = await page.goto(
+            `${host}/docs/this-page-does-not-exist-for-404-test`,
+            { waitUntil: 'domcontentloaded' }
+        )
+
+        expect(response?.status()).toBe(404)
+        await expect(
+            page.getByRole('heading', { name: 'Page not found' })
+        ).toBeVisible()
+    })
+})