Fix .md landing pages returning empty markdown (#60191)

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

Author: heiskr

src/article-api/middleware/article-body.ts

@@ -50,48 +50,23 @@ export async function getArticleBody(req: ExtendedRequestWithPageInfo) {
   // Extract apiVersion from query params if provided
   const apiVersion = req.query.apiVersion as string | undefined
 
-  // Check if there's a transformer for this page type (e.g., REST, webhooks, etc.)
+  // With the catch-all ArticleTransformer registered last,
+  // findTransformer always returns a transformer.
   const transformer = transformerRegistry.findTransformer(page)
+  if (!transformer) throw new Error(`No transformer found for page: ${pathname}`)
 
-  if (transformer) {
-    // Use the transformer for autogenerated pages
-    const renderingReq = await createContextualizedRenderingRequest(pathname, page)
-
-    // Determine the API version to use (provided or latest)
-    // Validation is handled by apiVersionValidationMiddleware
-    const currentVersion = renderingReq.context.currentVersion
-    let effectiveApiVersion = apiVersion
-
-    // Use latest version if not provided
-    if (!effectiveApiVersion && currentVersion && allVersions[currentVersion]) {
-      effectiveApiVersion = allVersions[currentVersion].latestApiVersion || undefined
-    }
-
-    return await transformer.transform(page, pathname, renderingReq.context, effectiveApiVersion)
-  }
-
-  // For regular articles (non-autogenerated)
-  if (page.documentType !== 'article') {
-    throw new Error(`Page ${pathname} isn't yet available in markdown.`)
-  }
-
-  // these parts allow us to render the page
+  // Use the transformer
   const renderingReq = await createContextualizedRenderingRequest(pathname, page)
-  renderingReq.context.markdownRequested = true
-  const content = await page.render(renderingReq.context)
 
-  // Get title and intro for consistency with transformer-based pages
-  const title = page.title
-  const intro = page.intro
-    ? await page.renderProp('intro', renderingReq.context, { textOnly: true })
-    : ''
+  // Determine the API version to use (provided or latest)
+  // Validation is handled by apiVersionValidationMiddleware
+  const currentVersion = renderingReq.context.currentVersion
+  let effectiveApiVersion = apiVersion
 
-  // Prepend title and intro to the content
-  let result = `# ${title}\n\n`
-  if (intro) {
-    result += `${intro}\n\n`
+  // Use latest version if not provided
+  if (!effectiveApiVersion && currentVersion && allVersions[currentVersion]) {
+    effectiveApiVersion = allVersions[currentVersion].latestApiVersion || undefined
   }
-  result += content
 
-  return result
+  return await transformer.transform(page, pathname, renderingReq.context, effectiveApiVersion)
 }

src/article-api/transformers/article-transformer.ts

@@ -0,0 +1,26 @@
+import type { Context, Page } from '@/types'
+import type { PageTransformer } from './types'
+
+/**
+ * Transformer for regular articles.
+ *
+ * This is a catch-all transformer registered last in the registry.
+ * It renders the page body as markdown and prepends the title and intro.
+ */
+export class ArticleTransformer implements PageTransformer {
+  canTransform(page: Page): boolean {
+    // Catch-all: handles any page not matched by a more specific transformer.
+    return page != null
+  }
+
+  async transform(page: Page, _pathname: string, context: Context): Promise<string> {
+    const body = await page.render({ ...context, markdownRequested: true })
+    const title = page.title
+    const intro = page.intro ? await page.renderProp('intro', context, { textOnly: true }) : ''
+
+    let result = `# ${title}\n\n`
+    if (intro) result += `${intro}\n\n`
+    result += body
+    return result
+  }
+}

src/article-api/transformers/index.ts

@@ -17,6 +17,7 @@ import { DiscoveryLandingTransformer } from './discovery-landing-transformer'
 import { ProductGuidesTransformer } from './product-guides-transformer'
 import { ProductLandingTransformer } from './product-landing-transformer'
 import { SearchPageTransformer } from './search-page-transformer'
+import { ArticleTransformer } from './article-transformer'
 
 /**
  * Global transformer registry
@@ -42,6 +43,8 @@ transformerRegistry.register(new DiscoveryLandingTransformer())
 transformerRegistry.register(new ProductGuidesTransformer())
 transformerRegistry.register(new ProductLandingTransformer())
 transformerRegistry.register(new SearchPageTransformer())
+// ArticleTransformer is the catch-all — must be registered last.
+transformerRegistry.register(new ArticleTransformer())
 
 export { TransformerRegistry } from './types'
 export type { PageTransformer } from './types'

src/frame/middleware/context/context.ts

@@ -43,6 +43,10 @@ export default async function contextualize(
     // req.pagePath is used later in the rendering pipeline to
     // locate the file in the tree so it cannot have .md
     req.pagePath = req.pagePath.replace(/\/index\.md$/, '').replace(/\.md$/, '')
+    req.context.markdownRequested = true
+    // Track that markdown was requested via URL suffix, not Accept header.
+    // This avoids adding a misleading Vary: accept cache header.
+    req.context.markdownViaUrl = true
   }
 
   // define each context property explicitly for code-search friendliness

src/frame/middleware/render-page.ts

@@ -9,6 +9,7 @@ import FailBot from '@/observability/lib/failbot'
 import statsd from '@/observability/lib/statsd'
 import type { ExtendedRequest } from '@/types'
 import { allVersions } from '@/versions/lib/all-versions'
+import { transformerRegistry } from '@/article-api/transformers'
 import { minimumNotFoundHtml } from '../lib/constants'
 import { contentTypeCacheControl, defaultCacheControl } from './cache-control'
 import { isConnectionDropped } from './halt-on-dropped-connection'
@@ -97,8 +98,20 @@ export default async function renderPage(req: ExtendedRequest, res: Response) {
   }
 
   if (!req.context) throw new Error('request not contextualized')
-  req.context.renderedPage = await buildRenderedPage(req)
-  req.context.miniTocItems = buildMiniTocItems(req)
+
+  if (context.markdownRequested) {
+    const transformer = transformerRegistry.findTransformer(page)
+    if (!transformer) throw new Error(`No transformer found for page: ${req.pagePath}`)
+    // Pass context without markdownRequested — transformers set it themselves
+    // when rendering templates. Having it set during prepareTemplateData()
+    // causes renderTitle/renderProp to output markdown instead of HTML,
+    // which breaks the cheerio-based unwrap logic.
+    const transformerContext = { ...context, markdownRequested: false }
+    req.context.renderedPage = await transformer.transform(page, path, transformerContext)
+  } else {
+    req.context.renderedPage = await buildRenderedPage(req)
+    req.context.miniTocItems = buildMiniTocItems(req)
+  }
 
   // Stop processing if the connection was already dropped
   if (isConnectionDropped(req, res)) return
@@ -151,7 +164,13 @@ export default async function renderPage(req: ExtendedRequest, res: Response) {
   }
 
   if (context.markdownRequested) {
-    contentTypeCacheControl(res)
+    if (context.markdownViaUrl) {
+      // .md URL suffix always returns markdown — Vary: accept would be misleading
+      defaultCacheControl(res)
+    } else {
+      // Accept header determines the representation — Vary: accept is correct
+      contentTypeCacheControl(res)
+    }
     return res.type('text/markdown').send(req.context.renderedPage)
   }
 

src/frame/tests/server.ts

@@ -311,6 +311,40 @@ describe('server', () => {
       expect(res.statusCode).toBe(200)
       expect(res.headers['content-type']).toContain('text/html')
     })
+
+    test('landing page returns non-empty markdown with title via Accept header', async () => {
+      const res = await get('/en/get-started', {
+        headers: {
+          accept: 'text/markdown',
+        },
+      })
+      expect(res.statusCode).toBe(200)
+      expect(res.headers['content-type']).toContain('text/markdown')
+      expect(res.body).toMatch(/^# .+/)
+      // Verify the landing page has content beyond just the title
+      expect(res.body).toMatch(/\n\n/)
+      expect(res.body.split('\n').length).toBeGreaterThan(3)
+    })
+
+    test('.md URL extension returns markdown with correct content type', async () => {
+      const res = await get('/en/get-started.md')
+      expect(res.statusCode).toBe(200)
+      expect(res.headers['content-type']).toContain('text/markdown')
+      expect(res.body).toMatch(/^# .+/)
+    })
+
+    test('/index.md redirects to the page without /index.md', async () => {
+      const res = await get('/en/get-started/index.md')
+      expect(res.statusCode).toBe(302)
+      expect(res.headers.location).toBe('/en/get-started')
+    })
+
+    test('regular article .md URL includes title and intro', async () => {
+      const res = await get('/en/get-started/start-your-journey/hello-world.md')
+      expect(res.statusCode).toBe(200)
+      expect(res.headers['content-type']).toContain('text/markdown')
+      expect(res.body).toMatch(/^# Hello World/)
+    })
   })
 })
 

src/shielding/middleware/handle-invalid-paths.ts

@@ -87,11 +87,7 @@ export default function handleInvalidPaths(
     defaultCacheControl(res)
     const newUrl = req.originalUrl.replace(req.path, req.path.replace(/\/index\.md$/, ''))
     return res.safeRedirect(newUrl)
-  } else if (req.path.endsWith('.md')) {
-    req.url = req.url.replace(/\.md($|\?)/, '$1')
-    req.originalUrl = req.originalUrl.replace(/\.md($|\?)/, '$1')
-    req.headers.accept = 'text/markdown'
-    return next()
   }
+
   return next()
 }

src/types/types.ts

@@ -190,6 +190,7 @@ export type Context = {
   renderedPage?: string
   miniTocItems?: MiniTocItem[]
   markdownRequested?: boolean
+  markdownViaUrl?: boolean
 }
 export type LearningTracks = {
   [group: string]: {