Merge pull request #1447 from datum-cloud/fix/gsc-crawled-not-indexed

AriaEdo · web-flow · commit bff16cce88f1 · 2026-07-02T10:59:57.000+07:00
fix: block Mintlify asset crawling and mark .md endpoints noindex
diff --git a/public/robots.txt b/public/robots.txt
@@ -2,6 +2,8 @@ User-agent: *
 Allow: /
 Disallow: /admin
 Disallow: /api
+Disallow: /docs/_next/
+Disallow: /mintlify-assets/_next/
 Crawl-delay: 10
 Content-Signal: ai-train=no, search=yes, ai-input=no
 
diff --git a/src/pages/[...mdslug].md.ts b/src/pages/[...mdslug].md.ts
@@ -12,7 +12,7 @@
 import type { APIRoute, GetStaticPaths } from 'astro';
 import { getEntry } from 'astro:content';
 import { getMarkdownRegistry } from '@utils/markdownRegistry';
-import { renderEntryMarkdown } from '@utils/pageMarkdown';
+import { markdownSeoHeaders, renderEntryMarkdown } from '@utils/pageMarkdown';
 
 export const getStaticPaths: GetStaticPaths = async () => {
   const registry = await getMarkdownRegistry();
@@ -37,17 +37,19 @@ export const GET: APIRoute = async ({ props }) => {
     );
     if (!entry) return new Response('Not found', { status: 404 });
 
+    const canonicalUrl = `https://www.datum.net${urlPath}/`;
     const body = renderEntryMarkdown(entry, {
       // Demote body headings so a body-level H1 doesn't compete with the
       // frontmatter title. Safe for all pages — pages without a body H1
       // are unaffected.
       demoteHeadings: true,
-      sourceUrl: `https://www.datum.net${urlPath}/`,
+      sourceUrl: canonicalUrl,
     });
     return new Response(body, {
       headers: {
         'Content-Type': 'text/markdown; charset=utf-8',
         'Cache-Control': 'public, max-age=300, s-maxage=300',
+        ...markdownSeoHeaders(canonicalUrl),
       },
     });
   } catch (error) {
diff --git a/src/pages/blog.md.ts b/src/pages/blog.md.ts
@@ -7,7 +7,7 @@ import type { APIRoute } from 'astro';
 import { getEntry } from 'astro:content';
 import { fetchStrapiArticles } from '@libs/strapi';
 import type { StrapiArticle } from '@libs/strapi';
-import { renderEntryMarkdown } from '@utils/pageMarkdown';
+import { markdownSeoHeaders, renderEntryMarkdown } from '@utils/pageMarkdown';
 import { STRAPI_SSR_CACHE_CONTROL } from '@libs/strapi/httpCache';
 
 function formatDate(d?: string): string {
@@ -35,14 +35,16 @@ export const GET: APIRoute = async () => {
       list.push(`- [${a.title}](${url})${meta}${a.description ? ` - ${a.description}` : ''}`);
     }
 
+    const canonicalUrl = 'https://www.datum.net/blog/';
     const body = renderEntryMarkdown(page ?? { data: { title: 'Blog' } }, {
       trailingSections: [list.join('\n')],
-      sourceUrl: 'https://www.datum.net/blog/',
+      sourceUrl: canonicalUrl,
     });
     return new Response(body, {
       headers: {
         'Content-Type': 'text/markdown; charset=utf-8',
         'Cache-Control': STRAPI_SSR_CACHE_CONTROL,
+        ...markdownSeoHeaders(canonicalUrl),
       },
     });
   } catch (error) {
diff --git a/src/pages/blog/[slug].md.ts b/src/pages/blog/[slug].md.ts
@@ -8,6 +8,7 @@ export const prerender = false;
 import type { APIRoute } from 'astro';
 import { ensureStrapiArticleDetail } from '@libs/strapi';
 import { STRAPI_SSR_CACHE_CONTROL } from '@libs/strapi/httpCache';
+import { markdownSeoHeaders } from '@utils/pageMarkdown';
 
 export const GET: APIRoute = async ({ params }) => {
   const slug = params.slug;
@@ -29,10 +30,12 @@ export const GET: APIRoute = async ({ params }) => {
     const title = article.title ? `# ${article.title}\n\n` : '';
     const description = article.description ? `> ${article.description}\n\n` : '';
 
+    const canonicalUrl = `https://www.datum.net/blog/${slug}/`;
     return new Response(`${title}${description}${body}`, {
       headers: {
         'Content-Type': 'text/markdown; charset=utf-8',
         'Cache-Control': STRAPI_SSR_CACHE_CONTROL,
+        ...markdownSeoHeaders(canonicalUrl),
       },
     });
   } catch (error) {
diff --git a/src/pages/brand.md.ts b/src/pages/brand.md.ts
@@ -1,6 +1,6 @@
 import type { APIRoute } from 'astro';
 import { getCollection, getEntry } from 'astro:content';
-import { renderEntryMarkdown } from '@utils/pageMarkdown';
+import { markdownSeoHeaders, renderEntryMarkdown } from '@utils/pageMarkdown';
 
 interface BrandSection {
   id: string;
@@ -22,15 +22,17 @@ export const GET: APIRoute = async () => {
       list.push(`- [${s.data.title ?? slug}](/brand/${slug}/)${desc}`);
     }
 
+    const canonicalUrl = 'https://www.datum.net/brand/';
     const body = renderEntryMarkdown(page ?? { data: { title: 'Brand' } }, {
       skipBody: true,
       trailingSections: sections.length ? [list.join('\n')] : [],
-      sourceUrl: 'https://www.datum.net/brand/',
+      sourceUrl: canonicalUrl,
     });
     return new Response(body, {
       headers: {
         'Content-Type': 'text/markdown; charset=utf-8',
         'Cache-Control': 'public, max-age=300, s-maxage=300',
+        ...markdownSeoHeaders(canonicalUrl),
       },
     });
   } catch (error) {
diff --git a/src/pages/changelog.md.ts b/src/pages/changelog.md.ts
@@ -1,6 +1,6 @@
 import type { APIRoute } from 'astro';
 import { getCollection, getEntry } from 'astro:content';
-import { renderEntryMarkdown, stripMdxToMarkdown } from '@utils/pageMarkdown';
+import { markdownSeoHeaders, renderEntryMarkdown, stripMdxToMarkdown } from '@utils/pageMarkdown';
 
 interface ChangelogEntry {
   data: { title?: string; description?: string; date?: string | Date };
@@ -38,14 +38,16 @@ export const GET: APIRoute = async () => {
       sections.push('');
     }
 
+    const canonicalUrl = 'https://www.datum.net/changelog/';
     const body = renderEntryMarkdown(index ?? { data: { title: 'Changelog' } }, {
       trailingSections: sections.length ? [sections.join('\n').trim()] : [],
-      sourceUrl: 'https://www.datum.net/changelog/',
+      sourceUrl: canonicalUrl,
     });
     return new Response(body, {
       headers: {
         'Content-Type': 'text/markdown; charset=utf-8',
         'Cache-Control': 'public, max-age=300, s-maxage=300',
+        ...markdownSeoHeaders(canonicalUrl),
       },
     });
   } catch (error) {
diff --git a/src/pages/download/[slug].md.ts b/src/pages/download/[slug].md.ts
@@ -5,7 +5,7 @@
 // route the ".md" variant specifically.
 import type { APIRoute, GetStaticPaths } from 'astro';
 import { getEntry, getCollection } from 'astro:content';
-import { renderEntryMarkdown } from '@utils/pageMarkdown';
+import { markdownSeoHeaders, renderEntryMarkdown } from '@utils/pageMarkdown';
 
 export const getStaticPaths: GetStaticPaths = async () => {
   const entries = (await getCollection('download')) as Array<{ id: string }>;
@@ -17,14 +17,16 @@ export const GET: APIRoute = async ({ props }) => {
   try {
     const entry = await getEntry('download', entryId);
     if (!entry) return new Response('Not found', { status: 404 });
+    const canonicalUrl = `https://www.datum.net/download/${entryId}/`;
     const body = renderEntryMarkdown(entry, {
       demoteHeadings: true,
-      sourceUrl: `https://www.datum.net/download/${entryId}/`,
+      sourceUrl: canonicalUrl,
     });
     return new Response(body, {
       headers: {
         'Content-Type': 'text/markdown; charset=utf-8',
         'Cache-Control': 'public, max-age=300, s-maxage=300',
+        ...markdownSeoHeaders(canonicalUrl),
       },
     });
   } catch (err) {
diff --git a/src/pages/events.md.ts b/src/pages/events.md.ts
@@ -1,6 +1,6 @@
 import type { APIRoute } from 'astro';
 import { getCollection, getEntry } from 'astro:content';
-import { renderEntryMarkdown } from '@utils/pageMarkdown';
+import { markdownSeoHeaders, renderEntryMarkdown } from '@utils/pageMarkdown';
 
 interface EventEntry {
   id: string;
@@ -61,14 +61,16 @@ export const GET: APIRoute = async () => {
       '- [Alt Cloud Meetups](/events/alt-cloud-meetups/)'
     );
 
+    const canonicalUrl = 'https://www.datum.net/events/';
     const body = renderEntryMarkdown(page ?? { data: { title: 'Events' } }, {
       trailingSections: [sections.join('\n')],
-      sourceUrl: 'https://www.datum.net/events/',
+      sourceUrl: canonicalUrl,
     });
     return new Response(body, {
       headers: {
         'Content-Type': 'text/markdown; charset=utf-8',
         'Cache-Control': 'public, max-age=300, s-maxage=300',
+        ...markdownSeoHeaders(canonicalUrl),
       },
     });
   } catch (error) {
diff --git a/src/pages/handbook.md.ts b/src/pages/handbook.md.ts
@@ -1,6 +1,6 @@
 import type { APIRoute } from 'astro';
 import { getEntry } from 'astro:content';
-import { renderEntryMarkdown } from '@utils/pageMarkdown';
+import { markdownSeoHeaders, renderEntryMarkdown } from '@utils/pageMarkdown';
 
 interface HandbookSection {
   slug: string;
@@ -21,14 +21,16 @@ export const GET: APIRoute = async () => {
       }
     }
 
+    const canonicalUrl = 'https://www.datum.net/handbook/';
     const body = renderEntryMarkdown(index ?? { data: { title: 'Handbook' } }, {
       trailingSections: sections.length ? [sections.join('\n')] : [],
-      sourceUrl: 'https://www.datum.net/handbook/',
+      sourceUrl: canonicalUrl,
     });
     return new Response(body, {
       headers: {
         'Content-Type': 'text/markdown; charset=utf-8',
         'Cache-Control': 'public, max-age=300, s-maxage=300',
+        ...markdownSeoHeaders(canonicalUrl),
       },
     });
   } catch (error) {
diff --git a/src/pages/locations.md.ts b/src/pages/locations.md.ts
@@ -4,6 +4,7 @@ import type { APIRoute } from 'astro';
 import { getEntry } from 'astro:content';
 import locations from '@data/locations.json';
 import { toAsciiMarkdown } from '@utils/markdownExport';
+import { markdownSeoHeaders } from '@utils/pageMarkdown';
 
 interface Location {
   regionCode: string;
@@ -65,6 +66,7 @@ export const GET: APIRoute = async () => {
       }
     }
 
+    const canonicalUrl = 'https://www.datum.net/locations/';
     sections.push(
       '',
       '## Why these cities',
@@ -73,7 +75,7 @@ export const GET: APIRoute = async () => {
       '',
       '---',
       '',
-      'Source: <https://www.datum.net/locations/>',
+      `Source: <${canonicalUrl}>`,
       ''
     );
 
@@ -82,6 +84,7 @@ export const GET: APIRoute = async () => {
       headers: {
         'Content-Type': 'text/markdown; charset=utf-8',
         'Cache-Control': 'public, max-age=300, s-maxage=300',
+        ...markdownSeoHeaders(canonicalUrl),
       },
     });
   } catch (error) {
diff --git a/src/pages/pricing.md.ts b/src/pages/pricing.md.ts
@@ -7,6 +7,7 @@
 import type { APIRoute } from 'astro';
 import { getCollection, getEntry } from 'astro:content';
 import { toAsciiMarkdown } from '@utils/markdownExport';
+import { markdownSeoHeaders } from '@utils/pageMarkdown';
 
 interface PricingTier {
   title: string;
@@ -92,13 +93,15 @@ export const GET: APIRoute = async () => {
       }
     }
 
-    sections.push('', '---', '', 'Source: <https://www.datum.net/pricing/>', '');
+    const canonicalUrl = 'https://www.datum.net/pricing/';
+    sections.push('', '---', '', `Source: <${canonicalUrl}>`, '');
 
     const body = toAsciiMarkdown(sections.join('\n'));
     return new Response(body, {
       headers: {
         'Content-Type': 'text/markdown; charset=utf-8',
         'Cache-Control': 'public, max-age=300, s-maxage=300',
+        ...markdownSeoHeaders(canonicalUrl),
       },
     });
   } catch (error) {
diff --git a/src/pages/roadmap.md.ts b/src/pages/roadmap.md.ts
@@ -14,6 +14,7 @@ import {
 import type { StrapiRoadmap } from '@libs/strapi';
 import { STRAPI_SSR_CACHE_CONTROL } from '@libs/strapi/httpCache';
 import { toAsciiMarkdown } from '@utils/markdownExport';
+import { markdownSeoHeaders } from '@utils/pageMarkdown';
 
 function renderMilestone(roadmap: StrapiRoadmap, includeDetail: boolean): string {
   const month = getMonthAbbreviation(roadmap.releaseDate);
@@ -67,6 +68,7 @@ export const GET: APIRoute = async () => {
       sections.push('', 'No roadmap milestones available yet.');
     }
 
+    const canonicalUrl = 'https://www.datum.net/roadmap/';
     sections.push(
       '',
       '## How we plan',
@@ -77,7 +79,7 @@ export const GET: APIRoute = async () => {
       '',
       '---',
       '',
-      'Source: <https://www.datum.net/roadmap/>',
+      `Source: <${canonicalUrl}>`,
       ''
     );
 
@@ -86,6 +88,7 @@ export const GET: APIRoute = async () => {
       headers: {
         'Content-Type': 'text/markdown; charset=utf-8',
         'Cache-Control': STRAPI_SSR_CACHE_CONTROL,
+        ...markdownSeoHeaders(canonicalUrl),
       },
     });
   } catch (error) {
diff --git a/src/pages/roadmap/[slug].md.ts b/src/pages/roadmap/[slug].md.ts
@@ -12,6 +12,7 @@ import {
 } from '@libs/strapi';
 import { STRAPI_SSR_CACHE_CONTROL } from '@libs/strapi/httpCache';
 import { toAsciiMarkdown } from '@utils/markdownExport';
+import { markdownSeoHeaders } from '@utils/pageMarkdown';
 
 export const GET: APIRoute = async ({ params }) => {
   const slug = params.slug;
@@ -66,13 +67,15 @@ export const GET: APIRoute = async ({ params }) => {
       sections.push(`[View on GitHub](${roadmap.githubUrl})`, '');
     }
 
-    sections.push('---', '', `Source: <https://www.datum.net/roadmap/${slug}/>`, '');
+    const canonicalUrl = `https://www.datum.net/roadmap/${slug}/`;
+    sections.push('---', '', `Source: <${canonicalUrl}>`, '');
 
     const body = toAsciiMarkdown(sections.join('\n'));
     return new Response(body, {
       headers: {
         'Content-Type': 'text/markdown; charset=utf-8',
         'Cache-Control': STRAPI_SSR_CACHE_CONTROL,
+        ...markdownSeoHeaders(canonicalUrl),
       },
     });
   } catch (error) {
diff --git a/src/pages/roadmap/backlog.md.ts b/src/pages/roadmap/backlog.md.ts
@@ -8,6 +8,7 @@ import { getCollectionEntry } from '@utils/collectionUtils';
 import { fetchGitHubBacklog } from '@libs/githubBacklog';
 import type { GitHubBacklogItem } from '@libs/githubBacklog';
 import { toAsciiMarkdown } from '@utils/markdownExport';
+import { markdownSeoHeaders } from '@utils/pageMarkdown';
 
 function renderItem(item: GitHubBacklogItem): string {
   const labels = item.labels.length > 0 ? ` _(${item.labels.join(', ')})_` : '';
@@ -48,13 +49,14 @@ export const GET: APIRoute = async () => {
       }
     }
 
+    const canonicalUrl = 'https://www.datum.net/roadmap/backlog/';
     sections.push(
       '',
       '---',
       '',
       'Want to propose a new feature? [Start a discussion on GitHub](https://github.com/orgs/datum-cloud/discussions/categories/feature-requests).',
       '',
-      'Source: <https://www.datum.net/roadmap/backlog/>',
+      `Source: <${canonicalUrl}>`,
       ''
     );
 
@@ -64,6 +66,7 @@ export const GET: APIRoute = async () => {
         'Content-Type': 'text/markdown; charset=utf-8',
         // Backlog is cached for 24 h on GitHub's side; match that TTL.
         'Cache-Control': 'public, max-age=86400, s-maxage=86400',
+        ...markdownSeoHeaders(canonicalUrl),
       },
     });
   } catch (error) {
diff --git a/src/utils/pageMarkdown.ts b/src/utils/pageMarkdown.ts
@@ -24,6 +24,19 @@ export function stripMdxToMarkdown(body: string): string {
     .trim();
 }
 
+/**
+ * Response headers marking a `.md` endpoint as a non-canonical alternate of
+ * its HTML page: `X-Robots-Tag: noindex` keeps it out of the search index,
+ * and the `Link` canonical header points crawlers back at the HTML version.
+ * Without these, search engines index `.md` and HTML as duplicate content.
+ */
+export function markdownSeoHeaders(canonicalUrl: string): HeadersInit {
+  return {
+    'X-Robots-Tag': 'noindex',
+    Link: `<${canonicalUrl}>; rel="canonical"`,
+  };
+}
+
 interface RenderOptions {
   /** Optional override for the H1. Defaults to entry.data.title. */
   title?: string;
@@ -124,6 +137,7 @@ export function makeEntryMarkdownRoute(
         headers: {
           'Content-Type': 'text/markdown; charset=utf-8',
           'Cache-Control': 'public, max-age=300, s-maxage=300',
+          ...(opts.sourceUrl ? markdownSeoHeaders(opts.sourceUrl) : {}),
         },
       });
     } catch (error) {
