From 4783a05972ca9d1f737ea18ee69feed6977fe5d8 Mon Sep 17 00:00:00 2001 From: Maarten Balliauw Date: Mon, 27 Apr 2026 10:55:15 +0200 Subject: [PATCH 1/9] Detect duplicate redirects and optimize file writing process --- astro/package.json | 2 +- astro/src/plugins/static-redirects.ts | 106 ++++++++++++++++++++++++-- 2 files changed, 100 insertions(+), 8 deletions(-) diff --git a/astro/package.json b/astro/package.json index b6a46108..8580a6fa 100644 --- a/astro/package.json +++ b/astro/package.json @@ -17,7 +17,7 @@ "scripts": { "dev": "astro dev", "start": "astro dev", - "build": "astro build", + "build": "NODE_OPTIONS=--max_old_space_size=8192 astro build", "preview": "astro preview", "astro": "astro", "linkchecker": "npm run build && lychee --skip-missing --no-progress --max-concurrency 16 --require-https --exclude-path dist/_llms-txt --exclude-path dist/llms.txt --exclude-path dist/llms-full.txt --exclude-path dist/llms-small.txt --exclude sample.duendesoftware.com --exclude docs.duendesoftware.com --exclude sitemap --exclude github --root-dir \"$PWD/dist\" dist/**", diff --git a/astro/src/plugins/static-redirects.ts b/astro/src/plugins/static-redirects.ts index c08758a3..166a58d2 100644 --- a/astro/src/plugins/static-redirects.ts +++ b/astro/src/plugins/static-redirects.ts @@ -2,8 +2,10 @@ import url from "node:url"; import type { AstroConfig, AstroIntegrationLogger } from "astro"; import path from "node:path"; import fs from "node:fs/promises"; +import { globby } from "globby"; +import matter from "gray-matter"; -const redirectMap: Record = {}; +const redirectMap = new Map(); export async function configurePlugin(hookOptions: any) { const buildOutput: string = hookOptions.buildOutput; @@ -26,41 +28,131 @@ export async function configurePlugin(hookOptions: any) { // Build redirect map logger.info("Generating static redirects file..."); - Object.keys(redirects).forEach((from) => { + for (const from in redirects) { const redirect = redirects[from]; const destination = typeof redirect === "string" ? redirect : redirect.destination; // Normalize: strip trailing slash from source for consistent matching const normalizedFrom = from.endsWith("/") ? from.slice(0, -1) : from; + // Ensure destination has trailing slash const normalizedTo = destination.endsWith("/") ? destination : destination + "/"; - redirectMap[normalizedFrom] = normalizedTo; + redirectMap.set(normalizedFrom, normalizedTo); + } + + // Detect duplicate redirect_from entries across content files. + // We cannot detect duplicates from config.redirects alone: astro-redirect-from + // builds a plain JS object from frontmatter, so when two files declare the same + // redirect_from path, the second silently overwrites the first before our hook + // ever runs. Scanning frontmatter directly is the only way to catch these. + await detectDuplicateRedirects(logger); +} + +async function detectDuplicateRedirects(logger: AstroIntegrationLogger) { + const contentDir = path.join(process.cwd(), "src/content/docs"); + const files = await globby("./**/*.{md,mdx}", { + cwd: contentDir, + gitignore: true, }); + + // Map each redirect source path to the list of files that claim it + const sourceToFiles = new Map(); + + for (const file of files) { + const filePath = path.join(contentDir, file); + + // Read only enough to extract frontmatter — avoid loading full file bodies + const handle = await fs.open(filePath, "r"); + try { + const buf = Buffer.alloc(8192); + const { bytesRead } = await handle.read(buf, 0, 8192, 0); + const head = buf.toString("utf-8", 0, bytesRead); + + // Quick check: skip files without redirect_from in the frontmatter region + if (!head.includes("redirect_from")) continue; + + // Only now read the full file for proper YAML parsing + const content = bytesRead < 8192 + ? head + : await fs.readFile(filePath, { encoding: "utf-8" }); + const { data: frontmatter } = matter(content); + + if (!frontmatter?.redirect_from) continue; + + const redirectFrom: string[] = Array.isArray(frontmatter.redirect_from) + ? frontmatter.redirect_from + : [frontmatter.redirect_from]; + + for (const source of redirectFrom) { + const normalized = source.endsWith("/") ? source.slice(0, -1) : source; + const existing = sourceToFiles.get(normalized); + if (existing) { + existing.push(file); + } else { + sourceToFiles.set(normalized, [file]); + } + } + } finally { + await handle.close(); + } + } + + let duplicateCount = 0; + for (const [source, claimingFiles] of sourceToFiles) { + if (claimingFiles.length > 1) { + if (duplicateCount === 0) { + logger.error("Duplicate redirect_from entries detected:"); + } + duplicateCount++; + logger.error( + ` "${source}" is claimed by ${claimingFiles.length} files: ${claimingFiles.join(", ")}`, + ); + } + } + + if (duplicateCount > 0) { + throw new Error( + `Build failed: ${duplicateCount} duplicate redirect_from source(s) detected. See log above for details.`, + ); + } } export async function writeToOutput(hookOptions: any) { const outDir: string = hookOptions.dir; const logger: AstroIntegrationLogger = hookOptions.logger; - if (!Object.keys(redirectMap).length) { + if (redirectMap.size === 0) { logger.warn( `Skip generating static redirects file: no redirects were generated.`, ); return; } - // Write redirects.json + // Write redirects.json by streaming to avoid a full in-memory JSON string const jsonDestinationPath = path.join( url.fileURLToPath(outDir), "redirects.json", ); - await fs.writeFile(jsonDestinationPath, JSON.stringify(redirectMap, null, 2)); + const handle = await fs.open(jsonDestinationPath, "w"); + try { + let first = true; + await handle.write("{\n"); + for (const [key, value] of redirectMap) { + if (!first) await handle.write(",\n"); + await handle.write(` ${JSON.stringify(key)}: ${JSON.stringify(value)}`); + first = false; + } + await handle.write("\n}\n"); + } finally { + await handle.close(); + } + logger.info( - `Generated ${Object.keys(redirectMap).length} redirects: ${jsonDestinationPath}`, + `Generated ${redirectMap.size} redirects: ${jsonDestinationPath}`, ); } From 744c4683364d9baf844bf04f24244f1b2c69cbef Mon Sep 17 00:00:00 2001 From: Maarten Balliauw Date: Mon, 27 Apr 2026 11:04:19 +0200 Subject: [PATCH 2/9] Update astro/src/plugins/static-redirects.ts Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- astro/src/plugins/static-redirects.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/astro/src/plugins/static-redirects.ts b/astro/src/plugins/static-redirects.ts index 166a58d2..2e943d6f 100644 --- a/astro/src/plugins/static-redirects.ts +++ b/astro/src/plugins/static-redirects.ts @@ -28,8 +28,7 @@ export async function configurePlugin(hookOptions: any) { // Build redirect map logger.info("Generating static redirects file..."); - for (const from in redirects) { - const redirect = redirects[from]; + for (const [from, redirect] of Object.entries(redirects)) { const destination = typeof redirect === "string" ? redirect : redirect.destination; From 7cd9bbb2ef93de2c5c059dc61870a55265ac3444 Mon Sep 17 00:00:00 2001 From: Maarten Balliauw Date: Mon, 27 Apr 2026 11:04:37 +0200 Subject: [PATCH 3/9] Update astro/src/plugins/static-redirects.ts Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- astro/src/plugins/static-redirects.ts | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/astro/src/plugins/static-redirects.ts b/astro/src/plugins/static-redirects.ts index 2e943d6f..0c6dc2bc 100644 --- a/astro/src/plugins/static-redirects.ts +++ b/astro/src/plugins/static-redirects.ts @@ -43,16 +43,24 @@ export async function configurePlugin(hookOptions: any) { redirectMap.set(normalizedFrom, normalizedTo); } + const contentDir = path.join( + url.fileURLToPath(config.srcDir), + "content", + "docs", + ); + // Detect duplicate redirect_from entries across content files. // We cannot detect duplicates from config.redirects alone: astro-redirect-from // builds a plain JS object from frontmatter, so when two files declare the same // redirect_from path, the second silently overwrites the first before our hook // ever runs. Scanning frontmatter directly is the only way to catch these. - await detectDuplicateRedirects(logger); + await detectDuplicateRedirects(logger, contentDir); } -async function detectDuplicateRedirects(logger: AstroIntegrationLogger) { - const contentDir = path.join(process.cwd(), "src/content/docs"); +async function detectDuplicateRedirects( + logger: AstroIntegrationLogger, + contentDir: string, +) { const files = await globby("./**/*.{md,mdx}", { cwd: contentDir, gitignore: true, From 9e322027791b86f74ee8204ccb306992556d5f17 Mon Sep 17 00:00:00 2001 From: Maarten Balliauw Date: Mon, 27 Apr 2026 11:06:00 +0200 Subject: [PATCH 4/9] Apply suggestion from @maartenba --- astro/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/astro/package.json b/astro/package.json index 8580a6fa..b6a46108 100644 --- a/astro/package.json +++ b/astro/package.json @@ -17,7 +17,7 @@ "scripts": { "dev": "astro dev", "start": "astro dev", - "build": "NODE_OPTIONS=--max_old_space_size=8192 astro build", + "build": "astro build", "preview": "astro preview", "astro": "astro", "linkchecker": "npm run build && lychee --skip-missing --no-progress --max-concurrency 16 --require-https --exclude-path dist/_llms-txt --exclude-path dist/llms.txt --exclude-path dist/llms-full.txt --exclude-path dist/llms-small.txt --exclude sample.duendesoftware.com --exclude docs.duendesoftware.com --exclude sitemap --exclude github --root-dir \"$PWD/dist\" dist/**", From beec6c939200763714eb00fbc6440a4c9bf376e4 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 27 Apr 2026 09:11:26 +0000 Subject: [PATCH 5/9] Fix frontmatter reading to scan beyond 8KB limit in duplicate redirect detection Agent-Logs-Url: https://github.com/DuendeSoftware/docs.duendesoftware.com/sessions/f7504ee4-3bc0-44b4-b032-be855d3802fd Co-authored-by: maartenba <485230+maartenba@users.noreply.github.com> --- astro/src/plugins/static-redirects.ts | 112 ++++++++++++++++++-------- 1 file changed, 80 insertions(+), 32 deletions(-) diff --git a/astro/src/plugins/static-redirects.ts b/astro/src/plugins/static-redirects.ts index 0c6dc2bc..6f836669 100644 --- a/astro/src/plugins/static-redirects.ts +++ b/astro/src/plugins/static-redirects.ts @@ -57,6 +57,65 @@ export async function configurePlugin(hookOptions: any) { await detectDuplicateRedirects(logger, contentDir); } +/** + * Reads a file in chunks until the closing `---` of the YAML frontmatter block + * is found, then returns only those bytes. This avoids loading the full file + * body while also being immune to the 8 KB truncation bug: the loop keeps + * reading until the delimiter is seen or EOF is reached. + */ +async function extractFrontmatterBlock(filePath: string): Promise { + const CHUNK_SIZE = 4096; + const handle = await fs.open(filePath, "r"); + try { + let accumulated = ""; + let offset = 0; + let firstChunk = true; + // searchFrom tracks how far we've already scanned for the closing delimiter + // so each chunk addition only rescans the newly added bytes. + let searchFrom = 0; + + while (true) { + const buf = Buffer.alloc(CHUNK_SIZE); + const { bytesRead } = await handle.read(buf, 0, CHUNK_SIZE, offset); + if (bytesRead === 0) break; + + const chunk = buf.toString("utf-8", 0, bytesRead); + accumulated += chunk; + offset += bytesRead; + + // After reading the first chunk, bail out early for files without frontmatter + if (firstChunk) { + firstChunk = false; + if (!accumulated.startsWith("---")) return ""; + // Skip past the opening --- line before searching for the closing one + searchFrom = accumulated.indexOf("\n") + 1; + } + + // Search for the closing --- delimiter starting where we left off + const closingIdx = accumulated.indexOf("\n---", searchFrom); + if (closingIdx !== -1) { + // Include the closing delimiter line in the returned block + const endIdx = accumulated.indexOf("\n", closingIdx + 1); + return endIdx === -1 + ? accumulated.slice(0, closingIdx + 4) + : accumulated.slice(0, endIdx + 1); + } + + // Advance searchFrom so the next iteration only scans the new chunk, + // minus a small overlap to avoid splitting a \n--- across chunk boundaries. + searchFrom = Math.max(searchFrom, accumulated.length - 4); + + if (bytesRead < CHUNK_SIZE) break; // reached EOF before closing delimiter + } + + // No closing --- found. gray-matter returns empty data for malformed + // frontmatter, so the caller's redirect_from check will safely skip this file. + return accumulated; + } finally { + await handle.close(); + } +} + async function detectDuplicateRedirects( logger: AstroIntegrationLogger, contentDir: string, @@ -72,39 +131,28 @@ async function detectDuplicateRedirects( for (const file of files) { const filePath = path.join(contentDir, file); - // Read only enough to extract frontmatter — avoid loading full file bodies - const handle = await fs.open(filePath, "r"); - try { - const buf = Buffer.alloc(8192); - const { bytesRead } = await handle.read(buf, 0, 8192, 0); - const head = buf.toString("utf-8", 0, bytesRead); - - // Quick check: skip files without redirect_from in the frontmatter region - if (!head.includes("redirect_from")) continue; - - // Only now read the full file for proper YAML parsing - const content = bytesRead < 8192 - ? head - : await fs.readFile(filePath, { encoding: "utf-8" }); - const { data: frontmatter } = matter(content); - - if (!frontmatter?.redirect_from) continue; - - const redirectFrom: string[] = Array.isArray(frontmatter.redirect_from) - ? frontmatter.redirect_from - : [frontmatter.redirect_from]; - - for (const source of redirectFrom) { - const normalized = source.endsWith("/") ? source.slice(0, -1) : source; - const existing = sourceToFiles.get(normalized); - if (existing) { - existing.push(file); - } else { - sourceToFiles.set(normalized, [file]); - } + // Read only the frontmatter block, scanning until the closing --- delimiter + // regardless of how large the frontmatter may be. + const frontmatterBlock = await extractFrontmatterBlock(filePath); + + if (!frontmatterBlock.includes("redirect_from")) continue; + + const { data: frontmatter } = matter(frontmatterBlock); + + if (!frontmatter?.redirect_from) continue; + + const redirectFrom: string[] = Array.isArray(frontmatter.redirect_from) + ? frontmatter.redirect_from + : [frontmatter.redirect_from]; + + for (const source of redirectFrom) { + const normalized = source.endsWith("/") ? source.slice(0, -1) : source; + const existing = sourceToFiles.get(normalized); + if (existing) { + existing.push(file); + } else { + sourceToFiles.set(normalized, [file]); } - } finally { - await handle.close(); } } From 65ed56f6e17a1bbeca0098375f2fa959a24e6ef6 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 27 Apr 2026 09:14:20 +0000 Subject: [PATCH 6/9] Add globby and gray-matter as explicit dependencies Agent-Logs-Url: https://github.com/DuendeSoftware/docs.duendesoftware.com/sessions/73255815-7960-4d5f-ab87-9e7f260cb533 Co-authored-by: maartenba <485230+maartenba@users.noreply.github.com> --- astro/package.json | 2 ++ 1 file changed, 2 insertions(+) diff --git a/astro/package.json b/astro/package.json index b6a46108..0499951f 100644 --- a/astro/package.json +++ b/astro/package.json @@ -33,6 +33,8 @@ "astro": "^6.1.0", "astro-opengraph-images": "^1.14.3", "astro-redirect-from": "^1.3.5", + "globby": "^16.1.1", + "gray-matter": "^4.0.3", "astro-rehype-relative-markdown-links": "^0.19.0", "hast-util-to-text": "^4.0.2", "jsdom": "^29.0.2", From 078e2f1fc0166aea017cbe4a13968287fbc650be Mon Sep 17 00:00:00 2001 From: Maarten Balliauw Date: Mon, 27 Apr 2026 11:18:24 +0200 Subject: [PATCH 7/9] Apply suggestion from @maartenba --- astro/src/plugins/static-redirects.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/astro/src/plugins/static-redirects.ts b/astro/src/plugins/static-redirects.ts index 6f836669..6514a748 100644 --- a/astro/src/plugins/static-redirects.ts +++ b/astro/src/plugins/static-redirects.ts @@ -60,8 +60,7 @@ export async function configurePlugin(hookOptions: any) { /** * Reads a file in chunks until the closing `---` of the YAML frontmatter block * is found, then returns only those bytes. This avoids loading the full file - * body while also being immune to the 8 KB truncation bug: the loop keeps - * reading until the delimiter is seen or EOF is reached. + * body. The loop keeps reading until the delimiter is seen or EOF is reached. */ async function extractFrontmatterBlock(filePath: string): Promise { const CHUNK_SIZE = 4096; From 80f7cfe01f4ee14544fa1380218b2eaa9f164a48 Mon Sep 17 00:00:00 2001 From: Maarten Balliauw Date: Mon, 27 Apr 2026 12:02:40 +0200 Subject: [PATCH 8/9] Apply suggestion from @Copilot Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- astro/src/plugins/static-redirects.ts | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/astro/src/plugins/static-redirects.ts b/astro/src/plugins/static-redirects.ts index 6514a748..9d088eb0 100644 --- a/astro/src/plugins/static-redirects.ts +++ b/astro/src/plugins/static-redirects.ts @@ -144,8 +144,18 @@ async function detectDuplicateRedirects( ? frontmatter.redirect_from : [frontmatter.redirect_from]; + const normalizeRedirectSource = (source: string) => { + const trimmed = source.trim(); + if (trimmed !== source) { + logger.warn( + `Trimmed whitespace from redirect_from entry in ${file}: ${JSON.stringify(source)} -> ${JSON.stringify(trimmed)}`, + ); + } + return trimmed.endsWith("/") ? trimmed.slice(0, -1) : trimmed; + }; + for (const source of redirectFrom) { - const normalized = source.endsWith("/") ? source.slice(0, -1) : source; + const normalized = normalizeRedirectSource(source); const existing = sourceToFiles.get(normalized); if (existing) { existing.push(file); From f242213f0fbbbf9bd6e9ffaeb2e15b9318d5b5f8 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 27 Apr 2026 10:11:22 +0000 Subject: [PATCH 9/9] Replace incremental handle.write() with fs.writeFile() to avoid partial writes Agent-Logs-Url: https://github.com/DuendeSoftware/docs.duendesoftware.com/sessions/227e15e2-f727-434d-8cb5-edce0021356f Co-authored-by: maartenba <485230+maartenba@users.noreply.github.com> --- astro/src/plugins/static-redirects.ts | 19 +++++-------------- 1 file changed, 5 insertions(+), 14 deletions(-) diff --git a/astro/src/plugins/static-redirects.ts b/astro/src/plugins/static-redirects.ts index 9d088eb0..4a8b4d25 100644 --- a/astro/src/plugins/static-redirects.ts +++ b/astro/src/plugins/static-redirects.ts @@ -196,24 +196,15 @@ export async function writeToOutput(hookOptions: any) { return; } - // Write redirects.json by streaming to avoid a full in-memory JSON string const jsonDestinationPath = path.join( url.fileURLToPath(outDir), "redirects.json", ); - const handle = await fs.open(jsonDestinationPath, "w"); - try { - let first = true; - await handle.write("{\n"); - for (const [key, value] of redirectMap) { - if (!first) await handle.write(",\n"); - await handle.write(` ${JSON.stringify(key)}: ${JSON.stringify(value)}`); - first = false; - } - await handle.write("\n}\n"); - } finally { - await handle.close(); - } + await fs.writeFile( + jsonDestinationPath, + JSON.stringify(Object.fromEntries(redirectMap), null, 2), + "utf-8", + ); logger.info( `Generated ${redirectMap.size} redirects: ${jsonDestinationPath}`,