From fac1a08d7ae274aeab3a383220b0b81b4302dac3 Mon Sep 17 00:00:00 2001 From: Marcos Caceres Date: Mon, 13 Apr 2026 12:23:19 +1000 Subject: [PATCH 01/10] feat(xref): add headings lookup API for cross-spec section links Extends the xref infrastructure to read and serve section heading data from WebRef's ed/headings/ directory. This enables ReSpec's [[[SPEC#id]]] syntax to display actual heading text instead of just the spec title. New endpoint: POST /xref/headings Request: { queries: [{ spec: 'fetch', id: 'cookie-header' }] } Response: { result: [{ spec, id, title, number, href, level, specTitle }] } Changes: - scraper.ts: reads ed/headings/*.json during update, writes headings.json - store.ts: loads headings data, adds getHeading(spec, id) lookup method - headings.post.ts: new route handler - index.ts: registers the /xref/headings POST endpoint - update.ts: also triggers on ed/headings/ changes in webref webhook --- routes/xref/headings.post.ts | 43 ++++++++++++++++++++++++++ routes/xref/index.ts | 4 +++ routes/xref/lib/scraper.ts | 60 ++++++++++++++++++++++++++++++++++++ routes/xref/lib/store.ts | 24 ++++++++++++++- routes/xref/update.ts | 4 ++- 5 files changed, 133 insertions(+), 2 deletions(-) create mode 100644 routes/xref/headings.post.ts diff --git a/routes/xref/headings.post.ts b/routes/xref/headings.post.ts new file mode 100644 index 00000000..3a12f8ef --- /dev/null +++ b/routes/xref/headings.post.ts @@ -0,0 +1,43 @@ +import { Request, Response } from "express"; +import { store } from "./lib/store-init.js"; + +interface HeadingsQuery { + spec: string; + id: string; +} + +interface RequestBody { + queries: HeadingsQuery[]; +} + +type IRequest = Request; + +/** + * POST /xref/headings + * + * Looks up section headings by spec shortname and fragment id. + * Used by ReSpec's [[[SPEC#id]]] syntax to get heading text for + * cross-spec section links. + * + * Request body: { queries: [{ spec: "fetch", id: "cookie-header" }] } + * Response: { result: [{ spec: "fetch", id: "cookie-header", ... }] } + */ +export default function route(req: IRequest, res: Response) { + const { queries = [] } = req.body; + const result = queries.map(({ spec, id }) => { + const heading = store.getHeading(spec, id); + if (!heading) { + return { spec, id, error: "not found" }; + } + return { + spec, + id, + title: heading.title, + number: heading.number || null, + href: heading.href, + level: heading.level, + specTitle: heading.specTitle, + }; + }); + res.json({ result }); +} diff --git a/routes/xref/index.ts b/routes/xref/index.ts index 7faf4ed9..b327cc4a 100644 --- a/routes/xref/index.ts +++ b/routes/xref/index.ts @@ -10,6 +10,7 @@ import { env, ms } from "../../utils/misc.js"; import { store } from "./lib/store-init.js"; import searchRouteGet from "./search.get.js"; import searchRoutePost from "./search.post.js"; +import headingsRoutePost from "./headings.post.js"; import metaRoute from "./meta.js"; import updateRoute from "./update.js"; import { search, Options, Query } from "./lib/search.js"; @@ -26,6 +27,9 @@ xref .get("/search", cors(), searchRouteGet) .post("/search", express.json({ limit: "2mb" }), cors(), searchRoutePost); xref.get("/meta{/:field}", cors(), metaRoute); +xref + .options("/headings", cors({ methods: ["POST"], maxAge: ms("1day") })) + .post("/headings", express.json({ limit: "1mb" }), cors(), headingsRoutePost); xref.post("/update", authGithubWebhook(env("W3C_WEBREF_SECRET")), updateRoute); xref.use("/data", express.static(path.join(DATA_DIR, "xref"))); diff --git a/routes/xref/lib/scraper.ts b/routes/xref/lib/scraper.ts index ca5a4d82..0ffcc192 100644 --- a/routes/xref/lib/scraper.ts +++ b/routes/xref/lib/scraper.ts @@ -25,6 +25,7 @@ const OUT_DIR_BASE = path.join(DATA_DIR, "xref"); const OUTFILE_BY_TERM = path.resolve(OUT_DIR_BASE, "./xref.json"); const OUTFILE_BY_SPEC = path.resolve(OUT_DIR_BASE, "./specs.json"); const OUTFILE_SPECMAP = path.resolve(OUT_DIR_BASE, "./specmap.json"); +const OUTFILE_HEADINGS = path.resolve(OUT_DIR_BASE, "./headings.json"); type Status = "current" | "snapshot"; const dirToStatus = [ @@ -41,6 +42,18 @@ interface DataBySpec { [shortname: string]: Omit[]; } +export interface HeadingEntry { + id: string; + href: string; + title: string; + number?: string; + level: number; +} + +export interface HeadingsBySpec { + [shortname: string]: HeadingEntry[]; +} + const defaultOptions = { forceUpdate: false }; type Options = typeof defaultOptions; @@ -83,12 +96,18 @@ export default async function main(options: Partial = {}) { dataByTerm[term] = uniq(dataByTerm[term]); } + // Read headings data from webref (ed/ only, same as xref default) + const headingsBySpec = await readAllHeadings( + path.join(INPUT_DIR_BASE, "ed", "headings"), + ); + console.log("Writing processed data files..."); await mkdir(OUT_DIR_BASE, { recursive: true }); await Promise.all([ writeFile(OUTFILE_BY_TERM, JSON.stringify(dataByTerm, null, 2)), writeFile(OUTFILE_BY_SPEC, JSON.stringify(dataBySpec, null, 2)), writeFile(OUTFILE_SPECMAP, JSON.stringify(specificationsMap, null, 2)), + writeFile(OUTFILE_HEADINGS, JSON.stringify(headingsBySpec, null, 2)), ]); return true; } @@ -225,3 +244,44 @@ async function readJSON(filePath: string) { const text = await readFile(filePath, "utf-8"); return JSON.parse(text); } + +/** + * Read all headings data from webref's ed/headings/ directory. + * Builds a map of { shortname: { id: HeadingEntry } } for fast lookup. + */ +async function readAllHeadings( + headingsDir: string, +): Promise { + const result: HeadingsBySpec = Object.create(null); + if (!existsSync(headingsDir)) { + console.warn(`Headings directory not found: ${headingsDir}`); + return result; + } + + const { readdir } = await import("fs/promises"); + const files = await readdir(headingsDir); + const jsonFiles = files.filter(f => f.endsWith(".json")); + + console.log(`Processing ${jsonFiles.length} heading files...`); + for (const file of jsonFiles) { + try { + const data = await readJSON(path.join(headingsDir, file)); + const shortname = data.spec?.shortname?.toLowerCase() + || file.replace(/\.json$/, "").toLowerCase(); + const headings: HeadingEntry[] = (data.headings || []).map( + (h: { id: string; href: string; title: string; number?: string; level: number }) => ({ + id: h.id, + href: h.href, + title: h.title, + number: h.number, + level: h.level, + }), + ); + result[shortname] = headings; + } catch (error) { + console.error(`Error reading headings from ${file}:`, error); + } + } + + return result; +} diff --git a/routes/xref/lib/store.ts b/routes/xref/lib/store.ts index 076cc2cc..6920ed86 100644 --- a/routes/xref/lib/store.ts +++ b/routes/xref/lib/store.ts @@ -1,8 +1,9 @@ import path from "path"; -import { readFileSync } from "fs"; +import { readFileSync, existsSync } from "fs"; import { env } from "../../../utils/misc.js"; import { DataEntry } from "./search.js"; +import { HeadingEntry, HeadingsBySpec } from "./scraper.js"; export class Store { version = -1; @@ -15,6 +16,7 @@ export class Store { title: string; }; } = {}; + headings: HeadingsBySpec = {}; constructor() { this.fill(); @@ -25,13 +27,33 @@ export class Store { this.byTerm = readJson("xref.json"); this.bySpec = readJson("specs.json"); this.specmap = readJson("specmap.json"); + this.headings = readJson("headings.json") || {}; this.version = Date.now(); } + + /** Look up a heading by spec shortname and fragment id. */ + getHeading(spec: string, id: string): (HeadingEntry & { specTitle: string }) | null { + const normalizedSpec = spec.toLowerCase(); + const headings = this.headings[normalizedSpec]; + if (!headings) return null; + + const heading = headings.find(h => h.id === id); + if (!heading) return null; + + const specInfo = Object.values(this.specmap).find( + s => s.shortname === normalizedSpec || s.url?.includes(normalizedSpec), + ); + return { + ...heading, + specTitle: specInfo?.title || spec, + }; + } } function readJson(filename: string) { const DATA_DIR = env("DATA_DIR"); const dataFile = path.resolve(DATA_DIR, `./xref/${filename}`); + if (!existsSync(dataFile)) return {}; const text = readFileSync(dataFile, "utf8"); return JSON.parse(text); } diff --git a/routes/xref/update.ts b/routes/xref/update.ts index 07c086ec..af34502c 100644 --- a/routes/xref/update.ts +++ b/routes/xref/update.ts @@ -58,5 +58,7 @@ function hasRelevantUpdate(commits: Commit[]) { const changedFiles = commits .map(commit => [commit.added, commit.removed, commit.modified]) .flat(2); - return changedFiles.some(file => file?.startsWith("ed/dfns/")); + return changedFiles.some( + file => file?.startsWith("ed/dfns/") || file?.startsWith("ed/headings/"), + ); } From 1b8b796cae66665242260b2782f0e862da75d90a Mon Sep 17 00:00:00 2001 From: Marcos Caceres Date: Mon, 13 Apr 2026 13:19:15 +1000 Subject: [PATCH 02/10] fix: address Copilot review feedback on headings API - store.ts: index headings by id for O(1) lookup instead of linear scan; precompute specTitleByShortname map; split readJson into required/optional variants so missing xref.json still throws - scraper.ts: fix doc comment on readAllHeadings - headings.post.ts: validate queries is an array before mapping --- routes/xref/headings.post.ts | 6 +++- routes/xref/lib/scraper.ts | 2 +- routes/xref/lib/store.ts | 63 ++++++++++++++++++++++++++++++------ 3 files changed, 59 insertions(+), 12 deletions(-) diff --git a/routes/xref/headings.post.ts b/routes/xref/headings.post.ts index 3a12f8ef..95424000 100644 --- a/routes/xref/headings.post.ts +++ b/routes/xref/headings.post.ts @@ -23,7 +23,11 @@ type IRequest = Request; * Response: { result: [{ spec: "fetch", id: "cookie-header", ... }] } */ export default function route(req: IRequest, res: Response) { - const { queries = [] } = req.body; + const { queries } = req.body; + if (!Array.isArray(queries)) { + res.status(400).json({ error: "queries must be an array" }); + return; + } const result = queries.map(({ spec, id }) => { const heading = store.getHeading(spec, id); if (!heading) { diff --git a/routes/xref/lib/scraper.ts b/routes/xref/lib/scraper.ts index 0ffcc192..df4798e1 100644 --- a/routes/xref/lib/scraper.ts +++ b/routes/xref/lib/scraper.ts @@ -247,7 +247,7 @@ async function readJSON(filePath: string) { /** * Read all headings data from webref's ed/headings/ directory. - * Builds a map of { shortname: { id: HeadingEntry } } for fast lookup. + * Returns { shortname: HeadingEntry[] }. Indexed by id in the Store. */ async function readAllHeadings( headingsDir: string, diff --git a/routes/xref/lib/store.ts b/routes/xref/lib/store.ts index 6920ed86..fc364523 100644 --- a/routes/xref/lib/store.ts +++ b/routes/xref/lib/store.ts @@ -5,6 +5,11 @@ import { env } from "../../../utils/misc.js"; import { DataEntry } from "./search.js"; import { HeadingEntry, HeadingsBySpec } from "./scraper.js"; +/** Headings indexed by id for O(1) lookup. */ +type HeadingsIndex = { + [shortname: string]: { [id: string]: HeadingEntry }; +}; + export class Store { version = -1; bySpec: { [shortname: string]: DataEntry[] } = {}; @@ -16,7 +21,10 @@ export class Store { title: string; }; } = {}; - headings: HeadingsBySpec = {}; + /** Headings indexed by spec shortname, then by fragment id. */ + headings: HeadingsIndex = {}; + /** Reverse lookup: shortname → spec title. */ + private specTitleByShortname: { [shortname: string]: string } = {}; constructor() { this.fill(); @@ -27,33 +35,68 @@ export class Store { this.byTerm = readJson("xref.json"); this.bySpec = readJson("specs.json"); this.specmap = readJson("specmap.json"); - this.headings = readJson("headings.json") || {}; + const rawHeadings: HeadingsBySpec = readJsonOptional("headings.json"); + this.headings = indexHeadings(rawHeadings); + this.specTitleByShortname = buildSpecTitleMap(this.specmap); this.version = Date.now(); } /** Look up a heading by spec shortname and fragment id. */ - getHeading(spec: string, id: string): (HeadingEntry & { specTitle: string }) | null { + getHeading( + spec: string, + id: string, + ): (HeadingEntry & { specTitle: string }) | null { const normalizedSpec = spec.toLowerCase(); - const headings = this.headings[normalizedSpec]; - if (!headings) return null; + const specHeadings = this.headings[normalizedSpec]; + if (!specHeadings) return null; - const heading = headings.find(h => h.id === id); + const heading = specHeadings[id]; if (!heading) return null; - const specInfo = Object.values(this.specmap).find( - s => s.shortname === normalizedSpec || s.url?.includes(normalizedSpec), - ); return { ...heading, - specTitle: specInfo?.title || spec, + specTitle: this.specTitleByShortname[normalizedSpec] || spec, }; } } +/** Read a required JSON data file. Throws if missing. */ function readJson(filename: string) { + const DATA_DIR = env("DATA_DIR"); + const dataFile = path.resolve(DATA_DIR, `./xref/${filename}`); + const text = readFileSync(dataFile, "utf8"); + return JSON.parse(text); +} + +/** Read an optional JSON data file. Returns {} if missing. */ +function readJsonOptional(filename: string) { const DATA_DIR = env("DATA_DIR"); const dataFile = path.resolve(DATA_DIR, `./xref/${filename}`); if (!existsSync(dataFile)) return {}; const text = readFileSync(dataFile, "utf8"); return JSON.parse(text); } + +/** Index headings arrays by id for O(1) lookup per spec. */ +function indexHeadings(raw: HeadingsBySpec): HeadingsIndex { + const indexed: HeadingsIndex = Object.create(null); + for (const [shortname, headings] of Object.entries(raw)) { + const byId: { [id: string]: HeadingEntry } = Object.create(null); + for (const h of headings) { + byId[h.id] = h; + } + indexed[shortname] = byId; + } + return indexed; +} + +/** Build a shortname → title map from the specmap for O(1) title lookup. */ +function buildSpecTitleMap( + specmap: Store["specmap"], +): { [shortname: string]: string } { + const result: { [shortname: string]: string } = Object.create(null); + for (const entry of Object.values(specmap)) { + result[entry.shortname] = entry.title; + } + return result; +} From 44856c13574b69926fb72d69c957e64718cb959d Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 14 Apr 2026 10:41:30 +0000 Subject: [PATCH 03/10] refactor(xref): address reviewer feedback on headings API - Move `readdir` to top-level fs/promises import in scraper.ts - Add generic type `readJSON` with HeadingsJSON interface to reduce `any` - Use typed readJSON calls in getAllData and readAllHeadings - Change specTitleByShortname to Map in store.ts - Add per-item validation (spec/id are strings) in headings.post.ts Agent-Logs-Url: https://github.com/speced/respec-web-services/sessions/e14c5758-5d6f-4324-bb98-77839d96f660 --- routes/xref/headings.post.ts | 8 ++++++++ routes/xref/lib/scraper.ts | 34 ++++++++++++++++++---------------- routes/xref/lib/store.ts | 12 +++++------- 3 files changed, 31 insertions(+), 23 deletions(-) diff --git a/routes/xref/headings.post.ts b/routes/xref/headings.post.ts index 95424000..356094ae 100644 --- a/routes/xref/headings.post.ts +++ b/routes/xref/headings.post.ts @@ -28,6 +28,14 @@ export default function route(req: IRequest, res: Response) { res.status(400).json({ error: "queries must be an array" }); return; } + for (const item of queries) { + if (typeof item?.spec !== "string" || typeof item?.id !== "string") { + res + .status(400) + .json({ error: "each query must have string fields: spec, id" }); + return; + } + } const result = queries.map(({ spec, id }) => { const heading = store.getHeading(spec, id); if (!heading) { diff --git a/routes/xref/lib/scraper.ts b/routes/xref/lib/scraper.ts index df4798e1..ad220066 100644 --- a/routes/xref/lib/scraper.ts +++ b/routes/xref/lib/scraper.ts @@ -5,7 +5,7 @@ import path from "path"; import { existsSync } from "fs"; -import { mkdir, readFile, writeFile } from "fs/promises"; +import { mkdir, readFile, readdir, writeFile } from "fs/promises"; import { Definition as InputDfn, DfnsJSON, SpecsJSON } from "webref"; @@ -54,6 +54,11 @@ export interface HeadingsBySpec { [shortname: string]: HeadingEntry[]; } +interface HeadingsJSON { + spec?: { shortname?: string }; + headings?: HeadingEntry[]; +} + const defaultOptions = { forceUpdate: false }; type Options = typeof defaultOptions; @@ -212,7 +217,7 @@ function normalizeTerm(term: string, type: string) { async function getAllData(baseDir: string) { const SPECS_JSON = path.resolve(baseDir, "./index.json"); console.log(`Getting data from ${SPECS_JSON}`); - const urlFileContent = await readJSON(SPECS_JSON); + const urlFileContent = await readJSON<{ results: SpecsJSON[] }>(SPECS_JSON); const data: SpecsJSON[] = urlFileContent.results; const specMap: Store["specmap"] = Object.create(null); @@ -220,7 +225,7 @@ async function getAllData(baseDir: string) { for (const entry of data) { if (entry.dfns) { - const dfnsData = await readJSON(path.join(baseDir, entry.dfns)); + const dfnsData = await readJSON<{ dfns: InputDfn[] }>(path.join(baseDir, entry.dfns)); const dfns: InputDfn[] = dfnsData.dfns; dfnSources.push({ series: entry.series.shortname, @@ -240,9 +245,9 @@ async function getAllData(baseDir: string) { return { specMap, dfnSources }; } -async function readJSON(filePath: string) { +async function readJSON(filePath: string): Promise { const text = await readFile(filePath, "utf-8"); - return JSON.parse(text); + return JSON.parse(text) as T; } /** @@ -258,25 +263,22 @@ async function readAllHeadings( return result; } - const { readdir } = await import("fs/promises"); const files = await readdir(headingsDir); const jsonFiles = files.filter(f => f.endsWith(".json")); console.log(`Processing ${jsonFiles.length} heading files...`); for (const file of jsonFiles) { try { - const data = await readJSON(path.join(headingsDir, file)); + const data = await readJSON(path.join(headingsDir, file)); const shortname = data.spec?.shortname?.toLowerCase() || file.replace(/\.json$/, "").toLowerCase(); - const headings: HeadingEntry[] = (data.headings || []).map( - (h: { id: string; href: string; title: string; number?: string; level: number }) => ({ - id: h.id, - href: h.href, - title: h.title, - number: h.number, - level: h.level, - }), - ); + const headings: HeadingEntry[] = (data.headings || []).map(h => ({ + id: h.id, + href: h.href, + title: h.title, + number: h.number, + level: h.level, + })); result[shortname] = headings; } catch (error) { console.error(`Error reading headings from ${file}:`, error); diff --git a/routes/xref/lib/store.ts b/routes/xref/lib/store.ts index fc364523..3d024f6c 100644 --- a/routes/xref/lib/store.ts +++ b/routes/xref/lib/store.ts @@ -24,7 +24,7 @@ export class Store { /** Headings indexed by spec shortname, then by fragment id. */ headings: HeadingsIndex = {}; /** Reverse lookup: shortname → spec title. */ - private specTitleByShortname: { [shortname: string]: string } = {}; + private specTitleByShortname: Map = new Map(); constructor() { this.fill(); @@ -55,7 +55,7 @@ export class Store { return { ...heading, - specTitle: this.specTitleByShortname[normalizedSpec] || spec, + specTitle: this.specTitleByShortname.get(normalizedSpec) || spec, }; } } @@ -91,12 +91,10 @@ function indexHeadings(raw: HeadingsBySpec): HeadingsIndex { } /** Build a shortname → title map from the specmap for O(1) title lookup. */ -function buildSpecTitleMap( - specmap: Store["specmap"], -): { [shortname: string]: string } { - const result: { [shortname: string]: string } = Object.create(null); +function buildSpecTitleMap(specmap: Store["specmap"]): Map { + const result = new Map(); for (const entry of Object.values(specmap)) { - result[entry.shortname] = entry.title; + result.set(entry.shortname, entry.title); } return result; } From abb38b35d17941caeac64f75354002bac061227b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcos=20C=C3=A1ceres?= Date: Mon, 20 Apr 2026 07:40:04 +1000 Subject: [PATCH 04/10] Apply suggestion from @sidvishnoi Co-authored-by: Sid Vishnoi <8426945+sidvishnoi@users.noreply.github.com> --- routes/xref/lib/store.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/routes/xref/lib/store.ts b/routes/xref/lib/store.ts index 3d024f6c..6546c918 100644 --- a/routes/xref/lib/store.ts +++ b/routes/xref/lib/store.ts @@ -70,11 +70,11 @@ function readJson(filename: string) { /** Read an optional JSON data file. Returns {} if missing. */ function readJsonOptional(filename: string) { - const DATA_DIR = env("DATA_DIR"); - const dataFile = path.resolve(DATA_DIR, `./xref/${filename}`); - if (!existsSync(dataFile)) return {}; - const text = readFileSync(dataFile, "utf8"); - return JSON.parse(text); + try { + return readJson(filename); + } catch { + return {} + } } /** Index headings arrays by id for O(1) lookup per spec. */ From 08813599fd43441cb32d3a2f67ddbae7c0e2f972 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcos=20C=C3=A1ceres?= Date: Mon, 20 Apr 2026 07:41:09 +1000 Subject: [PATCH 05/10] Update routes/xref/index.ts Co-authored-by: Sid Vishnoi <8426945+sidvishnoi@users.noreply.github.com> --- routes/xref/index.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/routes/xref/index.ts b/routes/xref/index.ts index b327cc4a..9bc774a2 100644 --- a/routes/xref/index.ts +++ b/routes/xref/index.ts @@ -28,8 +28,8 @@ xref .post("/search", express.json({ limit: "2mb" }), cors(), searchRoutePost); xref.get("/meta{/:field}", cors(), metaRoute); xref - .options("/headings", cors({ methods: ["POST"], maxAge: ms("1day") })) - .post("/headings", express.json({ limit: "1mb" }), cors(), headingsRoutePost); + .options("/search/headings", cors({ methods: ["POST"], maxAge: ms("1day") })) + .post("/search/headings", express.json({ limit: "1mb" }), cors(), headingsRoutePost); xref.post("/update", authGithubWebhook(env("W3C_WEBREF_SECRET")), updateRoute); xref.use("/data", express.static(path.join(DATA_DIR, "xref"))); From 6f769f423733ddd946bd01cb90ad79e659e4d2c8 Mon Sep 17 00:00:00 2001 From: Marcos Caceres Date: Thu, 23 Apr 2026 22:58:47 +1000 Subject: [PATCH 06/10] fix(xref/headings): fix specTitleMap, add query limit, harden error handling - Fix buildSpecTitleMap to iterate nested specmap structure correctly (was iterating top-level {current, snapshot} objects instead of entries) - Add 1000-query limit and empty-string validation to headings endpoint - Only catch ENOENT in readJsonOptional (was swallowing all errors) - Remove dead spec?.shortname branch from scraper (webref doesn't include it) - Update JSDoc to reflect actual route path (/xref/search/headings) --- routes/xref/headings.post.ts | 12 +++++++++++- routes/xref/lib/scraper.ts | 5 ++--- routes/xref/lib/store.ts | 15 +++++++++++---- 3 files changed, 24 insertions(+), 8 deletions(-) diff --git a/routes/xref/headings.post.ts b/routes/xref/headings.post.ts index 356094ae..a7855687 100644 --- a/routes/xref/headings.post.ts +++ b/routes/xref/headings.post.ts @@ -13,7 +13,7 @@ interface RequestBody { type IRequest = Request; /** - * POST /xref/headings + * POST /xref/search/headings * * Looks up section headings by spec shortname and fragment id. * Used by ReSpec's [[[SPEC#id]]] syntax to get heading text for @@ -28,6 +28,10 @@ export default function route(req: IRequest, res: Response) { res.status(400).json({ error: "queries must be an array" }); return; } + if (queries.length > 1000) { + res.status(400).json({ error: "too many queries (max 1000)" }); + return; + } for (const item of queries) { if (typeof item?.spec !== "string" || typeof item?.id !== "string") { res @@ -35,6 +39,12 @@ export default function route(req: IRequest, res: Response) { .json({ error: "each query must have string fields: spec, id" }); return; } + if (!item.spec.trim() || !item.id.trim()) { + res + .status(400) + .json({ error: "spec and id must be non-empty strings" }); + return; + } } const result = queries.map(({ spec, id }) => { const heading = store.getHeading(spec, id); diff --git a/routes/xref/lib/scraper.ts b/routes/xref/lib/scraper.ts index ad220066..da2c09d0 100644 --- a/routes/xref/lib/scraper.ts +++ b/routes/xref/lib/scraper.ts @@ -55,7 +55,6 @@ export interface HeadingsBySpec { } interface HeadingsJSON { - spec?: { shortname?: string }; headings?: HeadingEntry[]; } @@ -270,8 +269,8 @@ async function readAllHeadings( for (const file of jsonFiles) { try { const data = await readJSON(path.join(headingsDir, file)); - const shortname = data.spec?.shortname?.toLowerCase() - || file.replace(/\.json$/, "").toLowerCase(); + // Shortname derived from filename (webref doesn't include it in the JSON) + const shortname = file.replace(/\.json$/, "").toLowerCase(); const headings: HeadingEntry[] = (data.headings || []).map(h => ({ id: h.id, href: h.href, diff --git a/routes/xref/lib/store.ts b/routes/xref/lib/store.ts index 6546c918..2aa59d37 100644 --- a/routes/xref/lib/store.ts +++ b/routes/xref/lib/store.ts @@ -72,8 +72,12 @@ function readJson(filename: string) { function readJsonOptional(filename: string) { try { return readJson(filename); - } catch { - return {} + } catch (err: any) { + if (err?.code === "ENOENT") { + console.warn(`Optional data file not found: ${filename}`); + return {}; + } + throw err; } } @@ -93,8 +97,11 @@ function indexHeadings(raw: HeadingsBySpec): HeadingsIndex { /** Build a shortname → title map from the specmap for O(1) title lookup. */ function buildSpecTitleMap(specmap: Store["specmap"]): Map { const result = new Map(); - for (const entry of Object.values(specmap)) { - result.set(entry.shortname, entry.title); + // specmap is { current: { [specid]: entry }, snapshot: { [specid]: entry } } + for (const group of Object.values(specmap)) { + for (const entry of Object.values(group as Record)) { + result.set(entry.shortname, entry.title); + } } return result; } From db0b1a7e0a6bd0323b03ade5499ec29e34ac3c87 Mon Sep 17 00:00:00 2001 From: Marcos Caceres Date: Fri, 24 Apr 2026 01:25:32 +1000 Subject: [PATCH 07/10] refactor(xref/headings): pre-index headings by ID at scrape time Per sidvishnoi's review: index headings by ID during scraping rather than at store load time. Removes the redundant indexHeadings() runtime function. --- routes/xref/lib/scraper.ts | 23 +++++++++++++---------- routes/xref/lib/store.ts | 27 ++++----------------------- 2 files changed, 17 insertions(+), 33 deletions(-) diff --git a/routes/xref/lib/scraper.ts b/routes/xref/lib/scraper.ts index da2c09d0..837cf9b0 100644 --- a/routes/xref/lib/scraper.ts +++ b/routes/xref/lib/scraper.ts @@ -51,7 +51,7 @@ export interface HeadingEntry { } export interface HeadingsBySpec { - [shortname: string]: HeadingEntry[]; + [shortname: string]: { [id: string]: HeadingEntry }; } interface HeadingsJSON { @@ -251,7 +251,7 @@ async function readJSON(filePath: string): Promise { /** * Read all headings data from webref's ed/headings/ directory. - * Returns { shortname: HeadingEntry[] }. Indexed by id in the Store. + * Returns { shortname: { id: HeadingEntry } } — pre-indexed for O(1) lookup. */ async function readAllHeadings( headingsDir: string, @@ -271,14 +271,17 @@ async function readAllHeadings( const data = await readJSON(path.join(headingsDir, file)); // Shortname derived from filename (webref doesn't include it in the JSON) const shortname = file.replace(/\.json$/, "").toLowerCase(); - const headings: HeadingEntry[] = (data.headings || []).map(h => ({ - id: h.id, - href: h.href, - title: h.title, - number: h.number, - level: h.level, - })); - result[shortname] = headings; + const byId: { [id: string]: HeadingEntry } = Object.create(null); + for (const h of data.headings || []) { + byId[h.id] = { + id: h.id, + href: h.href, + title: h.title, + number: h.number, + level: h.level, + }; + } + result[shortname] = byId; } catch (error) { console.error(`Error reading headings from ${file}:`, error); } diff --git a/routes/xref/lib/store.ts b/routes/xref/lib/store.ts index 2aa59d37..caeec429 100644 --- a/routes/xref/lib/store.ts +++ b/routes/xref/lib/store.ts @@ -1,15 +1,10 @@ import path from "path"; -import { readFileSync, existsSync } from "fs"; +import { readFileSync } from "fs"; import { env } from "../../../utils/misc.js"; import { DataEntry } from "./search.js"; import { HeadingEntry, HeadingsBySpec } from "./scraper.js"; -/** Headings indexed by id for O(1) lookup. */ -type HeadingsIndex = { - [shortname: string]: { [id: string]: HeadingEntry }; -}; - export class Store { version = -1; bySpec: { [shortname: string]: DataEntry[] } = {}; @@ -21,8 +16,8 @@ export class Store { title: string; }; } = {}; - /** Headings indexed by spec shortname, then by fragment id. */ - headings: HeadingsIndex = {}; + /** Headings pre-indexed by spec shortname, then by fragment id. */ + headings: HeadingsBySpec = {}; /** Reverse lookup: shortname → spec title. */ private specTitleByShortname: Map = new Map(); @@ -35,8 +30,7 @@ export class Store { this.byTerm = readJson("xref.json"); this.bySpec = readJson("specs.json"); this.specmap = readJson("specmap.json"); - const rawHeadings: HeadingsBySpec = readJsonOptional("headings.json"); - this.headings = indexHeadings(rawHeadings); + this.headings = readJsonOptional("headings.json"); this.specTitleByShortname = buildSpecTitleMap(this.specmap); this.version = Date.now(); } @@ -81,19 +75,6 @@ function readJsonOptional(filename: string) { } } -/** Index headings arrays by id for O(1) lookup per spec. */ -function indexHeadings(raw: HeadingsBySpec): HeadingsIndex { - const indexed: HeadingsIndex = Object.create(null); - for (const [shortname, headings] of Object.entries(raw)) { - const byId: { [id: string]: HeadingEntry } = Object.create(null); - for (const h of headings) { - byId[h.id] = h; - } - indexed[shortname] = byId; - } - return indexed; -} - /** Build a shortname → title map from the specmap for O(1) title lookup. */ function buildSpecTitleMap(specmap: Store["specmap"]): Map { const result = new Map(); From ed28a35c0579fce5a43bb4371a546db544b9568d Mon Sep 17 00:00:00 2001 From: Marcos Caceres Date: Fri, 24 Apr 2026 15:25:49 +1000 Subject: [PATCH 08/10] fix(xref/headings): trim inputs, add version/series fallback in getHeading Per Copilot review: - Trim spec and id before lookup (validation checked trim but passed raw) - Add version-stripped and series-shortname fallback in getHeading so both "cssom-view-1" and "cssom-view" resolve headings correctly - Also resolves specTitle via stripped form as fallback --- routes/xref/headings.post.ts | 2 +- routes/xref/lib/store.ts | 45 ++++++++++++++++++++++++++++++------ 2 files changed, 39 insertions(+), 8 deletions(-) diff --git a/routes/xref/headings.post.ts b/routes/xref/headings.post.ts index a7855687..2835c72c 100644 --- a/routes/xref/headings.post.ts +++ b/routes/xref/headings.post.ts @@ -47,7 +47,7 @@ export default function route(req: IRequest, res: Response) { } } const result = queries.map(({ spec, id }) => { - const heading = store.getHeading(spec, id); + const heading = store.getHeading(spec.trim(), id.trim()); if (!heading) { return { spec, id, error: "not found" }; } diff --git a/routes/xref/lib/store.ts b/routes/xref/lib/store.ts index caeec429..d70d81e3 100644 --- a/routes/xref/lib/store.ts +++ b/routes/xref/lib/store.ts @@ -10,10 +10,12 @@ export class Store { bySpec: { [shortname: string]: DataEntry[] } = {}; byTerm: { [term: string]: DataEntry[] } = {}; specmap: { - [specid: string]: { - url: string; - shortname: string; - title: string; + [group: string]: { + [specid: string]: { + url: string; + shortname: string; + title: string; + }; }; } = {}; /** Headings pre-indexed by spec shortname, then by fragment id. */ @@ -41,7 +43,24 @@ export class Store { id: string, ): (HeadingEntry & { specTitle: string }) | null { const normalizedSpec = spec.toLowerCase(); - const specHeadings = this.headings[normalizedSpec]; + let specHeadings = this.headings[normalizedSpec]; + // Fallback: try stripping version suffix (e.g., cssom-view → cssom-view-1) + // or adding it via specmap lookup + if (!specHeadings) { + const stripped = normalizedSpec.replace(/-\d+$/, ""); + if (stripped !== normalizedSpec) { + specHeadings = this.headings[stripped]; + } + if (!specHeadings) { + // Try resolving series shortname to versioned via specmap + for (const [specId, entry] of this.specmapEntries()) { + if (entry.shortname === normalizedSpec || entry.shortname === stripped) { + specHeadings = this.headings[specId]; + if (specHeadings) break; + } + } + } + } if (!specHeadings) return null; const heading = specHeadings[id]; @@ -49,9 +68,21 @@ export class Store { return { ...heading, - specTitle: this.specTitleByShortname.get(normalizedSpec) || spec, + specTitle: this.specTitleByShortname.get(normalizedSpec) + || this.specTitleByShortname.get(normalizedSpec.replace(/-\d+$/, "")) + || spec, }; } + + private *specmapEntries() { + for (const group of Object.values(this.specmap)) { + for (const [specId, entry] of Object.entries( + group as unknown as Record + )) { + yield [specId, entry] as const; + } + } + } } /** Read a required JSON data file. Throws if missing. */ @@ -80,7 +111,7 @@ function buildSpecTitleMap(specmap: Store["specmap"]): Map { const result = new Map(); // specmap is { current: { [specid]: entry }, snapshot: { [specid]: entry } } for (const group of Object.values(specmap)) { - for (const entry of Object.values(group as Record)) { + for (const entry of Object.values(group as unknown as Record)) { result.set(entry.shortname, entry.title); } } From a722cce33eff6e6e16914090dda128b7a517e669 Mon Sep 17 00:00:00 2001 From: Marcos Caceres Date: Sat, 25 Apr 2026 17:28:30 +1000 Subject: [PATCH 09/10] fix: correct specmap type to match nested JSON structure The specmap type previously described a flat map, but the actual data is nested with current/snapshot groups. This caused TS2352 errors after rebasing onto main's stricter type checking. --- routes/xref/lib/scraper.ts | 8 ++++---- routes/xref/lib/store.ts | 24 +++++++++++------------- 2 files changed, 15 insertions(+), 17 deletions(-) diff --git a/routes/xref/lib/scraper.ts b/routes/xref/lib/scraper.ts index 837cf9b0..3e6e280f 100644 --- a/routes/xref/lib/scraper.ts +++ b/routes/xref/lib/scraper.ts @@ -11,7 +11,7 @@ import { Definition as InputDfn, DfnsJSON, SpecsJSON } from "webref"; import { SUPPORTED_TYPES, CSS_TYPES_INPUT } from "./constants.js"; import { uniq } from "./utils.js"; -import { Store } from "./store.js"; +import { Store, SpecMapGroup } from "./store.js"; import { env } from "../../../utils/misc.js"; import sh from "../../../utils/sh.js"; @@ -72,8 +72,8 @@ export default async function main(options: Partial = {}) { const dataByTerm: DataByTerm = Object.create(null); const dataBySpec: DataBySpec = Object.create(null); const specificationsMap = { - current: {} as Store["specmap"], - snapshot: {} as Store["specmap"], + current: {} as SpecMapGroup, + snapshot: {} as SpecMapGroup, }; for (const [dir, status] of dirToStatus) { @@ -219,7 +219,7 @@ async function getAllData(baseDir: string) { const urlFileContent = await readJSON<{ results: SpecsJSON[] }>(SPECS_JSON); const data: SpecsJSON[] = urlFileContent.results; - const specMap: Store["specmap"] = Object.create(null); + const specMap: SpecMapGroup = Object.create(null); const dfnSources: DfnsJSON[] = []; for (const entry of data) { diff --git a/routes/xref/lib/store.ts b/routes/xref/lib/store.ts index d70d81e3..6f287bc7 100644 --- a/routes/xref/lib/store.ts +++ b/routes/xref/lib/store.ts @@ -5,19 +5,19 @@ import { env } from "../../../utils/misc.js"; import { DataEntry } from "./search.js"; import { HeadingEntry, HeadingsBySpec } from "./scraper.js"; +export type SpecMapGroup = { + [specid: string]: { + url: string; + shortname: string; + title: string; + }; +}; + export class Store { version = -1; bySpec: { [shortname: string]: DataEntry[] } = {}; byTerm: { [term: string]: DataEntry[] } = {}; - specmap: { - [group: string]: { - [specid: string]: { - url: string; - shortname: string; - title: string; - }; - }; - } = {}; + specmap: { [group: string]: SpecMapGroup } = {}; /** Headings pre-indexed by spec shortname, then by fragment id. */ headings: HeadingsBySpec = {}; /** Reverse lookup: shortname → spec title. */ @@ -76,9 +76,7 @@ export class Store { private *specmapEntries() { for (const group of Object.values(this.specmap)) { - for (const [specId, entry] of Object.entries( - group as unknown as Record - )) { + for (const [specId, entry] of Object.entries(group)) { yield [specId, entry] as const; } } @@ -111,7 +109,7 @@ function buildSpecTitleMap(specmap: Store["specmap"]): Map { const result = new Map(); // specmap is { current: { [specid]: entry }, snapshot: { [specid]: entry } } for (const group of Object.values(specmap)) { - for (const entry of Object.values(group as unknown as Record)) { + for (const entry of Object.values(group)) { result.set(entry.shortname, entry.title); } } From a6a9f2393492d7b61481027e027652503bc561f5 Mon Sep 17 00:00:00 2001 From: Marcos Caceres Date: Sat, 25 Apr 2026 20:00:10 +1000 Subject: [PATCH 10/10] fix: address Sid's review comments on PR #469 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Extract heading resolution fallback into resolveHeadings() method - Remove unnecessary generator (specmapEntries) — iterate directly - Use separate res.status()/res.json() calls instead of chaining --- routes/xref/headings.post.ts | 10 ++++----- routes/xref/lib/store.ts | 39 ++++++++++++++++++------------------ 2 files changed, 23 insertions(+), 26 deletions(-) diff --git a/routes/xref/headings.post.ts b/routes/xref/headings.post.ts index 2835c72c..592efc01 100644 --- a/routes/xref/headings.post.ts +++ b/routes/xref/headings.post.ts @@ -34,15 +34,13 @@ export default function route(req: IRequest, res: Response) { } for (const item of queries) { if (typeof item?.spec !== "string" || typeof item?.id !== "string") { - res - .status(400) - .json({ error: "each query must have string fields: spec, id" }); + res.status(400); + res.json({ error: "each query must have string fields: spec, id" }); return; } if (!item.spec.trim() || !item.id.trim()) { - res - .status(400) - .json({ error: "spec and id must be non-empty strings" }); + res.status(400); + res.json({ error: "spec and id must be non-empty strings" }); return; } } diff --git a/routes/xref/lib/store.ts b/routes/xref/lib/store.ts index 6f287bc7..41c536eb 100644 --- a/routes/xref/lib/store.ts +++ b/routes/xref/lib/store.ts @@ -43,24 +43,7 @@ export class Store { id: string, ): (HeadingEntry & { specTitle: string }) | null { const normalizedSpec = spec.toLowerCase(); - let specHeadings = this.headings[normalizedSpec]; - // Fallback: try stripping version suffix (e.g., cssom-view → cssom-view-1) - // or adding it via specmap lookup - if (!specHeadings) { - const stripped = normalizedSpec.replace(/-\d+$/, ""); - if (stripped !== normalizedSpec) { - specHeadings = this.headings[stripped]; - } - if (!specHeadings) { - // Try resolving series shortname to versioned via specmap - for (const [specId, entry] of this.specmapEntries()) { - if (entry.shortname === normalizedSpec || entry.shortname === stripped) { - specHeadings = this.headings[specId]; - if (specHeadings) break; - } - } - } - } + const specHeadings = this.resolveHeadings(normalizedSpec); if (!specHeadings) return null; const heading = specHeadings[id]; @@ -74,12 +57,28 @@ export class Store { }; } - private *specmapEntries() { + private resolveHeadings(spec: string): Record | null { + const direct = this.headings[spec]; + if (direct) return direct; + + // Try stripping version suffix (e.g., cssom-view-1 → cssom-view) + const stripped = spec.replace(/-\d+$/, ""); + if (stripped !== spec) { + const unversioned = this.headings[stripped]; + if (unversioned) return unversioned; + } + + // Try resolving series shortname to versioned (or vice versa) via specmap for (const group of Object.values(this.specmap)) { for (const [specId, entry] of Object.entries(group)) { - yield [specId, entry] as const; + if (entry.shortname === spec || entry.shortname === stripped) { + const resolved = this.headings[specId]; + if (resolved) return resolved; + } } } + + return null; } }