diff --git a/routes/xref/headings.post.ts b/routes/xref/headings.post.ts new file mode 100644 index 00000000..592efc01 --- /dev/null +++ b/routes/xref/headings.post.ts @@ -0,0 +1,63 @@ +import { Request, Response } from "express"; +import { store } from "./lib/store-init.js"; + +interface HeadingsQuery { + spec: string; + id: string; +} + +interface RequestBody { + queries: HeadingsQuery[]; +} + +type IRequest = Request; + +/** + * POST /xref/search/headings + * + * Looks up section headings by spec shortname and fragment id. + * Used by ReSpec's [[[SPEC#id]]] syntax to get heading text for + * cross-spec section links. + * + * Request body: { queries: [{ spec: "fetch", id: "cookie-header" }] } + * Response: { result: [{ spec: "fetch", id: "cookie-header", ... }] } + */ +export default function route(req: IRequest, res: Response) { + const { queries } = req.body; + if (!Array.isArray(queries)) { + res.status(400).json({ error: "queries must be an array" }); + return; + } + if (queries.length > 1000) { + res.status(400).json({ error: "too many queries (max 1000)" }); + return; + } + for (const item of queries) { + if (typeof item?.spec !== "string" || typeof item?.id !== "string") { + res.status(400); + res.json({ error: "each query must have string fields: spec, id" }); + return; + } + if (!item.spec.trim() || !item.id.trim()) { + res.status(400); + res.json({ error: "spec and id must be non-empty strings" }); + return; + } + } + const result = queries.map(({ spec, id }) => { + const heading = store.getHeading(spec.trim(), id.trim()); + if (!heading) { + return { spec, id, error: "not found" }; + } + return { + spec, + id, + title: heading.title, + number: heading.number || null, + href: heading.href, + level: heading.level, + specTitle: heading.specTitle, + }; + }); + res.json({ result }); +} diff --git a/routes/xref/index.ts b/routes/xref/index.ts index 7faf4ed9..9bc774a2 100644 --- a/routes/xref/index.ts +++ b/routes/xref/index.ts @@ -10,6 +10,7 @@ import { env, ms } from "../../utils/misc.js"; import { store } from "./lib/store-init.js"; import searchRouteGet from "./search.get.js"; import searchRoutePost from "./search.post.js"; +import headingsRoutePost from "./headings.post.js"; import metaRoute from "./meta.js"; import updateRoute from "./update.js"; import { search, Options, Query } from "./lib/search.js"; @@ -26,6 +27,9 @@ xref .get("/search", cors(), searchRouteGet) .post("/search", express.json({ limit: "2mb" }), cors(), searchRoutePost); xref.get("/meta{/:field}", cors(), metaRoute); +xref + .options("/search/headings", cors({ methods: ["POST"], maxAge: ms("1day") })) + .post("/search/headings", express.json({ limit: "1mb" }), cors(), headingsRoutePost); xref.post("/update", authGithubWebhook(env("W3C_WEBREF_SECRET")), updateRoute); xref.use("/data", express.static(path.join(DATA_DIR, "xref"))); diff --git a/routes/xref/lib/scraper.ts b/routes/xref/lib/scraper.ts index ca5a4d82..3e6e280f 100644 --- a/routes/xref/lib/scraper.ts +++ b/routes/xref/lib/scraper.ts @@ -5,13 +5,13 @@ import path from "path"; import { existsSync } from "fs"; -import { mkdir, readFile, writeFile } from "fs/promises"; +import { mkdir, readFile, readdir, writeFile } from "fs/promises"; import { Definition as InputDfn, DfnsJSON, SpecsJSON } from "webref"; import { SUPPORTED_TYPES, CSS_TYPES_INPUT } from "./constants.js"; import { uniq } from "./utils.js"; -import { Store } from "./store.js"; +import { Store, SpecMapGroup } from "./store.js"; import { env } from "../../../utils/misc.js"; import sh from "../../../utils/sh.js"; @@ -25,6 +25,7 @@ const OUT_DIR_BASE = path.join(DATA_DIR, "xref"); const OUTFILE_BY_TERM = path.resolve(OUT_DIR_BASE, "./xref.json"); const OUTFILE_BY_SPEC = path.resolve(OUT_DIR_BASE, "./specs.json"); const OUTFILE_SPECMAP = path.resolve(OUT_DIR_BASE, "./specmap.json"); +const OUTFILE_HEADINGS = path.resolve(OUT_DIR_BASE, "./headings.json"); type Status = "current" | "snapshot"; const dirToStatus = [ @@ -41,6 +42,22 @@ interface DataBySpec { [shortname: string]: Omit[]; } +export interface HeadingEntry { + id: string; + href: string; + title: string; + number?: string; + level: number; +} + +export interface HeadingsBySpec { + [shortname: string]: { [id: string]: HeadingEntry }; +} + +interface HeadingsJSON { + headings?: HeadingEntry[]; +} + const defaultOptions = { forceUpdate: false }; type Options = typeof defaultOptions; @@ -55,8 +72,8 @@ export default async function main(options: Partial = {}) { const dataByTerm: DataByTerm = Object.create(null); const dataBySpec: DataBySpec = Object.create(null); const specificationsMap = { - current: {} as Store["specmap"], - snapshot: {} as Store["specmap"], + current: {} as SpecMapGroup, + snapshot: {} as SpecMapGroup, }; for (const [dir, status] of dirToStatus) { @@ -83,12 +100,18 @@ export default async function main(options: Partial = {}) { dataByTerm[term] = uniq(dataByTerm[term]); } + // Read headings data from webref (ed/ only, same as xref default) + const headingsBySpec = await readAllHeadings( + path.join(INPUT_DIR_BASE, "ed", "headings"), + ); + console.log("Writing processed data files..."); await mkdir(OUT_DIR_BASE, { recursive: true }); await Promise.all([ writeFile(OUTFILE_BY_TERM, JSON.stringify(dataByTerm, null, 2)), writeFile(OUTFILE_BY_SPEC, JSON.stringify(dataBySpec, null, 2)), writeFile(OUTFILE_SPECMAP, JSON.stringify(specificationsMap, null, 2)), + writeFile(OUTFILE_HEADINGS, JSON.stringify(headingsBySpec, null, 2)), ]); return true; } @@ -193,15 +216,15 @@ function normalizeTerm(term: string, type: string) { async function getAllData(baseDir: string) { const SPECS_JSON = path.resolve(baseDir, "./index.json"); console.log(`Getting data from ${SPECS_JSON}`); - const urlFileContent = await readJSON(SPECS_JSON); + const urlFileContent = await readJSON<{ results: SpecsJSON[] }>(SPECS_JSON); const data: SpecsJSON[] = urlFileContent.results; - const specMap: Store["specmap"] = Object.create(null); + const specMap: SpecMapGroup = Object.create(null); const dfnSources: DfnsJSON[] = []; for (const entry of data) { if (entry.dfns) { - const dfnsData = await readJSON(path.join(baseDir, entry.dfns)); + const dfnsData = await readJSON<{ dfns: InputDfn[] }>(path.join(baseDir, entry.dfns)); const dfns: InputDfn[] = dfnsData.dfns; dfnSources.push({ series: entry.series.shortname, @@ -221,7 +244,48 @@ async function getAllData(baseDir: string) { return { specMap, dfnSources }; } -async function readJSON(filePath: string) { +async function readJSON(filePath: string): Promise { const text = await readFile(filePath, "utf-8"); - return JSON.parse(text); + return JSON.parse(text) as T; +} + +/** + * Read all headings data from webref's ed/headings/ directory. + * Returns { shortname: { id: HeadingEntry } } — pre-indexed for O(1) lookup. + */ +async function readAllHeadings( + headingsDir: string, +): Promise { + const result: HeadingsBySpec = Object.create(null); + if (!existsSync(headingsDir)) { + console.warn(`Headings directory not found: ${headingsDir}`); + return result; + } + + const files = await readdir(headingsDir); + const jsonFiles = files.filter(f => f.endsWith(".json")); + + console.log(`Processing ${jsonFiles.length} heading files...`); + for (const file of jsonFiles) { + try { + const data = await readJSON(path.join(headingsDir, file)); + // Shortname derived from filename (webref doesn't include it in the JSON) + const shortname = file.replace(/\.json$/, "").toLowerCase(); + const byId: { [id: string]: HeadingEntry } = Object.create(null); + for (const h of data.headings || []) { + byId[h.id] = { + id: h.id, + href: h.href, + title: h.title, + number: h.number, + level: h.level, + }; + } + result[shortname] = byId; + } catch (error) { + console.error(`Error reading headings from ${file}:`, error); + } + } + + return result; } diff --git a/routes/xref/lib/store.ts b/routes/xref/lib/store.ts index 076cc2cc..41c536eb 100644 --- a/routes/xref/lib/store.ts +++ b/routes/xref/lib/store.ts @@ -3,18 +3,25 @@ import { readFileSync } from "fs"; import { env } from "../../../utils/misc.js"; import { DataEntry } from "./search.js"; +import { HeadingEntry, HeadingsBySpec } from "./scraper.js"; + +export type SpecMapGroup = { + [specid: string]: { + url: string; + shortname: string; + title: string; + }; +}; export class Store { version = -1; bySpec: { [shortname: string]: DataEntry[] } = {}; byTerm: { [term: string]: DataEntry[] } = {}; - specmap: { - [specid: string]: { - url: string; - shortname: string; - title: string; - }; - } = {}; + specmap: { [group: string]: SpecMapGroup } = {}; + /** Headings pre-indexed by spec shortname, then by fragment id. */ + headings: HeadingsBySpec = {}; + /** Reverse lookup: shortname → spec title. */ + private specTitleByShortname: Map = new Map(); constructor() { this.fill(); @@ -25,13 +32,85 @@ export class Store { this.byTerm = readJson("xref.json"); this.bySpec = readJson("specs.json"); this.specmap = readJson("specmap.json"); + this.headings = readJsonOptional("headings.json"); + this.specTitleByShortname = buildSpecTitleMap(this.specmap); this.version = Date.now(); } + + /** Look up a heading by spec shortname and fragment id. */ + getHeading( + spec: string, + id: string, + ): (HeadingEntry & { specTitle: string }) | null { + const normalizedSpec = spec.toLowerCase(); + const specHeadings = this.resolveHeadings(normalizedSpec); + if (!specHeadings) return null; + + const heading = specHeadings[id]; + if (!heading) return null; + + return { + ...heading, + specTitle: this.specTitleByShortname.get(normalizedSpec) + || this.specTitleByShortname.get(normalizedSpec.replace(/-\d+$/, "")) + || spec, + }; + } + + private resolveHeadings(spec: string): Record | null { + const direct = this.headings[spec]; + if (direct) return direct; + + // Try stripping version suffix (e.g., cssom-view-1 → cssom-view) + const stripped = spec.replace(/-\d+$/, ""); + if (stripped !== spec) { + const unversioned = this.headings[stripped]; + if (unversioned) return unversioned; + } + + // Try resolving series shortname to versioned (or vice versa) via specmap + for (const group of Object.values(this.specmap)) { + for (const [specId, entry] of Object.entries(group)) { + if (entry.shortname === spec || entry.shortname === stripped) { + const resolved = this.headings[specId]; + if (resolved) return resolved; + } + } + } + + return null; + } } +/** Read a required JSON data file. Throws if missing. */ function readJson(filename: string) { const DATA_DIR = env("DATA_DIR"); const dataFile = path.resolve(DATA_DIR, `./xref/${filename}`); const text = readFileSync(dataFile, "utf8"); return JSON.parse(text); } + +/** Read an optional JSON data file. Returns {} if missing. */ +function readJsonOptional(filename: string) { + try { + return readJson(filename); + } catch (err: any) { + if (err?.code === "ENOENT") { + console.warn(`Optional data file not found: ${filename}`); + return {}; + } + throw err; + } +} + +/** Build a shortname → title map from the specmap for O(1) title lookup. */ +function buildSpecTitleMap(specmap: Store["specmap"]): Map { + const result = new Map(); + // specmap is { current: { [specid]: entry }, snapshot: { [specid]: entry } } + for (const group of Object.values(specmap)) { + for (const entry of Object.values(group)) { + result.set(entry.shortname, entry.title); + } + } + return result; +} diff --git a/routes/xref/update.ts b/routes/xref/update.ts index 07c086ec..af34502c 100644 --- a/routes/xref/update.ts +++ b/routes/xref/update.ts @@ -58,5 +58,7 @@ function hasRelevantUpdate(commits: Commit[]) { const changedFiles = commits .map(commit => [commit.added, commit.removed, commit.modified]) .flat(2); - return changedFiles.some(file => file?.startsWith("ed/dfns/")); + return changedFiles.some( + file => file?.startsWith("ed/dfns/") || file?.startsWith("ed/headings/"), + ); }