feat(xref): add headings lookup API for cross-spec section links

routes/xref/headings.post.ts

@@ -0,0 +1,63 @@
+import { Request, Response } from "express";
+import { store } from "./lib/store-init.js";
+
+interface HeadingsQuery {
+  spec: string;
+  id: string;
+}
+
+interface RequestBody {
+  queries: HeadingsQuery[];
+}
+
+type IRequest = Request<never, any, RequestBody>;
+
+/**
+ * POST /xref/search/headings
+ *
+ * Looks up section headings by spec shortname and fragment id.
+ * Used by ReSpec's [[[SPEC#id]]] syntax to get heading text for
+ * cross-spec section links.
+ *
+ * Request body: { queries: [{ spec: "fetch", id: "cookie-header" }] }
+ * Response: { result: [{ spec: "fetch", id: "cookie-header", ... }] }
+ */
+export default function route(req: IRequest, res: Response) {
+  const { queries } = req.body;
+  if (!Array.isArray(queries)) {
+    res.status(400).json({ error: "queries must be an array" });
+    return;
+  }
+  if (queries.length > 1000) {
+    res.status(400).json({ error: "too many queries (max 1000)" });
+    return;
+  }
+  for (const item of queries) {
+    if (typeof item?.spec !== "string" || typeof item?.id !== "string") {
+      res.status(400);
+      res.json({ error: "each query must have string fields: spec, id" });
+      return;
+    }
+    if (!item.spec.trim() || !item.id.trim()) {
+      res.status(400);
+      res.json({ error: "spec and id must be non-empty strings" });
+      return;
+    }
+  }
+  const result = queries.map(({ spec, id }) => {
+    const heading = store.getHeading(spec.trim(), id.trim());
+    if (!heading) {
+      return { spec, id, error: "not found" };
+    }
+    return {
+      spec,
+      id,
+      title: heading.title,
+      number: heading.number || null,
+      href: heading.href,
+      level: heading.level,
+      specTitle: heading.specTitle,
+    };
+  });
+  res.json({ result });
+}

routes/xref/index.ts

@@ -10,6 +10,7 @@ import { env, ms } from "../../utils/misc.js";
 import { store } from "./lib/store-init.js";
 import searchRouteGet from "./search.get.js";
 import searchRoutePost from "./search.post.js";
+import headingsRoutePost from "./headings.post.js";
 import metaRoute from "./meta.js";
 import updateRoute from "./update.js";
 import { search, Options, Query } from "./lib/search.js";
@@ -26,6 +27,9 @@ xref
   .get("/search", cors(), searchRouteGet)
   .post("/search", express.json({ limit: "2mb" }), cors(), searchRoutePost);
 xref.get("/meta{/:field}", cors(), metaRoute);
+xref
+  .options("/search/headings", cors({ methods: ["POST"], maxAge: ms("1day") }))
+  .post("/search/headings", express.json({ limit: "1mb" }), cors(), headingsRoutePost);
 xref.post("/update", authGithubWebhook(env("W3C_WEBREF_SECRET")), updateRoute);
 xref.use("/data", express.static(path.join(DATA_DIR, "xref")));
 

routes/xref/lib/scraper.ts

@@ -5,13 +5,13 @@
 
 import path from "path";
 import { existsSync } from "fs";
-import { mkdir, readFile, writeFile } from "fs/promises";
+import { mkdir, readFile, readdir, writeFile } from "fs/promises";
 
 import { Definition as InputDfn, DfnsJSON, SpecsJSON } from "webref";
 
 import { SUPPORTED_TYPES, CSS_TYPES_INPUT } from "./constants.js";
 import { uniq } from "./utils.js";
-import { Store } from "./store.js";
+import { Store, SpecMapGroup } from "./store.js";
 import { env } from "../../../utils/misc.js";
 import sh from "../../../utils/sh.js";
 
@@ -25,6 +25,7 @@ const OUT_DIR_BASE = path.join(DATA_DIR, "xref");
 const OUTFILE_BY_TERM = path.resolve(OUT_DIR_BASE, "./xref.json");
 const OUTFILE_BY_SPEC = path.resolve(OUT_DIR_BASE, "./specs.json");
 const OUTFILE_SPECMAP = path.resolve(OUT_DIR_BASE, "./specmap.json");
+const OUTFILE_HEADINGS = path.resolve(OUT_DIR_BASE, "./headings.json");
 
 type Status = "current" | "snapshot";
 const dirToStatus = [
@@ -41,6 +42,22 @@ interface DataBySpec {
   [shortname: string]: Omit<ParsedDataEntry, "shortname" | "isExported">[];
 }
 
+export interface HeadingEntry {
+  id: string;
+  href: string;
+  title: string;
+  number?: string;
+  level: number;
+}
+
+export interface HeadingsBySpec {
+  [shortname: string]: { [id: string]: HeadingEntry };
+}
+
+interface HeadingsJSON {
+  headings?: HeadingEntry[];
+}
+
 const defaultOptions = { forceUpdate: false };
 type Options = typeof defaultOptions;
 
@@ -55,8 +72,8 @@ export default async function main(options: Partial<Options> = {}) {
   const dataByTerm: DataByTerm = Object.create(null);
   const dataBySpec: DataBySpec = Object.create(null);
   const specificationsMap = {
-    current: {} as Store["specmap"],
-    snapshot: {} as Store["specmap"],
+    current: {} as SpecMapGroup,
+    snapshot: {} as SpecMapGroup,
   };
 
   for (const [dir, status] of dirToStatus) {
@@ -83,12 +100,18 @@ export default async function main(options: Partial<Options> = {}) {
     dataByTerm[term] = uniq(dataByTerm[term]);
   }
 
+  // Read headings data from webref (ed/ only, same as xref default)
+  const headingsBySpec = await readAllHeadings(
+    path.join(INPUT_DIR_BASE, "ed", "headings"),
+  );
+
   console.log("Writing processed data files...");
   await mkdir(OUT_DIR_BASE, { recursive: true });
   await Promise.all([
     writeFile(OUTFILE_BY_TERM, JSON.stringify(dataByTerm, null, 2)),
     writeFile(OUTFILE_BY_SPEC, JSON.stringify(dataBySpec, null, 2)),
     writeFile(OUTFILE_SPECMAP, JSON.stringify(specificationsMap, null, 2)),
+    writeFile(OUTFILE_HEADINGS, JSON.stringify(headingsBySpec, null, 2)),
   ]);
   return true;
 }
@@ -193,15 +216,15 @@ function normalizeTerm(term: string, type: string) {
 async function getAllData(baseDir: string) {
   const SPECS_JSON = path.resolve(baseDir, "./index.json");
   console.log(`Getting data from ${SPECS_JSON}`);
-  const urlFileContent = await readJSON(SPECS_JSON);
+  const urlFileContent = await readJSON<{ results: SpecsJSON[] }>(SPECS_JSON);
   const data: SpecsJSON[] = urlFileContent.results;
 
-  const specMap: Store["specmap"] = Object.create(null);
+  const specMap: SpecMapGroup = Object.create(null);
   const dfnSources: DfnsJSON[] = [];
 
   for (const entry of data) {
     if (entry.dfns) {
-      const dfnsData = await readJSON(path.join(baseDir, entry.dfns));
+      const dfnsData = await readJSON<{ dfns: InputDfn[] }>(path.join(baseDir, entry.dfns));
       const dfns: InputDfn[] = dfnsData.dfns;
       dfnSources.push({
         series: entry.series.shortname,
@@ -221,7 +244,48 @@ async function getAllData(baseDir: string) {
   return { specMap, dfnSources };
 }
 
-async function readJSON(filePath: string) {
+async function readJSON<T = unknown>(filePath: string): Promise<T> {
   const text = await readFile(filePath, "utf-8");
-  return JSON.parse(text);
+  return JSON.parse(text) as T;
+}
+
+/**
+ * Read all headings data from webref's ed/headings/ directory.
+ * Returns { shortname: { id: HeadingEntry } } — pre-indexed for O(1) lookup.
+ */
+async function readAllHeadings(
+  headingsDir: string,
+): Promise<HeadingsBySpec> {
+  const result: HeadingsBySpec = Object.create(null);
+  if (!existsSync(headingsDir)) {
+    console.warn(`Headings directory not found: ${headingsDir}`);
+    return result;
+  }
+
+  const files = await readdir(headingsDir);
+  const jsonFiles = files.filter(f => f.endsWith(".json"));
+
+  console.log(`Processing ${jsonFiles.length} heading files...`);
+  for (const file of jsonFiles) {
+    try {
+      const data = await readJSON<HeadingsJSON>(path.join(headingsDir, file));
+      // Shortname derived from filename (webref doesn't include it in the JSON)
+      const shortname = file.replace(/\.json$/, "").toLowerCase();
+      const byId: { [id: string]: HeadingEntry } = Object.create(null);
+      for (const h of data.headings || []) {
+        byId[h.id] = {
+          id: h.id,
+          href: h.href,
+          title: h.title,
+          number: h.number,
+          level: h.level,
+        };
+      }
+      result[shortname] = byId;
+    } catch (error) {
+      console.error(`Error reading headings from ${file}:`, error);
+    }
+  }
+
+  return result;
 }

routes/xref/lib/store.ts

@@ -3,18 +3,25 @@ import { readFileSync } from "fs";
 
 import { env } from "../../../utils/misc.js";
 import { DataEntry } from "./search.js";
+import { HeadingEntry, HeadingsBySpec } from "./scraper.js";
+
+export type SpecMapGroup = {
+  [specid: string]: {
+    url: string;
+    shortname: string;
+    title: string;
+  };
+};
 
 export class Store {
   version = -1;
   bySpec: { [shortname: string]: DataEntry[] } = {};
   byTerm: { [term: string]: DataEntry[] } = {};
-  specmap: {
-    [specid: string]: {
-      url: string;
-      shortname: string;
-      title: string;
-    };
-  } = {};
+  specmap: { [group: string]: SpecMapGroup } = {};
+  /** Headings pre-indexed by spec shortname, then by fragment id. */
+  headings: HeadingsBySpec = {};
+  /** Reverse lookup: shortname → spec title. */
+  private specTitleByShortname: Map<string, string> = new Map();
 
   constructor() {
     this.fill();
@@ -25,13 +32,85 @@ export class Store {
     this.byTerm = readJson("xref.json");
     this.bySpec = readJson("specs.json");
     this.specmap = readJson("specmap.json");
+    this.headings = readJsonOptional("headings.json");
+    this.specTitleByShortname = buildSpecTitleMap(this.specmap);
     this.version = Date.now();
   }
+
+  /** Look up a heading by spec shortname and fragment id. */
+  getHeading(
+    spec: string,
+    id: string,
+  ): (HeadingEntry & { specTitle: string }) | null {
+    const normalizedSpec = spec.toLowerCase();
+    const specHeadings = this.resolveHeadings(normalizedSpec);
+    if (!specHeadings) return null;
+
+    const heading = specHeadings[id];
+    if (!heading) return null;
+
+    return {
+      ...heading,
+      specTitle: this.specTitleByShortname.get(normalizedSpec)
+        || this.specTitleByShortname.get(normalizedSpec.replace(/-\d+$/, ""))
+        || spec,
+    };
+  }
+
+  private resolveHeadings(spec: string): Record<string, HeadingEntry> | null {
+    const direct = this.headings[spec];
+    if (direct) return direct;
+
+    // Try stripping version suffix (e.g., cssom-view-1 → cssom-view)
+    const stripped = spec.replace(/-\d+$/, "");
+    if (stripped !== spec) {
+      const unversioned = this.headings[stripped];
+      if (unversioned) return unversioned;
+    }
+
+    // Try resolving series shortname to versioned (or vice versa) via specmap
+    for (const group of Object.values(this.specmap)) {
+      for (const [specId, entry] of Object.entries(group)) {
+        if (entry.shortname === spec || entry.shortname === stripped) {
+          const resolved = this.headings[specId];
+          if (resolved) return resolved;
+        }
+      }
+    }
+
+    return null;
+  }
 }
 
+/** Read a required JSON data file. Throws if missing. */
 function readJson(filename: string) {
   const DATA_DIR = env("DATA_DIR");
   const dataFile = path.resolve(DATA_DIR, `./xref/${filename}`);
   const text = readFileSync(dataFile, "utf8");
   return JSON.parse(text);
 }
+
+/** Read an optional JSON data file. Returns {} if missing. */
+function readJsonOptional(filename: string) {
+  try {
+    return readJson(filename);
+  } catch (err: any) {
+    if (err?.code === "ENOENT") {
+      console.warn(`Optional data file not found: ${filename}`);
+      return {};
+    }
+    throw err;
+  }
+}
+
+/** Build a shortname → title map from the specmap for O(1) title lookup. */
+function buildSpecTitleMap(specmap: Store["specmap"]): Map<string, string> {
+  const result = new Map<string, string>();
+  // specmap is { current: { [specid]: entry }, snapshot: { [specid]: entry } }
+  for (const group of Object.values(specmap)) {
+    for (const entry of Object.values(group)) {
+      result.set(entry.shortname, entry.title);
+    }
+  }
+  return result;
+}

routes/xref/update.ts

@@ -58,5 +58,7 @@ function hasRelevantUpdate(commits: Commit[]) {
   const changedFiles = commits
     .map(commit => [commit.added, commit.removed, commit.modified])
     .flat(2);
-  return changedFiles.some(file => file?.startsWith("ed/dfns/"));
+  return changedFiles.some(
+    file => file?.startsWith("ed/dfns/") || file?.startsWith("ed/headings/"),
+  );
 }