feat(xref): add headings lookup API for cross-spec section links (#469)

* feat(xref): add headings lookup API for cross-spec section links

Extends the xref infrastructure to read and serve section heading data
from WebRef's ed/headings/ directory. This enables ReSpec's [[[SPEC#id]]]
syntax to display actual heading text instead of just the spec title.

New endpoint: POST /xref/headings
Request:  { queries: [{ spec: 'fetch', id: 'cookie-header' }] }
Response: { result: [{ spec, id, title, number, href, level, specTitle }] }

Changes:
- scraper.ts: reads ed/headings/*.json during update, writes headings.json
- store.ts: loads headings data, adds getHeading(spec, id) lookup method
- headings.post.ts: new route handler
- index.ts: registers the /xref/headings POST endpoint
- update.ts: also triggers on ed/headings/ changes in webref webhook

* fix: address Copilot review feedback on headings API

- store.ts: index headings by id for O(1) lookup instead of linear scan;
  precompute specTitleByShortname map; split readJson into required/optional
  variants so missing xref.json still throws
- scraper.ts: fix doc comment on readAllHeadings
- headings.post.ts: validate queries is an array before mapping

* refactor(xref): address reviewer feedback on headings API

- Move `readdir` to top-level fs/promises import in scraper.ts
- Add generic type `readJSON<T>` with HeadingsJSON interface to reduce `any`
- Use typed readJSON<T> calls in getAllData and readAllHeadings
- Change specTitleByShortname to Map<string, string> in store.ts
- Add per-item validation (spec/id are strings) in headings.post.ts

Agent-Logs-Url: https://github.com/speced/respec-web-services/sessions/e14c5758-5d6f-4324-bb98-77839d96f660

* Apply suggestion from @sidvishnoi

Co-authored-by: Sid Vishnoi <8426945+sidvishnoi@users.noreply.github.com>

* Update routes/xref/index.ts

Co-authored-by: Sid Vishnoi <8426945+sidvishnoi@users.noreply.github.com>

* fix(xref/headings): fix specTitleMap, add query limit, harden error handling

- Fix buildSpecTitleMap to iterate nested specmap structure correctly
  (was iterating top-level {current, snapshot} objects instead of entries)
- Add 1000-query limit and empty-string validation to headings endpoint
- Only catch ENOENT in readJsonOptional (was swallowing all errors)
- Remove dead spec?.shortname branch from scraper (webref doesn't include it)
- Update JSDoc to reflect actual route path (/xref/search/headings)

* refactor(xref/headings): pre-index headings by ID at scrape time

Per sidvishnoi's review: index headings by ID during scraping rather than
at store load time. Removes the redundant indexHeadings() runtime function.

* fix(xref/headings): trim inputs, add version/series fallback in getHeading

Per Copilot review:
- Trim spec and id before lookup (validation checked trim but passed raw)
- Add version-stripped and series-shortname fallback in getHeading so
  both "cssom-view-1" and "cssom-view" resolve headings correctly
- Also resolves specTitle via stripped form as fallback

* fix: correct specmap type to match nested JSON structure

The specmap type previously described a flat map, but the actual data
is nested with current/snapshot groups. This caused TS2352 errors
after rebasing onto main's stricter type checking.

* fix: address Sid's review comments on PR #469

- Extract heading resolution fallback into resolveHeadings() method
- Remove unnecessary generator (specmapEntries) — iterate directly
- Use separate res.status()/res.json() calls instead of chaining

---------

Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com>
Co-authored-by: Sid Vishnoi <8426945+sidvishnoi@users.noreply.github.com>

Author: 3 people

routes/xref/headings.post.ts

@@ -0,0 +1,63 @@
+import { Request, Response } from "express";
+import { store } from "./lib/store-init.js";
+
+interface HeadingsQuery {
+  spec: string;
+  id: string;
+}
+
+interface RequestBody {
+  queries: HeadingsQuery[];
+}
+
+type IRequest = Request<never, any, RequestBody>;
+
+/**
+ * POST /xref/search/headings
+ *
+ * Looks up section headings by spec shortname and fragment id.
+ * Used by ReSpec's [[[SPEC#id]]] syntax to get heading text for
+ * cross-spec section links.
+ *
+ * Request body: { queries: [{ spec: "fetch", id: "cookie-header" }] }
+ * Response: { result: [{ spec: "fetch", id: "cookie-header", ... }] }
+ */
+export default function route(req: IRequest, res: Response) {
+  const { queries } = req.body;
+  if (!Array.isArray(queries)) {
+    res.status(400).json({ error: "queries must be an array" });
+    return;
+  }
+  if (queries.length > 1000) {
+    res.status(400).json({ error: "too many queries (max 1000)" });
+    return;
+  }
+  for (const item of queries) {
+    if (typeof item?.spec !== "string" || typeof item?.id !== "string") {
+      res.status(400);
+      res.json({ error: "each query must have string fields: spec, id" });
+      return;
+    }
+    if (!item.spec.trim() || !item.id.trim()) {
+      res.status(400);
+      res.json({ error: "spec and id must be non-empty strings" });
+      return;
+    }
+  }
+  const result = queries.map(({ spec, id }) => {
+    const heading = store.getHeading(spec.trim(), id.trim());
+    if (!heading) {
+      return { spec, id, error: "not found" };
+    }
+    return {
+      spec,
+      id,
+      title: heading.title,
+      number: heading.number || null,
+      href: heading.href,
+      level: heading.level,
+      specTitle: heading.specTitle,
+    };
+  });
+  res.json({ result });
+}

routes/xref/index.ts

@@ -10,6 +10,7 @@ import { env, ms } from "../../utils/misc.js";
 import { store } from "./lib/store-init.js";
 import searchRouteGet from "./search.get.js";
 import searchRoutePost from "./search.post.js";
+import headingsRoutePost from "./headings.post.js";
 import metaRoute from "./meta.js";
 import updateRoute from "./update.js";
 import { search, Options, Query } from "./lib/search.js";
@@ -26,6 +27,9 @@ xref
   .get("/search", cors(), searchRouteGet)
   .post("/search", express.json({ limit: "2mb" }), cors(), searchRoutePost);
 xref.get("/meta{/:field}", cors(), metaRoute);
+xref
+  .options("/search/headings", cors({ methods: ["POST"], maxAge: ms("1day") }))
+  .post("/search/headings", express.json({ limit: "1mb" }), cors(), headingsRoutePost);
 xref.post("/update", authGithubWebhook(env("W3C_WEBREF_SECRET")), updateRoute);
 xref.use("/data", express.static(path.join(DATA_DIR, "xref")));
 

routes/xref/lib/scraper.ts

@@ -5,13 +5,13 @@
 
 import path from "path";
 import { existsSync } from "fs";
-import { mkdir, readFile, writeFile } from "fs/promises";
+import { mkdir, readFile, readdir, writeFile } from "fs/promises";
 
 import { Definition as InputDfn, DfnsJSON, SpecsJSON } from "webref";
 
 import { SUPPORTED_TYPES, CSS_TYPES_INPUT } from "./constants.js";
 import { uniq } from "./utils.js";
-import { Store } from "./store.js";
+import { Store, SpecMapGroup } from "./store.js";
 import { env } from "../../../utils/misc.js";
 import sh from "../../../utils/sh.js";
 
@@ -25,6 +25,7 @@ const OUT_DIR_BASE = path.join(DATA_DIR, "xref");
 const OUTFILE_BY_TERM = path.resolve(OUT_DIR_BASE, "./xref.json");
 const OUTFILE_BY_SPEC = path.resolve(OUT_DIR_BASE, "./specs.json");
 const OUTFILE_SPECMAP = path.resolve(OUT_DIR_BASE, "./specmap.json");
+const OUTFILE_HEADINGS = path.resolve(OUT_DIR_BASE, "./headings.json");
 
 type Status = "current" | "snapshot";
 const dirToStatus = [
@@ -41,6 +42,22 @@ interface DataBySpec {
   [shortname: string]: Omit<ParsedDataEntry, "shortname" | "isExported">[];
 }
 
+export interface HeadingEntry {
+  id: string;
+  href: string;
+  title: string;
+  number?: string;
+  level: number;
+}
+
+export interface HeadingsBySpec {
+  [shortname: string]: { [id: string]: HeadingEntry };
+}
+
+interface HeadingsJSON {
+  headings?: HeadingEntry[];
+}
+
 const defaultOptions = { forceUpdate: false };
 type Options = typeof defaultOptions;
 
@@ -55,8 +72,8 @@ export default async function main(options: Partial<Options> = {}) {
   const dataByTerm: DataByTerm = Object.create(null);
   const dataBySpec: DataBySpec = Object.create(null);
   const specificationsMap = {
-    current: {} as Store["specmap"],
-    snapshot: {} as Store["specmap"],
+    current: {} as SpecMapGroup,
+    snapshot: {} as SpecMapGroup,
   };
 
   for (const [dir, status] of dirToStatus) {
@@ -83,12 +100,18 @@ export default async function main(options: Partial<Options> = {}) {
     dataByTerm[term] = uniq(dataByTerm[term]);
   }
 
+  // Read headings data from webref (ed/ only, same as xref default)
+  const headingsBySpec = await readAllHeadings(
+    path.join(INPUT_DIR_BASE, "ed", "headings"),
+  );
+
   console.log("Writing processed data files...");
   await mkdir(OUT_DIR_BASE, { recursive: true });
   await Promise.all([
     writeFile(OUTFILE_BY_TERM, JSON.stringify(dataByTerm, null, 2)),
     writeFile(OUTFILE_BY_SPEC, JSON.stringify(dataBySpec, null, 2)),
     writeFile(OUTFILE_SPECMAP, JSON.stringify(specificationsMap, null, 2)),
+    writeFile(OUTFILE_HEADINGS, JSON.stringify(headingsBySpec, null, 2)),
   ]);
   return true;
 }
@@ -193,15 +216,15 @@ function normalizeTerm(term: string, type: string) {
 async function getAllData(baseDir: string) {
   const SPECS_JSON = path.resolve(baseDir, "./index.json");
   console.log(`Getting data from ${SPECS_JSON}`);
-  const urlFileContent = await readJSON(SPECS_JSON);
+  const urlFileContent = await readJSON<{ results: SpecsJSON[] }>(SPECS_JSON);
   const data: SpecsJSON[] = urlFileContent.results;
 
-  const specMap: Store["specmap"] = Object.create(null);
+  const specMap: SpecMapGroup = Object.create(null);
   const dfnSources: DfnsJSON[] = [];
 
   for (const entry of data) {
     if (entry.dfns) {
-      const dfnsData = await readJSON(path.join(baseDir, entry.dfns));
+      const dfnsData = await readJSON<{ dfns: InputDfn[] }>(path.join(baseDir, entry.dfns));
       const dfns: InputDfn[] = dfnsData.dfns;
       dfnSources.push({
         series: entry.series.shortname,
@@ -221,7 +244,48 @@ async function getAllData(baseDir: string) {
   return { specMap, dfnSources };
 }
 
-async function readJSON(filePath: string) {
+async function readJSON<T = unknown>(filePath: string): Promise<T> {
   const text = await readFile(filePath, "utf-8");
-  return JSON.parse(text);
+  return JSON.parse(text) as T;
+}
+
+/**
+ * Read all headings data from webref's ed/headings/ directory.
+ * Returns { shortname: { id: HeadingEntry } } — pre-indexed for O(1) lookup.
+ */
+async function readAllHeadings(
+  headingsDir: string,
+): Promise<HeadingsBySpec> {
+  const result: HeadingsBySpec = Object.create(null);
+  if (!existsSync(headingsDir)) {
+    console.warn(`Headings directory not found: ${headingsDir}`);
+    return result;
+  }
+
+  const files = await readdir(headingsDir);
+  const jsonFiles = files.filter(f => f.endsWith(".json"));
+
+  console.log(`Processing ${jsonFiles.length} heading files...`);
+  for (const file of jsonFiles) {
+    try {
+      const data = await readJSON<HeadingsJSON>(path.join(headingsDir, file));
+      // Shortname derived from filename (webref doesn't include it in the JSON)
+      const shortname = file.replace(/\.json$/, "").toLowerCase();
+      const byId: { [id: string]: HeadingEntry } = Object.create(null);
+      for (const h of data.headings || []) {
+        byId[h.id] = {
+          id: h.id,
+          href: h.href,
+          title: h.title,
+          number: h.number,
+          level: h.level,
+        };
+      }
+      result[shortname] = byId;
+    } catch (error) {
+      console.error(`Error reading headings from ${file}:`, error);
+    }
+  }
+
+  return result;
 }

routes/xref/lib/store.ts

@@ -3,18 +3,25 @@ import { readFileSync } from "fs";
 
 import { env } from "../../../utils/misc.js";
 import { DataEntry } from "./search.js";
+import { HeadingEntry, HeadingsBySpec } from "./scraper.js";
+
+export type SpecMapGroup = {
+  [specid: string]: {
+    url: string;
+    shortname: string;
+    title: string;
+  };
+};
 
 export class Store {
   version = -1;
   bySpec: { [shortname: string]: DataEntry[] } = {};
   byTerm: { [term: string]: DataEntry[] } = {};
-  specmap: {
-    [specid: string]: {
-      url: string;
-      shortname: string;
-      title: string;
-    };
-  } = {};
+  specmap: { [group: string]: SpecMapGroup } = {};
+  /** Headings pre-indexed by spec shortname, then by fragment id. */
+  headings: HeadingsBySpec = {};
+  /** Reverse lookup: shortname → spec title. */
+  private specTitleByShortname: Map<string, string> = new Map();
 
   constructor() {
     this.fill();
@@ -25,13 +32,85 @@ export class Store {
     this.byTerm = readJson("xref.json");
     this.bySpec = readJson("specs.json");
     this.specmap = readJson("specmap.json");
+    this.headings = readJsonOptional("headings.json");
+    this.specTitleByShortname = buildSpecTitleMap(this.specmap);
     this.version = Date.now();
   }
+
+  /** Look up a heading by spec shortname and fragment id. */
+  getHeading(
+    spec: string,
+    id: string,
+  ): (HeadingEntry & { specTitle: string }) | null {
+    const normalizedSpec = spec.toLowerCase();
+    const specHeadings = this.resolveHeadings(normalizedSpec);
+    if (!specHeadings) return null;
+
+    const heading = specHeadings[id];
+    if (!heading) return null;
+
+    return {
+      ...heading,
+      specTitle: this.specTitleByShortname.get(normalizedSpec)
+        || this.specTitleByShortname.get(normalizedSpec.replace(/-\d+$/, ""))
+        || spec,
+    };
+  }
+
+  private resolveHeadings(spec: string): Record<string, HeadingEntry> | null {
+    const direct = this.headings[spec];
+    if (direct) return direct;
+
+    // Try stripping version suffix (e.g., cssom-view-1 → cssom-view)
+    const stripped = spec.replace(/-\d+$/, "");
+    if (stripped !== spec) {
+      const unversioned = this.headings[stripped];
+      if (unversioned) return unversioned;
+    }
+
+    // Try resolving series shortname to versioned (or vice versa) via specmap
+    for (const group of Object.values(this.specmap)) {
+      for (const [specId, entry] of Object.entries(group)) {
+        if (entry.shortname === spec || entry.shortname === stripped) {
+          const resolved = this.headings[specId];
+          if (resolved) return resolved;
+        }
+      }
+    }
+
+    return null;
+  }
 }
 
+/** Read a required JSON data file. Throws if missing. */
 function readJson(filename: string) {
   const DATA_DIR = env("DATA_DIR");
   const dataFile = path.resolve(DATA_DIR, `./xref/${filename}`);
   const text = readFileSync(dataFile, "utf8");
   return JSON.parse(text);
 }
+
+/** Read an optional JSON data file. Returns {} if missing. */
+function readJsonOptional(filename: string) {
+  try {
+    return readJson(filename);
+  } catch (err: any) {
+    if (err?.code === "ENOENT") {
+      console.warn(`Optional data file not found: ${filename}`);
+      return {};
+    }
+    throw err;
+  }
+}
+
+/** Build a shortname → title map from the specmap for O(1) title lookup. */
+function buildSpecTitleMap(specmap: Store["specmap"]): Map<string, string> {
+  const result = new Map<string, string>();
+  // specmap is { current: { [specid]: entry }, snapshot: { [specid]: entry } }
+  for (const group of Object.values(specmap)) {
+    for (const entry of Object.values(group)) {
+      result.set(entry.shortname, entry.title);
+    }
+  }
+  return result;
+}

routes/xref/update.ts

@@ -58,5 +58,7 @@ function hasRelevantUpdate(commits: Commit[]) {
   const changedFiles = commits
     .map(commit => [commit.added, commit.removed, commit.modified])
     .flat(2);
-  return changedFiles.some(file => file?.startsWith("ed/dfns/"));
+  return changedFiles.some(
+    file => file?.startsWith("ed/dfns/") || file?.startsWith("ed/headings/"),
+  );
 }