feat(api/baseline): add web-features/baseline data endpoint

.env.sample

@@ -10,6 +10,7 @@ DATA_DIR=/home/respec.org/data
 W3C_WEBREF_SECRET=xxxxxxxxxxxxxxxxxxxxxx
 CANIUSE_SECRET=xxxxxxxxxxxxxxxxxxxxxx
 RESPEC_SECRET=xxxxxxxxxxxxxxxxxxxxxx
+WEB_FEATURES_SECRET=xxxxxxxxxxxxxxxxxxxxxx
 
 # GitHub Action secret on speced/respec
 RESPEC_GH_ACTION_SECRET=xxxxxxxxxxxxxxxxxxxxxx

app.ts

@@ -14,6 +14,7 @@ import caniuseRouter from "./routes/caniuse/index.js";
 import githubRouter from "./routes/github/index.js";
 import respecRouter from "./routes/respec/index.js";
 import w3cRouter from "./routes/w3c/index.js";
+import baselineRouter from "./routes/api/baseline/index.js";
 import wellKnownRouter from "./routes/well-known/index.js";
 import docsRouter from "./routes/docs/index.js";
 
@@ -46,6 +47,7 @@ app.use(
 
 app.use("/xref", xrefRouter);
 app.use("/caniuse", caniuseRouter);
+app.use("/api/baseline", baselineRouter);
 app.use("/github/:org/:repo", githubRouter);
 app.use("/respec", respecRouter);
 app.use("/w3c", w3cRouter);

routes/api/baseline/all.ts

@@ -0,0 +1,14 @@
+import { Request, Response } from "express";
+
+import { seconds } from "../../../utils/misc.js";
+import { store } from "./lib/store-init.js";
+
+export default function route(_req: Request, res: Response) {
+  if (!store.data) {
+    res.sendStatus(404);
+    return;
+  }
+
+  res.set("Cache-Control", `max-age=${seconds("24h")}`);
+  res.json(store.data);
+}

routes/api/baseline/feature.ts

@@ -0,0 +1,49 @@
+import { Request, Response } from "express";
+
+import { seconds } from "../../../utils/misc.js";
+import { store } from "./lib/store-init.js";
+
+type Params = { feature: string };
+type IRequest = Request<Params>;
+
+export default function route(req: IRequest, res: Response) {
+  const { feature: featureId } = req.params;
+  const cacheControl = `max-age=${seconds("24h")}`;
+
+  const featureData = store.byFeature.get(featureId);
+  if (featureData) {
+    res.set("Cache-Control", cacheControl);
+    res.json({ id: featureId, ...featureData });
+    return;
+  }
+
+  // Check if it's a moved or split feature in the raw data
+  const rawFeature = store.data?.features[featureId];
+  if (!rawFeature) {
+    res.sendStatus(404);
+    return;
+  }
+
+  if (rawFeature.kind === "moved" && rawFeature.redirect_target) {
+    const target = store.byFeature.get(rawFeature.redirect_target);
+    if (target) {
+      res.set("Cache-Control", cacheControl);
+      res.json({ id: rawFeature.redirect_target, redirected_from: featureId, ...target });
+      return;
+    }
+  }
+
+  if (rawFeature.kind === "split" && rawFeature.redirect_targets) {
+    const targets = rawFeature.redirect_targets
+      .map(targetId => {
+        const target = store.byFeature.get(targetId);
+        return target ? { id: targetId, ...target } : null;
+      })
+      .filter(Boolean);
+    res.set("Cache-Control", cacheControl);
+    res.json({ id: featureId, kind: "split", split_into: targets });
+    return;
+  }
+
+  res.sendStatus(404);
+}

routes/api/baseline/index.ts

@@ -0,0 +1,32 @@
+import express from "express";
+import cors from "cors";
+
+import authGithubWebhook from "../../../utils/auth-github-webhook.js";
+import { env, ms } from "../../../utils/misc.js";
+
+import allRoute from "./all.js";
+import featureRoute from "./feature.js";
+import searchRoute from "./search.post.js";
+import updateRoute from "./update.js";
+
+const baseline = express.Router({ mergeParams: true });
+
+baseline
+  .options("/", cors({ methods: ["GET"], maxAge: ms("1day") }))
+  .get("/", cors(), allRoute);
+
+baseline
+  .options("/search", cors({ methods: ["POST"], maxAge: ms("1day") }))
+  .post("/search", express.json({ limit: "1mb" }), cors(), searchRoute);
+
+baseline
+  .options("/:feature", cors({ methods: ["GET"], maxAge: ms("1day") }))
+  .get("/:feature", cors(), featureRoute);
+
+baseline.post(
+  "/update",
+  authGithubWebhook(env("WEB_FEATURES_SECRET")),
+  updateRoute,
+);
+
+export default baseline;

routes/api/baseline/lib/scraper.ts

@@ -0,0 +1,53 @@
+import path from "path";
+import { mkdir, readFile, rename, writeFile } from "fs/promises";
+
+import { env } from "../../../../utils/misc.js";
+
+const DATA_DIR = env("DATA_DIR");
+const LATEST_DATA_URL =
+  "https://github.com/web-platform-dx/web-features/releases/latest/download/data.json";
+
+export default async function main(): Promise<boolean> {
+  const outputDir = path.join(DATA_DIR, "baseline");
+  const dataFile = path.join(outputDir, "baseline.json");
+  const etagFile = path.join(outputDir, "baseline.etag");
+
+  const headers: Record<string, string> = {};
+  try {
+    const savedEtag = (await readFile(etagFile, "utf8")).trim();
+    if (savedEtag) {
+      headers["If-None-Match"] = savedEtag;
+    }
+  } catch {
+    // No saved ETag yet; proceed without conditional request
+  }
+
+  const dataRes = await fetch(LATEST_DATA_URL, { headers });
+
+  if (dataRes.status === 304) {
+    // Data is already up to date; no download needed.
+    return false;
+  }
+
+  if (!dataRes.ok) {
+    throw new Error(
+      `Failed to download data.json: ${dataRes.status} ${dataRes.statusText}`,
+    );
+  }
+
+  const data = await dataRes.text();
+
+  await mkdir(outputDir, { recursive: true });
+
+  const tempFile = `${dataFile}.tmp`;
+  await writeFile(tempFile, data);
+  await rename(tempFile, dataFile);
+
+  const etag = dataRes.headers.get("etag");
+  if (etag) {
+    await writeFile(etagFile, etag);
+  }
+
+  // Returns true when new data was downloaded, false when already up to date.
+  return true;
+}

routes/api/baseline/lib/store-init.ts

@@ -0,0 +1,3 @@
+import { BaselineStore } from "./store.js";
+
+export const store = new BaselineStore();

routes/api/baseline/lib/store.ts

@@ -0,0 +1,108 @@
+import path from "path";
+import { existsSync, readFileSync } from "fs";
+
+import { env } from "../../../../utils/misc.js";
+
+interface SupportData {
+  chrome?: string;
+  chrome_android?: string;
+  edge?: string;
+  firefox?: string;
+  firefox_android?: string;
+  safari?: string;
+  safari_ios?: string;
+}
+
+interface StatusData {
+  baseline: "high" | "low" | false;
+  baseline_low_date?: string;
+  baseline_high_date?: string;
+  support: SupportData;
+}
+
+export interface FeatureData {
+  kind: "feature" | "moved" | "split";
+  name?: string;
+  description?: string;
+  description_html?: string;
+  spec?: string[];
+  status?: StatusData;
+  caniuse?: string[];
+  compat_features?: string[];
+  group?: string[];
+  snapshot?: string[];
+  redirect_target?: string;
+  redirect_targets?: string[];
+}
+
+interface WebFeaturesData {
+  browsers: Record<string, unknown>;
+  features: Record<string, FeatureData>;
+  groups: Record<string, unknown>;
+  snapshots: Record<string, unknown>;
+}
+
+export function normalizeUrl(url: string): string {
+  const parsed = URL.parse(url);
+  if (!parsed) return url;
+  parsed.hash = "";
+  parsed.search = "";
+  if (!parsed.pathname.endsWith("/") && !parsed.pathname.includes(".")) {
+    parsed.pathname += "/";
+  }
+  return parsed.href;
+}
+
+export class BaselineStore {
+  version = -1;
+  data: WebFeaturesData | null = null;
+  byFeature = new Map<string, FeatureData>();
+  bySpecUrl = new Map<string, string[]>();
+
+  constructor() {
+    this.fill();
+  }
+
+  fill() {
+    const dataFile = path.resolve(env("DATA_DIR"), "baseline/baseline.json");
+
+    if (!existsSync(dataFile)) {
+      console.warn("baseline: data file not found, store is empty.");
+      this.data = null;
+      this.byFeature = new Map();
+      this.bySpecUrl = new Map();
+      this.version = Date.now();
+      return;
+    }
+
+    try {
+      const data = JSON.parse(readFileSync(dataFile, "utf8")) as WebFeaturesData;
+
+      const features = Object.entries(data.features).filter(
+        ([, feature]) => feature.kind === "feature",
+      );
+
+      const byFeature = new Map(features);
+      const bySpecUrl = new Map<string, string[]>();
+      for (const [featureId, feature] of features) {
+        for (const specUrl of feature.spec ?? []) {
+          const normalized = normalizeUrl(specUrl);
+          const ids = bySpecUrl.get(normalized) ?? [];
+          ids.push(featureId);
+          bySpecUrl.set(normalized, ids);
+        }
+      }
+
+      this.data = data;
+      this.byFeature = byFeature;
+      this.bySpecUrl = bySpecUrl;
+      this.version = Date.now();
+    } catch (error) {
+      console.warn(
+        "baseline: failed to read or parse data file, keeping existing store data.",
+        error,
+      );
+      return;
+    }
+  }
+}

routes/api/baseline/search.post.ts

@@ -0,0 +1,58 @@
+import { Request, Response } from "express";
+
+import { seconds } from "../../../utils/misc.js";
+import { store } from "./lib/store-init.js";
+import { normalizeUrl } from "./lib/store.js";
+
+const MAX_SPECS = 50;
+
+interface SearchBody {
+  specs: string[];
+}
+
+type IRequest = Request<unknown, unknown, SearchBody>;
+
+export default function route(req: IRequest, res: Response) {
+  if (!store.data) {
+    res.sendStatus(503);
+    return;
+  }
+
+  const { specs } = req.body;
+
+  if (!Array.isArray(specs) || specs.length === 0) {
+    res.status(400);
+    res.json({ error: "Request body must contain a `specs` array." });
+    return;
+  }
+
+  if (specs.length > MAX_SPECS) {
+    res.status(400);
+    res.json({ error: `Too many spec URLs. Maximum is ${MAX_SPECS}, got ${specs.length}.` });
+    return;
+  }
+
+  if (!specs.every(s => typeof s === "string" && s.trim().length > 0)) {
+    res.status(400);
+    res.json({ error: "Each spec must be a non-empty string." });
+    return;
+  }
+
+  const normalizedSpecs = specs.map(normalizeUrl);
+
+  const matchingIds = new Set<string>();
+  for (const [url, ids] of store.bySpecUrl) {
+    if (normalizedSpecs.some(spec => url.startsWith(spec))) {
+      for (const id of ids) {
+        matchingIds.add(id);
+      }
+    }
+  }
+
+  const result = [...matchingIds]
+    .map(id => ({ id, ...store.byFeature.get(id)! }))
+    .filter(entry => entry.name);
+
+  res.set("Cache-Control", `max-age=${seconds("30m")}`);
+  res.json({ result });
+}

routes/api/baseline/update.ts

@@ -0,0 +1,34 @@
+import path from "path";
+
+import { Request, Response } from "express";
+
+import { BackgroundTaskQueue } from "../../../utils/background-task-queue.js";
+import { store } from "./lib/store-init.js";
+
+const workerFile = path.join(import.meta.dirname, "update.worker.js");
+const taskQueue = new BackgroundTaskQueue<typeof import("./update.worker")>(
+  workerFile,
+  "baseline_update",
+);
+
+export default async function route(req: Request, res: Response) {
+  if (req.body.action !== "published") {
+    res.status(400);
+    res.locals.reason = "action-not-published";
+    res.send("Webhook action ignored (expected 'published').");
+    return;
+  }
+
+  const job = taskQueue.add({ webhookId: req.get("X-GitHub-Delivery") || "" });
+  try {
+    const { updated } = await job.run();
+    if (updated) {
+      store.fill();
+    }
+  } catch {
+    res.status(500);
+  } finally {
+    res.locals.job = job.id;
+    res.send(job.id);
+  }
+}

routes/api/baseline/update.worker.ts

@@ -0,0 +1,10 @@
+import baselineScraper from "./lib/scraper.js";
+
+interface Input {
+  webhookId: string;
+}
+
+export default async function baselineUpdate(_input: Input) {
+  const updated = await baselineScraper();
+  return { updated };
+}