From e57b4b0584b38a974ac1ebe383739c3aeee73c08 Mon Sep 17 00:00:00 2001 From: Marcos Caceres Date: Sat, 25 Apr 2026 17:22:50 +1000 Subject: [PATCH 1/7] feat(api/baseline): add web-features/baseline data endpoint MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds endpoints for serving Baseline browser availability data: - GET /api/baseline — full web-features dataset - GET /api/baseline/:feature — single feature lookup - POST /api/baseline/search — spec-URL-based feature search Data sourced from web-platform-dx/web-features GitHub releases. Enables ReSpec's implementationStatus feature (speced/respec#5174) without requiring unpkg.com dependency. Closes #490 --- .env.sample | 1 + app.ts | 2 + routes/api/baseline/all.ts | 14 ++++ routes/api/baseline/feature.ts | 55 +++++++++++++ routes/api/baseline/index.ts | 32 ++++++++ routes/api/baseline/lib/scraper.ts | 56 ++++++++++++++ routes/api/baseline/lib/store-init.ts | 3 + routes/api/baseline/lib/store.ts | 106 ++++++++++++++++++++++++++ routes/api/baseline/search.post.ts | 53 +++++++++++++ routes/api/baseline/update.ts | 27 +++++++ routes/api/baseline/update.worker.ts | 10 +++ scripts/update-data-sources.ts | 5 ++ 12 files changed, 364 insertions(+) create mode 100644 routes/api/baseline/all.ts create mode 100644 routes/api/baseline/feature.ts create mode 100644 routes/api/baseline/index.ts create mode 100644 routes/api/baseline/lib/scraper.ts create mode 100644 routes/api/baseline/lib/store-init.ts create mode 100644 routes/api/baseline/lib/store.ts create mode 100644 routes/api/baseline/search.post.ts create mode 100644 routes/api/baseline/update.ts create mode 100644 routes/api/baseline/update.worker.ts diff --git a/.env.sample b/.env.sample index 01df87a9..12ca5abc 100644 --- a/.env.sample +++ b/.env.sample @@ -10,6 +10,7 @@ DATA_DIR=/home/respec.org/data W3C_WEBREF_SECRET=xxxxxxxxxxxxxxxxxxxxxx CANIUSE_SECRET=xxxxxxxxxxxxxxxxxxxxxx RESPEC_SECRET=xxxxxxxxxxxxxxxxxxxxxx +WEB_FEATURES_SECRET=xxxxxxxxxxxxxxxxxxxxxx # GitHub Action secret on speced/respec RESPEC_GH_ACTION_SECRET=xxxxxxxxxxxxxxxxxxxxxx diff --git a/app.ts b/app.ts index 30264dfb..0853e9f2 100644 --- a/app.ts +++ b/app.ts @@ -14,6 +14,7 @@ import caniuseRouter from "./routes/caniuse/index.js"; import githubRouter from "./routes/github/index.js"; import respecRouter from "./routes/respec/index.js"; import w3cRouter from "./routes/w3c/index.js"; +import baselineRouter from "./routes/api/baseline/index.js"; import wellKnownRouter from "./routes/well-known/index.js"; import docsRouter from "./routes/docs/index.js"; @@ -46,6 +47,7 @@ app.use( app.use("/xref", xrefRouter); app.use("/caniuse", caniuseRouter); +app.use("/api/baseline", baselineRouter); app.use("/github/:org/:repo", githubRouter); app.use("/respec", respecRouter); app.use("/w3c", w3cRouter); diff --git a/routes/api/baseline/all.ts b/routes/api/baseline/all.ts new file mode 100644 index 00000000..798927ea --- /dev/null +++ b/routes/api/baseline/all.ts @@ -0,0 +1,14 @@ +import { Request, Response } from "express"; + +import { seconds } from "../../../utils/misc.js"; +import { store } from "./lib/store-init.js"; + +export default function route(_req: Request, res: Response) { + if (!store.data) { + res.sendStatus(404); + return; + } + + res.set("Cache-Control", `max-age=${seconds("24h")}`); + res.json(store.data); +} diff --git a/routes/api/baseline/feature.ts b/routes/api/baseline/feature.ts new file mode 100644 index 00000000..4eef2a2c --- /dev/null +++ b/routes/api/baseline/feature.ts @@ -0,0 +1,55 @@ +import { Request, Response } from "express"; + +import { seconds } from "../../../utils/misc.js"; +import { store } from "./lib/store-init.js"; + +type Params = { feature: string }; +type IRequest = Request; + +export default function route(req: IRequest, res: Response) { + const { feature: featureId } = req.params; + const featureData = store.byFeature.get(featureId); + + if (featureData) { + res.set("Cache-Control", `max-age=${seconds("24h")}`); + res.json({ id: featureId, ...featureData }); + return; + } + + // Check if it's a moved or split feature in the raw data + if (store.data) { + const rawFeature = store.data.features[featureId]; + if (rawFeature) { + if (rawFeature.kind === "moved" && rawFeature.redirect_target) { + const target = store.byFeature.get(rawFeature.redirect_target); + if (target) { + res.set("Cache-Control", `max-age=${seconds("24h")}`); + res.json({ + id: rawFeature.redirect_target, + redirected_from: featureId, + ...target, + }); + return; + } + } + + if (rawFeature.kind === "split" && rawFeature.redirect_targets) { + const targets = rawFeature.redirect_targets + .map(targetId => { + const target = store.byFeature.get(targetId); + return target ? { id: targetId, ...target } : null; + }) + .filter(Boolean); + res.set("Cache-Control", `max-age=${seconds("24h")}`); + res.json({ + id: featureId, + kind: "split", + split_into: targets, + }); + return; + } + } + } + + res.sendStatus(404); +} diff --git a/routes/api/baseline/index.ts b/routes/api/baseline/index.ts new file mode 100644 index 00000000..d89f3ffa --- /dev/null +++ b/routes/api/baseline/index.ts @@ -0,0 +1,32 @@ +import express from "express"; +import cors from "cors"; + +import authGithubWebhook from "../../../utils/auth-github-webhook.js"; +import { env, ms } from "../../../utils/misc.js"; + +import allRoute from "./all.js"; +import featureRoute from "./feature.js"; +import searchRoute from "./search.post.js"; +import updateRoute from "./update.js"; + +const baseline = express.Router({ mergeParams: true }); + +baseline + .options("/", cors({ methods: ["GET"], maxAge: ms("1day") })) + .get("/", cors(), allRoute); + +baseline + .options("/search", cors({ methods: ["POST"], maxAge: ms("1day") })) + .post("/search", express.json({ limit: "1mb" }), cors(), searchRoute); + +baseline + .options("/:feature", cors({ methods: ["GET"], maxAge: ms("1day") })) + .get("/:feature", cors(), featureRoute); + +baseline.post( + "/update", + authGithubWebhook(env("WEB_FEATURES_SECRET")), + updateRoute, +); + +export default baseline; diff --git a/routes/api/baseline/lib/scraper.ts b/routes/api/baseline/lib/scraper.ts new file mode 100644 index 00000000..4a6bdb03 --- /dev/null +++ b/routes/api/baseline/lib/scraper.ts @@ -0,0 +1,56 @@ +import path from "path"; +import { mkdir, writeFile } from "fs/promises"; + +import { env } from "../../../../utils/misc.js"; + +const DATA_DIR = env("DATA_DIR"); + +interface ReleaseAsset { + name: string; + browser_download_url: string; +} + +interface ReleaseResponse { + assets: ReleaseAsset[]; +} + +export default async function main() { + const releaseUrl = + "https://api.github.com/repos/web-platform-dx/web-features/releases/latest"; + + const headers: Record = { + Accept: "application/vnd.github+json", + }; + + const ghToken = process.env.GH_TOKEN; + if (ghToken) { + headers.Authorization = `Bearer ${ghToken}`; + } + + const releaseRes = await fetch(releaseUrl, { headers }); + if (!releaseRes.ok) { + throw new Error( + `Failed to fetch latest release: ${releaseRes.status} ${releaseRes.statusText}`, + ); + } + const release: ReleaseResponse = await releaseRes.json(); + + const asset = release.assets.find(a => a.name === "data.json"); + if (!asset) { + throw new Error("No data.json asset in latest web-features release"); + } + + const dataRes = await fetch(asset.browser_download_url); + if (!dataRes.ok) { + throw new Error( + `Failed to download data.json: ${dataRes.status} ${dataRes.statusText}`, + ); + } + const data = await dataRes.text(); + + const outputDir = path.join(DATA_DIR, "baseline"); + await mkdir(outputDir, { recursive: true }); + await writeFile(path.join(outputDir, "baseline.json"), data); + + return true; +} diff --git a/routes/api/baseline/lib/store-init.ts b/routes/api/baseline/lib/store-init.ts new file mode 100644 index 00000000..8443b260 --- /dev/null +++ b/routes/api/baseline/lib/store-init.ts @@ -0,0 +1,3 @@ +import { BaselineStore } from "./store.js"; + +export const store = new BaselineStore(); diff --git a/routes/api/baseline/lib/store.ts b/routes/api/baseline/lib/store.ts new file mode 100644 index 00000000..228bfb6f --- /dev/null +++ b/routes/api/baseline/lib/store.ts @@ -0,0 +1,106 @@ +import path from "path"; +import { existsSync, readFileSync } from "fs"; + +import { env } from "../../../../utils/misc.js"; + +interface SupportData { + chrome?: string; + chrome_android?: string; + edge?: string; + firefox?: string; + firefox_android?: string; + safari?: string; + safari_ios?: string; +} + +interface StatusData { + baseline: "high" | "low" | false; + baseline_low_date?: string; + baseline_high_date?: string; + support: SupportData; +} + +export interface FeatureData { + kind: "feature" | "moved" | "split"; + name?: string; + description?: string; + description_html?: string; + spec?: string[]; + status?: StatusData; + caniuse?: string[]; + compat_features?: string[]; + group?: string[]; + snapshot?: string[]; + redirect_target?: string; + redirect_targets?: string[]; +} + +interface WebFeaturesData { + browsers: Record; + features: Record; + groups: Record; + snapshots: Record; +} + +export function normalizeUrl(url: string): string { + try { + const u = new URL(url); + u.hash = ""; + if (!u.pathname.endsWith("/") && !u.pathname.includes(".")) { + u.pathname += "/"; + } + return u.href; + } catch { + return url; + } +} + +export class BaselineStore { + version = -1; + data: WebFeaturesData | null = null; + byFeature: Map = new Map(); + bySpecUrl: Map = new Map(); + + constructor() { + this.fill(); + } + + /** Fill the store with its contents from the filesystem. */ + fill() { + const DATA_DIR = env("DATA_DIR"); + const dataFile = path.resolve(DATA_DIR, "baseline/baseline.json"); + + if (!existsSync(dataFile)) { + console.warn("baseline: data file not found, store is empty."); + this.data = null; + this.byFeature = new Map(); + this.bySpecUrl = new Map(); + this.version = Date.now(); + return; + } + + const text = readFileSync(dataFile, "utf8"); + const data: WebFeaturesData = JSON.parse(text); + this.data = data; + + this.byFeature = new Map(); + this.bySpecUrl = new Map(); + + for (const [id, feature] of Object.entries(data.features)) { + if (feature.kind !== "feature") continue; + + this.byFeature.set(id, feature); + + if (feature.spec) { + for (const specUrl of feature.spec) { + const normalized = normalizeUrl(specUrl); + const existing = this.bySpecUrl.get(normalized) || []; + existing.push(id); + this.bySpecUrl.set(normalized, existing); + } + } + } + + this.version = Date.now(); + } +} diff --git a/routes/api/baseline/search.post.ts b/routes/api/baseline/search.post.ts new file mode 100644 index 00000000..63358136 --- /dev/null +++ b/routes/api/baseline/search.post.ts @@ -0,0 +1,53 @@ +import { Request, Response } from "express"; + +import { seconds } from "../../../utils/misc.js"; +import { store } from "./lib/store-init.js"; +import { normalizeUrl, FeatureData } from "./lib/store.js"; + +const MAX_SPECS = 50; + +interface SearchBody { + specs: string[]; +} + +type IRequest = Request; + +export default function route(req: IRequest, res: Response) { + const { specs } = req.body; + + if (!Array.isArray(specs) || specs.length === 0) { + res.status(400).json({ error: "Request body must contain a `specs` array." }); + return; + } + + if (specs.length > MAX_SPECS) { + res.status(400).json({ + error: `Too many spec URLs. Maximum is ${MAX_SPECS}, got ${specs.length}.`, + }); + return; + } + + const results: Record = {}; + + for (const specUrl of specs) { + const normalized = normalizeUrl(specUrl); + const matched: { id: string; feature: FeatureData }[] = []; + + // Find all features whose spec URLs start with the normalized URL + for (const [storedUrl, featureIds] of store.bySpecUrl) { + if (storedUrl.startsWith(normalized)) { + for (const id of featureIds) { + const feature = store.byFeature.get(id); + if (feature) { + matched.push({ id, feature }); + } + } + } + } + + results[specUrl] = matched; + } + + res.set("Cache-Control", `max-age=${seconds("30m")}`); + res.json(results); +} diff --git a/routes/api/baseline/update.ts b/routes/api/baseline/update.ts new file mode 100644 index 00000000..dd9cfb7a --- /dev/null +++ b/routes/api/baseline/update.ts @@ -0,0 +1,27 @@ +import path from "path"; + +import { Request, Response } from "express"; + +import { BackgroundTaskQueue } from "../../../utils/background-task-queue.js"; +import { store } from "./lib/store-init.js"; + +const workerFile = path.join(import.meta.dirname, "update.worker.js"); +const taskQueue = new BackgroundTaskQueue( + workerFile, + "baseline_update", +); + +export default async function route(req: Request, res: Response) { + const job = taskQueue.add({ webhookId: req.get("X-GitHub-Delivery") || "" }); + try { + const { updated } = await job.run(); + if (updated) { + store.fill(); + } + } catch { + res.status(500); + } finally { + res.locals.job = job.id; + res.send(job.id); + } +} diff --git a/routes/api/baseline/update.worker.ts b/routes/api/baseline/update.worker.ts new file mode 100644 index 00000000..d3706c11 --- /dev/null +++ b/routes/api/baseline/update.worker.ts @@ -0,0 +1,10 @@ +import baselineScraper from "./lib/scraper.js"; + +interface Input { + webhookId: string; +} + +export default async function baselineUpdate(_input: Input) { + const updated = await baselineScraper(); + return { updated }; +} diff --git a/scripts/update-data-sources.ts b/scripts/update-data-sources.ts index 2bd87959..1194b247 100644 --- a/scripts/update-data-sources.ts +++ b/scripts/update-data-sources.ts @@ -3,6 +3,7 @@ import { mkdir } from "fs/promises"; import { env } from "../utils/misc.js"; import caniuse from "../routes/caniuse/lib/scraper.js"; import xref from "../routes/xref/lib/scraper.js"; +import baseline from "../routes/api/baseline/lib/scraper.js"; import { pullRelease } from "../routes/respec/builds/update.js"; import w3cGroupsList from "./update-w3c-groups-list.js"; @@ -17,6 +18,10 @@ console.group("xref"); await xref({ forceUpdate: true }); console.groupEnd(); +console.group("baseline"); +await baseline(); +console.groupEnd(); + console.group("W3C Groups List"); await w3cGroupsList(); console.groupEnd(); From a8d58be1a42f5d0d1422bb9e43306727771c9f35 Mon Sep 17 00:00:00 2001 From: Marcos Caceres Date: Sat, 25 Apr 2026 20:48:13 +1000 Subject: [PATCH 2/7] fix(api/baseline): return flat {result: [...]} from search endpoint Client expects {result: WebFeatureEntry[]}, server was returning Record. Flatten features, deduplicate by ID, and wrap in {result: [...]}. --- routes/api/baseline/feature.ts | 60 ++++++++++++++---------------- routes/api/baseline/lib/scraper.ts | 8 ++-- routes/api/baseline/lib/store.ts | 50 ++++++++++--------------- routes/api/baseline/search.post.ts | 45 ++++++++++------------ routes/api/baseline/update.ts | 5 +++ 5 files changed, 76 insertions(+), 92 deletions(-) diff --git a/routes/api/baseline/feature.ts b/routes/api/baseline/feature.ts index 4eef2a2c..48234323 100644 --- a/routes/api/baseline/feature.ts +++ b/routes/api/baseline/feature.ts @@ -8,48 +8,42 @@ type IRequest = Request; export default function route(req: IRequest, res: Response) { const { feature: featureId } = req.params; - const featureData = store.byFeature.get(featureId); + const cacheControl = `max-age=${seconds("24h")}`; + const featureData = store.byFeature.get(featureId); if (featureData) { - res.set("Cache-Control", `max-age=${seconds("24h")}`); + res.set("Cache-Control", cacheControl); res.json({ id: featureId, ...featureData }); return; } // Check if it's a moved or split feature in the raw data - if (store.data) { - const rawFeature = store.data.features[featureId]; - if (rawFeature) { - if (rawFeature.kind === "moved" && rawFeature.redirect_target) { - const target = store.byFeature.get(rawFeature.redirect_target); - if (target) { - res.set("Cache-Control", `max-age=${seconds("24h")}`); - res.json({ - id: rawFeature.redirect_target, - redirected_from: featureId, - ...target, - }); - return; - } - } - - if (rawFeature.kind === "split" && rawFeature.redirect_targets) { - const targets = rawFeature.redirect_targets - .map(targetId => { - const target = store.byFeature.get(targetId); - return target ? { id: targetId, ...target } : null; - }) - .filter(Boolean); - res.set("Cache-Control", `max-age=${seconds("24h")}`); - res.json({ - id: featureId, - kind: "split", - split_into: targets, - }); - return; - } + const rawFeature = store.data?.features[featureId]; + if (!rawFeature) { + res.sendStatus(404); + return; + } + + if (rawFeature.kind === "moved" && rawFeature.redirect_target) { + const target = store.byFeature.get(rawFeature.redirect_target); + if (target) { + res.set("Cache-Control", cacheControl); + res.json({ id: rawFeature.redirect_target, redirected_from: featureId, ...target }); + return; } } + if (rawFeature.kind === "split" && rawFeature.redirect_targets) { + const targets = rawFeature.redirect_targets + .map(targetId => { + const target = store.byFeature.get(targetId); + return target ? { id: targetId, ...target } : null; + }) + .filter(Boolean); + res.set("Cache-Control", cacheControl); + res.json({ id: featureId, kind: "split", split_into: targets }); + return; + } + res.sendStatus(404); } diff --git a/routes/api/baseline/lib/scraper.ts b/routes/api/baseline/lib/scraper.ts index 4a6bdb03..5973c586 100644 --- a/routes/api/baseline/lib/scraper.ts +++ b/routes/api/baseline/lib/scraper.ts @@ -15,9 +15,6 @@ interface ReleaseResponse { } export default async function main() { - const releaseUrl = - "https://api.github.com/repos/web-platform-dx/web-features/releases/latest"; - const headers: Record = { Accept: "application/vnd.github+json", }; @@ -27,7 +24,10 @@ export default async function main() { headers.Authorization = `Bearer ${ghToken}`; } - const releaseRes = await fetch(releaseUrl, { headers }); + const releaseRes = await fetch( + "https://api.github.com/repos/web-platform-dx/web-features/releases/latest", + { headers }, + ); if (!releaseRes.ok) { throw new Error( `Failed to fetch latest release: ${releaseRes.status} ${releaseRes.statusText}`, diff --git a/routes/api/baseline/lib/store.ts b/routes/api/baseline/lib/store.ts index 228bfb6f..2fff9550 100644 --- a/routes/api/baseline/lib/store.ts +++ b/routes/api/baseline/lib/store.ts @@ -43,32 +43,27 @@ interface WebFeaturesData { } export function normalizeUrl(url: string): string { - try { - const u = new URL(url); - u.hash = ""; - if (!u.pathname.endsWith("/") && !u.pathname.includes(".")) { - u.pathname += "/"; - } - return u.href; - } catch { - return url; + const parsed = URL.parse(url); + if (!parsed) return url; + parsed.hash = ""; + if (!parsed.pathname.endsWith("/") && !parsed.pathname.includes(".")) { + parsed.pathname += "/"; } + return parsed.href; } export class BaselineStore { version = -1; data: WebFeaturesData | null = null; - byFeature: Map = new Map(); - bySpecUrl: Map = new Map(); + byFeature = new Map(); + bySpecUrl = new Map(); constructor() { this.fill(); } - /** Fill the store with its contents from the filesystem. */ fill() { - const DATA_DIR = env("DATA_DIR"); - const dataFile = path.resolve(DATA_DIR, "baseline/baseline.json"); + const dataFile = path.resolve(env("DATA_DIR"), "baseline/baseline.json"); if (!existsSync(dataFile)) { console.warn("baseline: data file not found, store is empty."); @@ -79,25 +74,20 @@ export class BaselineStore { return; } - const text = readFileSync(dataFile, "utf8"); - const data: WebFeaturesData = JSON.parse(text); - this.data = data; - - this.byFeature = new Map(); - this.bySpecUrl = new Map(); + this.data = JSON.parse(readFileSync(dataFile, "utf8")); - for (const [id, feature] of Object.entries(data.features)) { - if (feature.kind !== "feature") continue; + const features = Object.entries(this.data!.features) + .filter(([, feature]) => feature.kind === "feature"); - this.byFeature.set(id, feature); + this.byFeature = new Map(features); - if (feature.spec) { - for (const specUrl of feature.spec) { - const normalized = normalizeUrl(specUrl); - const existing = this.bySpecUrl.get(normalized) || []; - existing.push(id); - this.bySpecUrl.set(normalized, existing); - } + this.bySpecUrl = new Map(); + for (const [featureId, feature] of features) { + for (const specUrl of feature.spec ?? []) { + const normalized = normalizeUrl(specUrl); + const ids = this.bySpecUrl.get(normalized) ?? []; + ids.push(featureId); + this.bySpecUrl.set(normalized, ids); } } diff --git a/routes/api/baseline/search.post.ts b/routes/api/baseline/search.post.ts index 63358136..d6b1955f 100644 --- a/routes/api/baseline/search.post.ts +++ b/routes/api/baseline/search.post.ts @@ -2,7 +2,7 @@ import { Request, Response } from "express"; import { seconds } from "../../../utils/misc.js"; import { store } from "./lib/store-init.js"; -import { normalizeUrl, FeatureData } from "./lib/store.js"; +import { normalizeUrl } from "./lib/store.js"; const MAX_SPECS = 50; @@ -16,38 +16,33 @@ export default function route(req: IRequest, res: Response) { const { specs } = req.body; if (!Array.isArray(specs) || specs.length === 0) { - res.status(400).json({ error: "Request body must contain a `specs` array." }); + res.status(400); + res.json({ error: "Request body must contain a `specs` array." }); return; } if (specs.length > MAX_SPECS) { - res.status(400).json({ - error: `Too many spec URLs. Maximum is ${MAX_SPECS}, got ${specs.length}.`, - }); + res.status(400); + res.json({ error: `Too many spec URLs. Maximum is ${MAX_SPECS}, got ${specs.length}.` }); return; } - const results: Record = {}; - - for (const specUrl of specs) { - const normalized = normalizeUrl(specUrl); - const matched: { id: string; feature: FeatureData }[] = []; - - // Find all features whose spec URLs start with the normalized URL - for (const [storedUrl, featureIds] of store.bySpecUrl) { - if (storedUrl.startsWith(normalized)) { - for (const id of featureIds) { - const feature = store.byFeature.get(id); - if (feature) { - matched.push({ id, feature }); - } - } - } - } - - results[specUrl] = matched; + if (!specs.every(s => typeof s === "string")) { + res.status(400); + res.json({ error: "Each spec must be a string URL." }); + return; } + const normalizedSpecs = specs.map(normalizeUrl); + + const matchingIds = [...store.bySpecUrl.entries()] + .filter(([url]) => normalizedSpecs.some(spec => url.startsWith(spec))) + .flatMap(([, ids]) => ids); + + const result = [...new Set(matchingIds)] + .map(id => ({ id, ...store.byFeature.get(id)! })) + .filter(entry => entry.name); + res.set("Cache-Control", `max-age=${seconds("30m")}`); - res.json(results); + res.json({ result }); } diff --git a/routes/api/baseline/update.ts b/routes/api/baseline/update.ts index dd9cfb7a..9389974c 100644 --- a/routes/api/baseline/update.ts +++ b/routes/api/baseline/update.ts @@ -12,6 +12,11 @@ const taskQueue = new BackgroundTaskQueue( ); export default async function route(req: Request, res: Response) { + if (req.body.action !== "published") { + res.status(200).send("Ignored non-publish action"); + return; + } + const job = taskQueue.add({ webhookId: req.get("X-GitHub-Delivery") || "" }); try { const { updated } = await job.run(); From 5668a502072009f6da713ab46612d95c09a651a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcos=20C=C3=A1ceres?= Date: Mon, 27 Apr 2026 18:30:23 +1000 Subject: [PATCH 3/7] Apply suggestion from @Copilot Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- routes/api/baseline/lib/scraper.ts | 38 +++--------------------------- 1 file changed, 3 insertions(+), 35 deletions(-) diff --git a/routes/api/baseline/lib/scraper.ts b/routes/api/baseline/lib/scraper.ts index 5973c586..70c91ae0 100644 --- a/routes/api/baseline/lib/scraper.ts +++ b/routes/api/baseline/lib/scraper.ts @@ -4,43 +4,11 @@ import { mkdir, writeFile } from "fs/promises"; import { env } from "../../../../utils/misc.js"; const DATA_DIR = env("DATA_DIR"); - -interface ReleaseAsset { - name: string; - browser_download_url: string; -} - -interface ReleaseResponse { - assets: ReleaseAsset[]; -} +const LATEST_DATA_URL = + "https://github.com/web-platform-dx/web-features/releases/latest/download/data.json"; export default async function main() { - const headers: Record = { - Accept: "application/vnd.github+json", - }; - - const ghToken = process.env.GH_TOKEN; - if (ghToken) { - headers.Authorization = `Bearer ${ghToken}`; - } - - const releaseRes = await fetch( - "https://api.github.com/repos/web-platform-dx/web-features/releases/latest", - { headers }, - ); - if (!releaseRes.ok) { - throw new Error( - `Failed to fetch latest release: ${releaseRes.status} ${releaseRes.statusText}`, - ); - } - const release: ReleaseResponse = await releaseRes.json(); - - const asset = release.assets.find(a => a.name === "data.json"); - if (!asset) { - throw new Error("No data.json asset in latest web-features release"); - } - - const dataRes = await fetch(asset.browser_download_url); + const dataRes = await fetch(LATEST_DATA_URL); if (!dataRes.ok) { throw new Error( `Failed to download data.json: ${dataRes.status} ${dataRes.statusText}`, From d3a0682f6827220bce780cf1601df2dbac08e3b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcos=20C=C3=A1ceres?= Date: Mon, 27 Apr 2026 18:32:22 +1000 Subject: [PATCH 4/7] Apply suggestions from code review Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- routes/api/baseline/lib/store.ts | 39 +++++++++++++++++++----------- routes/api/baseline/search.post.ts | 15 ++++++++---- 2 files changed, 35 insertions(+), 19 deletions(-) diff --git a/routes/api/baseline/lib/store.ts b/routes/api/baseline/lib/store.ts index 2fff9550..0014a0eb 100644 --- a/routes/api/baseline/lib/store.ts +++ b/routes/api/baseline/lib/store.ts @@ -74,23 +74,34 @@ export class BaselineStore { return; } - this.data = JSON.parse(readFileSync(dataFile, "utf8")); + try { + const data = JSON.parse(readFileSync(dataFile, "utf8")) as WebFeaturesData; - const features = Object.entries(this.data!.features) - .filter(([, feature]) => feature.kind === "feature"); + const features = Object.entries(data.features).filter( + ([, feature]) => feature.kind === "feature", + ); - this.byFeature = new Map(features); - - this.bySpecUrl = new Map(); - for (const [featureId, feature] of features) { - for (const specUrl of feature.spec ?? []) { - const normalized = normalizeUrl(specUrl); - const ids = this.bySpecUrl.get(normalized) ?? []; - ids.push(featureId); - this.bySpecUrl.set(normalized, ids); + const byFeature = new Map(features); + const bySpecUrl = new Map(); + for (const [featureId, feature] of features) { + for (const specUrl of feature.spec ?? []) { + const normalized = normalizeUrl(specUrl); + const ids = bySpecUrl.get(normalized) ?? []; + ids.push(featureId); + bySpecUrl.set(normalized, ids); + } } - } - this.version = Date.now(); + this.data = data; + this.byFeature = byFeature; + this.bySpecUrl = bySpecUrl; + this.version = Date.now(); + } catch (error) { + console.warn( + "baseline: failed to read or parse data file, keeping existing store data.", + error, + ); + return; + } } } diff --git a/routes/api/baseline/search.post.ts b/routes/api/baseline/search.post.ts index d6b1955f..986d394a 100644 --- a/routes/api/baseline/search.post.ts +++ b/routes/api/baseline/search.post.ts @@ -29,17 +29,22 @@ export default function route(req: IRequest, res: Response) { if (!specs.every(s => typeof s === "string")) { res.status(400); - res.json({ error: "Each spec must be a string URL." }); + res.json({ error: "Each spec must be a string." }); return; } const normalizedSpecs = specs.map(normalizeUrl); - const matchingIds = [...store.bySpecUrl.entries()] - .filter(([url]) => normalizedSpecs.some(spec => url.startsWith(spec))) - .flatMap(([, ids]) => ids); + const matchingIds = new Set(); + for (const [url, ids] of store.bySpecUrl) { + if (normalizedSpecs.some(spec => url.startsWith(spec))) { + for (const id of ids) { + matchingIds.add(id); + } + } + } - const result = [...new Set(matchingIds)] + const result = [...matchingIds] .map(id => ({ id, ...store.byFeature.get(id)! })) .filter(entry => entry.name); From 9e87c3a42950361be02c3a6504bb2802ec9b9efa Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 27 Apr 2026 08:43:35 +0000 Subject: [PATCH 5/7] fix(api/baseline): address unresolved review comments - search.post.ts: return 503 when store.data is null (service unavailable) - search.post.ts: reject empty/whitespace-only strings in specs array - store.ts: normalizeUrl() now also clears search params (query string) - scraper.ts: add ETag/If-None-Match support to skip unnecessary re-downloads - scraper.ts: use atomic write (temp file + rename) to prevent corrupt data on interrupted writes - Add tests: normalizeUrl, feature route (moved/split/404), search.post validation Agent-Logs-Url: https://github.com/speced/respec-web-services/sessions/41823caf-5a5e-40fc-8bf7-fca753fb37ab --- routes/api/baseline/lib/scraper.ts | 34 +++++- routes/api/baseline/lib/store.ts | 1 + routes/api/baseline/search.post.ts | 9 +- tests/routes/baseline/feature.test.js | 121 ++++++++++++++++++++ tests/routes/baseline/lib/store.test.js | 43 ++++++++ tests/routes/baseline/search.test.js | 141 ++++++++++++++++++++++++ 6 files changed, 343 insertions(+), 6 deletions(-) create mode 100644 tests/routes/baseline/feature.test.js create mode 100644 tests/routes/baseline/lib/store.test.js create mode 100644 tests/routes/baseline/search.test.js diff --git a/routes/api/baseline/lib/scraper.ts b/routes/api/baseline/lib/scraper.ts index 70c91ae0..cd1bbe11 100644 --- a/routes/api/baseline/lib/scraper.ts +++ b/routes/api/baseline/lib/scraper.ts @@ -1,5 +1,6 @@ import path from "path"; -import { mkdir, writeFile } from "fs/promises"; +import { mkdir, readFile, rename, writeFile } from "fs/promises"; +import { existsSync } from "fs"; import { env } from "../../../../utils/misc.js"; @@ -8,17 +9,42 @@ const LATEST_DATA_URL = "https://github.com/web-platform-dx/web-features/releases/latest/download/data.json"; export default async function main() { - const dataRes = await fetch(LATEST_DATA_URL); + const outputDir = path.join(DATA_DIR, "baseline"); + const dataFile = path.join(outputDir, "baseline.json"); + const etagFile = path.join(outputDir, "baseline.etag"); + + const headers: Record = {}; + if (existsSync(etagFile)) { + const savedEtag = (await readFile(etagFile, "utf8")).trim(); + if (savedEtag) { + headers["If-None-Match"] = savedEtag; + } + } + + const dataRes = await fetch(LATEST_DATA_URL, { headers }); + + if (dataRes.status === 304) { + return false; + } + if (!dataRes.ok) { throw new Error( `Failed to download data.json: ${dataRes.status} ${dataRes.statusText}`, ); } + const data = await dataRes.text(); - const outputDir = path.join(DATA_DIR, "baseline"); await mkdir(outputDir, { recursive: true }); - await writeFile(path.join(outputDir, "baseline.json"), data); + + const tempFile = `${dataFile}.tmp`; + await writeFile(tempFile, data); + await rename(tempFile, dataFile); + + const etag = dataRes.headers.get("etag"); + if (etag) { + await writeFile(etagFile, etag); + } return true; } diff --git a/routes/api/baseline/lib/store.ts b/routes/api/baseline/lib/store.ts index 0014a0eb..44ec3788 100644 --- a/routes/api/baseline/lib/store.ts +++ b/routes/api/baseline/lib/store.ts @@ -46,6 +46,7 @@ export function normalizeUrl(url: string): string { const parsed = URL.parse(url); if (!parsed) return url; parsed.hash = ""; + parsed.search = ""; if (!parsed.pathname.endsWith("/") && !parsed.pathname.includes(".")) { parsed.pathname += "/"; } diff --git a/routes/api/baseline/search.post.ts b/routes/api/baseline/search.post.ts index 986d394a..2880986d 100644 --- a/routes/api/baseline/search.post.ts +++ b/routes/api/baseline/search.post.ts @@ -13,6 +13,11 @@ interface SearchBody { type IRequest = Request; export default function route(req: IRequest, res: Response) { + if (!store.data) { + res.sendStatus(503); + return; + } + const { specs } = req.body; if (!Array.isArray(specs) || specs.length === 0) { @@ -27,9 +32,9 @@ export default function route(req: IRequest, res: Response) { return; } - if (!specs.every(s => typeof s === "string")) { + if (!specs.every(s => typeof s === "string" && s.trim().length > 0)) { res.status(400); - res.json({ error: "Each spec must be a string." }); + res.json({ error: "Each spec must be a non-empty string." }); return; } diff --git a/tests/routes/baseline/feature.test.js b/tests/routes/baseline/feature.test.js new file mode 100644 index 00000000..7e074773 --- /dev/null +++ b/tests/routes/baseline/feature.test.js @@ -0,0 +1,121 @@ +import featureRoute from "../../../build/routes/api/baseline/feature.js"; +import { store } from "../../../build/routes/api/baseline/lib/store-init.js"; + +/** @returns {import("express").Response} */ +function makeRes() { + const res = { + _status: 200, + _body: undefined, + _headers: {}, + status(code) { + this._status = code; + return this; + }, + sendStatus(code) { + this._status = code; + return this; + }, + set(name, value) { + this._headers[name] = value; + return this; + }, + json(data) { + this._body = data; + return this; + }, + }; + return res; +} + +const FEATURE_DATA = { + kind: "feature", + name: "CSS Animations", + spec: ["https://drafts.csswg.org/css-animations/"], + status: { baseline: "high", support: {} }, +}; + +describe("routes/api/baseline/feature", () => { + beforeEach(() => { + store.data = { + features: { + "css-animations": FEATURE_DATA, + "old-animations": { + kind: "moved", + redirect_target: "css-animations", + }, + "mega-feature": { + kind: "split", + redirect_targets: ["css-animations"], + }, + }, + browsers: {}, + groups: {}, + snapshots: {}, + }; + store.byFeature = new Map([["css-animations", FEATURE_DATA]]); + }); + + afterEach(() => { + store.data = null; + store.byFeature = new Map(); + store.bySpecUrl = new Map(); + }); + + it("returns feature data for a known feature", () => { + const req = { params: { feature: "css-animations" } }; + const res = makeRes(); + featureRoute(req, res); + expect(res._status).toBe(200); + expect(res._body.id).toBe("css-animations"); + expect(res._body.name).toBe("CSS Animations"); + }); + + it("returns 404 when feature is unknown", () => { + const req = { params: { feature: "unknown-feature" } }; + const res = makeRes(); + featureRoute(req, res); + expect(res._status).toBe(404); + }); + + it("returns 404 when store has no data", () => { + store.data = null; + store.byFeature = new Map(); + const req = { params: { feature: "css-animations" } }; + const res = makeRes(); + featureRoute(req, res); + expect(res._status).toBe(404); + }); + + it("resolves a moved feature to its redirect target", () => { + const req = { params: { feature: "old-animations" } }; + const res = makeRes(); + featureRoute(req, res); + expect(res._status).toBe(200); + expect(res._body.id).toBe("css-animations"); + expect(res._body.redirected_from).toBe("old-animations"); + expect(res._body.name).toBe("CSS Animations"); + }); + + it("returns 404 for a moved feature whose target is missing", () => { + store.data.features["orphan-moved"] = { + kind: "moved", + redirect_target: "nonexistent", + }; + const req = { params: { feature: "orphan-moved" } }; + const res = makeRes(); + featureRoute(req, res); + expect(res._status).toBe(404); + }); + + it("resolves a split feature to its redirect targets", () => { + const req = { params: { feature: "mega-feature" } }; + const res = makeRes(); + featureRoute(req, res); + expect(res._status).toBe(200); + expect(res._body.id).toBe("mega-feature"); + expect(res._body.kind).toBe("split"); + expect(Array.isArray(res._body.split_into)).toBeTrue(); + expect(res._body.split_into.length).toBe(1); + expect(res._body.split_into[0].id).toBe("css-animations"); + }); +}); diff --git a/tests/routes/baseline/lib/store.test.js b/tests/routes/baseline/lib/store.test.js new file mode 100644 index 00000000..65a7e00d --- /dev/null +++ b/tests/routes/baseline/lib/store.test.js @@ -0,0 +1,43 @@ +import { normalizeUrl } from "../../../../build/routes/api/baseline/lib/store.js"; + +describe("routes/api/baseline/lib/store - normalizeUrl", () => { + it("removes URL fragment (hash)", () => { + expect(normalizeUrl("https://example.com/path#section")).toBe( + "https://example.com/path/", + ); + }); + + it("removes URL search params (query string)", () => { + expect(normalizeUrl("https://example.com/path?foo=bar&baz=1")).toBe( + "https://example.com/path/", + ); + }); + + it("removes both fragment and search params", () => { + expect(normalizeUrl("https://example.com/path?q=1#section")).toBe( + "https://example.com/path/", + ); + }); + + it("adds trailing slash for directory-like paths", () => { + expect(normalizeUrl("https://example.com/path")).toBe( + "https://example.com/path/", + ); + }); + + it("does not add trailing slash for file paths with extension", () => { + expect(normalizeUrl("https://example.com/spec.html")).toBe( + "https://example.com/spec.html", + ); + }); + + it("preserves existing trailing slash", () => { + expect(normalizeUrl("https://example.com/path/")).toBe( + "https://example.com/path/", + ); + }); + + it("returns unchanged input for invalid URLs", () => { + expect(normalizeUrl("not-a-url")).toBe("not-a-url"); + }); +}); diff --git a/tests/routes/baseline/search.test.js b/tests/routes/baseline/search.test.js new file mode 100644 index 00000000..ef1b4c5d --- /dev/null +++ b/tests/routes/baseline/search.test.js @@ -0,0 +1,141 @@ +import searchRoute from "../../../build/routes/api/baseline/search.post.js"; +import { store } from "../../../build/routes/api/baseline/lib/store-init.js"; + +/** @returns {import("express").Response} */ +function makeRes() { + const res = { + _status: 200, + _body: undefined, + status(code) { + this._status = code; + return this; + }, + sendStatus(code) { + this._status = code; + return this; + }, + set() { + return this; + }, + json(data) { + this._body = data; + return this; + }, + }; + return res; +} + +const EMPTY_STORE_DATA = { + features: {}, + browsers: {}, + groups: {}, + snapshots: {}, +}; + +describe("routes/api/baseline/search.post", () => { + afterEach(() => { + store.data = null; + store.byFeature = new Map(); + store.bySpecUrl = new Map(); + }); + + describe("when store data is unavailable", () => { + it("returns 503 when store.data is null", () => { + store.data = null; + const req = { body: { specs: ["https://example.com/spec/"] } }; + const res = makeRes(); + searchRoute(req, res); + expect(res._status).toBe(503); + }); + }); + + describe("input validation", () => { + beforeEach(() => { + store.data = EMPTY_STORE_DATA; + }); + + it("returns 400 when specs is missing", () => { + const req = { body: {} }; + const res = makeRes(); + searchRoute(req, res); + expect(res._status).toBe(400); + }); + + it("returns 400 when specs is not an array", () => { + const req = { body: { specs: "https://example.com/" } }; + const res = makeRes(); + searchRoute(req, res); + expect(res._status).toBe(400); + }); + + it("returns 400 when specs is an empty array", () => { + const req = { body: { specs: [] } }; + const res = makeRes(); + searchRoute(req, res); + expect(res._status).toBe(400); + }); + + it("returns 400 when specs contains an empty string", () => { + const req = { body: { specs: [""] } }; + const res = makeRes(); + searchRoute(req, res); + expect(res._status).toBe(400); + }); + + it("returns 400 when specs contains a whitespace-only string", () => { + const req = { body: { specs: [" "] } }; + const res = makeRes(); + searchRoute(req, res); + expect(res._status).toBe(400); + }); + + it("returns 400 when specs contains a non-string value", () => { + const req = { body: { specs: [42] } }; + const res = makeRes(); + searchRoute(req, res); + expect(res._status).toBe(400); + }); + }); + + describe("matching", () => { + beforeEach(() => { + store.data = EMPTY_STORE_DATA; + store.bySpecUrl = new Map([ + [ + "https://drafts.csswg.org/css-animations/", + ["css-animations"], + ], + ]); + store.byFeature = new Map([ + [ + "css-animations", + { + kind: "feature", + name: "CSS Animations", + status: { baseline: "high", support: {} }, + }, + ], + ]); + }); + + it("returns matching features for a given spec URL", () => { + const req = { + body: { specs: ["https://drafts.csswg.org/css-animations/"] }, + }; + const res = makeRes(); + searchRoute(req, res); + expect(res._status).toBe(200); + expect(res._body.result).toBeInstanceOf(Array); + expect(res._body.result.length).toBe(1); + expect(res._body.result[0].id).toBe("css-animations"); + }); + + it("returns empty result for non-matching spec URL", () => { + const req = { body: { specs: ["https://example.com/unknown/"] } }; + const res = makeRes(); + searchRoute(req, res); + expect(res._status).toBe(200); + expect(res._body.result).toEqual([]); + }); + }); +}); From 1e3cafc6ea19b217590179d824319cdaf51e0903 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 27 Apr 2026 08:45:21 +0000 Subject: [PATCH 6/7] fix(api/baseline): replace existsSync with try-catch for ETag read, add return type doc Agent-Logs-Url: https://github.com/speced/respec-web-services/sessions/41823caf-5a5e-40fc-8bf7-fca753fb37ab --- routes/api/baseline/lib/scraper.ts | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/routes/api/baseline/lib/scraper.ts b/routes/api/baseline/lib/scraper.ts index cd1bbe11..25d422aa 100644 --- a/routes/api/baseline/lib/scraper.ts +++ b/routes/api/baseline/lib/scraper.ts @@ -1,6 +1,5 @@ import path from "path"; import { mkdir, readFile, rename, writeFile } from "fs/promises"; -import { existsSync } from "fs"; import { env } from "../../../../utils/misc.js"; @@ -8,22 +7,25 @@ const DATA_DIR = env("DATA_DIR"); const LATEST_DATA_URL = "https://github.com/web-platform-dx/web-features/releases/latest/download/data.json"; -export default async function main() { +export default async function main(): Promise { const outputDir = path.join(DATA_DIR, "baseline"); const dataFile = path.join(outputDir, "baseline.json"); const etagFile = path.join(outputDir, "baseline.etag"); const headers: Record = {}; - if (existsSync(etagFile)) { + try { const savedEtag = (await readFile(etagFile, "utf8")).trim(); if (savedEtag) { headers["If-None-Match"] = savedEtag; } + } catch { + // No saved ETag yet; proceed without conditional request } const dataRes = await fetch(LATEST_DATA_URL, { headers }); if (dataRes.status === 304) { + // Data is already up to date; no download needed. return false; } @@ -46,5 +48,6 @@ export default async function main() { await writeFile(etagFile, etag); } + // Returns true when new data was downloaded, false when already up to date. return true; } From f4501139045836a63a9f05134996a0a408a6fc81 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 27 Apr 2026 20:24:35 +0000 Subject: [PATCH 7/7] fix(api/baseline): update.ts returns 400 for non-published webhook actions, fix XSS - Return 400 (not 200) for non-published webhook actions, consistent with caniuse/update.ts and respec/builds/update.ts patterns - Set res.locals.reason = "action-not-published" for structured logging - Use a static message instead of echoing req.body.action to avoid reflected XSS (CodeQL js/reflected-xss) Agent-Logs-Url: https://github.com/speced/respec-web-services/sessions/5af37242-1fe0-4bdd-b54b-c45ab0c4120f --- routes/api/baseline/update.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/routes/api/baseline/update.ts b/routes/api/baseline/update.ts index 9389974c..887ba97a 100644 --- a/routes/api/baseline/update.ts +++ b/routes/api/baseline/update.ts @@ -13,7 +13,9 @@ const taskQueue = new BackgroundTaskQueue( export default async function route(req: Request, res: Response) { if (req.body.action !== "published") { - res.status(200).send("Ignored non-publish action"); + res.status(400); + res.locals.reason = "action-not-published"; + res.send("Webhook action ignored (expected 'published')."); return; }