diff --git a/.env.sample b/.env.sample index 01df87a9..12ca5abc 100644 --- a/.env.sample +++ b/.env.sample @@ -10,6 +10,7 @@ DATA_DIR=/home/respec.org/data W3C_WEBREF_SECRET=xxxxxxxxxxxxxxxxxxxxxx CANIUSE_SECRET=xxxxxxxxxxxxxxxxxxxxxx RESPEC_SECRET=xxxxxxxxxxxxxxxxxxxxxx +WEB_FEATURES_SECRET=xxxxxxxxxxxxxxxxxxxxxx # GitHub Action secret on speced/respec RESPEC_GH_ACTION_SECRET=xxxxxxxxxxxxxxxxxxxxxx diff --git a/app.ts b/app.ts index e36a49db..1c508770 100644 --- a/app.ts +++ b/app.ts @@ -14,6 +14,7 @@ import caniuseRouter from "./routes/caniuse/index.js"; import githubRouter from "./routes/github/index.js"; import respecRouter from "./routes/respec/index.js"; import w3cRouter from "./routes/w3c/index.js"; +import baselineRouter from "./routes/api/baseline/index.js"; import wellKnownRouter from "./routes/well-known/index.js"; import docsRouter from "./routes/docs/index.js"; @@ -46,6 +47,7 @@ app.use( app.use("/xref", xrefRouter); app.use("/caniuse", caniuseRouter); +app.use("/api/baseline", baselineRouter); app.use("/github/:org/:repo", githubRouter); app.use("/respec", respecRouter); app.use("/w3c", w3cRouter); diff --git a/routes/api/baseline/all.ts b/routes/api/baseline/all.ts new file mode 100644 index 00000000..798927ea --- /dev/null +++ b/routes/api/baseline/all.ts @@ -0,0 +1,14 @@ +import { Request, Response } from "express"; + +import { seconds } from "../../../utils/misc.js"; +import { store } from "./lib/store-init.js"; + +export default function route(_req: Request, res: Response) { + if (!store.data) { + res.sendStatus(404); + return; + } + + res.set("Cache-Control", `max-age=${seconds("24h")}`); + res.json(store.data); +} diff --git a/routes/api/baseline/feature.ts b/routes/api/baseline/feature.ts new file mode 100644 index 00000000..48234323 --- /dev/null +++ b/routes/api/baseline/feature.ts @@ -0,0 +1,49 @@ +import { Request, Response } from "express"; + +import { seconds } from "../../../utils/misc.js"; +import { store } from "./lib/store-init.js"; + +type Params = { feature: string }; +type IRequest = Request; + +export default function route(req: IRequest, res: Response) { + const { feature: featureId } = req.params; + const cacheControl = `max-age=${seconds("24h")}`; + + const featureData = store.byFeature.get(featureId); + if (featureData) { + res.set("Cache-Control", cacheControl); + res.json({ id: featureId, ...featureData }); + return; + } + + // Check if it's a moved or split feature in the raw data + const rawFeature = store.data?.features[featureId]; + if (!rawFeature) { + res.sendStatus(404); + return; + } + + if (rawFeature.kind === "moved" && rawFeature.redirect_target) { + const target = store.byFeature.get(rawFeature.redirect_target); + if (target) { + res.set("Cache-Control", cacheControl); + res.json({ id: rawFeature.redirect_target, redirected_from: featureId, ...target }); + return; + } + } + + if (rawFeature.kind === "split" && rawFeature.redirect_targets) { + const targets = rawFeature.redirect_targets + .map(targetId => { + const target = store.byFeature.get(targetId); + return target ? { id: targetId, ...target } : null; + }) + .filter(Boolean); + res.set("Cache-Control", cacheControl); + res.json({ id: featureId, kind: "split", split_into: targets }); + return; + } + + res.sendStatus(404); +} diff --git a/routes/api/baseline/index.ts b/routes/api/baseline/index.ts new file mode 100644 index 00000000..d89f3ffa --- /dev/null +++ b/routes/api/baseline/index.ts @@ -0,0 +1,32 @@ +import express from "express"; +import cors from "cors"; + +import authGithubWebhook from "../../../utils/auth-github-webhook.js"; +import { env, ms } from "../../../utils/misc.js"; + +import allRoute from "./all.js"; +import featureRoute from "./feature.js"; +import searchRoute from "./search.post.js"; +import updateRoute from "./update.js"; + +const baseline = express.Router({ mergeParams: true }); + +baseline + .options("/", cors({ methods: ["GET"], maxAge: ms("1day") })) + .get("/", cors(), allRoute); + +baseline + .options("/search", cors({ methods: ["POST"], maxAge: ms("1day") })) + .post("/search", express.json({ limit: "1mb" }), cors(), searchRoute); + +baseline + .options("/:feature", cors({ methods: ["GET"], maxAge: ms("1day") })) + .get("/:feature", cors(), featureRoute); + +baseline.post( + "/update", + authGithubWebhook(env("WEB_FEATURES_SECRET")), + updateRoute, +); + +export default baseline; diff --git a/routes/api/baseline/lib/scraper.ts b/routes/api/baseline/lib/scraper.ts new file mode 100644 index 00000000..25d422aa --- /dev/null +++ b/routes/api/baseline/lib/scraper.ts @@ -0,0 +1,53 @@ +import path from "path"; +import { mkdir, readFile, rename, writeFile } from "fs/promises"; + +import { env } from "../../../../utils/misc.js"; + +const DATA_DIR = env("DATA_DIR"); +const LATEST_DATA_URL = + "https://github.com/web-platform-dx/web-features/releases/latest/download/data.json"; + +export default async function main(): Promise { + const outputDir = path.join(DATA_DIR, "baseline"); + const dataFile = path.join(outputDir, "baseline.json"); + const etagFile = path.join(outputDir, "baseline.etag"); + + const headers: Record = {}; + try { + const savedEtag = (await readFile(etagFile, "utf8")).trim(); + if (savedEtag) { + headers["If-None-Match"] = savedEtag; + } + } catch { + // No saved ETag yet; proceed without conditional request + } + + const dataRes = await fetch(LATEST_DATA_URL, { headers }); + + if (dataRes.status === 304) { + // Data is already up to date; no download needed. + return false; + } + + if (!dataRes.ok) { + throw new Error( + `Failed to download data.json: ${dataRes.status} ${dataRes.statusText}`, + ); + } + + const data = await dataRes.text(); + + await mkdir(outputDir, { recursive: true }); + + const tempFile = `${dataFile}.tmp`; + await writeFile(tempFile, data); + await rename(tempFile, dataFile); + + const etag = dataRes.headers.get("etag"); + if (etag) { + await writeFile(etagFile, etag); + } + + // Returns true when new data was downloaded, false when already up to date. + return true; +} diff --git a/routes/api/baseline/lib/store-init.ts b/routes/api/baseline/lib/store-init.ts new file mode 100644 index 00000000..8443b260 --- /dev/null +++ b/routes/api/baseline/lib/store-init.ts @@ -0,0 +1,3 @@ +import { BaselineStore } from "./store.js"; + +export const store = new BaselineStore(); diff --git a/routes/api/baseline/lib/store.ts b/routes/api/baseline/lib/store.ts new file mode 100644 index 00000000..44ec3788 --- /dev/null +++ b/routes/api/baseline/lib/store.ts @@ -0,0 +1,108 @@ +import path from "path"; +import { existsSync, readFileSync } from "fs"; + +import { env } from "../../../../utils/misc.js"; + +interface SupportData { + chrome?: string; + chrome_android?: string; + edge?: string; + firefox?: string; + firefox_android?: string; + safari?: string; + safari_ios?: string; +} + +interface StatusData { + baseline: "high" | "low" | false; + baseline_low_date?: string; + baseline_high_date?: string; + support: SupportData; +} + +export interface FeatureData { + kind: "feature" | "moved" | "split"; + name?: string; + description?: string; + description_html?: string; + spec?: string[]; + status?: StatusData; + caniuse?: string[]; + compat_features?: string[]; + group?: string[]; + snapshot?: string[]; + redirect_target?: string; + redirect_targets?: string[]; +} + +interface WebFeaturesData { + browsers: Record; + features: Record; + groups: Record; + snapshots: Record; +} + +export function normalizeUrl(url: string): string { + const parsed = URL.parse(url); + if (!parsed) return url; + parsed.hash = ""; + parsed.search = ""; + if (!parsed.pathname.endsWith("/") && !parsed.pathname.includes(".")) { + parsed.pathname += "/"; + } + return parsed.href; +} + +export class BaselineStore { + version = -1; + data: WebFeaturesData | null = null; + byFeature = new Map(); + bySpecUrl = new Map(); + + constructor() { + this.fill(); + } + + fill() { + const dataFile = path.resolve(env("DATA_DIR"), "baseline/baseline.json"); + + if (!existsSync(dataFile)) { + console.warn("baseline: data file not found, store is empty."); + this.data = null; + this.byFeature = new Map(); + this.bySpecUrl = new Map(); + this.version = Date.now(); + return; + } + + try { + const data = JSON.parse(readFileSync(dataFile, "utf8")) as WebFeaturesData; + + const features = Object.entries(data.features).filter( + ([, feature]) => feature.kind === "feature", + ); + + const byFeature = new Map(features); + const bySpecUrl = new Map(); + for (const [featureId, feature] of features) { + for (const specUrl of feature.spec ?? []) { + const normalized = normalizeUrl(specUrl); + const ids = bySpecUrl.get(normalized) ?? []; + ids.push(featureId); + bySpecUrl.set(normalized, ids); + } + } + + this.data = data; + this.byFeature = byFeature; + this.bySpecUrl = bySpecUrl; + this.version = Date.now(); + } catch (error) { + console.warn( + "baseline: failed to read or parse data file, keeping existing store data.", + error, + ); + return; + } + } +} diff --git a/routes/api/baseline/search.post.ts b/routes/api/baseline/search.post.ts new file mode 100644 index 00000000..2880986d --- /dev/null +++ b/routes/api/baseline/search.post.ts @@ -0,0 +1,58 @@ +import { Request, Response } from "express"; + +import { seconds } from "../../../utils/misc.js"; +import { store } from "./lib/store-init.js"; +import { normalizeUrl } from "./lib/store.js"; + +const MAX_SPECS = 50; + +interface SearchBody { + specs: string[]; +} + +type IRequest = Request; + +export default function route(req: IRequest, res: Response) { + if (!store.data) { + res.sendStatus(503); + return; + } + + const { specs } = req.body; + + if (!Array.isArray(specs) || specs.length === 0) { + res.status(400); + res.json({ error: "Request body must contain a `specs` array." }); + return; + } + + if (specs.length > MAX_SPECS) { + res.status(400); + res.json({ error: `Too many spec URLs. Maximum is ${MAX_SPECS}, got ${specs.length}.` }); + return; + } + + if (!specs.every(s => typeof s === "string" && s.trim().length > 0)) { + res.status(400); + res.json({ error: "Each spec must be a non-empty string." }); + return; + } + + const normalizedSpecs = specs.map(normalizeUrl); + + const matchingIds = new Set(); + for (const [url, ids] of store.bySpecUrl) { + if (normalizedSpecs.some(spec => url.startsWith(spec))) { + for (const id of ids) { + matchingIds.add(id); + } + } + } + + const result = [...matchingIds] + .map(id => ({ id, ...store.byFeature.get(id)! })) + .filter(entry => entry.name); + + res.set("Cache-Control", `max-age=${seconds("30m")}`); + res.json({ result }); +} diff --git a/routes/api/baseline/update.ts b/routes/api/baseline/update.ts new file mode 100644 index 00000000..887ba97a --- /dev/null +++ b/routes/api/baseline/update.ts @@ -0,0 +1,34 @@ +import path from "path"; + +import { Request, Response } from "express"; + +import { BackgroundTaskQueue } from "../../../utils/background-task-queue.js"; +import { store } from "./lib/store-init.js"; + +const workerFile = path.join(import.meta.dirname, "update.worker.js"); +const taskQueue = new BackgroundTaskQueue( + workerFile, + "baseline_update", +); + +export default async function route(req: Request, res: Response) { + if (req.body.action !== "published") { + res.status(400); + res.locals.reason = "action-not-published"; + res.send("Webhook action ignored (expected 'published')."); + return; + } + + const job = taskQueue.add({ webhookId: req.get("X-GitHub-Delivery") || "" }); + try { + const { updated } = await job.run(); + if (updated) { + store.fill(); + } + } catch { + res.status(500); + } finally { + res.locals.job = job.id; + res.send(job.id); + } +} diff --git a/routes/api/baseline/update.worker.ts b/routes/api/baseline/update.worker.ts new file mode 100644 index 00000000..d3706c11 --- /dev/null +++ b/routes/api/baseline/update.worker.ts @@ -0,0 +1,10 @@ +import baselineScraper from "./lib/scraper.js"; + +interface Input { + webhookId: string; +} + +export default async function baselineUpdate(_input: Input) { + const updated = await baselineScraper(); + return { updated }; +} diff --git a/scripts/update-data-sources.ts b/scripts/update-data-sources.ts index 2bd87959..1194b247 100644 --- a/scripts/update-data-sources.ts +++ b/scripts/update-data-sources.ts @@ -3,6 +3,7 @@ import { mkdir } from "fs/promises"; import { env } from "../utils/misc.js"; import caniuse from "../routes/caniuse/lib/scraper.js"; import xref from "../routes/xref/lib/scraper.js"; +import baseline from "../routes/api/baseline/lib/scraper.js"; import { pullRelease } from "../routes/respec/builds/update.js"; import w3cGroupsList from "./update-w3c-groups-list.js"; @@ -17,6 +18,10 @@ console.group("xref"); await xref({ forceUpdate: true }); console.groupEnd(); +console.group("baseline"); +await baseline(); +console.groupEnd(); + console.group("W3C Groups List"); await w3cGroupsList(); console.groupEnd(); diff --git a/tests/routes/baseline/feature.test.js b/tests/routes/baseline/feature.test.js new file mode 100644 index 00000000..7e074773 --- /dev/null +++ b/tests/routes/baseline/feature.test.js @@ -0,0 +1,121 @@ +import featureRoute from "../../../build/routes/api/baseline/feature.js"; +import { store } from "../../../build/routes/api/baseline/lib/store-init.js"; + +/** @returns {import("express").Response} */ +function makeRes() { + const res = { + _status: 200, + _body: undefined, + _headers: {}, + status(code) { + this._status = code; + return this; + }, + sendStatus(code) { + this._status = code; + return this; + }, + set(name, value) { + this._headers[name] = value; + return this; + }, + json(data) { + this._body = data; + return this; + }, + }; + return res; +} + +const FEATURE_DATA = { + kind: "feature", + name: "CSS Animations", + spec: ["https://drafts.csswg.org/css-animations/"], + status: { baseline: "high", support: {} }, +}; + +describe("routes/api/baseline/feature", () => { + beforeEach(() => { + store.data = { + features: { + "css-animations": FEATURE_DATA, + "old-animations": { + kind: "moved", + redirect_target: "css-animations", + }, + "mega-feature": { + kind: "split", + redirect_targets: ["css-animations"], + }, + }, + browsers: {}, + groups: {}, + snapshots: {}, + }; + store.byFeature = new Map([["css-animations", FEATURE_DATA]]); + }); + + afterEach(() => { + store.data = null; + store.byFeature = new Map(); + store.bySpecUrl = new Map(); + }); + + it("returns feature data for a known feature", () => { + const req = { params: { feature: "css-animations" } }; + const res = makeRes(); + featureRoute(req, res); + expect(res._status).toBe(200); + expect(res._body.id).toBe("css-animations"); + expect(res._body.name).toBe("CSS Animations"); + }); + + it("returns 404 when feature is unknown", () => { + const req = { params: { feature: "unknown-feature" } }; + const res = makeRes(); + featureRoute(req, res); + expect(res._status).toBe(404); + }); + + it("returns 404 when store has no data", () => { + store.data = null; + store.byFeature = new Map(); + const req = { params: { feature: "css-animations" } }; + const res = makeRes(); + featureRoute(req, res); + expect(res._status).toBe(404); + }); + + it("resolves a moved feature to its redirect target", () => { + const req = { params: { feature: "old-animations" } }; + const res = makeRes(); + featureRoute(req, res); + expect(res._status).toBe(200); + expect(res._body.id).toBe("css-animations"); + expect(res._body.redirected_from).toBe("old-animations"); + expect(res._body.name).toBe("CSS Animations"); + }); + + it("returns 404 for a moved feature whose target is missing", () => { + store.data.features["orphan-moved"] = { + kind: "moved", + redirect_target: "nonexistent", + }; + const req = { params: { feature: "orphan-moved" } }; + const res = makeRes(); + featureRoute(req, res); + expect(res._status).toBe(404); + }); + + it("resolves a split feature to its redirect targets", () => { + const req = { params: { feature: "mega-feature" } }; + const res = makeRes(); + featureRoute(req, res); + expect(res._status).toBe(200); + expect(res._body.id).toBe("mega-feature"); + expect(res._body.kind).toBe("split"); + expect(Array.isArray(res._body.split_into)).toBeTrue(); + expect(res._body.split_into.length).toBe(1); + expect(res._body.split_into[0].id).toBe("css-animations"); + }); +}); diff --git a/tests/routes/baseline/lib/store.test.js b/tests/routes/baseline/lib/store.test.js new file mode 100644 index 00000000..65a7e00d --- /dev/null +++ b/tests/routes/baseline/lib/store.test.js @@ -0,0 +1,43 @@ +import { normalizeUrl } from "../../../../build/routes/api/baseline/lib/store.js"; + +describe("routes/api/baseline/lib/store - normalizeUrl", () => { + it("removes URL fragment (hash)", () => { + expect(normalizeUrl("https://example.com/path#section")).toBe( + "https://example.com/path/", + ); + }); + + it("removes URL search params (query string)", () => { + expect(normalizeUrl("https://example.com/path?foo=bar&baz=1")).toBe( + "https://example.com/path/", + ); + }); + + it("removes both fragment and search params", () => { + expect(normalizeUrl("https://example.com/path?q=1#section")).toBe( + "https://example.com/path/", + ); + }); + + it("adds trailing slash for directory-like paths", () => { + expect(normalizeUrl("https://example.com/path")).toBe( + "https://example.com/path/", + ); + }); + + it("does not add trailing slash for file paths with extension", () => { + expect(normalizeUrl("https://example.com/spec.html")).toBe( + "https://example.com/spec.html", + ); + }); + + it("preserves existing trailing slash", () => { + expect(normalizeUrl("https://example.com/path/")).toBe( + "https://example.com/path/", + ); + }); + + it("returns unchanged input for invalid URLs", () => { + expect(normalizeUrl("not-a-url")).toBe("not-a-url"); + }); +}); diff --git a/tests/routes/baseline/search.test.js b/tests/routes/baseline/search.test.js new file mode 100644 index 00000000..ef1b4c5d --- /dev/null +++ b/tests/routes/baseline/search.test.js @@ -0,0 +1,141 @@ +import searchRoute from "../../../build/routes/api/baseline/search.post.js"; +import { store } from "../../../build/routes/api/baseline/lib/store-init.js"; + +/** @returns {import("express").Response} */ +function makeRes() { + const res = { + _status: 200, + _body: undefined, + status(code) { + this._status = code; + return this; + }, + sendStatus(code) { + this._status = code; + return this; + }, + set() { + return this; + }, + json(data) { + this._body = data; + return this; + }, + }; + return res; +} + +const EMPTY_STORE_DATA = { + features: {}, + browsers: {}, + groups: {}, + snapshots: {}, +}; + +describe("routes/api/baseline/search.post", () => { + afterEach(() => { + store.data = null; + store.byFeature = new Map(); + store.bySpecUrl = new Map(); + }); + + describe("when store data is unavailable", () => { + it("returns 503 when store.data is null", () => { + store.data = null; + const req = { body: { specs: ["https://example.com/spec/"] } }; + const res = makeRes(); + searchRoute(req, res); + expect(res._status).toBe(503); + }); + }); + + describe("input validation", () => { + beforeEach(() => { + store.data = EMPTY_STORE_DATA; + }); + + it("returns 400 when specs is missing", () => { + const req = { body: {} }; + const res = makeRes(); + searchRoute(req, res); + expect(res._status).toBe(400); + }); + + it("returns 400 when specs is not an array", () => { + const req = { body: { specs: "https://example.com/" } }; + const res = makeRes(); + searchRoute(req, res); + expect(res._status).toBe(400); + }); + + it("returns 400 when specs is an empty array", () => { + const req = { body: { specs: [] } }; + const res = makeRes(); + searchRoute(req, res); + expect(res._status).toBe(400); + }); + + it("returns 400 when specs contains an empty string", () => { + const req = { body: { specs: [""] } }; + const res = makeRes(); + searchRoute(req, res); + expect(res._status).toBe(400); + }); + + it("returns 400 when specs contains a whitespace-only string", () => { + const req = { body: { specs: [" "] } }; + const res = makeRes(); + searchRoute(req, res); + expect(res._status).toBe(400); + }); + + it("returns 400 when specs contains a non-string value", () => { + const req = { body: { specs: [42] } }; + const res = makeRes(); + searchRoute(req, res); + expect(res._status).toBe(400); + }); + }); + + describe("matching", () => { + beforeEach(() => { + store.data = EMPTY_STORE_DATA; + store.bySpecUrl = new Map([ + [ + "https://drafts.csswg.org/css-animations/", + ["css-animations"], + ], + ]); + store.byFeature = new Map([ + [ + "css-animations", + { + kind: "feature", + name: "CSS Animations", + status: { baseline: "high", support: {} }, + }, + ], + ]); + }); + + it("returns matching features for a given spec URL", () => { + const req = { + body: { specs: ["https://drafts.csswg.org/css-animations/"] }, + }; + const res = makeRes(); + searchRoute(req, res); + expect(res._status).toBe(200); + expect(res._body.result).toBeInstanceOf(Array); + expect(res._body.result.length).toBe(1); + expect(res._body.result[0].id).toBe("css-animations"); + }); + + it("returns empty result for non-matching spec URL", () => { + const req = { body: { specs: ["https://example.com/unknown/"] } }; + const res = makeRes(); + searchRoute(req, res); + expect(res._status).toBe(200); + expect(res._body.result).toEqual([]); + }); + }); +});