test(release): sync qa evidence manifests

Author: iexitdev

scripts/check-release-gates.mjs

@@ -187,6 +187,30 @@ const getIds = (items) =>
 
 const isExternalEvidenceLink = (value) => /^https?:\/\//.test(value);
 
+const getMatrixStatus = (rows = []) => {
+  if (rows.length === 0) {
+    return "pending";
+  }
+
+  if (rows.every((row) => row.status === "pass")) {
+    return "complete";
+  }
+
+  if (rows.some((row) => row.status === "fail")) {
+    return "fail";
+  }
+
+  if (rows.some((row) => row.status === "blocked")) {
+    return "blocked";
+  }
+
+  if (rows.some((row) => row.status === "pass")) {
+    return "partial";
+  }
+
+  return "pending";
+};
+
 const validateEvidenceMatrix = async (matrix) => {
   const errors = [];
   const pageIds = getIds(matrix.pages);
@@ -200,6 +224,14 @@ const validateEvidenceMatrix = async (matrix) => {
     errors.push("matrix must define at least one row");
   }
 
+  const derivedStatus = getMatrixStatus(matrix.rows ?? []);
+
+  if (matrix.status && matrix.status !== derivedStatus) {
+    errors.push(
+      `matrix status ${matrix.status} does not match row-derived status ${derivedStatus}`
+    );
+  }
+
   if (Array.isArray(matrix.pages)) {
     for (const page of matrix.pages) {
       const missingGroups = (page.requiredCheckGroups ?? []).filter(
@@ -406,6 +438,14 @@ const validateReleaseEvidenceManifest = async ({
     if (!matrix && completedEntries.length === 0) {
       errors.push("complete manifest must include completedEntries");
     }
+  } else if (
+    matrix &&
+    Array.isArray(matrix.rows) &&
+    matrix.rows.every((row) => row.status === "pass")
+  ) {
+    errors.push(
+      "matrix-backed manifest must be complete when all matrix rows pass"
+    );
   } else if (missingEvidence.length === 0) {
     errors.push(`${status} manifest must list missingEvidence`);
   }

scripts/record-native-qa-evidence.mjs

@@ -17,18 +17,30 @@ const validStatuses = new Set([
 const matrixConfigs = {
   accessibility: {
     label: "Accessibility QA",
+    completeSummary:
+      "Native accessibility QA matrix is complete for required VoiceOver and TalkBack showcase pages.",
+    manifestPath: "docs/release/evidence/native-accessibility-qa.json",
     path: "docs/release/evidence/native-accessibility-matrix.json"
   },
   performance: {
     label: "Native Performance",
+    completeSummary:
+      "Native performance matrix is complete for required iOS and Android release scenarios.",
+    manifestPath: "docs/release/evidence/native-performance-benchmark.json",
     path: "docs/release/evidence/native-performance-matrix.json"
   },
   runtime: {
     label: "Runtime QA",
+    completeSummary:
+      "Native runtime QA matrix is complete for required iOS and Android showcase pages.",
+    manifestPath: "docs/release/evidence/native-runtime-qa.json",
     path: "docs/release/evidence/native-runtime-matrix.json"
   },
   skia: {
     label: "Skia Renderer",
+    completeSummary:
+      "Skia renderer native install, renderer parity, and performance evidence matrix is complete.",
+    manifestPath: "docs/release/evidence/skia-renderer-evidence.json",
     path: "docs/release/evidence/skia-renderer-matrix.json"
   }
 };
@@ -134,6 +146,47 @@ const getMatrixStatus = (rows) => {
   return "pending";
 };
 
+const getManifestStatus = (matrixStatus) => {
+  if (matrixStatus === "complete") {
+    return "complete";
+  }
+
+  if (matrixStatus === "blocked" || matrixStatus === "fail") {
+    return "blocked";
+  }
+
+  return "partial";
+};
+
+const getManifestMissingEvidence = (rows) =>
+  rows
+    .filter((row) => row.status !== "pass")
+    .map(
+      (row) =>
+        `${row.id} is ${row.status}; evidence is required before this matrix can be complete.`
+    );
+
+const syncEvidenceManifest = async ({
+  config,
+  matrix,
+  matrixStatus,
+  repoRoot,
+  updated
+}) => {
+  const manifest = await readJson(repoRoot, config.manifestPath);
+  const manifestStatus = getManifestStatus(matrixStatus);
+  const complete = manifestStatus === "complete";
+  const nextManifest = {
+    ...manifest,
+    lastUpdated: updated,
+    missingEvidence: complete ? [] : getManifestMissingEvidence(matrix.rows),
+    status: manifestStatus,
+    summary: complete ? config.completeSummary : manifest.summary
+  };
+
+  return nextManifest;
+};
+
 const getRowTarget = (matrix, row) => {
   const page = matrix.pages?.find((item) => item.id === row.pageId);
   const platform = matrix.platforms?.find((item) => item.id === row.platform);
@@ -279,9 +332,17 @@ export const recordNativeQaEvidence = async ({
     rows: nextRows,
     status: getMatrixStatus(nextRows)
   };
+  const nextManifest = await syncEvidenceManifest({
+    config,
+    matrix: nextMatrix,
+    matrixStatus: nextMatrix.status,
+    repoRoot,
+    updated
+  });
 
   if (!dryRun) {
     await writeJson(repoRoot, config.path, nextMatrix);
+    await writeJson(repoRoot, config.manifestPath, nextManifest);
     await writeFile(
       path.join(repoRoot, checklistPath),
       await generateNativeQaChecklist({ repoRoot }),
@@ -292,6 +353,8 @@ export const recordNativeQaEvidence = async ({
   return {
     checklistPath,
     dryRun,
+    manifestPath: config.manifestPath,
+    manifestStatus: nextManifest.status,
     matrixPath: config.path,
     row: nextRow,
     status: nextMatrix.status

scripts/record-native-qa-evidence.test.mjs

@@ -17,8 +17,12 @@ import {
 const repoRoot = process.cwd();
 const matrixFiles = [
   "native-accessibility-matrix.json",
+  "native-accessibility-qa.json",
+  "native-performance-benchmark.json",
   "native-performance-matrix.json",
   "native-runtime-matrix.json",
+  "native-runtime-qa.json",
+  "skia-renderer-evidence.json",
   "skia-renderer-matrix.json"
 ];
 
@@ -141,6 +145,8 @@ describe("native QA evidence recorder", () => {
 
     expect(result).toMatchObject({
       dryRun: false,
+      manifestPath: "docs/release/evidence/native-runtime-qa.json",
+      manifestStatus: "partial",
       matrixPath: "docs/release/evidence/native-runtime-matrix.json",
       status: "partial"
     });
@@ -154,6 +160,19 @@ describe("native QA evidence recorder", () => {
     expect(checklist).toContain(
       "`docs/release/artifacts/ios-line-charts-runtime.md`"
     );
+    const manifest = JSON.parse(
+      await readFile(
+        join(tempRepo, "docs/release/evidence/native-runtime-qa.json"),
+        "utf8"
+      )
+    );
+    expect(manifest).toMatchObject({
+      lastUpdated: "2026-05-06",
+      status: "partial"
+    });
+    expect(manifest.missingEvidence).toContain(
+      "ios-bar-charts is pending; evidence is required before this matrix can be complete."
+    );
   });
 
   it("supports dry-run without writing matrix changes", async () => {
@@ -213,6 +232,8 @@ describe("native QA evidence recorder", () => {
 
     expect(result).toMatchObject({
       checklistPath: "docs/release/native-qa-checklists.md",
+      manifestPath: "docs/release/evidence/skia-renderer-evidence.json",
+      manifestStatus: "partial",
       matrixPath: "docs/release/evidence/skia-renderer-matrix.json",
       status: "partial"
     });
@@ -223,4 +244,68 @@ describe("native QA evidence recorder", () => {
     expect(checklist).toContain("| Skia Renderer | 8 | 1 | 7 | 0 | 0 | 0 |");
     expect(checklist).toContain("`ios-skia-native-install`");
   });
+
+  it("marks aggregate evidence complete when the last matrix row passes", async () => {
+    const tempRepo = await createTempRepo();
+    await createArtifact(
+      tempRepo,
+      "docs/release/artifacts/ios-line-charts-runtime.md"
+    );
+
+    const matrixPath = join(
+      tempRepo,
+      "docs/release/evidence/native-runtime-matrix.json"
+    );
+    const matrix = JSON.parse(await readFile(matrixPath, "utf8"));
+    const preparedRows = matrix.rows.map((row) =>
+      row.id === "ios-line-charts"
+        ? row
+        : {
+            ...row,
+            evidence: [`https://example.test/${row.id}.mp4`],
+            status: "pass"
+          }
+    );
+
+    await writeFile(
+      matrixPath,
+      `${JSON.stringify(
+        {
+          ...matrix,
+          rows: preparedRows,
+          status: "partial"
+        },
+        null,
+        2
+      )}\n`,
+      "utf8"
+    );
+
+    const result = await recordNativeQaEvidence({
+      evidence: ["docs/release/artifacts/ios-line-charts-runtime.md"],
+      matrixName: "runtime",
+      repoRoot: tempRepo,
+      rowId: "ios-line-charts",
+      status: "pass",
+      updated: "2026-05-06"
+    });
+    const manifest = JSON.parse(
+      await readFile(
+        join(tempRepo, "docs/release/evidence/native-runtime-qa.json"),
+        "utf8"
+      )
+    );
+
+    expect(result).toMatchObject({
+      manifestStatus: "complete",
+      status: "complete"
+    });
+    expect(manifest).toMatchObject({
+      lastUpdated: "2026-05-06",
+      missingEvidence: [],
+      status: "complete",
+      summary:
+        "Native runtime QA matrix is complete for required iOS and Android showcase pages."
+    });
+  });
 });