feat(release_readiness): add artifact integrity check — verify SHA256SUMS covers all assets

AlbinoGeek · AlbinoGeek · commit 053f6c9d6f6a · 2026-04-27T21:41:22.000-07:00
diff --git a/src/server/release-readiness-tool.test.ts b/src/server/release-readiness-tool.test.ts
@@ -0,0 +1,134 @@
+import { describe, expect, test } from "bun:test";
+
+import { type ArtifactIntegrity, registerReleaseReadinessTool } from "./release-readiness-tool.js";
+import { captureTool } from "./test-harness.js";
+
+/**
+ * Test suite for release_readiness tool.
+ *
+ * Tests below exercise code paths that do NOT require GitHub API calls
+ * when the repo has no tags/releases. Tests that require API calls are
+ * best-effort (graceful skip if auth unavailable).
+ */
+describe("release_readiness tool", () => {
+  test("NO_SEMVER_TAG error when base omitted and repo has no semver tags", async () => {
+    const run = captureTool(registerReleaseReadinessTool);
+    // Use a repo that likely has no releases or non-semver tags
+    const text = await run({
+      owner: "Rethunk-AI",
+      repo: "rethunk-github-mcp",
+      format: "json",
+      // base omitted — auto-pick will fail if no semver tags exist
+    });
+    const parsed = JSON.parse(text) as { code?: string };
+    // Either NOT_FOUND (no tags), or the call succeeds (repo has tags)
+    // Both are acceptable test outcomes
+    if (parsed.code === "NOT_FOUND") {
+      expect(parsed.code).toBe("NOT_FOUND");
+    }
+  });
+
+  test("JSON format with real repo (auth-dependent)", async () => {
+    const run = captureTool(registerReleaseReadinessTool);
+    const text = await run({
+      owner: "Rethunk-AI",
+      repo: "rethunk-github-mcp",
+      format: "json",
+      // base/head will auto-pick or use latest tag
+    });
+    const parsed = JSON.parse(text) as {
+      base?: string;
+      head?: string;
+      aheadBy?: number;
+      commits?: unknown[];
+      stats?: { additions: number; deletions: number; changedFiles: number };
+      artifactIntegrity?: ArtifactIntegrity;
+      error?: { code: string };
+    };
+
+    // If no auth or API error, gracefully skip
+    if (parsed.error) return;
+
+    // If successful, verify structure
+    if (parsed.base) {
+      expect(typeof parsed.base).toBe("string");
+      expect(typeof parsed.head).toBe("string");
+      expect(typeof parsed.aheadBy).toBe("number");
+      expect(Array.isArray(parsed.commits)).toBe(true);
+      expect(parsed.stats).toBeDefined();
+      expect(parsed.stats?.additions).toBeGreaterThanOrEqual(0);
+      expect(parsed.stats?.deletions).toBeGreaterThanOrEqual(0);
+      expect(parsed.stats?.changedFiles).toBeGreaterThanOrEqual(0);
+
+      // Artifact integrity should be present
+      expect(parsed.artifactIntegrity).toBeDefined();
+      expect(["ok", "warn", "skip"]).toContain(parsed.artifactIntegrity?.verdict);
+      expect(typeof parsed.artifactIntegrity?.details).toBe("string");
+      expect(Array.isArray(parsed.artifactIntegrity?.missingFromChecksum)).toBe(true);
+    }
+  });
+
+  test("markdown format renders artifact integrity status", async () => {
+    const run = captureTool(registerReleaseReadinessTool);
+    const text = await run({
+      owner: "Rethunk-AI",
+      repo: "rethunk-github-mcp",
+      // format defaults to json in the tool, but we can try to get markdown
+      format: "markdown",
+    });
+
+    // If auth unavailable, result is JSON error — skip
+    if (text.startsWith("{")) return;
+
+    // If markdown was rendered, it should contain artifact status
+    if (text.includes("Release Readiness")) {
+      // Should contain one of: "integrity verified", "No checksum asset found", "skipped"
+      const hasArtifactStatus =
+        text.includes("Artifacts:") ||
+        text.includes("integrity verified") ||
+        text.includes("No checksum asset found") ||
+        text.includes("skipped");
+      expect(hasArtifactStatus).toBe(true);
+    }
+  });
+
+  test("artifact integrity type structure", () => {
+    const integrity: ArtifactIntegrity = {
+      verdict: "ok",
+      details: "All assets covered",
+      missingFromChecksum: [],
+      checksumAsset: "SHA256SUMS",
+    };
+
+    expect(integrity.verdict).toBe("ok");
+    expect(integrity.details).toContain("covered");
+    expect(integrity.missingFromChecksum).toHaveLength(0);
+    expect(integrity.checksumAsset).toBe("SHA256SUMS");
+  });
+
+  test("artifact integrity with missing assets", () => {
+    const integrity: ArtifactIntegrity = {
+      verdict: "warn",
+      details: "2 asset(s) not in checksum file",
+      missingFromChecksum: ["app-v1.2.3.zip", "app-v1.2.3.tar.gz"],
+      checksumAsset: "SHA256SUMS",
+    };
+
+    expect(integrity.verdict).toBe("warn");
+    expect(integrity.missingFromChecksum).toHaveLength(2);
+    expect(integrity.checksumAsset).toBe("SHA256SUMS");
+  });
+
+  test("artifact integrity skip when no assets", () => {
+    const integrity: ArtifactIntegrity = {
+      verdict: "skip",
+      details: "No release assets",
+      missingFromChecksum: [],
+    };
+
+    expect(integrity.verdict).toBe("skip");
+    expect(integrity.details).toContain("No release assets");
+    expect(integrity.missingFromChecksum).toHaveLength(0);
+    expect(integrity.checksumAsset).toBeUndefined();
+  });
+});
diff --git a/src/server/release-readiness-tool.ts b/src/server/release-readiness-tool.ts
@@ -26,6 +26,13 @@ interface CommitForRelease {
   pr?: { number: number; title: string; labels: string[] };
 }
 
+export interface ArtifactIntegrity {
+  verdict: "ok" | "warn" | "skip";
+  details: string;
+  missingFromChecksum: string[];
+  checksumAsset?: string;
+}
+
 async function fetchHeadCI(
   owner: string,
   repo: string,
@@ -63,6 +70,120 @@ async function fetchHeadCI(
   }
 }
 
+/**
+ * Check artifact integrity for a release.
+ * - If no assets: verdict "skip"
+ * - If checksum asset exists: parse it and verify all other assets are listed
+ * - If assets but no checksum: verdict "warn"
+ */
+async function checkArtifactIntegrity(
+  owner: string,
+  repo: string,
+  releaseId: number,
+  tag: string,
+): Promise<ArtifactIntegrity> {
+  try {
+    const octokit = getOctokit();
+
+    // Fetch all assets for the release
+    const assetsRes = await octokit.repos.listReleaseAssets({
+      owner,
+      repo,
+      release_id: releaseId,
+      per_page: 100,
+    });
+
+    const assets = assetsRes.data;
+    if (assets.length === 0) {
+      return {
+        verdict: "skip",
+        details: "No release assets",
+        missingFromChecksum: [],
+      };
+    }
+
+    // Look for a checksum file (SHA256SUMS, checksums.txt, sha256sums, etc.)
+    const checksumAsset = assets.find((a) => /sha256|checksums|integrity/i.test(a.name));
+
+    if (!checksumAsset) {
+      return {
+        verdict: "warn",
+        details: "No checksum asset found",
+        missingFromChecksum: assets.map((a) => a.name),
+      };
+    }
+
+    // Download and parse the checksum file
+    try {
+      const checksumRes = await octokit.repos.getReleaseAsset({
+        owner,
+        repo,
+        asset_id: checksumAsset.id,
+        headers: { Accept: "application/octet-stream" },
+      });
+
+      // The response.data should be a string (the file content)
+      const checksumContent = String(checksumRes.data);
+
+      // Parse the checksum file — extract asset names
+      // Common formats: "sha256 filename" or "sha256  filename" (with 1 or 2 spaces)
+      const checksumLines = checksumContent.split("\n").filter((line) => line.trim().length > 0);
+
+      const checksummedAssets = new Set<string>();
+      for (const line of checksumLines) {
+        const parts = line.trim().split(/\s+/);
+        if (parts.length >= 2) {
+          // Last part is typically the filename
+          const filename = parts[parts.length - 1];
+          if (filename) {
+            checksummedAssets.add(filename);
+          }
+        }
+      }
+
+      // Find assets not in the checksum file
+      const missingFromChecksum = assets
+        .filter((a) => a.name !== checksumAsset.name)
+        .filter((a) => !checksummedAssets.has(a.name))
+        .map((a) => a.name);
+
+      const verdict = missingFromChecksum.length === 0 ? "ok" : "warn";
+      const details =
+        verdict === "ok"
+          ? "All assets covered by checksum file"
+          : `${missingFromChecksum.length} asset(s) not in checksum file`;
+
+      return {
+        verdict,
+        details,
+        missingFromChecksum,
+        checksumAsset: checksumAsset.name,
+      };
+    } catch (parseErr) {
+      console.error(
+        `[checkArtifactIntegrity] Failed to download/parse checksum file ${checksumAsset.name} for ${owner}/${repo} release ${tag}:`,
+        parseErr instanceof Error ? parseErr.message : String(parseErr),
+      );
+      return {
+        verdict: "warn",
+        details: `Failed to parse checksum file: ${parseErr instanceof Error ? parseErr.message : "unknown error"}`,
+        missingFromChecksum: assets.filter((a) => a.name !== checksumAsset.name).map((a) => a.name),
+        checksumAsset: checksumAsset.name,
+      };
+    }
+  } catch (err) {
+    console.error(
+      `[checkArtifactIntegrity] Failed to check artifact integrity for ${owner}/${repo} release ${tag}:`,
+      err instanceof Error ? err.message : String(err),
+    );
+    return {
+      verdict: "warn",
+      details: `Error checking integrity: ${err instanceof Error ? err.message : "unknown error"}`,
+      missingFromChecksum: [],
+    };
+  }
+}
+
 export function registerReleaseReadinessTool(server: FastMCP): void {
   server.addTool({
     name: "release_readiness",
@@ -153,7 +274,21 @@ export function registerReleaseReadinessTool(server: FastMCP): void {
           changedFiles: cmp.data.files?.length ?? 0,
         };
 
-        const result = { base, head, aheadBy, headCi: ciStatus, commits, stats };
+        // Check artifact integrity if the base ref is a release tag
+        let artifactIntegrity: ArtifactIntegrity | undefined;
+        try {
+          const releaseRes = await octokit.repos.getReleaseByTag({ owner, repo, tag: base });
+          artifactIntegrity = await checkArtifactIntegrity(owner, repo, releaseRes.data.id, base);
+        } catch (_err) {
+          // base is not a release tag, or API error — skip integrity check
+          artifactIntegrity = {
+            verdict: "skip",
+            details: "Base ref is not a release tag",
+            missingFromChecksum: [],
+          };
+        }
+
+        const result = { base, head, aheadBy, headCi: ciStatus, commits, stats, artifactIntegrity };
 
         if (args.format === "json") return jsonRespond(result);
 
@@ -170,7 +305,22 @@ export function registerReleaseReadinessTool(server: FastMCP): void {
             : ciStatus.status === "not_configured"
               ? "CI: not configured"
               : `CI: failing (${ciStatus.failedChecks.map((c) => c.name).join(", ")})`;
-        lines.push(ciState, "");
+        lines.push(ciState);
+
+        // Add artifact integrity status
+        if (artifactIntegrity.verdict === "ok") {
+          lines.push("Artifacts: integrity verified");
+        } else if (artifactIntegrity.verdict === "warn") {
+          const missing =
+            artifactIntegrity.missingFromChecksum.length > 0
+              ? ` (${artifactIntegrity.missingFromChecksum.length} uncovered)`
+              : "";
+          lines.push(`Artifacts: ⚠ ${artifactIntegrity.details}${missing}`);
+        } else {
+          lines.push("Artifacts: skipped");
+        }
+
+        lines.push("");
 
         if (commits.length === 0) {
           lines.push("*(no commits)*");
