[CRCR] Implement on-call bot and allowlist functionality for downstream CI failures

Author: KarhouTam

torchci/lib/bot/crcrOncallBot.ts

@@ -0,0 +1,159 @@
+import { Probot } from "probot";
+import { fetchCrcrAllowlist } from "../crcrAllowlist";
+import { isPyTorchbotSupportedOrg } from "./utils";
+
+const FAILURE_CONCLUSIONS: ReadonlySet<string> = new Set([
+  "failure",
+  "cancelled",
+  "timed_out",
+  "action_required",
+]);
+
+const MARKER = "<!-- crcr-oncall -->";
+
+/**
+ * Parse the downstream ``owner/repo`` out of a CRCR check run name.
+ *
+ * Check runs are named ``crcr/<owner>/<repo>/<workflow_name>`` (see the
+ * Python gh_helper.check_run_name), so the downstream repo is the first two
+ * path segments after the ``crcr/`` prefix.
+ */
+function downstreamRepoFromCheckRunName(name: string): string | null {
+  const prefix = "crcr/";
+  if (!name.startsWith(prefix)) {
+    return null;
+  }
+  const parts = name.slice(prefix.length).split("/");
+  if (parts.length < 3 || !parts[0] || !parts[1]) {
+    return null;
+  }
+  return `${parts[0]}/${parts[1]}`;
+}
+
+/**
+ * Search existing PR comments for one with the CRCR on-call marker.
+ * Returns its ID (or 0 if none found).
+ */
+async function findExistingComment(
+  octokit: any,
+  owner: string,
+  repo: string,
+  prNumber: number
+): Promise<number> {
+  const commentsRes = await octokit.rest.issues.listComments({
+    owner,
+    repo,
+    issue_number: prNumber,
+  });
+  for (const comment of commentsRes.data) {
+    if (comment.body?.includes(MARKER)) {
+      return comment.id;
+    }
+  }
+  return 0;
+}
+
+export default function crcrOncallBot(app: Probot): void {
+  app.on("check_run.completed", async (ctx) => {
+    const owner = ctx.payload.repository.owner.login;
+    if (!isPyTorchbotSupportedOrg(owner)) {
+      return;
+    }
+
+    const repo = ctx.payload.repository.name;
+    const checkRun = ctx.payload.check_run;
+
+    // Only act on CRCR-created check runs
+    const downstreamRepo = downstreamRepoFromCheckRunName(checkRun.name);
+    if (!downstreamRepo) {
+      return;
+    }
+
+    // Only comment on failures
+    const conclusion = checkRun.conclusion ?? "";
+    if (!FAILURE_CONCLUSIONS.has(conclusion)) {
+      return;
+    }
+
+    // Get the PR this check run belongs to
+    const prs = checkRun.pull_requests ?? [];
+    if (prs.length === 0) {
+      ctx.log(
+        `crcrOncall: no PR associated with check run ${checkRun.name}, skipping`
+      );
+      return;
+    }
+
+    const headSha = checkRun.head_sha;
+    const checkRunUrl = checkRun.html_url ?? "";
+
+    // Load oncalls from the allowlist
+    let oncalls: string[];
+    try {
+      const allowlist = await fetchCrcrAllowlist(ctx.octokit);
+      oncalls = allowlist.getOncallsForRepo(downstreamRepo);
+    } catch (err) {
+      ctx.log({ err }, "crcrOncall: failed to load allowlist, skipping");
+      return;
+    }
+
+    if (oncalls.length === 0) {
+      ctx.log(
+        `crcrOncall: no oncalls configured for ${downstreamRepo}, skipping`
+      );
+      return;
+    }
+
+    const mentions = oncalls.map((o) => `@${o}`).join(" ");
+
+    // Post a comment on each associated PR (typically just one)
+    for (const pr of prs) {
+      try {
+        const prNumber = pr.number;
+
+        // Dedup by marker: only comment once per PR.
+        const existingId = await findExistingComment(
+          ctx.octokit,
+          owner,
+          repo,
+          prNumber
+        );
+        if (existingId !== 0) {
+          ctx.log(
+            `crcrOncall: comment already exists on PR #${prNumber}, skipping`
+          );
+          continue;
+        }
+
+        const commentBody = `${MARKER}
+## :x: CRCR downstream CI failure
+
+The downstream CI workflow in **${downstreamRepo}** has failed on commit \`${headSha.slice(0, 7)}\`.
+
+${mentions} please investigate.
+
+### Details
+
+| Field | Value |
+|---|---|
+| **Check Run** | [${checkRun.name}](${checkRunUrl}) |
+| **Commit** | \`${headSha}\` |
+| **Conclusion** | \`${conclusion}\` |
+`;
+
+        await ctx.octokit.issues.createComment({
+          owner,
+          repo,
+          issue_number: prNumber,
+          body: commentBody,
+        });
+
+        ctx.log(
+          `crcrOncall: commented on PR #${prNumber} for ${downstreamRepo} (${conclusion})`
+        );
+      } catch (err) {
+        ctx.log({ err }, `crcrOncall: failed to comment on PR for ${downstreamRepo}`);
+      }
+    }
+  });
+}

torchci/lib/bot/index.ts

@@ -6,6 +6,7 @@ import cancelWorkflowsOnCloseBot from "./cancelWorkflowsOnCloseBot";
 import checkLabelsBot from "./checkLabelsBot";
 import ciflowPushTrigger from "./ciflowPushTrigger";
 import codevNoWritePerm from "./codevNoWritePermBot";
+import crcrOncallBot from "./crcrOncallBot";
 import drciBot from "./drciBot";
 import nitpickBot from "./nitpickBot";
 import pytorchBot from "./pytorchBot";
@@ -22,6 +23,7 @@ export default function bot(app: Probot) {
   checkLabelsBot(app);
   ciflowPushTrigger(app);
   codevNoWritePerm(app);
+  crcrOncallBot(app);
   drciBot(app);
   nitpickBot(app);
   pytorchBot(app);

torchci/lib/bot/pytorchBotHandler.ts

@@ -5,6 +5,7 @@ import shlex from "shlex";
 import { queryClickhouseSaved } from "../clickhouse";
 import { getHelp, getParser } from "./cliParser";
 import { cherryPickClassifications } from "./Constants";
+import { fetchCrcrAllowlist } from "../crcrAllowlist";
 import PytorchBotLogger from "./pytorchbotLogger";
 import {
   hasWritePermissions as _hasWP,
@@ -319,6 +320,28 @@ The explanation needs to be clear on why this is needed. Here are some good exam
     }
 
     await this.logger.log("merge", extra_data);
+
+    // Check for L4 CRCR blocking failures (L3 failures are non-blocking).
+    // Force merge (-f) bypasses this check, consistent with the existing
+    // force-merge semantics for in-repo CI failures.
+    if (!forceRequested) {
+      try {
+        const blockingRepos = await this.getCrcrBlockingFailures();
+        if (blockingRepos.length > 0) {
+          const repoList = blockingRepos.join(", ");
+          await this.addComment(
+            `The following L4 downstream CI workflows have failed and are blocking this merge:\n\n` +
+              blockingRepos.map((r) => `- \`${r}\``).join("\n") +
+              `\n\nPlease investigate or use \`@pytorchbot merge -f\` to bypass.`
+          );
+          return;
+        }
+      } catch (err) {
+        // If the blocking check itself fails, log and proceed (fail open).
+        this.ctx.log({ err }, "CRCR blocking check failed, allowing merge");
+      }
+    }
+
     if (!forceRequested && isPyTorchPyTorch(this.owner, this.repo)) {
       let labels: string[] = this.ctx.payload?.issue?.labels.map(
         (e: any) => e["name"]
@@ -635,6 +658,82 @@ The explanation needs to be clear on why this is needed. Here are some good exam
       headSha: this.headSha,
     });
   }
+
+  /**
+   * Return the list of L4 downstream repos whose CRCR check runs have failed
+   * on this PR's head commit.  L3 failures are intentionally omitted — they
+   * are non-blocking.
+   */
+  async getCrcrBlockingFailures(): Promise<string[]> {
+    // Lazy-load the head SHA if not already cached
+    if (this.headSha === undefined) {
+      const pullRequest = await this.ctx.octokit.pulls.get({
+        owner: this.owner,
+        repo: this.repo,
+        pull_number: this.prNum,
+      });
+      this.headSha = pullRequest.data.head.sha;
+    }
+
+    // Query GitHub Check Runs API for all check runs on this commit
+    let checkRuns: any[] = [];
+    try {
+      const res = await this.ctx.octokit.checks.listForRef({
+        owner: this.owner,
+        repo: this.repo,
+        ref: this.headSha!,
+        filter: "latest",
+        per_page: 100,
+      });
+      checkRuns = res.data.check_runs;
+    } catch {
+      // If we can't fetch check runs, fail open (don't block merge on
+      // an infrastructure error)
+      this.ctx.log("getCrcrBlockingFailures: failed to list check runs");
+      return [];
+    }
+
+    // Filter for CRCR check runs with a blocking conclusion
+    const FAILURE_CONCLUSIONS = new Set([
+      "failure",
+      "cancelled",
+      "timed_out",
+      "action_required",
+    ]);
+
+    const crcrFailures = checkRuns.filter(
+      (cr: any) =>
+        cr.name.startsWith("crcr/") &&
+        FAILURE_CONCLUSIONS.has(cr.conclusion ?? "")
+    );
+
+    if (crcrFailures.length === 0) {
+      return [];
+    }
+
+    // Load the allowlist to classify each failed repo as L3 or L4
+    let allowlist;
+    try {
+      allowlist = await fetchCrcrAllowlist(this.ctx.octokit);
+    } catch {
+      this.ctx.log("getCrcrBlockingFailures: failed to load allowlist");
+      return [];
+    }
+
+    const blocking: string[] = [];
+    for (const cr of crcrFailures) {
+      // Parse downstream repo from check run name: crcr/<owner>/<repo>/...
+      const nameParts = cr.name.slice(5).split("/"); // remove "crcr/"
+      if (nameParts.length >= 2) {
+        const downstreamRepo = `${nameParts[0]}/${nameParts[1]}`;
+        if (allowlist.isBlocking(downstreamRepo)) {
+          blocking.push(downstreamRepo);
+        }
+      }
+    }
+
+    return blocking;
+  }
 }
 
 export default PytorchBotHandler;