Skip to content

Commit 3715e2d

Browse files
GedochaoGedochao
committed
Post a comment under PRs indicating which tests are ran/skipped by the CI
1 parent 2bee48c commit 3715e2d

6 files changed

Lines changed: 583 additions & 46 deletions

File tree

‎.github/scripts/build-pr-classification-comment.sc‎

Lines changed: 228 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,228 @@
1+
#!/usr/bin/env -S scala-cli shebang
2+
//> using scala 3
3+
//> using toolkit default
4+
//> using options -Werror -Wunused:all
5+
6+
// Builds the Markdown body for the PR sticky comment that summarizes how the
7+
// `changes` job classified the diff and which suite groups will run / be skipped.
8+
//
9+
// Inputs (env vars):
10+
// CLASSIFY_OUTPUT_FILE - KEY=VALUE file produced by classify-changes.sh.
11+
// OVERRIDE_OUTPUT_FILE - KEY=VALUE file produced by check-override-keywords.sh.
12+
// COMMENT_OUTPUT_FILE - Path to write the resulting Markdown comment to
13+
// (default: ./comment.md).
14+
// CLASSIFY_RUN_ID - Run ID of the classification workflow (optional).
15+
// CLASSIFY_RUN_URL - URL to the classification workflow run (optional).
16+
// CI_RUN_ID - Run ID of the matching CI workflow run (optional;
17+
// empty means "not yet resolved", link falls back).
18+
// CI_RUN_URL - URL to the matching CI workflow run, or a fallback
19+
// link (e.g. the PR's /checks page).
20+
21+
import java.nio.file.Paths
22+
23+
def envOpt(name: String): Option[String] =
24+
sys.env.get(name).filter(_.nonEmpty)
25+
26+
def envRequiredFile(name: String): os.Path =
27+
val raw = envOpt(name).getOrElse:
28+
System.err.println(s"::error::$name is missing")
29+
sys.exit(1)
30+
val path = toAbsolutePath(raw)
31+
if !os.exists(path) then
32+
System.err.println(s"::error::$name points to non-existent file: $path")
33+
sys.exit(1)
34+
path
35+
36+
def toAbsolutePath(s: String): os.Path =
37+
if Paths.get(s).isAbsolute then os.Path(s) else os.Path(s, os.pwd)
38+
39+
def readKeyValueFile(path: os.Path): Map[String, String] =
40+
os.read.lines(path).iterator.flatMap { line =>
41+
val trimmed = line.trim
42+
if trimmed.isEmpty || trimmed.startsWith("#") then None
43+
else
44+
trimmed.split("=", 2) match
45+
case Array(k, v) => Some(k.trim -> v.trim)
46+
case _ => None
47+
}.toMap
48+
49+
enum SuiteGroup(val label: String):
50+
case UnitAndMill extends SuiteGroup("Unit tests & fish shell")
51+
case JvmIntegration extends SuiteGroup("JVM integration tests")
52+
case NativeIntegration extends SuiteGroup("Native integration tests")
53+
case DocsTests extends SuiteGroup("Docs tests")
54+
case Checks extends SuiteGroup("Checks")
55+
case Format extends SuiteGroup("Format / scalafix")
56+
case ReferenceDoc extends SuiteGroup("Reference docs")
57+
case BloopMemoryFootprint extends SuiteGroup("Bloop memory footprint")
58+
case SbtExportVcRedist extends SuiteGroup("Sbt export / vc-redist")
59+
60+
enum OverrideKey(val keyword: String):
61+
case TestAll extends OverrideKey("test_all")
62+
case TestNative extends OverrideKey("test_native")
63+
case TestIntegration extends OverrideKey("test_integration")
64+
case TestDocs extends OverrideKey("test_docs")
65+
case TestFormat extends OverrideKey("test_format")
66+
67+
// Signals provided to suite-group expressions — keep the boolean predicates in
68+
// sync with the SHOULD_RUN expressions used in .github/workflows/ci.yml.
69+
case class Signals(
70+
code: Boolean,
71+
docs: Boolean,
72+
ci: Boolean,
73+
formatConfig: Boolean,
74+
benchmark: Boolean,
75+
gifs: Boolean,
76+
millWrapper: Boolean,
77+
testAll: Boolean,
78+
testNative: Boolean,
79+
testIntegration: Boolean,
80+
testDocs: Boolean,
81+
testFormat: Boolean
82+
):
83+
def withoutOverrides: Signals = copy(
84+
testAll = false,
85+
testNative = false,
86+
testIntegration = false,
87+
testDocs = false,
88+
testFormat = false
89+
)
90+
91+
def withOverride(key: OverrideKey, enabled: Boolean): Signals = key match
92+
case OverrideKey.TestAll => copy(testAll = enabled)
93+
case OverrideKey.TestNative => copy(testNative = enabled)
94+
case OverrideKey.TestIntegration => copy(testIntegration = enabled)
95+
case OverrideKey.TestDocs => copy(testDocs = enabled)
96+
case OverrideKey.TestFormat => copy(testFormat = enabled)
97+
98+
def shouldRun(group: SuiteGroup): Boolean = group match
99+
case SuiteGroup.UnitAndMill =>
100+
code || ci || millWrapper || testAll
101+
case SuiteGroup.JvmIntegration =>
102+
code || ci || testAll || testIntegration
103+
case SuiteGroup.NativeIntegration =>
104+
code || ci || testAll || testNative
105+
case SuiteGroup.DocsTests =>
106+
code || docs || ci || gifs || testAll || testDocs
107+
case SuiteGroup.Checks =>
108+
code || docs || ci || formatConfig || testAll
109+
case SuiteGroup.Format =>
110+
code || docs || ci || formatConfig || testAll || testFormat
111+
case SuiteGroup.ReferenceDoc =>
112+
code || docs || ci || testAll
113+
case SuiteGroup.BloopMemoryFootprint =>
114+
code || ci || benchmark || testAll
115+
case SuiteGroup.SbtExportVcRedist =>
116+
code || ci || testAll
117+
118+
object Signals:
119+
def apply(categories: Map[String, String], overrides: Map[String, String]): Signals =
120+
def boolAt(map: Map[String, String], key: String): Boolean =
121+
map.get(key).exists(_.equalsIgnoreCase("true"))
122+
Signals(
123+
code = boolAt(categories, "code"),
124+
docs = boolAt(categories, "docs"),
125+
ci = boolAt(categories, "ci"),
126+
formatConfig = boolAt(categories, "format_config"),
127+
benchmark = boolAt(categories, "benchmark"),
128+
gifs = boolAt(categories, "gifs"),
129+
millWrapper = boolAt(categories, "mill_wrapper"),
130+
testAll = boolAt(overrides, "test_all"),
131+
testNative = boolAt(overrides, "test_native"),
132+
testIntegration = boolAt(overrides, "test_integration"),
133+
testDocs = boolAt(overrides, "test_docs"),
134+
testFormat = boolAt(overrides, "test_format")
135+
)
136+
137+
/** Which individual override keywords (among those currently active) actually
138+
* add suite groups that wouldn't otherwise run. Used to list only the
139+
* overrides that matter.
140+
*/
141+
extension (signals: Signals)
142+
def isOverrideActive(key: OverrideKey): Boolean = key match
143+
case OverrideKey.TestAll => signals.testAll
144+
case OverrideKey.TestNative => signals.testNative
145+
case OverrideKey.TestIntegration => signals.testIntegration
146+
case OverrideKey.TestDocs => signals.testDocs
147+
case OverrideKey.TestFormat => signals.testFormat
148+
149+
def overrideContributions(signals: Signals): Seq[(OverrideKey, Seq[SuiteGroup])] =
150+
val baseline = signals.withoutOverrides
151+
val baselineRuns = SuiteGroup.values.filter(baseline.shouldRun).toSet
152+
OverrideKey.values.toIndexedSeq.flatMap { key =>
153+
if !signals.isOverrideActive(key) then None
154+
else
155+
val probe = baseline.withOverride(key, enabled = true)
156+
val added = SuiteGroup.values.toIndexedSeq
157+
.filter(g => probe.shouldRun(g) && !baselineRuns.contains(g))
158+
if added.isEmpty then None else Some(key -> added)
159+
}
160+
161+
def renderComment(
162+
signals: Signals,
163+
classifyRunId: Option[String],
164+
classifyRunUrl: Option[String],
165+
ciRunId: Option[String],
166+
ciRunUrl: Option[String]
167+
): String =
168+
val groups = SuiteGroup.values.toIndexedSeq
169+
val (runGroups, skipGroups) = groups.partition(signals.shouldRun)
170+
val contributions = overrideContributions(signals)
171+
172+
val sb = StringBuilder()
173+
sb ++= "### CI change classification\n\n"
174+
sb ++= "**Change categories**\n\n"
175+
sb ++= "| Category | Changed |\n"
176+
sb ++= "| --- | --- |\n"
177+
sb ++= s"| code | ${signals.code} |\n"
178+
sb ++= s"| docs | ${signals.docs} |\n"
179+
sb ++= s"| ci | ${signals.ci} |\n"
180+
sb ++= s"| format_config | ${signals.formatConfig} |\n"
181+
sb ++= s"| benchmark | ${signals.benchmark} |\n"
182+
sb ++= s"| gifs | ${signals.gifs} |\n"
183+
sb ++= s"| mill_wrapper | ${signals.millWrapper} |\n"
184+
sb ++= "\n**Suite groups**\n\n"
185+
sb ++= (
186+
if runGroups.isEmpty then "- Will run: _(none)_\n"
187+
else s"- Will run: ${runGroups.map(_.label).mkString(", ")}\n"
188+
)
189+
sb ++= (
190+
if skipGroups.isEmpty then "- Will be skipped: _(none)_\n"
191+
else s"- Will be skipped: ${skipGroups.map(_.label).mkString(", ")}\n"
192+
)
193+
194+
if contributions.nonEmpty then
195+
sb ++= "\n**Override keywords affecting the run set**\n\n"
196+
for (key, added) <- contributions do
197+
sb ++= s"- `[${key.keyword}]` forces on: ${added.map(_.label).mkString(", ")}\n"
198+
199+
ciRunId match
200+
case Some(id) =>
201+
val url = ciRunUrl.getOrElse("")
202+
sb ++= s"\nFull CI run: [#$id]($url)\n"
203+
case None =>
204+
ciRunUrl.foreach { url => sb ++= s"\nFull CI run: $url\n" }
205+
206+
classifyRunId.foreach { id =>
207+
val url = classifyRunUrl.getOrElse("")
208+
sb ++= s"\n_Classified in run [#$id]($url)._\n"
209+
}
210+
211+
sb.result()
212+
213+
val categories = readKeyValueFile(envRequiredFile("CLASSIFY_OUTPUT_FILE"))
214+
val overrides = readKeyValueFile(envRequiredFile("OVERRIDE_OUTPUT_FILE"))
215+
val signals = Signals(categories, overrides)
216+
217+
val commentPath = toAbsolutePath(envOpt("COMMENT_OUTPUT_FILE").getOrElse("comment.md"))
218+
219+
val body = renderComment(
220+
signals,
221+
classifyRunId = envOpt("CLASSIFY_RUN_ID"),
222+
classifyRunUrl = envOpt("CLASSIFY_RUN_URL"),
223+
ciRunId = envOpt("CI_RUN_ID"),
224+
ciRunUrl = envOpt("CI_RUN_URL")
225+
)
226+
227+
os.write.over(commentPath, body, createFolders = true)
228+
println(s"Wrote comment to $commentPath")

‎.github/scripts/check-override-keywords.sh‎

Lines changed: 44 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -2,13 +2,37 @@
22
set -euo pipefail
33

44
# Checks the PR body for [test_*] override keywords.
5-
# Inputs (env vars): EVENT_NAME, PR_BODY
6-
# Outputs: writes override=true/false pairs to $GITHUB_OUTPUT and a summary table to $GITHUB_STEP_SUMMARY
5+
# Inputs (env vars):
6+
# EVENT_NAME - GitHub event name (pull_request, push, ...).
7+
# PR_BODY - The PR body to scan for override keywords.
8+
# OVERRIDE_OUTPUT_FILE - Optional path of a KEY=VALUE file to also write
9+
# override results into (in addition to $GITHUB_OUTPUT).
10+
# Outputs: writes override=true/false pairs to $GITHUB_OUTPUT (when set) and
11+
# a summary table to $GITHUB_STEP_SUMMARY (when set). When OVERRIDE_OUTPUT_FILE
12+
# is provided, also writes the same KEY=VALUE pairs there.
713

8-
if [[ "$EVENT_NAME" != "pull_request" ]]; then
14+
OVERRIDES=(test_all test_native test_integration test_docs test_format)
15+
16+
write_output() {
17+
local key="$1"
18+
local val="$2"
19+
if [[ -n "${GITHUB_OUTPUT:-}" ]]; then
20+
echo "$key=$val" >> "$GITHUB_OUTPUT"
21+
fi
22+
if [[ -n "${OVERRIDE_OUTPUT_FILE:-}" ]]; then
23+
echo "$key=$val" >> "$OVERRIDE_OUTPUT_FILE"
24+
fi
25+
}
26+
27+
write_summary() {
28+
[[ -n "${GITHUB_STEP_SUMMARY:-}" ]] || return 0
29+
echo "$1" >> "$GITHUB_STEP_SUMMARY"
30+
}
31+
32+
if [[ "${EVENT_NAME:-}" != "pull_request" ]]; then
933
echo "Non-PR event, setting all overrides to true"
10-
for override in test_all test_native test_integration test_docs test_format; do
11-
echo "$override=true" >> "$GITHUB_OUTPUT"
34+
for override in "${OVERRIDES[@]}"; do
35+
write_output "$override" "true"
1236
done
1337
exit 0
1438
fi
@@ -18,7 +42,7 @@ TEST_ALL=false; TEST_NATIVE=false; TEST_INTEGRATION=false; TEST_DOCS=false; TEST
1842
check_override() {
1943
local keyword="$1"
2044
local var_name="$2"
21-
if printf '%s' "$PR_BODY" | grep -qF "$keyword"; then
45+
if printf '%s' "${PR_BODY:-}" | grep -qF "$keyword"; then
2246
eval "$var_name=true"
2347
echo "Override $keyword found"
2448
fi
@@ -37,17 +61,17 @@ echo " test_integration=$TEST_INTEGRATION"
3761
echo " test_docs=$TEST_DOCS"
3862
echo " test_format=$TEST_FORMAT"
3963

40-
echo "test_all=$TEST_ALL" >> "$GITHUB_OUTPUT"
41-
echo "test_native=$TEST_NATIVE" >> "$GITHUB_OUTPUT"
42-
echo "test_integration=$TEST_INTEGRATION" >> "$GITHUB_OUTPUT"
43-
echo "test_docs=$TEST_DOCS" >> "$GITHUB_OUTPUT"
44-
echo "test_format=$TEST_FORMAT" >> "$GITHUB_OUTPUT"
45-
46-
echo "## Override keywords" >> "$GITHUB_STEP_SUMMARY"
47-
echo "| Keyword | Active |" >> "$GITHUB_STEP_SUMMARY"
48-
echo "|---------|--------|" >> "$GITHUB_STEP_SUMMARY"
49-
echo "| [test_all] | $TEST_ALL |" >> "$GITHUB_STEP_SUMMARY"
50-
echo "| [test_native] | $TEST_NATIVE |" >> "$GITHUB_STEP_SUMMARY"
51-
echo "| [test_integration] | $TEST_INTEGRATION |" >> "$GITHUB_STEP_SUMMARY"
52-
echo "| [test_docs] | $TEST_DOCS |" >> "$GITHUB_STEP_SUMMARY"
53-
echo "| [test_format] | $TEST_FORMAT |" >> "$GITHUB_STEP_SUMMARY"
64+
write_output test_all "$TEST_ALL"
65+
write_output test_native "$TEST_NATIVE"
66+
write_output test_integration "$TEST_INTEGRATION"
67+
write_output test_docs "$TEST_DOCS"
68+
write_output test_format "$TEST_FORMAT"
69+
70+
write_summary "## Override keywords"
71+
write_summary "| Keyword | Active |"
72+
write_summary "|---------|--------|"
73+
write_summary "| [test_all] | $TEST_ALL |"
74+
write_summary "| [test_native] | $TEST_NATIVE |"
75+
write_summary "| [test_integration] | $TEST_INTEGRATION |"
76+
write_summary "| [test_docs] | $TEST_DOCS |"
77+
write_summary "| [test_format] | $TEST_FORMAT |"

0 commit comments

Comments
 (0)