Skip to content

Commit 2c47ea9

Browse files
pocopepesylvestre
pocopepe
authored and
sylvestre
committed
feat: add gnu testsuite tracking workflows
1 parent 9a7a727 commit 2c47ea9

4 files changed

Lines changed: 489 additions & 31 deletions

File tree

‎.github/workflows/GnuComment.yml‎

Lines changed: 100 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,100 @@
1+
name: GnuComment
2+
3+
on:
4+
workflow_run:
5+
workflows: ["GnuTests"]
6+
types:
7+
- completed
8+
9+
permissions: {}
10+
11+
jobs:
12+
post-comment:
13+
permissions:
14+
actions: read # to list workflow runs artifacts
15+
pull-requests: write # to comment on pr
16+
17+
runs-on: ubuntu-latest
18+
if: >
19+
github.event.workflow_run.event == 'pull_request'
20+
steps:
21+
- name: 'Download artifact'
22+
uses: actions/github-script@v9
23+
with:
24+
script: |
25+
// List all artifacts from GnuTests
26+
var artifacts = await github.rest.actions.listWorkflowRunArtifacts({
27+
owner: context.repo.owner,
28+
repo: context.repo.repo,
29+
run_id: ${{ github.event.workflow_run.id }},
30+
});
31+
32+
// Download the "comment" artifact, which contains a PR number (NR) and result.txt
33+
var matchArtifact = artifacts.data.artifacts.filter((artifact) => {
34+
return artifact.name == "comment"
35+
})[0];
36+
37+
if (!matchArtifact) {
38+
console.log('No comment artifact found');
39+
return;
40+
}
41+
42+
var download = await github.rest.actions.downloadArtifact({
43+
owner: context.repo.owner,
44+
repo: context.repo.repo,
45+
artifact_id: matchArtifact.id,
46+
archive_format: 'zip',
47+
});
48+
var fs = require('fs');
49+
fs.writeFileSync('${{ github.workspace }}/comment.zip', Buffer.from(download.data));
50+
51+
- run: unzip comment.zip || echo "Failed to unzip comment artifact"
52+
53+
- name: 'Comment on PR'
54+
uses: actions/github-script@v9
55+
with:
56+
github-token: ${{ secrets.GITHUB_TOKEN }}
57+
script: |
58+
var fs = require('fs');
59+
60+
// Check if files exist
61+
if (!fs.existsSync('./NR')) {
62+
console.log('No NR file found, skipping comment');
63+
return;
64+
}
65+
if (!fs.existsSync('./result.txt')) {
66+
console.log('No result.txt file found, skipping comment');
67+
return;
68+
}
69+
70+
var issue_number = Number(fs.readFileSync('./NR'));
71+
var content = fs.readFileSync('./result.txt');
72+
73+
if (content.toString().trim().length > 7) { // 7 because we have backquote + \n
74+
// Update existing comment if present, otherwise create a new one
75+
var marker = '<!-- gnu-tests-bot -->';
76+
var body = marker + '\nGNU diffutils testsuite comparison:\n```\n' + content + '```';
77+
var comments = await github.rest.issues.listComments({
78+
owner: context.repo.owner,
79+
repo: context.repo.repo,
80+
issue_number: issue_number,
81+
});
82+
var existing = comments.data.filter(c => c.body.includes(marker))[0];
83+
if (existing) {
84+
await github.rest.issues.updateComment({
85+
owner: context.repo.owner,
86+
repo: context.repo.repo,
87+
comment_id: existing.id,
88+
body: body,
89+
});
90+
} else {
91+
await github.rest.issues.createComment({
92+
owner: context.repo.owner,
93+
repo: context.repo.repo,
94+
issue_number: issue_number,
95+
body: body,
96+
});
97+
}
98+
} else {
99+
console.log('Comment content too short, skipping');
100+
}

‎.github/workflows/GnuTests.yml‎

Lines changed: 231 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,231 @@
1+
name: GnuTests
2+
3+
# Run GNU diffutils testsuite against the Rust diffutils implementation
4+
# and compare results against the main branch to catch regressions
5+
6+
on:
7+
pull_request:
8+
push:
9+
branches:
10+
- '*'
11+
12+
permissions:
13+
contents: write # Publish diffutils instead of discarding
14+
15+
# End the current execution if there is a new changeset in the PR
16+
concurrency:
17+
group: ${{ github.workflow }}-${{ github.ref }}
18+
cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}
19+
20+
env:
21+
DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}
22+
TEST_FULL_SUMMARY_FILE: 'diffutils-gnu-full-result.json'
23+
24+
jobs:
25+
native:
26+
name: Run GNU diffutils testsuite
27+
runs-on: ubuntu-24.04
28+
steps:
29+
- name: Checkout code
30+
uses: actions/checkout@v4
31+
with:
32+
persist-credentials: false
33+
34+
- uses: dtolnay/rust-toolchain@master
35+
with:
36+
toolchain: stable
37+
38+
- uses: Swatinem/rust-cache@v2
39+
40+
### Build
41+
- name: Build Rust diffutils binary
42+
shell: bash
43+
run: |
44+
## Build Rust diffutils binary
45+
cargo build --config=profile.release.strip=true --profile=release
46+
zstd -19 target/release/diffutils -o diffutils-x86_64-unknown-linux-gnu.zst
47+
48+
- name: Publish latest commit
49+
uses: softprops/action-gh-release@v3
50+
if: github.event_name == 'push' && github.ref == 'refs/heads/main'
51+
with:
52+
tag_name: latest-commit
53+
body: |
54+
commit: ${{ github.sha }}
55+
draft: false
56+
prerelease: true
57+
files: |
58+
diffutils-x86_64-unknown-linux-gnu.zst
59+
env:
60+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
61+
62+
### Run tests
63+
- name: Run GNU diffutils testsuite
64+
shell: bash
65+
run: |
66+
## Run GNU diffutils testsuite
67+
./tests/run-upstream-testsuite.sh release || true
68+
env:
69+
TERM: xterm
70+
71+
- name: Upload full json results
72+
uses: actions/upload-artifact@v4
73+
with:
74+
name: diffutils-gnu-full-result
75+
path: tests/test-results.json
76+
if-no-files-found: warn
77+
78+
aggregate:
79+
needs: [native]
80+
permissions:
81+
actions: read
82+
contents: read
83+
pull-requests: read
84+
name: Aggregate GNU test results
85+
runs-on: ubuntu-24.04
86+
steps:
87+
- name: Initialize workflow variables
88+
id: vars
89+
shell: bash
90+
run: |
91+
## VARs setup
92+
outputs() { step_id="${{ github.action }}"; for var in "$@" ; do echo steps.${step_id}.outputs.${var}="${!var}"; echo "${var}=${!var}" >> $GITHUB_OUTPUT; done; }
93+
94+
TEST_SUMMARY_FILE='diffutils-gnu-result.json'
95+
outputs TEST_SUMMARY_FILE
96+
97+
- name: Checkout code
98+
uses: actions/checkout@v4
99+
with:
100+
persist-credentials: false
101+
102+
- name: Retrieve reference artifacts
103+
uses: dawidd6/action-download-artifact@v20
104+
continue-on-error: true
105+
with:
106+
workflow: GnuTests.yml
107+
branch: "${{ env.DEFAULT_BRANCH }}"
108+
workflow_conclusion: completed
109+
path: "reference"
110+
if_no_artifact_found: warn
111+
112+
- name: Download full json results
113+
uses: actions/download-artifact@v4
114+
with:
115+
name: diffutils-gnu-full-result
116+
path: results
117+
118+
- name: Extract/summarize testing info
119+
id: summary
120+
shell: bash
121+
run: |
122+
## Extract/summarize testing info
123+
outputs() { step_id="${{ github.action }}"; for var in "$@" ; do echo steps.${step_id}.outputs.${var}="${!var}"; echo "${var}=${!var}" >> $GITHUB_OUTPUT; done; }
124+
125+
RESULT_FILE="results/test-results.json"
126+
if [[ ! -f "$RESULT_FILE" ]]; then
127+
echo "::error ::Missing test results at $RESULT_FILE"
128+
exit 1
129+
fi
130+
131+
TOTAL=$(jq '[.tests[]] | length' "$RESULT_FILE")
132+
PASS=$(jq '[.tests[] | select(.result=="PASS")] | length' "$RESULT_FILE")
133+
FAIL=$(jq '[.tests[] | select(.result=="FAIL")] | length' "$RESULT_FILE")
134+
SKIP=$(jq '[.tests[] | select(.result=="SKIP")] | length' "$RESULT_FILE")
135+
ERROR=0
136+
137+
output="GNU diffutils tests summary = TOTAL: $TOTAL / PASS: $PASS / FAIL: $FAIL / SKIP: $SKIP"
138+
echo "${output}"
139+
140+
if [[ "$FAIL" -gt 0 ]]; then
141+
echo "::warning ::${output}"
142+
fi
143+
144+
jq -n \
145+
--arg date "$(date --rfc-email)" \
146+
--arg sha "$GITHUB_SHA" \
147+
--arg total "$TOTAL" \
148+
--arg pass "$PASS" \
149+
--arg skip "$SKIP" \
150+
--arg fail "$FAIL" \
151+
--arg error "$ERROR" \
152+
'{($date): { sha: $sha, total: $total, pass: $pass, skip: $skip, fail: $fail, error: $error }}' > '${{ steps.vars.outputs.TEST_SUMMARY_FILE }}'
153+
154+
HASH=$(sha1sum '${{ steps.vars.outputs.TEST_SUMMARY_FILE }}' | cut --delim=" " -f 1)
155+
outputs HASH TOTAL PASS FAIL SKIP
156+
157+
- name: Upload SHA1/ID of 'test-summary'
158+
uses: actions/upload-artifact@v4
159+
with:
160+
name: "${{ steps.summary.outputs.HASH }}"
161+
path: "${{ steps.vars.outputs.TEST_SUMMARY_FILE }}"
162+
163+
- name: Upload test results summary
164+
uses: actions/upload-artifact@v4
165+
with:
166+
name: test-summary
167+
path: "${{ steps.vars.outputs.TEST_SUMMARY_FILE }}"
168+
169+
- name: Compare test failures VS reference
170+
shell: bash
171+
run: |
172+
## Compare test failures VS reference
173+
REF_SUMMARY_FILE='reference/diffutils-gnu-full-result/test-results.json'
174+
CURRENT_SUMMARY_FILE="results/test-results.json"
175+
176+
IGNORE_INTERMITTENT=".github/workflows/ignore-intermittent.txt"
177+
178+
COMMENT_DIR="reference/comment"
179+
mkdir -p ${COMMENT_DIR}
180+
echo ${{ github.event.number }} > ${COMMENT_DIR}/NR
181+
COMMENT_LOG="${COMMENT_DIR}/result.txt"
182+
183+
COMPARISON_RESULT=0
184+
if test -f "${CURRENT_SUMMARY_FILE}"; then
185+
if test -f "${REF_SUMMARY_FILE}"; then
186+
echo "Reference summary SHA1/ID: $(sha1sum -- "${REF_SUMMARY_FILE}")"
187+
echo "Current summary SHA1/ID: $(sha1sum -- "${CURRENT_SUMMARY_FILE}")"
188+
189+
python3 util/compare_test_results.py \
190+
--ignore-file "${IGNORE_INTERMITTENT}" \
191+
--output "${COMMENT_LOG}" \
192+
"${CURRENT_SUMMARY_FILE}" "${REF_SUMMARY_FILE}"
193+
194+
COMPARISON_RESULT=$?
195+
else
196+
echo "::warning ::Skipping test comparison; no prior reference summary is available at '${REF_SUMMARY_FILE}'."
197+
fi
198+
else
199+
echo "::error ::Failed to find summary of test results (missing '${CURRENT_SUMMARY_FILE}'); failing early"
200+
exit 1
201+
fi
202+
203+
if [ ${COMPARISON_RESULT} -eq 1 ]; then
204+
echo "::error ::Found new non-intermittent test failures"
205+
exit 1
206+
else
207+
echo "::notice ::No new test failures detected"
208+
fi
209+
210+
- name: Upload comparison log (for GnuComment workflow)
211+
if: success() || failure()
212+
uses: actions/upload-artifact@v4
213+
with:
214+
name: comment
215+
path: reference/comment/
216+
217+
- name: Report test results
218+
if: success() || failure()
219+
shell: bash
220+
run: |
221+
## Report final results
222+
echo "::notice ::GNU diffutils testsuite results:"
223+
echo "::notice :: Total tests: ${{ steps.summary.outputs.TOTAL }}"
224+
echo "::notice :: Passed: ${{ steps.summary.outputs.PASS }}"
225+
echo "::notice :: Failed: ${{ steps.summary.outputs.FAIL }}"
226+
echo "::notice :: Skipped: ${{ steps.summary.outputs.SKIP }}"
227+
228+
if [[ "${{ steps.summary.outputs.FAIL }}" -gt 0 ]]; then
229+
PASS_RATE=$(( ${{ steps.summary.outputs.PASS }} * 100 / (${{ steps.summary.outputs.PASS }} + ${{ steps.summary.outputs.FAIL }}) ))
230+
echo "::notice :: Pass rate: ${PASS_RATE}%"
231+
fi

‎.github/workflows/ci.yml‎

Lines changed: 0 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -56,37 +56,6 @@ jobs:
5656
- uses: actions/checkout@v4
5757
- run: cargo clippy -- -D warnings
5858

59-
gnu-testsuite:
60-
permissions:
61-
contents: write # Publish diffutils instead of discarding
62-
name: GNU test suite
63-
runs-on: ubuntu-latest
64-
steps:
65-
- uses: actions/checkout@v4
66-
- run: |
67-
cargo build --config=profile.release.strip=true --profile=release #-fast
68-
zstd -19 target/release/diffutils -o diffutils-x86_64-unknown-linux-gnu.zst
69-
# do not fail, the report is merely informative (at least until all tests pass reliably)
70-
- run: ./tests/run-upstream-testsuite.sh release || true
71-
env:
72-
TERM: xterm
73-
- uses: actions/upload-artifact@v4
74-
with:
75-
name: test-results.json
76-
path: tests/test-results.json
77-
- run: ./tests/print-test-results.sh tests/test-results.json
78-
- name: Publish latest commit
79-
uses: softprops/action-gh-release@v3
80-
if: github.event_name == 'push' && github.ref == 'refs/heads/main'
81-
with:
82-
tag_name: latest-commit
83-
draft: false
84-
prerelease: true
85-
files: |
86-
diffutils-x86_64-unknown-linux-gnu.zst
87-
env:
88-
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
89-
9059
coverage:
9160
name: Code Coverage
9261
env:

0 commit comments

Comments
 (0)