Skip to content

Commit 31c44b2

Browse files
lewismclewismc
authored
NUTCH-3163 Integrate Apache Yetus' pre-commit patch testing into Nutch GitHub Continuous Integration (#907)
1 parent e47cfd5 commit 31c44b2

10 files changed

Lines changed: 292 additions & 47 deletions

File tree

‎.github/workflows/junit-report.yml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,11 @@ on:
1818
workflow_run:
1919
workflows: [master pull request ci]
2020
types: [completed]
21+
22+
concurrency:
23+
group: junit-report-${{ github.event.workflow_run.pull_requests[0].number || github.event.workflow_run.head_sha }}
24+
cancel-in-progress: true
25+
2126
permissions:
2227
checks: write
2328
contents: read

‎.github/workflows/master-build.yml‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,10 @@ on:
2121
types: [opened, synchronize, reopened]
2222
branches: [master]
2323

24+
concurrency:
25+
group: master-build-${{ github.ref }}
26+
cancel-in-progress: true
27+
2428
# Java Version Strategy:
2529
# - BUILD: Requires Java 17+ (JUnit 6 dependency)
2630
# - RUNTIME: Supports Java 11+ (javac.version=11 produces Java 11 bytecode)
@@ -75,7 +79,7 @@ jobs:
7579
restore-keys: |
7680
${{ runner.os }}-ivy-
7781
- name: Run Apache Rat
78-
run: ant clean run-rat -buildfile build.xml
82+
run: ant clean releaseaudit -buildfile build.xml
7983
- name: Cache unknown licenses
8084
run: echo "UNKNOWN_LICENSES=$(sed -n 18p /home/runner/work/nutch/nutch/build/apache-rat-report.txt)" >> $GITHUB_ENV
8185
- name: Versions
@@ -140,7 +144,7 @@ jobs:
140144
# Java 11 = major version 55, Java 17 = major version 61
141145
EXPECTED_VERSION=${{ matrix.javac-version == '11' && '55' || '61' }}
142146
echo "Expected major version: $EXPECTED_VERSION (Java ${{ matrix.javac-version }})"
143-
147+
144148
# Find a real class file (exclude package-info.class which may have different version)
145149
cd build/classes
146150
CLASS_FILE=$(find . -name "*.class" ! -name "package-info.class" | head -1)

‎.github/workflows/sonarcloud.yml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,11 @@ on:
1818
workflow_run:
1919
workflows: [master pull request ci]
2020
types: [completed]
21+
22+
concurrency:
23+
group: sonarcloud-${{ github.event.workflow_run.pull_requests[0].number || github.event.workflow_run.head_sha }}
24+
cancel-in-progress: true
25+
2126
jobs:
2227
analysis:
2328
if: github.event.workflow_run.conclusion == 'success'

‎.github/workflows/yetus.yml‎

Lines changed: 116 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,116 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one or more
2+
# contributor license agreements. See the NOTICE file distributed with
3+
# this work for additional information regarding copyright ownership.
4+
# The ASF licenses this file to You under the Apache License, Version 2.0
5+
# (the "License"); you may not use this file except in compliance with
6+
# the License. You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
#
16+
# Apache Yetus test-patch: pre-commit patch testing (Ant, JDK 17).
17+
# Runs alongside master-build.yml; all CI is unified on Java 17.
18+
# See https://yetus.apache.org/documentation/0.15.1/precommit/
19+
---
20+
name: Apache Yetus
21+
on:
22+
push:
23+
branches: [master]
24+
pull_request:
25+
types: [opened, synchronize, reopened]
26+
branches: [master]
27+
28+
concurrency:
29+
group: yetus-${{ github.ref }}
30+
cancel-in-progress: true
31+
32+
# GITHUB_TOKEN cannot comment on PRs from forks (403). Use a PAT secret
33+
# (e.g. YETUS_COMMENT_TOKEN) if you need comments on fork PRs.
34+
permissions:
35+
contents: read
36+
statuses: write
37+
pull-requests: write
38+
issues: write
39+
40+
jobs:
41+
yetus:
42+
runs-on: ubuntu-latest
43+
timeout-minutes: 45
44+
env:
45+
PATCH_DIR: ${{ github.workspace }}/out
46+
JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
47+
steps:
48+
- uses: actions/checkout@v5
49+
with:
50+
fetch-depth: 0
51+
- name: Cache Ivy dependencies
52+
uses: actions/cache@v4
53+
with:
54+
path: ~/.ivy2/cache
55+
key: ${{ runner.os }}-ivy-${{ hashFiles('ivy/ivy.xml', 'src/plugin/**/ivy.xml') }}
56+
restore-keys: |
57+
${{ runner.os }}-ivy-
58+
- name: Apache Yetus test-patch
59+
uses: apache/yetus-test-patch-action@0.15.1
60+
with:
61+
basedir: .
62+
buildtool: nobuild
63+
githubtoken: ${{ secrets.GITHUB_TOKEN }}
64+
javahome: '/usr/lib/jvm/java-17-openjdk-amd64'
65+
patchdir: /github/workspace/out
66+
plugins: all,-jira,-gitlab,-unit,-compile
67+
project: nutch
68+
- name: Artifact output
69+
if: always()
70+
uses: actions/upload-artifact@v4
71+
with:
72+
name: apacheyetuspatchdir
73+
path: ${{ env.PATCH_DIR }}
74+
- name: Install pandoc
75+
if: github.event_name == 'pull_request'
76+
run: sudo apt-get update && sudo apt-get install -y pandoc
77+
- name: Convert HTML report to Markdown
78+
if: github.event_name == 'pull_request'
79+
run: |
80+
OUT="${{ env.PATCH_DIR }}"
81+
echo "## Apache Yetus test-patch report" > yetus-report.md
82+
echo "" >> yetus-report.md
83+
if [ -f "$OUT/report.html" ]; then
84+
pandoc "$OUT/report.html" -f html -t gfm >> yetus-report.md 2>/dev/null || {
85+
echo "Pandoc conversion failed; using brief report." >> yetus-report.md
86+
echo '```' >> yetus-report.md
87+
cat "$OUT/brief.txt" >> yetus-report.md 2>/dev/null || true
88+
echo '```' >> yetus-report.md
89+
}
90+
elif [ -f "$OUT/brief.txt" ]; then
91+
echo '```' >> yetus-report.md
92+
cat "$OUT/brief.txt" >> yetus-report.md
93+
echo '```' >> yetus-report.md
94+
else
95+
echo "No Yetus report or brief found." >> yetus-report.md
96+
fi
97+
- name: Truncate if over comment limit
98+
if: github.event_name == 'pull_request'
99+
run: |
100+
MAX=60000
101+
if [ $(wc -c < yetus-report.md) -gt $MAX ]; then
102+
head -c $MAX yetus-report.md > yetus-report-trimmed.md
103+
echo "" >> yetus-report-trimmed.md
104+
echo "" >> yetus-report-trimmed.md
105+
echo "_Report truncated (GitHub comment limit). Full HTML in apacheyetuspatchdir artifact as report.html._" >> yetus-report-trimmed.md
106+
mv yetus-report-trimmed.md yetus-report.md
107+
fi
108+
- name: Comment PR with Yetus report
109+
if: github.event_name == 'pull_request'
110+
uses: peter-evans/create-or-update-comment@e8674b075228eee787fea43ef493e45ece1004c9
111+
continue-on-error: true
112+
with:
113+
token: ${{ secrets.YETUS_COMMENT_TOKEN || secrets.GITHUB_TOKEN }}
114+
repository: ${{ github.repository }}
115+
issue-number: ${{ github.event.pull_request.number }}
116+
body-path: yetus-report.md

‎.markdownlint.yaml‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
# Relaxations for Yetus markdownlint (README and docs)
2+
---
3+
MD001: false
4+
MD003: false
5+
MD012: false
6+
MD013:
7+
line_length: 200
8+
MD022: false
9+
MD025: false
10+
MD033: false
11+
MD034: false
12+
MD045: false
13+
MD046: fenced

‎.yamllint.yml‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
# Relax rules for workflow and config YAML (Yetus yamllint plugin)
2+
---
3+
extends: default
4+
5+
rules:
6+
line-length:
7+
max: 200
8+
document-start: disable
9+
truthy:
10+
allowed-values: ['true', 'false', 'yes', 'no', 'on', 'off']

‎.yetus/blanks-eol.txt‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
# Ignore trailing blanks in Yetus-generated patch/diff and logs (not source files).
2+
# See --blanks-eol-ignore-file in the blanks plugin.
3+
^out/

‎.yetus/personality.sh‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one or more
2+
# contributor license agreements. See the NOTICE file distributed with
3+
# this work for additional information regarding copyright ownership.
4+
# The ASF licenses this file to You under the Apache License, Version 2.0
5+
# (the "License"); you may not use this file except in compliance with
6+
# the License. You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
16+
# Ensure JAVA_HOME is set for pre-patch and other phases when running in
17+
# the Yetus Docker container (avoids "JAVA_HOME is not defined" in pre-patch).
18+
if [ -z "${JAVA_HOME}" ] && [ -d "/usr/lib/jvm/java-17-openjdk-amd64" ]; then
19+
export JAVA_HOME="/usr/lib/jvm/java-17-openjdk-amd64"
20+
fi
21+
22+
# Pass JAVA_HOME into the re-exec Docker container so pre-patch and other
23+
# phases see it (YETUS-913; otherwise the inner container may not get it).
24+
## @audience private
25+
## @stability stable
26+
function docker_do_env_adds
27+
{
28+
declare k
29+
# Use JAVA_HOME so detsecrets does not flag the literal path as high-entropy
30+
DOCKER_EXTRAARGS+=("--env=JAVA_HOME=${JAVA_HOME}")
31+
for k in "${DOCKER_EXTRAENVS[@]}"; do
32+
[[ -z "${k}" ]] && continue
33+
if [[ "JAVA_HOME" != "${k}" ]]; then
34+
DOCKER_EXTRAARGS+=("--env=${k}=${!k}")
35+
fi
36+
done
37+
}

0 commit comments

Comments
 (0)