NUTCH-3167 Upgrade to Hadoop 3.5.0 (#911)

Author: lewismc

.github/workflows/junit-report.yml

@@ -24,21 +24,44 @@ concurrency:
   cancel-in-progress: true
 
 permissions:
+  actions: read
   checks: write
   contents: read
   issues: write
   pull-requests: write
 jobs:
   checks:
+    if: github.event.workflow_run.conclusion == 'success'
     runs-on: ubuntu-latest
     steps:
-      - name: Download Test Report (Ubuntu)
+      - name: Set up JDK 17
+        uses: actions/setup-java@v5
+        with:
+          java-version: '17'
+          distribution: 'temurin'
+      - name: Download Test Report (Ubuntu JDK 17)
         uses: dawidd6/action-download-artifact@v11
         with:
-          name: junit-test-results-ubuntu-latest
+          name: junit-test-results-ubuntu-latest-jdk17
           workflow: master-build.yml
           run_id: ${{ github.event.workflow_run.id }}
-        continue-on-error: true
+          path: ./junit-ubuntu-jdk17
+      - name: Verify JUnit XML layout
+        run: |
+          set -euo pipefail
+          shopt -s globstar nullglob
+          root="./junit-ubuntu-jdk17"
+          if [ ! -d "$root" ]; then
+            echo "::error::Download path $root is missing."
+            exit 1
+          fi
+          files=("$root"/build/**/TEST-*.xml)
+          if [ ${#files[@]} -eq 0 ] || [ ! -e "${files[0]}" ]; then
+            echo "::error::No TEST-*.xml under $root/build/ (artifact missing, wrong layout, or download failed)."
+            find "$root" -maxdepth 5 -type d -print 2>/dev/null | head -80 || true
+            exit 1
+          fi
+          echo "Found ${#files[@]} JUnit report file(s) under $root/build/."
       - name: Resolve PR number
         id: pr
         run: |
@@ -59,12 +82,8 @@ jobs:
       - name: Publish Test Report
         uses: mikepenz/action-junit-report@v6
         with:
-          report_paths: |-
-            ./test/TEST-*.xml
-            ./*/**/test/TEST-*.xml
-          check_name: |-
-            JUnit Test Report
-            JUnit Test Report Plugins
+          report_paths: ./junit-ubuntu-jdk17/build/**/TEST-*.xml
+          check_name: JUnit Test Report
           commit: ${{ github.event.workflow_run.head_sha }}
           fail_on_failure: false
           fail_on_parse_error: true

.github/workflows/master-build.yml

@@ -26,19 +26,15 @@ concurrency:
   cancel-in-progress: true
 
 # Java Version Strategy:
-# - BUILD: Requires Java 17+ (JUnit 6 dependency)
-# - RUNTIME: Supports Java 11+ (javac.version=11 produces Java 11 bytecode)
-#
-# The 'build' job verifies bytecode compilation for both Java 11 and 17 targets.
-# The 'runtime-java11' job verifies the built artifacts actually run on Java 11.
-# The 'tests' job runs on JDK 17 (required by JUnit 6) with the default
-# javac.version=11 bytecode target for backward compatibility.
+# - Requires Java 17+ to build, test, and run (Hadoop 3.5+ client + JUnit 6).
+# - Default bytecode: javac.version=17 (see default.properties).
+# - CI exercises Eclipse Temurin JDK 17 and JDK 21 on Ubuntu (and tests on macOS).
 
 jobs:
   javadoc:
     strategy:
       matrix:
-        java: ['17']
+        java: ['17', '21']
         os: [ubuntu-latest]
     runs-on: ${{ matrix.os }}
     steps:
@@ -61,7 +57,7 @@ jobs:
   rat:
     strategy:
       matrix:
-        java: ['17']
+        java: ['17', '21']
         os: [ubuntu-latest]
     runs-on: ${{ matrix.os }}
     steps:
@@ -112,22 +108,21 @@ jobs:
         if: steps.filter.outputs.openapi == 'true'
         run: ./node_modules/.bin/lint-openapi openapi.yaml
 
-  # Build verification with Java bytecode target matrix
-  # Verifies bytecode compatibility for both Java 11 and Java 17 targets
+  # Build verification on JDK 17 and 21 (bytecode target Java 17)
   build:
     strategy:
       fail-fast: false
       matrix:
-        javac-version: ['11', '17']
+        java: ['17', '21']
         os: [ubuntu-latest]
     runs-on: ${{ matrix.os }}
-    name: build (javac.version=${{ matrix.javac-version }})
+    name: build (jdk ${{ matrix.java }}, javac.version=17)
     steps:
       - uses: actions/checkout@v5
-      - name: Set up JDK 17
+      - name: Set up JDK ${{ matrix.java }}
         uses: actions/setup-java@v5
         with:
-          java-version: '17'
+          java-version: ${{ matrix.java }}
           distribution: 'temurin'
       - name: Cache Ivy dependencies
         uses: actions/cache@v4
@@ -136,14 +131,13 @@ jobs:
           key: ${{ runner.os }}-ivy-${{ hashFiles('ivy/ivy.xml', 'src/plugin/**/ivy.xml') }}
           restore-keys: |
             ${{ runner.os }}-ivy-
-      - name: Build with javac.version=${{ matrix.javac-version }}
-        run: ant clean runtime -Djavac.version=${{ matrix.javac-version }} -buildfile build.xml
+      - name: Build with javac.version=17
+        run: ant clean runtime -Djavac.version=17 -buildfile build.xml
       - name: Verify bytecode version
         run: |
-          # Extract and verify the bytecode version of compiled classes
-          # Java 11 = major version 55, Java 17 = major version 61
-          EXPECTED_VERSION=${{ matrix.javac-version == '11' && '55' || '61' }}
-          echo "Expected major version: $EXPECTED_VERSION (Java ${{ matrix.javac-version }})"
+          # Java 17 = major version 61
+          EXPECTED_VERSION=61
+          echo "Expected major version: $EXPECTED_VERSION (Java 17 bytecode)"
 
           # Find a real class file (exclude package-info.class which may have different version)
           cd build/classes
@@ -162,17 +156,21 @@ jobs:
             exit 1
           fi
 
-  # Verify runtime compatibility on Java 11
-  # This ensures the built artifacts can actually run on Java 11
-  runtime-java11:
+  # Smoke-test runtime on the same JDK used to build (17 and 21)
+  runtime-smoke:
     needs: build
+    strategy:
+      fail-fast: false
+      matrix:
+        java: ['17', '21']
     runs-on: ubuntu-latest
+    name: runtime-smoke (jdk ${{ matrix.java }})
     steps:
       - uses: actions/checkout@v5
-      - name: Set up JDK 17 for building
+      - name: Set up JDK ${{ matrix.java }}
         uses: actions/setup-java@v5
         with:
-          java-version: '17'
+          java-version: ${{ matrix.java }}
           distribution: 'temurin'
       - name: Cache Ivy dependencies
         uses: actions/cache@v4
@@ -181,38 +179,31 @@ jobs:
           key: ${{ runner.os }}-ivy-${{ hashFiles('ivy/ivy.xml', 'src/plugin/**/ivy.xml') }}
           restore-keys: |
             ${{ runner.os }}-ivy-
-      - name: Build with Java 11 target
-        run: ant clean runtime -Djavac.version=11 -buildfile build.xml
-      - name: Set up JDK 11 for runtime verification
-        uses: actions/setup-java@v5
-        with:
-          java-version: '11'
-          distribution: 'temurin'
-      - name: Verify runtime on Java 11
+      - name: Build with javac.version=17
+        run: ant clean runtime -Djavac.version=17 -buildfile build.xml
+      - name: Verify runtime on JDK ${{ matrix.java }}
         run: |
-          echo "Verifying Nutch can run on Java 11..."
+          echo "Verifying Nutch on JDK ${{ matrix.java }}..."
           java -version
           cd runtime/local
-          # Actually load Java classes by running showproperties
-          # This invokes org.apache.nutch.tools.ShowProperties and verifies the JAR loads
           bin/nutch showproperties | head -20
-          echo "Java 11 runtime verification complete"
+          echo "Runtime smoke test complete"
 
-  # Tests run on JDK 17 (required by JUnit 6) with default javac.version=11
-  # Java 11 runtime compatibility is verified by the runtime-java11 job
   tests:
     strategy:
       fail-fast: false
       matrix:
+        java: ['17', '21']
         os: [ubuntu-latest, macos-latest]
     runs-on: ${{ matrix.os }}
+    name: tests (jdk ${{ matrix.java }}, ${{ matrix.os }})
     timeout-minutes: 45
     steps:
       - uses: actions/checkout@v5
-      - name: Set up JDK 17
+      - name: Set up JDK ${{ matrix.java }}
         uses: actions/setup-java@v5
         with:
-          java-version: '17'
+          java-version: ${{ matrix.java }}
           distribution: 'temurin'
       - name: Cache Ivy dependencies
         uses: actions/cache@v4
@@ -266,9 +257,9 @@ jobs:
           fi
       - name: Upload Test Report
         uses: actions/upload-artifact@v4
-        if: always() && matrix.os == 'ubuntu-latest' && steps.check_tests.outputs.has_results == 'true'
+        if: always() && matrix.os == 'ubuntu-latest' && matrix.java == '17' && steps.check_tests.outputs.has_results == 'true'
         with:
-          name: junit-test-results-${{ matrix.os }}
+          name: junit-test-results-${{ matrix.os }}-jdk${{ matrix.java }}
           path: |
             ./build/test/TEST-*.xml
             ./build/**/test/TEST-*.xml
@@ -277,7 +268,7 @@ jobs:
         uses: actions/upload-artifact@v4
         if: always() && matrix.os == 'ubuntu-latest'
         with:
-          name: coverage-data
+          name: coverage-data-ubuntu-jdk${{ matrix.java }}
           path: ./build/coverage/*.exec
           retention-days: 1
           if-no-files-found: ignore

.github/workflows/sonarcloud.yml

@@ -52,26 +52,27 @@ jobs:
             ${{ runner.os }}-ivy-
       - name: Compile (no tests)
         run: ant compile compile-plugins resolve-test -buildfile build.xml
-      - name: Download coverage data
+      # Coverage and JUnit XML come only from the master-build Ubuntu JDK 17 matrix job.
+      - name: Download coverage data (Ubuntu JDK 17)
         uses: dawidd6/action-download-artifact@v11
         with:
-          name: coverage-data
+          name: coverage-data-ubuntu-jdk17
           workflow: master-build.yml
           run_id: ${{ github.event.workflow_run.id }}
           path: ./build/coverage/
         continue-on-error: true
-      - name: Download test reports
+      - name: Download test reports (Ubuntu JDK 17)
         uses: dawidd6/action-download-artifact@v11
         with:
-          name: junit-test-results-ubuntu-latest
+          name: junit-test-results-ubuntu-latest-jdk17
           workflow: master-build.yml
           run_id: ${{ github.event.workflow_run.id }}
-          path: ./build/test/
+          path: ./build/test-jdk17/
         continue-on-error: true
-      - name: Flatten test reports
+      - name: Flatten test reports (JDK 17 only)
         run: |
           mkdir -p ./build/test-reports
-          find ./build/test -name 'TEST-*.xml' -exec cp {} ./build/test-reports/ \;
+          find ./build/test-jdk17 -name 'TEST-*.xml' -exec cp {} ./build/test-reports/ \; 2>/dev/null || true
         continue-on-error: true
       - name: Generate JaCoCo XML report
         run: ant jacoco-report -buildfile build.xml
@@ -110,7 +111,7 @@ jobs:
           GH_TOKEN: ${{ github.token }}
       - name: SonarCloud Scan (PR)
         if: steps.pr.outputs.is_pr == 'true'
-        uses: SonarSource/sonarqube-scan-action@v6
+        uses: SonarSource/sonarqube-scan-action@299e4b793aaa83bf2aba7c9c14bedbb485688ec4
         with:
           args: >
             -Dsonar.pullrequest.key=${{ steps.pr.outputs.number }}
@@ -121,7 +122,7 @@ jobs:
           SONAR_HOST_URL: https://sonarcloud.io
       - name: SonarCloud Scan (branch)
         if: steps.pr.outputs.is_pr == 'false' && steps.pr.outputs.skip != 'true'
-        uses: SonarSource/sonarqube-scan-action@v6
+        uses: SonarSource/sonarqube-scan-action@299e4b793aaa83bf2aba7c9c14bedbb485688ec4
         with:
           args: >
             -Dsonar.branch.name=${{ github.event.workflow_run.head_branch }}

.github/workflows/yetus.yml

@@ -13,8 +13,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-# Apache Yetus test-patch: pre-commit patch testing (Ant, JDK 17).
-# Runs alongside master-build.yml; all CI is unified on Java 17.
+# Apache Yetus test-patch: pre-commit patch testing (apache/yetus-test-patch-action).
+# The action runs inside ghcr.io/apache/yetus:0.15.1 — javahome must be a JDK path
+# inside that image (OpenJDK 11 on amd64), not actions/setup-java on the runner.
 # See https://yetus.apache.org/documentation/0.15.1/precommit/
 ---
 name: Apache Yetus
@@ -43,7 +44,6 @@ jobs:
     timeout-minutes: 45
     env:
       PATCH_DIR: ${{ github.workspace }}/out
-      JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64
     steps:
       - uses: actions/checkout@v5
         with:
@@ -61,7 +61,7 @@ jobs:
           basedir: .
           buildtool: nobuild
           githubtoken: ${{ secrets.GITHUB_TOKEN }}
-          javahome: '/usr/lib/jvm/java-17-openjdk-amd64'
+          javahome: /usr/lib/jvm/java-11-openjdk-amd64
           patchdir: /github/workspace/out
           plugins: all,-jira,-gitlab,-unit,-compile
           project: nutch

.yetus/blanks-tabs.txt

@@ -0,0 +1,3 @@
+# Ignore tabs in Yetus-generated patch dir (not source files).
+# See --blanks-tabs-ignore-file in the blanks plugin.
+^out/

.yetus/personality.sh

@@ -13,10 +13,13 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# Ensure JAVA_HOME is set for pre-patch and other phases when running in
-# the Yetus Docker container (avoids "JAVA_HOME is not defined" in pre-patch).
-if [ -z "${JAVA_HOME}" ] && [ -d "/usr/lib/jvm/java-17-openjdk-amd64" ]; then
-  export JAVA_HOME="/usr/lib/jvm/java-17-openjdk-amd64"
+# Ensure JAVA_HOME is set for pre-patch and other phases. The Yetus GitHub Action
+# runs inside ghcr.io/apache/yetus (see yetus.yml javahome). This fallback matches
+# OpenJDK layouts on Debian/Ubuntu when JAVA_HOME is not already set.
+if [ -z "${JAVA_HOME}" ] && [ -d "/usr/lib/jvm/java-11-openjdk-amd64" ]; then
+  export JAVA_HOME="/usr/lib/jvm/java-11-openjdk-amd64"
+elif [ -z "${JAVA_HOME}" ] && [ -d "/usr/lib/jvm/java-11-openjdk-arm64" ]; then
+  export JAVA_HOME="/usr/lib/jvm/java-11-openjdk-arm64"
 fi
 
 # Pass JAVA_HOME into the re-exec Docker container so pre-patch and other

README.md

@@ -105,9 +105,9 @@ radio button and select "Eclipse". Click "Create". On the next screen the
 "Eclipse projects directory" should be already set to the nutch folder. Leave
 the "Create module files near .classpath files" radio button selected.
 
-Click "Next" on the next screens. On the project SDK screen select Java 11 and
-click "Create". **N.B.** On Mac with homebrew openjdk, use the directory under
-_libexec_: `<openjdk11_directory>/libexec/openjdk.jdk/Contents/Home`.
+Click "Next" on the next screens. On the project SDK screen select Java 17
+(or newer) and click "Create". **N.B.** On Mac with homebrew openjdk, use the
+directory under _libexec_: `<openjdk_directory>/libexec/openjdk.jdk/Contents/Home`.
 
 Once the project is imported, you will see a popup saying "Ant build scripts
 found", "Frameworks detected - IvyIDEA Framework detected". Click "Import". If

build.xml

@@ -56,17 +56,12 @@
   <property name="ant-eclipse.jar" value="${ivy.dir}/lib/ant-eclipse-1.0-jvm1.2.jar" />
 
   <!--
-    Java Version Strategy (see HADOOP-18887 for similar approach):
+    Java Version Strategy:
 
-    BUILD REQUIREMENT: Java 17+ is required to build Nutch and run unit tests
-    because JUnit 6 (Jupiter) requires Java 17+.
-
-    RUNTIME COMPATIBILITY: The compiled bytecode targets Java 11 by default
-    (javac.version=11 in default.properties), allowing the binary package
-    to run on Java 11+ environments. This is important for Hadoop clusters
-    that may not yet support Java 17 runtime.
-
-    To build with Java 17 bytecode target: ant -Djavac.version=17 ...
+    Java 17+ is required to build, run unit tests, and run Nutch with the
+    Ivy-resolved Hadoop 3.5+ client libraries (Java 17 bytecode).
+    Default bytecode level is javac.version=17 in default.properties.
+    Use an older Nutch release line if you must run on Java 11.
   -->
 
   <!-- the normal classpath -->