Merge remote-tracking branch 'upstream/master' into add-bookkeeper-deleted-in-ml-info

Author: druidliu

.asf.yaml

@@ -37,7 +37,7 @@ github:
   enabled_merge_buttons:
     squash:  true
     merge:   false
-    rebase:  true
+    rebase:  false
   protected_branches:
     master:
       required_status_checks:
@@ -80,6 +80,7 @@ github:
     branch-2.10: {}
     branch-2.11: {}
     branch-3.0: {}
+    branch-3.1: {}
 
 notifications:
   commits:      commits@pulsar.apache.org

.github/ISSUE_TEMPLATE/pip.md

@@ -31,7 +31,7 @@ runs:
           directories=(/usr/local/lib/android /opt/ghc)
           if [[ "${{ inputs.mode }}" == "full" ]]; then
             # remove these directories only when mode is 'full'
-            directories+=(/usr/share/dotnet)
+            directories+=(/usr/share/dotnet /opt/hostedtoolcache/CodeQL)
           fi
           emptydir=/tmp/empty$$/
           mkdir $emptydir

.github/actions/clean-disk/action.yml

@@ -40,6 +40,7 @@ jobs:
       matrix:
         include:
           - branch: master
+          - branch: branch-3.1
           - branch: branch-3.0
           - branch: branch-2.11
           - branch: branch-2.10

.github/workflows/ci-owasp-dependency-check.yaml

@@ -85,6 +85,7 @@ jobs:
             txn
             ws
             zk
+            pip
           # The pull request's title should be fulfilled the following pattern:
           #
           #     [<type>][<optional scope>] <description>

.github/workflows/ci-semantic-pull-request.yml

@@ -23,20 +23,47 @@ on:
     branches:
       - master
   schedule:
+    # scheduled job with JDK 17
     - cron: '0 12 * * *'
+    # scheduled job with JDK 21
+    # if cron expression is changed, make sure to update the expression in jdk_major_version step in preconditions job
+    - cron: '0 6 * * *'
   workflow_dispatch:
     inputs:
       collect_coverage:
         description: 'Collect test coverage and upload to Codecov'
         required: true
-        default: 'true'
+        type: boolean
+        default: true
+      jdk_major_version:
+        description: 'JDK major version to use for the build'
+        required: true
+        type: choice
+        options:
+          - '17'
+          - '21'
+        default: '17'
+      trace_test_resource_cleanup:
+        description: 'Collect thread & heap information before exiting a test JVM. When set to "on", thread dump and heap histogram will be collected. When set to "full", a heap dump will also be collected.'
+        required: true
+        type: choice
+        options:
+          - 'off'
+          - 'on'
+          - 'full'
+        default: 'off'
+      thread_leak_detector_wait_millis:
+        description: 'Duration in ms to wait for threads to exit in thread leak detection between test classes. It is necessary to wait for threads to complete before they are determined to be leaked threads.'
+        required: true
+        type: number
+        default: 10000
 
 concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
+  group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }}${{ github.event_name == 'workflow_dispatch' && github.event.inputs.jdk_major_version || '' }}
   cancel-in-progress: true
 
 env:
-  MAVEN_OPTS: -Xss1500k -Xmx1024m -Daether.connector.http.reuseConnections=false -Daether.connector.requestTimeout=60000 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true -Dmaven.wagon.http.serviceUnavailableRetryStrategy.class=standard -Dmaven.wagon.rto=60000
+  MAVEN_OPTS: -Xss1500k -Xmx1500m -Daether.connector.http.reuseConnections=false -Daether.connector.requestTimeout=60000 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true -Dmaven.wagon.http.serviceUnavailableRetryStrategy.class=standard -Dmaven.wagon.rto=60000
   # defines the retention period for the intermediate build artifacts needed for rerunning a failed build job
   # it's possible to rerun individual failed jobs when the build artifacts are available
   # if the artifacts have already been expired, the complete workflow can be rerun by closing and reopening the PR or by rebasing the PR
@@ -46,23 +73,47 @@ jobs:
   preconditions:
     name: Preconditions
     runs-on: ubuntu-22.04
-    if: (github.event_name != 'schedule') || (github.repository == 'apache/pulsar')
     outputs:
       docs_only: ${{ steps.check_changes.outputs.docs_only }}
       changed_tests: ${{ steps.changes.outputs.tests_files }}
+      need_owasp: ${{ steps.changes.outputs.need_owasp }}
       collect_coverage: ${{ steps.check_coverage.outputs.collect_coverage }}
+      jdk_major_version: ${{ steps.jdk_major_version.outputs.jdk_major_version }}
+
     steps:
+      - name: Cancel scheduled jobs in forks by default
+        if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'schedule' }}
+        uses: actions/github-script@v6
+        with:
+          script: |
+            await github.rest.actions.cancelWorkflowRun({owner: context.repo.owner, repo: context.repo.repo, run_id: context.runId});
+            process.exit(1);
+
+      - name: Select JDK major version
+        id: jdk_major_version
+        run: |
+          # use JDK 21 for the scheduled build with cron expression '0 6 * * *'
+          if [[ "${{ github.event_name == 'schedule' && github.event.schedule == '0 6 * * *' && 'true' || 'false' }}" == "true" ]]; then
+            echo "jdk_major_version=21" >> $GITHUB_OUTPUT
+            exit 0
+          fi
+          # use JDK 17 for build unless overridden with workflow_dispatch input
+          echo "jdk_major_version=${{ github.event_name == 'workflow_dispatch' && github.event.inputs.jdk_major_version || '17'}}" >> $GITHUB_OUTPUT
+
       - name: checkout
+        if: ${{ github.event_name == 'pull_request' }}
         uses: actions/checkout@v3
 
       - name: Detect changed files
+        if: ${{ github.event_name == 'pull_request' }}
         id:   changes
         uses: apache/pulsar-test-infra/paths-filter@master
         with:
           filters: .github/changes-filter.yaml
           list-files: csv
 
       - name: Check changed files
+        if: ${{ github.event_name == 'pull_request' }}
         id: check_changes
         run: |
           if [[ "${GITHUB_EVENT_NAME}" != "schedule" && "${GITHUB_EVENT_NAME}" != "workflow_dispatch" ]]; then
@@ -95,6 +146,11 @@ jobs:
       JOB_NAME: Flaky tests suite
       COLLECT_COVERAGE: "${{ needs.preconditions.outputs.collect_coverage }}"
       GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
+      CI_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }}
+      TRACE_TEST_RESOURCE_CLEANUP: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.trace_test_resource_cleanup || 'off' }}
+      TRACE_TEST_RESOURCE_CLEANUP_DIR: ${{ github.workspace }}/target/trace-test-resource-cleanup
+      THREAD_LEAK_DETECTOR_WAIT_MILLIS: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.thread_leak_detector_wait_millis || 10000 }}
+      THREAD_LEAK_DETECTOR_DIR: ${{ github.workspace }}/target/thread-leak-dumps
     runs-on: ubuntu-22.04
     timeout-minutes: 100
     if: ${{ needs.preconditions.outputs.docs_only != 'true' }}
@@ -105,6 +161,10 @@ jobs:
       - name: Tune Runner VM
         uses: ./.github/actions/tune-runner-vm
 
+      - name: Clean Disk when tracing test resource cleanup
+        if: ${{ env.TRACE_TEST_RESOURCE_CLEANUP != 'off' }}
+        uses: ./.github/actions/clean-disk
+
       - name: Setup ssh access to build runner VM
         # ssh access is enabled for builds in own forks
         if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }}
@@ -124,11 +184,11 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-m2-dependencies-core-modules-
 
-      - name: Set up JDK 17
+      - name: Set up JDK ${{ env.CI_JDK_MAJOR_VERSION }}
         uses: actions/setup-java@v3
         with:
           distribution: 'temurin'
-          java-version: 17
+          java-version: ${{ env.CI_JDK_MAJOR_VERSION }}
 
       - name: Build core-modules
         run: |
@@ -146,8 +206,24 @@ jobs:
         if: ${{ always() }}
         uses: ./.github/actions/copy-test-reports
 
+      - name: Publish Test Report
+        uses: apache/pulsar-test-infra/action-junit-report@master
+        if: ${{ always() }}
+        with:
+          report_paths: 'test-reports/TEST-*.xml'
+          annotate_only: 'true'
+
+      - name: Report detected thread leaks
+        if: ${{ always() }}
+        run: |
+          if [ -d "$THREAD_LEAK_DETECTOR_DIR" ]; then
+            cd "$THREAD_LEAK_DETECTOR_DIR"
+            cat threadleak*.txt | awk '/^Summary:/ {print "::warning::" $0 "\n"; next} {print}'  
+          fi
+
       - name: Create Jacoco reports
         if: ${{ needs.preconditions.outputs.collect_coverage == 'true' }}
+        continue-on-error: true
         run: |
           $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh create_test_coverage_report 
           cd $GITHUB_WORKSPACE/target
@@ -160,33 +236,34 @@ jobs:
           name: Jacoco-coverage-report-flaky
           path: target/jacoco_test_coverage_report_flaky.zip
           retention-days: 3
+          if-no-files-found: ignore
 
       - name: Upload to Codecov
         if: ${{ needs.preconditions.outputs.collect_coverage == 'true' }}
         uses: ./.github/actions/upload-coverage
+        continue-on-error: true
         with:
           flags: unittests
 
-      - name: Publish Test Report
-        uses: apache/pulsar-test-infra/action-junit-report@master
-        if: ${{ always() }}
-        with:
-          report_paths: 'test-reports/TEST-*.xml'
-          annotate_only: 'true'
-
       - name: Upload Surefire reports
         uses: actions/upload-artifact@v3
-        if: ${{ !success() }}
+        if: ${{ !success() || env.TRACE_TEST_RESOURCE_CLEANUP != 'off' }}
         with:
           name: Unit-BROKER_FLAKY-surefire-reports
           path: surefire-reports
           retention-days: 7
+          if-no-files-found: ignore
 
-      - name: Upload possible heap dump
+      - name: Upload possible heap dump, core dump or crash files
         uses: actions/upload-artifact@v3
         if: ${{ always() }}
         with:
-          name: Unit-BROKER_FLAKY-heapdump
-          path: /tmp/*.hprof
+          name: Unit-BROKER_FLAKY-dumps
+          path: |
+            /tmp/*.hprof
+            **/hs_err_*.log
+            **/core.*
+            ${{ env.TRACE_TEST_RESOURCE_CLEANUP_DIR }}/*
+            ${{ env.THREAD_LEAK_DETECTOR_DIR }}/*
           retention-days: 7
-          if-no-files-found: ignore
+          if-no-files-found: ignore