geobrix/.github/workflows/codecov-scala-parallel.yml at main · databrickslabs/geobrix · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
# Optional: Run Scala coverage by package in parallel, merge, then upload to Codecov.
# Faster wall-clock time than a single job running all tests (~10–15 min vs ~30–45 min).
# Trigger: workflow_dispatch only (or add schedule/on push if desired).
#
# Requires: CODECOV_TOKEN secret.
# Merge script: scripts/ci/merge_scoverage.py
# Design: .github/docs/scala-coverage-parallel-design.md

name: Scala coverage (parallel by package)

on:
  workflow_dispatch: {}

permissions:
  contents: read

jobs:
  coverage-package:
    name: Coverage ${{ matrix.package }}
    # Heavy per matrix shard: mvn scoverage:test for one package + GDAL native install. 6 shards in parallel; per-shard fan-out keeps wall-clock down.
    runs-on:
      group: larger-runners
      labels: larger
    # Checkout uses REPO_ACCESS_TOKEN (non-exempt secret), so gate behind the protected env.
    environment: runtime
    permissions:
      contents: read
      # Required by .github/actions/jfrog-auth: GitHub OIDC token exchange for pip/Maven via JFrog.
      id-token: write
    env:
      GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
    strategy:
      fail-fast: false
      matrix:
        package: [rasterx, gridx, vectorx, ds, expressions, util]
        python: [3.12.3]
        pytest: [8.4.2]
        numpy: [2.1.3]
        gdal: [3.11.4]
        spark: [4.0.0]
    steps:
      - name: Checkout
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
        with:
          token: ${{ secrets.REPO_ACCESS_TOKEN || secrets.GITHUB_TOKEN }}

      - name: Create pip cache key file
        run: |
          echo "${{ github.ref }}-${{ matrix.python }}-${{ matrix.numpy }}-${{ matrix.spark }}-${{ matrix.gdal }}" > .ci-pip-cache-key

      - name: Cache apt packages
        uses: actions/cache@cdf6c1fa76f9f475f3d7449005a359c84ca0f306  # v5.0.3
        with:
          path: .cache/apt-archives
          key: apt-${{ runner.os }}-${{ hashFiles('.github/actions/scala_build/action.yml') }}

      - name: Build Scala with coverage (one package)
        uses: ./.github/actions/scala_build
        with:
          skip_tests: "false"
          enable_coverage: "true"
          fail_on_scalastyle: "false"
          suite_pattern: "com.databricks.labs.gbx.${{ matrix.package }}.*"

      - name: Upload scoverage for package
        uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f  # v7.0.0
        with:
          name: scoverage-${{ matrix.package }}
          path: target/scoverage.xml
          if-no-files-found: warn

  merge-and-upload:
    name: Merge and upload to Codecov
    runs-on:
      group: databrickslabs-protected-runner-group
      labels: linux-ubuntu-latest
    needs: coverage-package
    if: always() && needs.coverage-package.result == 'success'
    # Checkout uses REPO_ACCESS_TOKEN (non-exempt secret), so gate behind the protected env.
    environment: runtime
    steps:
      - name: Checkout
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
        with:
          token: ${{ secrets.REPO_ACCESS_TOKEN || secrets.GITHUB_TOKEN }}

      - name: Download all package coverage artifacts
        uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3  # v8.0.0
        with:
          path: coverage-artifacts
          pattern: scoverage-*
          merge-multiple: false

      - name: Merge scoverage XMLs
        run: |
          mkdir -p merged
          python3 scripts/ci/merge_scoverage.py -o merged/scoverage.xml coverage-artifacts/
          ls -la merged/
      - name: Print Scala coverage summary
        run: |
          if [ -f merged/scoverage.xml ]; then
            rate=$(sed -n 's/.*statement-rate="\([^"]*\)".*/\1/p' merged/scoverage.xml | head -1)
            inv=$(sed -n 's/.*statements-invoked="\([^"]*\)".*/\1/p' merged/scoverage.xml | head -1)
            tot=$(sed -n 's/.*statement-count="\([^"]*\)".*/\1/p' merged/scoverage.xml | head -1)
            echo ""
            echo "============================== Scala coverage (merged) ==============================="
            echo "Scala coverage: ${rate}% (${inv}/${tot} statements)"
            echo "=============================================================================="
          fi

      - name: Upload to Codecov
        uses: codecov/codecov-action@671740ac38dd9b0130fbe1cec585b89eea48d3de  # v5.5.2
        with:
          token: ${{ secrets.CODECOV_TOKEN }}
          files: merged/scoverage.xml
          fail_ci_if_error: false