integration-spog.yml

# SPOG Integration Tests for dbt-databricks.
#
# Mirrors integration.yml's matrix (prepare-shards + 3 sharded functional
# jobs) on a weekly Sunday schedule, but with the host name and http_path
# wired to the SPOG vanity host + ?o= so every test path exercises SPOG
# routing. PR-targeting / status-reporting logic is omitted — this is a
# scheduled smoke, not a PR gate.
name: SPOG Integration Tests
on:
  workflow_dispatch:
  schedule:
    - cron: "30 21 * * 0"  # Weekly: Sunday 21:30 UTC (Monday 03:00 IST).

permissions:
  id-token: write
  contents: read

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

jobs:
  prepare-shards:
    runs-on:
      group: databricks-protected-runner-group
      labels: linux-ubuntu-latest
    env:
      UV_FROZEN: "1"
    steps:
      - name: Check out the repository
        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5  # v4

      - name: Setup JFrog PyPI Proxy
        uses: ./.github/actions/setup-jfrog-pypi

      - name: Set up Python
        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065  # v5
        with:
          python-version: "3.10"

      - name: Install uv
        uses: astral-sh/setup-uv@38f3f104447c67c051c4a08e39b64a148898af3a  # v4
        with:
          cache-local-path: ~/.cache/uv

      - name: Install Hatch
        uses: pypa/hatch@257e27e51a6a5616ed08a39a408a21c35c9931bc  # install

      - name: Collect tests and assign to shards
        run: |
          set -euo pipefail
          mkdir -p shard-assignments
          declare -A NUM_SHARDS=(
            [databricks_cluster]=2
            [databricks_uc_cluster]=3
            [databricks_uc_sql_endpoint]=3
          )
          for PROFILE in "${!NUM_SHARDS[@]}"; do
            (
              hatch run pytest --collect-only -q --profile "$PROFILE" tests/functional 2>&1 \
                | grep "::" \
                > "shard-assignments/${PROFILE}-collected.txt"
            ) &
          done
          wait
          for PROFILE in "${!NUM_SHARDS[@]}"; do
            python3 scripts/shard_assign.py \
              --profile "$PROFILE" \
              --num-shards "${NUM_SHARDS[$PROFILE]}" \
              --input "shard-assignments/${PROFILE}-collected.txt" \
              --output-dir shard-assignments \
              --algo lpt_historical_time \
              --timings .github/test_timings.json
          done

      - name: Upload shard assignments
        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02  # v4
        with:
          name: shard-assignments-spog
          path: shard-assignments/
          retention-days: 5

  uc-cluster:
    needs: prepare-shards
    strategy:
      fail-fast: false
      matrix:
        shard: [0, 1, 2]
    runs-on:
      group: databricks-protected-runner-group
      labels: linux-ubuntu-latest
    environment: azure-prod
    env:
      DBT_DATABRICKS_HOST_NAME: ${{ secrets.TEST_PECO_SPOG_HOST }}
      DBT_DATABRICKS_CLIENT_ID: ${{ secrets.TEST_PECO_SP_ID }}
      DBT_DATABRICKS_CLIENT_SECRET: ${{ secrets.TEST_PECO_SP_SECRET }}
      DBT_DATABRICKS_UC_INITIAL_CATALOG: peco
      DBT_DATABRICKS_LOCATION_ROOT: ${{ secrets.TEST_PECO_EXTERNAL_LOCATION }}test
      TEST_PECO_SPOG_WORKSPACE_ID: ${{ secrets.TEST_PECO_SPOG_WORKSPACE_ID }}
      TEST_PECO_UC_CLUSTER_ID: ${{ secrets.TEST_PECO_UC_CLUSTER_ID }}
      TEST_PECO_SPOG_NATIVE: "1"
      UV_FROZEN: "1"
    steps:
      - name: Check out repository
        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5  # v4

      - name: Setup JFrog PyPI Proxy
        uses: ./.github/actions/setup-jfrog-pypi

      - name: Set up python
        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065  # v5
        with:
          python-version: "3.10"

      - name: Get http path from environment
        run: python .github/workflows/build_cluster_http_path.py
        shell: sh

      - name: Install uv
        uses: astral-sh/setup-uv@38f3f104447c67c051c4a08e39b64a148898af3a  # v4
        with:
          cache-local-path: ~/.cache/uv

      - name: Install Hatch
        uses: pypa/hatch@257e27e51a6a5616ed08a39a408a21c35c9931bc  # install

      - name: Download shard assignments
        uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e  # v4.1.7
        with:
          name: shard-assignments-spog
          path: shard-assignments/

      - name: Resolve test list for this shard
        run: |
          set -euo pipefail
          SHARD_FILE="shard-assignments/databricks_uc_cluster-shard-${{ matrix.shard }}.txt"
          if [ ! -s "$SHARD_FILE" ]; then
            echo "::error::Shard file missing or empty: $SHARD_FILE"
            exit 1
          fi
          echo "SHARD_TESTS_FILE=$SHARD_FILE" >> "$GITHUB_ENV"
          echo "Files in shard ${{ matrix.shard }}: $(wc -l < "$SHARD_FILE")"

      - name: Run UC Cluster Functional Tests (shard ${{ matrix.shard }})
        run: |
          mkdir -p logs
          DBT_TEST_USER=notnecessaryformosttests@example.com \
          xargs -r hatch -v run pytest \
            --color=yes -v \
            --profile databricks_uc_cluster \
            -n 10 --dist=loadfile \
            --reruns 2 --reruns-delay 120 \
            --junitxml=logs/junit-uc-cluster-shard-${{ matrix.shard }}.xml \
            < "$SHARD_TESTS_FILE"

      - name: Upload UC Cluster Test Logs
        if: always()
        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02  # v4
        with:
          name: spog-uc-cluster-logs-shard-${{ matrix.shard }}
          path: logs/
          retention-days: 14

  uc-sql-endpoint:
    needs: prepare-shards
    strategy:
      fail-fast: false
      matrix:
        shard: [0, 1, 2]
    runs-on:
      group: databricks-protected-runner-group
      labels: linux-ubuntu-latest
    environment: azure-prod
    env:
      DBT_DATABRICKS_HOST_NAME: ${{ secrets.TEST_PECO_SPOG_HOST }}
      DBT_DATABRICKS_CLIENT_ID: ${{ secrets.TEST_PECO_SP_ID }}
      DBT_DATABRICKS_CLIENT_SECRET: ${{ secrets.TEST_PECO_SP_SECRET }}
      DBT_DATABRICKS_HTTP_PATH: ${{ secrets.TEST_PECO_WAREHOUSE_HTTP_PATH }}?o=${{ secrets.TEST_PECO_SPOG_WORKSPACE_ID }}
      DBT_DATABRICKS_UC_INITIAL_CATALOG: peco
      DBT_DATABRICKS_LOCATION_ROOT: ${{ secrets.TEST_PECO_EXTERNAL_LOCATION }}test
      TEST_PECO_SPOG_WORKSPACE_ID: ${{ secrets.TEST_PECO_SPOG_WORKSPACE_ID }}
      TEST_PECO_UC_CLUSTER_ID: ${{ secrets.TEST_PECO_UC_CLUSTER_ID }}
      TEST_PECO_SPOG_NATIVE: "1"
      UV_FROZEN: "1"
    steps:
      - name: Check out repository
        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5  # v4

      - name: Setup JFrog PyPI Proxy
        uses: ./.github/actions/setup-jfrog-pypi

      - name: Set up python
        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065  # v5
        with:
          python-version: "3.10"

      - name: Get http path from environment
        run: python .github/workflows/build_cluster_http_path.py
        shell: sh

      - name: Install uv
        uses: astral-sh/setup-uv@38f3f104447c67c051c4a08e39b64a148898af3a  # v4
        with:
          cache-local-path: ~/.cache/uv

      - name: Install Hatch
        uses: pypa/hatch@257e27e51a6a5616ed08a39a408a21c35c9931bc  # install

      - name: Download shard assignments
        uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e  # v4.1.7
        with:
          name: shard-assignments-spog
          path: shard-assignments/

      - name: Resolve test list for this shard
        run: |
          set -euo pipefail
          SHARD_FILE="shard-assignments/databricks_uc_sql_endpoint-shard-${{ matrix.shard }}.txt"
          if [ ! -s "$SHARD_FILE" ]; then
            echo "::error::Shard file missing or empty: $SHARD_FILE"
            exit 1
          fi
          echo "SHARD_TESTS_FILE=$SHARD_FILE" >> "$GITHUB_ENV"
          echo "Files in shard ${{ matrix.shard }}: $(wc -l < "$SHARD_FILE")"

      - name: Run Sql Endpoint Functional Tests (shard ${{ matrix.shard }})
        run: |
          mkdir -p logs
          DBT_TEST_USER=notnecessaryformosttests@example.com \
          xargs -r hatch -v run pytest \
            --color=yes -v \
            --profile databricks_uc_sql_endpoint \
            -n 10 --dist=loadfile \
            --reruns 2 --reruns-delay 120 \
            --junitxml=logs/junit-uc-sql-endpoint-shard-${{ matrix.shard }}.xml \
            < "$SHARD_TESTS_FILE"

      - name: Upload SQL Endpoint Test Logs
        if: always()
        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02  # v4
        with:
          name: spog-uc-sql-endpoint-logs-shard-${{ matrix.shard }}
          path: logs/
          retention-days: 14

  cluster:
    needs: prepare-shards
    strategy:
      fail-fast: false
      matrix:
        shard: [0, 1]
    runs-on:
      group: databricks-protected-runner-group
      labels: linux-ubuntu-latest
    environment: azure-prod
    env:
      DBT_DATABRICKS_HOST_NAME: ${{ secrets.TEST_PECO_SPOG_HOST }}
      DBT_DATABRICKS_CLIENT_ID: ${{ secrets.TEST_PECO_SP_ID }}
      DBT_DATABRICKS_CLIENT_SECRET: ${{ secrets.TEST_PECO_SP_SECRET }}
      DBT_DATABRICKS_LOCATION_ROOT: ${{ secrets.TEST_PECO_EXTERNAL_LOCATION }}test
      TEST_PECO_SPOG_WORKSPACE_ID: ${{ secrets.TEST_PECO_SPOG_WORKSPACE_ID }}
      TEST_PECO_CLUSTER_ID: ${{ secrets.TEST_PECO_CLUSTER_ID }}
      TEST_PECO_SPOG_NATIVE: "1"
      UV_FROZEN: "1"
    steps:
      - name: Check out repository
        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5  # v4

      - name: Setup JFrog PyPI Proxy
        uses: ./.github/actions/setup-jfrog-pypi

      - name: Set up python
        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065  # v5
        with:
          python-version: "3.10"

      - name: Get http path from environment
        run: python .github/workflows/build_cluster_http_path.py
        shell: sh

      - name: Install uv
        uses: astral-sh/setup-uv@38f3f104447c67c051c4a08e39b64a148898af3a  # v4
        with:
          cache-local-path: ~/.cache/uv

      - name: Install Hatch
        uses: pypa/hatch@257e27e51a6a5616ed08a39a408a21c35c9931bc  # install

      - name: Download shard assignments
        uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e  # v4.1.7
        with:
          name: shard-assignments-spog
          path: shard-assignments/

      - name: Resolve test list for this shard
        run: |
          set -euo pipefail
          SHARD_FILE="shard-assignments/databricks_cluster-shard-${{ matrix.shard }}.txt"
          if [ ! -s "$SHARD_FILE" ]; then
            echo "::error::Shard file missing or empty: $SHARD_FILE"
            exit 1
          fi
          echo "SHARD_TESTS_FILE=$SHARD_FILE" >> "$GITHUB_ENV"
          echo "Files in shard ${{ matrix.shard }}: $(wc -l < "$SHARD_FILE")"

      - name: Run Cluster Functional Tests (shard ${{ matrix.shard }})
        run: |
          mkdir -p logs
          DBT_TEST_USER=notnecessaryformosttests@example.com \
          DBT_DATABRICKS_HTTP_PATH="$DBT_DATABRICKS_CLUSTER_HTTP_PATH" \
          xargs -r hatch -v run pytest \
            --color=yes -v \
            --profile databricks_cluster \
            -n 10 --dist=loadfile \
            --reruns 2 --reruns-delay 120 \
            --junitxml=logs/junit-cluster-shard-${{ matrix.shard }}.xml \
            < "$SHARD_TESTS_FILE"

      - name: Upload Cluster Test Logs
        if: always()
        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02  # v4
        with:
          name: spog-cluster-logs-shard-${{ matrix.shard }}
          path: logs/
          retention-days: 14