Merge branch 'main' into feature/foreign-key-check-with-null-safe-support

Author: mwojtyczka

.github/actions/jfrog-auth/action.yml

@@ -48,3 +48,15 @@ runs:
         uv auth login "${UV_INDEX_URL}" --username gha-service-account --password "${JFROG_ACCESS_TOKEN}"
         printf "%s=%s\n" 'UV_INDEX_URL' "${UV_INDEX_URL}" >> "${GITHUB_ENV}"
         printf "%s=%s\n" 'UV_FROZEN' '1' >> "${GITHUB_ENV}"
+
+    - name: Configure npm/yarn for JFrog
+      shell: bash
+      env:
+        JFROG_ACCESS_TOKEN: "${{ steps.jfrog-auth.outputs.jfrog-access-token }}"
+      run: |
+        umask 077
+        cat > ~/.npmrc << EOF
+        registry=https://databricks.jfrog.io/artifactory/api/npm/db-npm/
+        //databricks.jfrog.io/artifactory/api/npm/db-npm/:_authToken=${JFROG_ACCESS_TOKEN}
+        always-auth=true
+        EOF

.github/actions/prebuild-wheel/action.yml

@@ -0,0 +1,33 @@
+name: 'Pre-build DQX wheel'
+description: >
+  Builds the DQX wheel once per CI job and exports the absolute path as DQX_PREBUILT_WHEEL. 
+  Tests that use PrebuiltWheels copy this artifact instead of running `pip wheel` per-test, 
+  which avoids JFrog auth failures after the 1h OIDC token TTL expires mid-suite.
+runs:
+  using: "composite"
+  steps:
+    - name: Build wheel
+      shell: bash
+      run: |
+        # Bump version per CI run so cluster library installs and pip caches
+        # don't reuse stale bytes. Delegate the scheme to blueprint so it
+        # matches what blueprint would compute per-test
+        NEW_VERSION="$(UV_FROZEN=1 uv run --quiet python -c '
+        from databricks.labs.blueprint.wheels import ProductInfo
+        from databricks.labs.dqx.config import WorkspaceConfig
+        print(ProductInfo.from_class(WorkspaceConfig).version())
+        ')"
+        test -n "$NEW_VERSION" || { echo "failed to compute unreleased version"; exit 1; }
+
+        VERSION_FILE="src/databricks/labs/dqx/__about__.py"
+        sed -i "s/^__version__ = .*/__version__ = \"${NEW_VERSION}\"/" "$VERSION_FILE"
+
+        make build
+
+        # Restore so later steps see a clean tree
+        git checkout -- "$VERSION_FILE"
+
+        WHEEL_PATH="$(find dist -maxdepth 1 -name '*.whl' -print -quit)"
+        test -n "$WHEEL_PATH" || { echo "no wheel produced in dist/"; ls -la dist/; exit 1; }
+
+        printf '%s=%s\n' 'DQX_PREBUILT_WHEEL' "$PWD/$WHEEL_PATH" >> "$GITHUB_ENV"

.github/actions/setup-env/action.yml

@@ -0,0 +1,13 @@
+name: 'Setup Environment'
+description: 'Setup uv, authenticate with JFrog, and configure package managers (pip, uv, npm).'
+runs:
+  using: "composite"
+  steps:
+    - name: Setup uv
+      uses: astral-sh/setup-uv@5a095e7a2014a4212f075830d4f7277575a9d098  # v7.3.1
+      with:
+        version: "0.11.2"
+        checksum: "7ac2ca0449c8d68dae9b99e635cd3bc9b22a4cb1de64b7c43716398447d42981"
+
+    - name: Setup for JFrog
+      uses: ./.github/actions/jfrog-auth

.github/pull_request_template.md

@@ -14,3 +14,9 @@ Resolves #..
 - [ ] added integration tests
 - [ ] added end-to-end tests
 - [ ] added performance tests
+
+### Documentation and Demos
+<!-- Any user facing changes require documentation and demos update -->
+
+- [ ] added/updated demos
+- [ ] added/updated docs

.github/workflows/acceptance.yml

@@ -5,9 +5,6 @@ on:
     types: [ opened, synchronize, ready_for_review ]
   merge_group:
     types: [ checks_requested ]
-  push:
-    branches:
-      - main
 
 permissions:
   contents: read
@@ -17,14 +14,23 @@ concurrency:
   cancel-in-progress: false  # don't cancel ongoing runs to ensure fixtures are completed and resources terminated
 
 jobs:
+  # Gate all downstream jobs behind a single check so PRs from forks (no access to the
+  # tool environment) and draft PRs do not trigger the expensive acceptance suite.
+  # PRs from forks are to be tested by the reviewer(s) / maintainer(s) before merging.
+  not-a-fork:
+    runs-on:
+      group: databrickslabs-protected-runner-group
+      labels: linux-ubuntu-latest
+    if: github.event_name == 'pull_request' && !github.event.pull_request.draft && !github.event.pull_request.head.repo.fork
+    steps:
+      - run: echo "Not a fork PR, proceeding"
 
   integration:
-    # Only run this job for PRs from branches on the main repository and not from forks.
-    # Workflows triggered by PRs from forks don't have access to the tool environment.
-    # PRs from forks to be tested by the reviewer(s) / maintainer(s) before merging.
-    if: github.event_name == 'pull_request' && !github.event.pull_request.draft && !github.event.pull_request.head.repo.fork
+    needs: not-a-fork
     environment: tool
-    runs-on: larger
+    runs-on:
+      group: larger-runners
+      labels: larger
     permissions:
       # Access to the integration testing infrastructure.
       id-token: write
@@ -36,17 +42,11 @@ jobs:
         with:
           fetch-depth: 0
 
-      - name: Setup uv
-        uses: astral-sh/setup-uv@5a095e7a2014a4212f075830d4f7277575a9d098  # v7.3.1
-        with:
-          version: "0.11.2"
-          checksum: "7ac2ca0449c8d68dae9b99e635cd3bc9b22a4cb1de64b7c43716398447d42981"
+      - name: Setup environment
+        uses: ./.github/actions/setup-env
 
-      - name: Setup for JFrog
-        uses: ./.github/actions/jfrog-auth
-
-      - name: Run unit tests and generate test coverage report
-        run: make test
+      - name: Pre-build DQX wheel
+        uses: ./.github/actions/prebuild-wheel
 
       # Integration tests are run from within tests/integration folder.
       # Create .coveragerc with correct relative path to source code.
@@ -56,13 +56,14 @@ jobs:
           [run]
           source = ../../src
           relative_files = true
+          parallel = true
           EOF
 
       # Run tests from `tests/integration` as defined in .codegen.json
       # and generate code coverage for modules defined in .coveragerc
       # Run 10 tests in parallel: https://github.com/databrickslabs/sandbox/blob/main/acceptance/ecosystem/pytest_run.py
       - name: Run integration tests and generate test coverage report
-        uses: databrickslabs/sandbox/acceptance@3313d06ce86227537b3f37f5974f7eecb2a8e59a # acceptance/v0.4.4
+        uses: databrickslabs/sandbox/acceptance@83461e5dd7021feabb1a9ca3ee10d6f46b72092a # acceptance/v0.4.6
         with:
           vault_uri: ${{ secrets.VAULT_URI }}
           timeout: 2h
@@ -72,24 +73,22 @@ jobs:
           ARM_CLIENT_ID: ${{ secrets.ARM_CLIENT_ID }}
           ARM_TENANT_ID: ${{ secrets.ARM_TENANT_ID }}
           COVERAGE_FILE: ${{ github.workspace }}/.coverage  # make sure the coverage report is preserved
-
       - name: Merge coverage reports and convert them to XML
         run: make combine-coverage
 
-      # Recursively search the entire workspace directory for all coverage reports.
-      # All uploaded test coverage reports will be used even if publish is done multiple time.
       - name: Publish test coverage
         uses: codecov/codecov-action@b9fd7d16f6d7d1b5d2bec1a2887e65ceed900238 # v4.6.0
         with:
           use_oidc: true
+          files: coverage-combined.xml
+          flags: integration
 
   integration_serverless:
-    # Only run this job for PRs from branches on the main repository and not from forks.
-    # Workflows triggered by PRs from forks don't have access to the tool environment.
-    # PRs from forks to be tested by the reviewer(s) / maintainer(s) before merging.
-    if: github.event_name == 'pull_request' && !github.event.pull_request.draft && !github.event.pull_request.head.repo.fork
+    needs: not-a-fork
     environment: tool
-    runs-on: larger
+    runs-on:
+      group: larger-runners
+      labels: larger
     permissions:
       id-token: write
       pull-requests: write
@@ -101,14 +100,11 @@ jobs:
         with:
           fetch-depth: 0
 
-      - name: Setup uv
-        uses: astral-sh/setup-uv@5a095e7a2014a4212f075830d4f7277575a9d098  # v7.3.1
-        with:
-          version: "0.11.2"
-          checksum: "7ac2ca0449c8d68dae9b99e635cd3bc9b22a4cb1de64b7c43716398447d42981"
+      - name: Setup environment
+        uses: ./.github/actions/setup-env
 
-      - name: Setup for JFrog
-        uses: ./.github/actions/jfrog-auth
+      - name: Pre-build DQX wheel
+        uses: ./.github/actions/prebuild-wheel
 
       # Integration tests are run from within tests/integration folder.
       # Create .coveragerc with correct relative path to source code.
@@ -118,10 +114,11 @@ jobs:
           [run]
           source = ../../src
           relative_files = true
+          parallel = true
           EOF
 
       - name: Run integration tests on serverless cluster
-        uses: databrickslabs/sandbox/acceptance@3313d06ce86227537b3f37f5974f7eecb2a8e59a # acceptance/v0.4.4
+        uses: databrickslabs/sandbox/acceptance@83461e5dd7021feabb1a9ca3ee10d6f46b72092a # acceptance/v0.4.6
         with:
           vault_uri: ${{ secrets.VAULT_URI }}
           timeout: 2h
@@ -132,20 +129,22 @@ jobs:
           ARM_TENANT_ID: ${{ secrets.ARM_TENANT_ID }}
           DATABRICKS_SERVERLESS_COMPUTE_ID: ${{ env.DATABRICKS_SERVERLESS_COMPUTE_ID }}
           COVERAGE_FILE: ${{ github.workspace }}/.coverage  # make sure the coverage report is preserved
-
       - name: Merge coverage reports and convert them to XML
         run: make combine-coverage
 
-      # collects all coverage reports
       - name: Publish test coverage
         uses: codecov/codecov-action@b9fd7d16f6d7d1b5d2bec1a2887e65ceed900238 # v4.6.0
         with:
           use_oidc: true
+          files: coverage-combined.xml
+          flags: integration-serverless
 
   e2e:
-    if: github.event_name == 'pull_request' && !github.event.pull_request.draft && !github.event.pull_request.head.repo.fork
+    needs: not-a-fork
     environment: tool
-    runs-on: larger
+    runs-on:
+      group: larger-runners
+      labels: larger
     permissions:
       id-token: write
       pull-requests: write
@@ -155,21 +154,18 @@ jobs:
         with:
           fetch-depth: 0
 
-      - name: Setup uv
-        uses: astral-sh/setup-uv@5a095e7a2014a4212f075830d4f7277575a9d098  # v7.3.1
-        with:
-          version: "0.11.2"
-          checksum: "7ac2ca0449c8d68dae9b99e635cd3bc9b22a4cb1de64b7c43716398447d42981"
+      - name: Setup environment
+        uses: ./.github/actions/setup-env
 
-      - name: Setup for JFrog
-        uses: ./.github/actions/jfrog-auth
+      - name: Pre-build DQX wheel
+        uses: ./.github/actions/prebuild-wheel
 
       # Required for DAB (Databricks Asset Bundle) e2e tests
       - name: Install Databricks CLI
-        uses: databricks/setup-cli@acd0e77a1ed7f15f528faca1e1f7f5590bcfdff8 # v0.296.0
+        uses: databricks/setup-cli@596b0a354ba14aa59921aca1b02bd67c2b0a81a5 # v0.297.2
 
       - name: Run e2e tests
-        uses: databrickslabs/sandbox/acceptance@3313d06ce86227537b3f37f5974f7eecb2a8e59a # acceptance/v0.4.4
+        uses: databrickslabs/sandbox/acceptance@83461e5dd7021feabb1a9ca3ee10d6f46b72092a # acceptance/v0.4.6
         with:
           vault_uri: ${{ secrets.VAULT_URI }}
           timeout: 2h
@@ -181,9 +177,11 @@ jobs:
           ARM_TENANT_ID: ${{ secrets.ARM_TENANT_ID }}
 
   e2e_serverless:
-    if: github.event_name == 'pull_request' && !github.event.pull_request.draft && !github.event.pull_request.head.repo.fork
+    needs: not-a-fork
     environment: tool
-    runs-on: larger
+    runs-on:
+      group: larger-runners
+      labels: larger
     permissions:
       id-token: write
       pull-requests: write
@@ -195,21 +193,18 @@ jobs:
         with:
           fetch-depth: 0
 
-      - name: Setup uv
-        uses: astral-sh/setup-uv@5a095e7a2014a4212f075830d4f7277575a9d098  # v7.3.1
-        with:
-          version: "0.11.2"
-          checksum: "7ac2ca0449c8d68dae9b99e635cd3bc9b22a4cb1de64b7c43716398447d42981"
+      - name: Setup environment
+        uses: ./.github/actions/setup-env
 
-      - name: Setup for JFrog
-        uses: ./.github/actions/jfrog-auth
+      - name: Pre-build DQX wheel
+        uses: ./.github/actions/prebuild-wheel
 
       # Required for DAB (Databricks Asset Bundle) e2e tests
       - name: Install Databricks CLI
-        uses: databricks/setup-cli@acd0e77a1ed7f15f528faca1e1f7f5590bcfdff8 # v0.296.0
+        uses: databricks/setup-cli@596b0a354ba14aa59921aca1b02bd67c2b0a81a5 # v0.297.2
 
       - name: Run e2e tests on serverless cluster
-        uses: databrickslabs/sandbox/acceptance@3313d06ce86227537b3f37f5974f7eecb2a8e59a # acceptance/v0.4.4
+        uses: databrickslabs/sandbox/acceptance@83461e5dd7021feabb1a9ca3ee10d6f46b72092a # acceptance/v0.4.6
         with:
           vault_uri: ${{ secrets.VAULT_URI }}
           timeout: 2h