Merge branch 'main' into feat/support-for-key-only-databricks-tags

Author: sd-db

.github/CODEOWNERS

@@ -2,4 +2,7 @@
 # the repo. Unless a later match takes precedence, these
 # users will be requested for review when someone opens a
 # pull request.
-*       @sd-db @tejassp-db @benc-db
+*       @sd-db @tejassp-db @benc-db @jprakash-db
+
+# Explicit rule for CI/CD workflow changes
+/.github/workflows/ @sd-db @tejassp-db @benc-db @jprakash-db

.github/ISSUE_TEMPLATE/dependabot.yml

@@ -0,0 +1,48 @@
+name: "Setup JFrog PyPI Proxy"
+description: "Authenticate with JFrog via OIDC and configure uv to use JFrog as PyPI proxy"
+
+runs:
+  using: "composite"
+  steps:
+    - name: Get JFrog OIDC token
+      shell: bash
+      run: |
+        set -euo pipefail
+
+        # Get GitHub OIDC ID token
+        ID_TOKEN=$(curl -sLS \
+          -H "User-Agent: actions/oidc-client" \
+          -H "Authorization: Bearer $ACTIONS_ID_TOKEN_REQUEST_TOKEN" \
+          "${ACTIONS_ID_TOKEN_REQUEST_URL}&audience=jfrog-github" | jq .value | tr -d '"')
+        echo "::add-mask::${ID_TOKEN}"
+
+        # Exchange for JFrog access token
+        ACCESS_TOKEN=$(curl -sLS -XPOST -H "Content-Type: application/json" \
+          "https://databricks.jfrog.io/access/api/v1/oidc/token" \
+          -d "{\"grant_type\": \"urn:ietf:params:oauth:grant-type:token-exchange\", \"subject_token_type\":\"urn:ietf:params:oauth:token-type:id_token\", \"subject_token\": \"${ID_TOKEN}\", \"provider_name\": \"github-actions\"}" | jq .access_token | tr -d '"')
+        echo "::add-mask::${ACCESS_TOKEN}"
+
+        if [ -z "$ACCESS_TOKEN" ] || [ "$ACCESS_TOKEN" = "null" ]; then
+          echo "FAIL: Could not extract JFrog access token"
+          exit 1
+        fi
+
+        echo "JFROG_ACCESS_TOKEN=${ACCESS_TOKEN}" >> "$GITHUB_ENV"
+        echo "JFrog OIDC token obtained successfully"
+
+    - name: Configure pip and uv to use JFrog PyPI proxy
+      shell: bash
+      run: |
+        set -euo pipefail
+        JFROG_PYPI_URL="https://gha-service-account:${JFROG_ACCESS_TOKEN}@databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple"
+        echo "PIP_INDEX_URL=${JFROG_PYPI_URL}" >> "$GITHUB_ENV"
+        echo "UV_INDEX_URL=${JFROG_PYPI_URL}" >> "$GITHUB_ENV"
+
+        # Write pip.conf so subprocesses (hatch, pre-commit, virtualenv) also use JFrog
+        mkdir -p ~/.config/pip
+        cat > ~/.config/pip/pip.conf << EOF
+        [global]
+        index-url = ${JFROG_PYPI_URL}
+        EOF
+
+        echo "pip and uv configured to use JFrog registry"

.github/actions/setup-jfrog-pypi/action.yml

@@ -0,0 +1,39 @@
+name: "Setup Python Dependencies"
+description: |
+  Restores pre-cached Python dependencies and enables offline mode.
+  Outputs cache-hit so callers can fall back to setup-jfrog-pypi on miss.
+
+outputs:
+  cache-hit:
+    description: "Whether the dependency cache was restored and offline mode enabled"
+    value: ${{ steps.uv-cache.outputs.cache-matched-key != '' }}
+
+runs:
+  using: "composite"
+  steps:
+    - name: Restore uv and pip cache
+      id: uv-cache
+      uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4
+      with:
+        path: |
+          ~/.cache/uv
+          ~/.cache/pip
+          ~/.cache/pip-wheelhouse
+        key: python-deps-${{ hashFiles('uv.lock', 'pyproject.toml') }}-latest
+        restore-keys: python-deps-${{ hashFiles('uv.lock', 'pyproject.toml') }}-
+
+    - name: Restore pre-commit cache
+      uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4
+      with:
+        path: ~/.cache/pre-commit
+        key: pre-commit-deps-${{ hashFiles('.pre-commit-config.yaml') }}-latest
+        restore-keys: pre-commit-deps-${{ hashFiles('.pre-commit-config.yaml') }}-
+
+    - name: Enable offline mode
+      if: steps.uv-cache.outputs.cache-matched-key != ''
+      shell: bash
+      run: |
+        echo "UV_OFFLINE=true" >> "$GITHUB_ENV"
+        echo "UV_INDEX_URL=https://databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple" >> "$GITHUB_ENV"
+        echo "PIP_NO_INDEX=1" >> "$GITHUB_ENV"
+        echo "PIP_FIND_LINKS=$HOME/.cache/pip-wheelhouse" >> "$GITHUB_ENV"

.github/actions/setup-python-deps/action.yml

@@ -0,0 +1,17 @@
+version: 2
+updates:
+  # Python dependencies — security updates only
+  - package-ecosystem: "pip"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+    open-pull-requests-limit: 0
+    rebase-strategy: "disabled"
+
+  # GitHub Actions — security updates only
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+    open-pull-requests-limit: 0
+    rebase-strategy: "disabled"

.github/dependabot.yml

@@ -13,36 +13,27 @@ on:
 
 jobs:
   pr-title:
-    runs-on: linux-ubuntu-latest
+    runs-on:
+      group: databricks-protected-runner-group
+      labels: linux-ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
-
-      - name: Setup node
-        uses: actions/setup-node@v4
-        with:
-          node-version: 20
-      - name: Install conventional commit parser
-        shell: bash
-        run: npm install --global conventional-commits-parser
-
       - name: Validate PR title
         id: pr-format
         shell: bash
         env:
           PR_TITLE: ${{ github.event.pull_request.title }}
-        # language=bash
         run: |
           echo "PR title: ${PR_TITLE}"
-                    
-          # check if PR title follows conventional commits format
-          # issue on parser does not support "!" for breaking change (https://github.com/conventional-changelog/conventional-changelog/issues/648)
-          # so we override the regex to support it
-          conventionalCommitResult=$(echo "${PR_TITLE}" | conventional-commits-parser -p "^(\w*)!?(?:\(([\w\$\.\-\* ]*)\))?\: (.*)$" | jq ".[].type")
-          if [[ "${conventionalCommitResult}" != "null" ]]; then
-            echo "Conventional commit type: ${conventionalCommitResult}"
+
+          # Validate PR title follows conventional commits format
+          # Pattern: type[!][(scope)]: description
+          # Examples: feat(JIRA-123): add feature, fix!: breaking change
+          REGEX='^[a-zA-Z]+!?(\([^)]*\))?\: .+'
+          if [[ "${PR_TITLE}" =~ $REGEX ]]; then
+            echo "Valid conventional commit format"
             exit 0
           fi
-          
+
           echo "Invalid PR title"
           exit 1
 

.github/workflows/ci-pr-linting.yml

@@ -10,7 +10,9 @@ on:
 jobs:
   test:
     name: Run tests & display coverage
-    runs-on: linux-ubuntu-latest
+    runs-on:
+      group: databricks-protected-runner-group
+      labels: linux-ubuntu-latest
     if: github.event.workflow_run.event == 'pull_request' && github.event.workflow_run.conclusion == 'success'
     permissions:
       # Gives the action the necessary permissions for publishing new
@@ -27,7 +29,7 @@ jobs:
       # DO NOT run actions/checkout here, for security reasons
       # For details, refer to https://securitylab.github.com/research/github-actions-preventing-pwn-requests/
       - name: Post comment
-        uses: py-cov-action/python-coverage-comment-action@v3
+        uses: py-cov-action/python-coverage-comment-action@7188638f871f721a365d644f505d1ff3df20d683  # v3
         with:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           GITHUB_PR_RUN_ID: ${{ github.event.workflow_run.id }}

.github/workflows/coverage.yml

@@ -35,13 +35,19 @@ on:
         required: false
         type: string
 
+permissions:
+  id-token: write
+  contents: read
+
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
 
 jobs:
   run-uc-cluster-e2e-tests:
-    runs-on: ubuntu-latest
+    runs-on:
+      group: databricks-protected-runner-group
+      labels: linux-ubuntu-latest
     environment: azure-prod
     # Only run on internal PRs or manual dispatch - skip external forks to avoid secret access failures
     if: github.event_name == 'workflow_dispatch' || github.event.pull_request.head.repo.full_name == github.repository
@@ -52,9 +58,10 @@ jobs:
       DBT_DATABRICKS_UC_INITIAL_CATALOG: peco
       DBT_DATABRICKS_LOCATION_ROOT: ${{ secrets.TEST_PECO_EXTERNAL_LOCATION }}test
       TEST_PECO_UC_CLUSTER_ID: ${{ secrets.TEST_PECO_UC_CLUSTER_ID }}
+      UV_FROZEN: "1"
     steps:
       - name: Check out repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5  # v4
         with:
           # For pull_request: checkout the PR head commit
           # For workflow_dispatch with pr_number: checkout that PR's head
@@ -64,9 +71,13 @@ jobs:
           # Fetch enough history for PR testing
           fetch-depth: 0
 
+      - name: Setup JFrog PyPI Proxy
+        uses: ./.github/actions/setup-jfrog-pypi
+
+
       - name: Set up python
         id: setup-python
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065  # v5
         with:
           python-version: "3.10"
 
@@ -75,25 +86,27 @@ jobs:
         shell: sh
 
       - name: Install uv
-        uses: astral-sh/setup-uv@v4
+        uses: astral-sh/setup-uv@38f3f104447c67c051c4a08e39b64a148898af3a  # v4
 
       - name: Install Hatch
         id: install-dependencies
-        uses: pypa/hatch@install
+        uses: pypa/hatch@257e27e51a6a5616ed08a39a408a21c35c9931bc  # install
 
       - name: Run UC Cluster Functional Tests
         run: DBT_TEST_USER=notnecessaryformosttests@example.com DBT_DATABRICKS_LOCATION_ROOT=$DBT_DATABRICKS_LOCATION_ROOT DBT_DATABRICKS_HOST_NAME=$DBT_DATABRICKS_HOST_NAME DBT_DATABRICKS_UC_CLUSTER_HTTP_PATH=$DBT_DATABRICKS_UC_CLUSTER_HTTP_PATH DBT_DATABRICKS_CLIENT_ID=$DBT_DATABRICKS_CLIENT_ID DBT_DATABRICKS_CLIENT_SECRET=$DBT_DATABRICKS_CLIENT_SECRET hatch -v run uc-cluster-e2e
 
       - name: Upload UC Cluster Test Logs
         if: always()
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02  # v4
         with:
           name: uc-cluster-test-logs
           path: logs/
           retention-days: 5
 
   run-sqlwarehouse-e2e-tests:
-    runs-on: ubuntu-latest
+    runs-on:
+      group: databricks-protected-runner-group
+      labels: linux-ubuntu-latest
     environment: azure-prod
     # Only run on internal PRs or manual dispatch - skip external forks to avoid secret access failures
     if: github.event_name == 'workflow_dispatch' || github.event.pull_request.head.repo.full_name == github.repository
@@ -105,9 +118,10 @@ jobs:
       DBT_DATABRICKS_UC_INITIAL_CATALOG: peco
       DBT_DATABRICKS_LOCATION_ROOT: ${{ secrets.TEST_PECO_EXTERNAL_LOCATION }}test
       TEST_PECO_UC_CLUSTER_ID: ${{ secrets.TEST_PECO_UC_CLUSTER_ID }}
+      UV_FROZEN: "1"
     steps:
       - name: Check out repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5  # v4
         with:
           # For pull_request: checkout the PR head commit
           # For workflow_dispatch with pr_number: checkout that PR's head
@@ -117,9 +131,13 @@ jobs:
           # Fetch enough history for PR testing
           fetch-depth: 0
 
+      - name: Setup JFrog PyPI Proxy
+        uses: ./.github/actions/setup-jfrog-pypi
+
+
       - name: Set up python
         id: setup-python
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065  # v5
         with:
           python-version: "3.10"
 
@@ -128,25 +146,27 @@ jobs:
         shell: sh
 
       - name: Install uv
-        uses: astral-sh/setup-uv@v4
+        uses: astral-sh/setup-uv@38f3f104447c67c051c4a08e39b64a148898af3a  # v4
 
       - name: Install Hatch
         id: install-dependencies
-        uses: pypa/hatch@install
+        uses: pypa/hatch@257e27e51a6a5616ed08a39a408a21c35c9931bc  # install
 
       - name: Run Sql Endpoint Functional Tests
         run: DBT_TEST_USER=notnecessaryformosttests@example.com DBT_DATABRICKS_LOCATION_ROOT=$DBT_DATABRICKS_LOCATION_ROOT DBT_DATABRICKS_HOST_NAME=$DBT_DATABRICKS_HOST_NAME DBT_DATABRICKS_UC_CLUSTER_HTTP_PATH=$DBT_DATABRICKS_UC_CLUSTER_HTTP_PATH DBT_DATABRICKS_CLIENT_ID=$DBT_DATABRICKS_CLIENT_ID DBT_DATABRICKS_CLIENT_SECRET=$DBT_DATABRICKS_CLIENT_SECRET hatch -v run sqlw-e2e
 
       - name: Upload SQL Endpoint Test Logs
         if: always()
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02  # v4
         with:
           name: sql-endpoint-test-logs
           path: logs/
           retention-days: 5
 
   run-cluster-e2e-tests:
-    runs-on: ubuntu-latest
+    runs-on:
+      group: databricks-protected-runner-group
+      labels: linux-ubuntu-latest
     environment: azure-prod
     # Only run on internal PRs or manual dispatch - skip external forks to avoid secret access failures
     if: github.event_name == 'workflow_dispatch' || github.event.pull_request.head.repo.full_name == github.repository
@@ -156,9 +176,10 @@ jobs:
       DBT_DATABRICKS_CLIENT_SECRET: ${{ secrets.TEST_PECO_SP_SECRET }}
       TEST_PECO_CLUSTER_ID: ${{ secrets.TEST_PECO_CLUSTER_ID }}
       DBT_DATABRICKS_LOCATION_ROOT: ${{ secrets.TEST_PECO_EXTERNAL_LOCATION }}test
+      UV_FROZEN: "1"
     steps:
       - name: Check out repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5  # v4
         with:
           # For pull_request: checkout the PR head commit
           # For workflow_dispatch with pr_number: checkout that PR's head
@@ -168,9 +189,13 @@ jobs:
           # Fetch enough history for PR testing
           fetch-depth: 0
 
+      - name: Setup JFrog PyPI Proxy
+        uses: ./.github/actions/setup-jfrog-pypi
+
+
       - name: Set up python
         id: setup-python
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065  # v5
         with:
           python-version: "3.10"
 
@@ -179,18 +204,18 @@ jobs:
         shell: sh
 
       - name: Install uv
-        uses: astral-sh/setup-uv@v4
+        uses: astral-sh/setup-uv@38f3f104447c67c051c4a08e39b64a148898af3a  # v4
 
       - name: Install Hatch
         id: install-dependencies
-        uses: pypa/hatch@install
+        uses: pypa/hatch@257e27e51a6a5616ed08a39a408a21c35c9931bc  # install
 
       - name: Run Cluster Functional Tests
         run: DBT_TEST_USER=notnecessaryformosttests@example.com DBT_DATABRICKS_LOCATION_ROOT=$DBT_DATABRICKS_LOCATION_ROOT DBT_DATABRICKS_HOST_NAME=$DBT_DATABRICKS_HOST_NAME DBT_DATABRICKS_HTTP_PATH=$DBT_DATABRICKS_CLUSTER_HTTP_PATH DBT_DATABRICKS_CLIENT_ID=$DBT_DATABRICKS_CLIENT_ID DBT_DATABRICKS_CLIENT_SECRET=$DBT_DATABRICKS_CLIENT_SECRET hatch -v run cluster-e2e
 
       - name: Upload Cluster Test Logs
         if: always()
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02  # v4
         with:
           name: cluster-test-logs
           path: logs/