Fixed AI-assisted sql_query generation and made has_valid_schema compatible with older Spark versions (#995)

## Changes
<!-- Summary of your changes that are easy to understand. Add
screenshots when necessary -->
* Improved Agent guidelines
* Make `has_valid_schema` check compatible with older spark versions (<
4)
* Fix issue with subqueries in sql `expression` check in Serverless v5
when check name is not provided (auto-derived)
* Added guidelines on how to configure DQX with LDP/DLT (Lakeflow
Declarative Pipelines) to enable incrementalization for Materialized
Views (MVs)
* Improved validation of required check function arguments
* Fix CI issues

### Linked issues
<!-- DOC: Link issue with a keyword: close, closes, closed, fix, fixes,
fixed, resolve, resolves, resolved. See
https://docs.github.com/en/issues/tracking-your-work-with-issues/linking-a-pull-request-to-an-issue#linking-a-pull-request-to-an-issue-using-a-keyword
-->

Resolves: #1053 

### Tests
<!-- How is this tested? Please see the checklist below and also
describe any other relevant tests -->

- [x] manually tested
- [x] added unit tests
- [x] added integration tests
- [ ] added end-to-end tests
- [ ] added performance tests

---------

Co-authored-by: Marcin Wojtyczka <marcin.wojtyczka@databricks.com>
Co-authored-by: mwojtyczka <mwojtyczka@users.noreply.github.com>

Author: ghanse

.github/actions/jfrog-auth/action.yml

@@ -48,3 +48,15 @@ runs:
         uv auth login "${UV_INDEX_URL}" --username gha-service-account --password "${JFROG_ACCESS_TOKEN}"
         printf "%s=%s\n" 'UV_INDEX_URL' "${UV_INDEX_URL}" >> "${GITHUB_ENV}"
         printf "%s=%s\n" 'UV_FROZEN' '1' >> "${GITHUB_ENV}"
+
+    - name: Configure npm/yarn for JFrog
+      shell: bash
+      env:
+        JFROG_ACCESS_TOKEN: "${{ steps.jfrog-auth.outputs.jfrog-access-token }}"
+      run: |
+        umask 077
+        cat > ~/.npmrc << EOF
+        registry=https://databricks.jfrog.io/artifactory/api/npm/db-npm/
+        //databricks.jfrog.io/artifactory/api/npm/db-npm/:_authToken=${JFROG_ACCESS_TOKEN}
+        always-auth=true
+        EOF

.github/actions/setup-env/action.yml

@@ -0,0 +1,13 @@
+name: 'Setup Environment'
+description: 'Setup uv, authenticate with JFrog, and configure package managers (pip, uv, npm).'
+runs:
+  using: "composite"
+  steps:
+    - name: Setup uv
+      uses: astral-sh/setup-uv@5a095e7a2014a4212f075830d4f7277575a9d098  # v7.3.1
+      with:
+        version: "0.11.2"
+        checksum: "7ac2ca0449c8d68dae9b99e635cd3bc9b22a4cb1de64b7c43716398447d42981"
+
+    - name: Setup for JFrog
+      uses: ./.github/actions/jfrog-auth

.github/pull_request_template.md

@@ -14,3 +14,9 @@ Resolves #..
 - [ ] added integration tests
 - [ ] added end-to-end tests
 - [ ] added performance tests
+
+### Documentation and Demos
+<!-- Any user facing changes require documentation and demos update -->
+
+- [ ] added/updated demos
+- [ ] added/updated docs

.github/workflows/acceptance.yml

@@ -5,9 +5,6 @@ on:
     types: [ opened, synchronize, ready_for_review ]
   merge_group:
     types: [ checks_requested ]
-  push:
-    branches:
-      - main
 
 permissions:
   contents: read
@@ -17,14 +14,23 @@ concurrency:
   cancel-in-progress: false  # don't cancel ongoing runs to ensure fixtures are completed and resources terminated
 
 jobs:
+  # Gate all downstream jobs behind a single check so PRs from forks (no access to the
+  # tool environment) and draft PRs do not trigger the expensive acceptance suite.
+  # PRs from forks are to be tested by the reviewer(s) / maintainer(s) before merging.
+  not-a-fork:
+    runs-on:
+      group: databrickslabs-protected-runner-group
+      labels: linux-ubuntu-latest
+    if: github.event_name == 'pull_request' && !github.event.pull_request.draft && !github.event.pull_request.head.repo.fork
+    steps:
+      - run: echo "Not a fork PR, proceeding"
 
   integration:
-    # Only run this job for PRs from branches on the main repository and not from forks.
-    # Workflows triggered by PRs from forks don't have access to the tool environment.
-    # PRs from forks to be tested by the reviewer(s) / maintainer(s) before merging.
-    if: github.event_name == 'pull_request' && !github.event.pull_request.draft && !github.event.pull_request.head.repo.fork
+    needs: not-a-fork
     environment: tool
-    runs-on: larger
+    runs-on:
+      group: larger-runners
+      labels: larger
     permissions:
       # Access to the integration testing infrastructure.
       id-token: write
@@ -36,14 +42,8 @@ jobs:
         with:
           fetch-depth: 0
 
-      - name: Setup uv
-        uses: astral-sh/setup-uv@5a095e7a2014a4212f075830d4f7277575a9d098  # v7.3.1
-        with:
-          version: "0.11.2"
-          checksum: "7ac2ca0449c8d68dae9b99e635cd3bc9b22a4cb1de64b7c43716398447d42981"
-
-      - name: Setup for JFrog
-        uses: ./.github/actions/jfrog-auth
+      - name: Setup environment
+        uses: ./.github/actions/setup-env
 
       - name: Run unit tests and generate test coverage report
         run: make test
@@ -56,6 +56,7 @@ jobs:
           [run]
           source = ../../src
           relative_files = true
+          parallel = true
           EOF
 
       # Run tests from `tests/integration` as defined in .codegen.json
@@ -84,12 +85,11 @@ jobs:
           use_oidc: true
 
   integration_serverless:
-    # Only run this job for PRs from branches on the main repository and not from forks.
-    # Workflows triggered by PRs from forks don't have access to the tool environment.
-    # PRs from forks to be tested by the reviewer(s) / maintainer(s) before merging.
-    if: github.event_name == 'pull_request' && !github.event.pull_request.draft && !github.event.pull_request.head.repo.fork
+    needs: not-a-fork
     environment: tool
-    runs-on: larger
+    runs-on:
+      group: larger-runners
+      labels: larger
     permissions:
       id-token: write
       pull-requests: write
@@ -101,14 +101,8 @@ jobs:
         with:
           fetch-depth: 0
 
-      - name: Setup uv
-        uses: astral-sh/setup-uv@5a095e7a2014a4212f075830d4f7277575a9d098  # v7.3.1
-        with:
-          version: "0.11.2"
-          checksum: "7ac2ca0449c8d68dae9b99e635cd3bc9b22a4cb1de64b7c43716398447d42981"
-
-      - name: Setup for JFrog
-        uses: ./.github/actions/jfrog-auth
+      - name: Setup environment
+        uses: ./.github/actions/setup-env
 
       # Integration tests are run from within tests/integration folder.
       # Create .coveragerc with correct relative path to source code.
@@ -118,6 +112,7 @@ jobs:
           [run]
           source = ../../src
           relative_files = true
+          parallel = true
           EOF
 
       - name: Run integration tests on serverless cluster
@@ -143,9 +138,11 @@ jobs:
           use_oidc: true
 
   e2e:
-    if: github.event_name == 'pull_request' && !github.event.pull_request.draft && !github.event.pull_request.head.repo.fork
+    needs: not-a-fork
     environment: tool
-    runs-on: larger
+    runs-on:
+      group: larger-runners
+      labels: larger
     permissions:
       id-token: write
       pull-requests: write
@@ -155,18 +152,12 @@ jobs:
         with:
           fetch-depth: 0
 
-      - name: Setup uv
-        uses: astral-sh/setup-uv@5a095e7a2014a4212f075830d4f7277575a9d098  # v7.3.1
-        with:
-          version: "0.11.2"
-          checksum: "7ac2ca0449c8d68dae9b99e635cd3bc9b22a4cb1de64b7c43716398447d42981"
-
-      - name: Setup for JFrog
-        uses: ./.github/actions/jfrog-auth
+      - name: Setup environment
+        uses: ./.github/actions/setup-env
 
       # Required for DAB (Databricks Asset Bundle) e2e tests
       - name: Install Databricks CLI
-        uses: databricks/setup-cli@acd0e77a1ed7f15f528faca1e1f7f5590bcfdff8 # v0.296.0
+        uses: databricks/setup-cli@596b0a354ba14aa59921aca1b02bd67c2b0a81a5 # v0.297.2
 
       - name: Run e2e tests
         uses: databrickslabs/sandbox/acceptance@3313d06ce86227537b3f37f5974f7eecb2a8e59a # acceptance/v0.4.4
@@ -181,9 +172,11 @@ jobs:
           ARM_TENANT_ID: ${{ secrets.ARM_TENANT_ID }}
 
   e2e_serverless:
-    if: github.event_name == 'pull_request' && !github.event.pull_request.draft && !github.event.pull_request.head.repo.fork
+    needs: not-a-fork
     environment: tool
-    runs-on: larger
+    runs-on:
+      group: larger-runners
+      labels: larger
     permissions:
       id-token: write
       pull-requests: write
@@ -195,18 +188,12 @@ jobs:
         with:
           fetch-depth: 0
 
-      - name: Setup uv
-        uses: astral-sh/setup-uv@5a095e7a2014a4212f075830d4f7277575a9d098  # v7.3.1
-        with:
-          version: "0.11.2"
-          checksum: "7ac2ca0449c8d68dae9b99e635cd3bc9b22a4cb1de64b7c43716398447d42981"
-
-      - name: Setup for JFrog
-        uses: ./.github/actions/jfrog-auth
+      - name: Setup environment
+        uses: ./.github/actions/setup-env
 
       # Required for DAB (Databricks Asset Bundle) e2e tests
       - name: Install Databricks CLI
-        uses: databricks/setup-cli@acd0e77a1ed7f15f528faca1e1f7f5590bcfdff8 # v0.296.0
+        uses: databricks/setup-cli@596b0a354ba14aa59921aca1b02bd67c2b0a81a5 # v0.297.2
 
       - name: Run e2e tests on serverless cluster
         uses: databrickslabs/sandbox/acceptance@3313d06ce86227537b3f37f5974f7eecb2a8e59a # acceptance/v0.4.4

.github/workflows/anomaly.yml

@@ -5,9 +5,6 @@ on:
     types: [ opened, synchronize, ready_for_review ]
   merge_group:
     types: [ checks_requested ]
-  push:
-    branches:
-      - main
 
 permissions:
   contents: read
@@ -17,10 +14,23 @@ concurrency:
   cancel-in-progress: false  # don't cancel ongoing runs to ensure fixtures are completed and resources terminated
 
 jobs:
-  anomaly-tests:
+  # Gate all downstream jobs behind a single check so PRs from forks (no access to the
+  # tool environment) and draft PRs do not trigger the expensive anomaly suite.
+  # PRs from forks are to be tested by the reviewer(s) / maintainer(s) before merging.
+  not-a-fork:
+    runs-on:
+      group: databrickslabs-protected-runner-group
+      labels: linux-ubuntu-latest
     if: github.event_name == 'pull_request' && !github.event.pull_request.draft && !github.event.pull_request.head.repo.fork
+    steps:
+      - run: echo "Not a fork PR, proceeding"
+
+  anomaly-tests:
+    needs: not-a-fork
     environment: tool
-    runs-on: larger
+    runs-on:
+      group: larger-runners
+      labels: larger
     permissions:
       id-token: write
       pull-requests: write
@@ -30,14 +40,8 @@ jobs:
         with:
           fetch-depth: 0
 
-      - name: Setup uv
-        uses: astral-sh/setup-uv@5a095e7a2014a4212f075830d4f7277575a9d098  # v7.3.1
-        with:
-          version: "0.11.2"
-          checksum: "7ac2ca0449c8d68dae9b99e635cd3bc9b22a4cb1de64b7c43716398447d42981"
-
-      - name: Setup for JFrog
-        uses: ./.github/actions/jfrog-auth
+      - name: Setup environment
+        uses: ./.github/actions/setup-env
 
       # Create .coveragerc with correct relative path to source code.
       - name: Prepare code coverage configuration for anomaly tests
@@ -46,6 +50,7 @@ jobs:
           [run]
           source = ../../src
           relative_files = true
+          parallel = true
           EOF
 
       - name: Run anomaly integration tests
@@ -76,9 +81,11 @@ jobs:
           use_oidc: true
 
   anomaly-tests-serverless:
-    if: github.event_name == 'pull_request' && !github.event.pull_request.draft && !github.event.pull_request.head.repo.fork
+    needs: not-a-fork
     environment: tool
-    runs-on: larger
+    runs-on:
+      group: larger-runners
+      labels: larger
     permissions:
       id-token: write
       pull-requests: write
@@ -88,14 +95,8 @@ jobs:
         with:
           fetch-depth: 0
 
-      - name: Setup uv
-        uses: astral-sh/setup-uv@5a095e7a2014a4212f075830d4f7277575a9d098  # v7.3.1
-        with:
-          version: "0.11.2"
-          checksum: "7ac2ca0449c8d68dae9b99e635cd3bc9b22a4cb1de64b7c43716398447d42981"
-
-      - name: Setup for JFrog
-        uses: ./.github/actions/jfrog-auth
+      - name: Setup environment
+        uses: ./.github/actions/setup-env
 
       # Create .coveragerc with correct relative path to source code.
       - name: Prepare code coverage configuration for anomaly tests
@@ -104,6 +105,7 @@ jobs:
           [run]
           source = ../../src
           relative_files = true
+          parallel = true
           EOF
 
       - name: Run anomaly integration tests on serverless cluster

.github/workflows/docs-release.yml

@@ -25,18 +25,12 @@ jobs:
         with:
           fetch-depth: 0
 
-      - name: Setup uv
-        uses: astral-sh/setup-uv@5a095e7a2014a4212f075830d4f7277575a9d098  # v7.3.1
-        with:
-          version: "0.11.2"
-          checksum: "7ac2ca0449c8d68dae9b99e635cd3bc9b22a4cb1de64b7c43716398447d42981"
-
-      - name: Setup for JFrog
-        uses: ./.github/actions/jfrog-auth
+      - name: Setup environment
+        uses: ./.github/actions/setup-env
 
       - uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0
         with:
-          node-version: 20
+          node-version: 22
           cache: yarn
           cache-dependency-path: docs/dqx/yarn.lock # need to put the lockfile path explicitly