demo: lakebox #26568
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: build | |
| on: | |
| pull_request: | |
| types: [opened, synchronize] | |
| merge_group: | |
| types: [checks_requested] | |
| push: | |
| # Always run on push to main. The build cache can only be reused | |
| # if it was saved by a run from the repository's default branch. | |
| # The run result will be identical to that from the merge queue | |
| # because the commit is identical, yet we need to perform it to | |
| # seed the build cache. | |
| branches: | |
| - main | |
| schedule: | |
| - cron: '0 0,12 * * *' # Runs at 00:00 and 12:00 UTC daily | |
| env: | |
| GOTESTSUM_FORMAT: github-actions | |
| jobs: | |
| cleanups: | |
| runs-on: | |
| group: databricks-deco-testing-runner-group | |
| labels: ubuntu-latest-deco | |
| steps: | |
| - name: Clean up cache if running on schedule | |
| if: ${{ github.event_name == 'schedule' }} | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| run: gh cache delete --all --repo databricks/cli || true | |
| testmask: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| targets: ${{ steps.mask1.outputs.targets || steps.mask2.outputs.targets || steps.mask3.outputs.targets }} | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 | |
| with: | |
| fetch-depth: 0 | |
| - name: Setup Go | |
| uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0 | |
| with: | |
| go-version-file: tools/go.mod | |
| - name: Run testmask (pull requests) | |
| if: ${{ github.event_name == 'pull_request' }} | |
| id: mask1 | |
| working-directory: tools/testmask | |
| run: | | |
| go run . ${{ github.event.pull_request.head.sha }} ${{ github.event.pull_request.base.sha }} | tee output.json | |
| echo "targets=$(jq -c '.' output.json)" >> $GITHUB_OUTPUT | |
| - name: Run testmask (merge group) | |
| if: ${{ github.event_name == 'merge_group' }} | |
| id: mask2 | |
| working-directory: tools/testmask | |
| run: | | |
| go run . ${{ github.event.merge_group.head_sha }} ${{ github.event.merge_group.base_sha }} | tee output.json | |
| echo "targets=$(jq -c '.' output.json)" >> $GITHUB_OUTPUT | |
| - name: Run testmask (other events) | |
| if: ${{ github.event_name != 'pull_request' && github.event_name != 'merge_group' }} | |
| id: mask3 | |
| working-directory: tools/testmask | |
| run: | | |
| # Always run all tests | |
| echo "targets=[\"test\"]" >> $GITHUB_OUTPUT | |
| test: | |
| needs: | |
| - cleanups | |
| - testmask | |
| # Only run if the target is in the list of targets from testmask | |
| if: ${{ contains(fromJSON(needs.testmask.outputs.targets), 'test') }} | |
| name: "task test (${{matrix.os.name}}, ${{matrix.deployment}})" | |
| runs-on: ${{ matrix.os.runner }} | |
| defaults: | |
| run: | |
| shell: bash | |
| permissions: | |
| id-token: write | |
| contents: read | |
| env: | |
| TASK_CONCURRENCY: ${{ matrix.os.name == 'windows' && '1' || '' }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| # Use separate fields for the OS name and runner configuration. | |
| # When combined in a single object, "runs-on" errors with "Unexpected value 'name'". | |
| os: | |
| - name: linux | |
| runner: | |
| group: databricks-protected-runner-group-large | |
| labels: linux-ubuntu-latest-large | |
| - name: windows | |
| runner: | |
| group: databricks-protected-runner-group-large | |
| labels: windows-server-latest-large | |
| - name: macos | |
| runner: | |
| labels: macos-latest | |
| deployment: | |
| - "terraform" | |
| - "direct" | |
| # Include "event_name" in the matrix so we can include/exclude based on it. | |
| event: | |
| - ${{ github.event_name }} | |
| # Run on Linux only in merge queue to reduce time to merge. | |
| exclude: | |
| - event: merge_group | |
| os: | |
| name: windows | |
| - event: merge_group | |
| os: | |
| name: macos | |
| steps: | |
| - name: Checkout repository and submodules | |
| uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 | |
| - name: Setup build environment | |
| uses: ./.github/actions/setup-build-environment | |
| with: | |
| cache-key: test-${{ matrix.deployment }} | |
| - name: Run tests without coverage | |
| # We run tests without coverage on PR, merge_group, and schedule because we don't make use of coverage information | |
| # and would like to run the tests as fast as possible. We run it on schedule as well, because that is what | |
| # populates the cache and cache may include test results. | |
| if: ${{ github.event_name == 'pull_request' || github.event_name == 'merge_group' || github.event_name == 'schedule' }} | |
| env: | |
| ENVFILTER: DATABRICKS_BUNDLE_ENGINE=${{ matrix.deployment }} | |
| run: go tool -modfile=tools/task/go.mod task test | |
| - name: Run tests with coverage | |
| # Only run 'task cover' on push to main to make sure it does not get broken. | |
| if: ${{ github.event_name == 'push' }} | |
| env: | |
| ENVFILTER: DATABRICKS_BUNDLE_ENGINE=${{ matrix.deployment }} | |
| run: go tool -modfile=tools/task/go.mod task cover | |
| - name: Upload gotestsum JSON output | |
| # Always upload so we can inspect timing even if tests fail. | |
| if: ${{ always() }} | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| with: | |
| name: test-output-${{ matrix.os.name }}-${{ matrix.deployment }} | |
| path: test-output.json | |
| if-no-files-found: warn | |
| retention-days: 7 | |
| - name: Check out.test.toml files are up to date | |
| run: | | |
| if ! git diff --exit-code; then | |
| echo "ERROR: detected changed files in the repository; Most likely you have out.test.toml files that are out of date. Run 'go test ./acceptance -run \"^TestAccept$\" -only-out-test-toml' to update." | |
| exit 1 | |
| fi | |
| test-exp-aitools: | |
| needs: | |
| - cleanups | |
| - testmask | |
| # Only run if the target is in the list of targets from testmask | |
| if: ${{ contains(fromJSON(needs.testmask.outputs.targets), 'test-exp-aitools') }} | |
| name: "task test-exp-aitools (${{matrix.os.name}})" | |
| runs-on: ${{ matrix.os.runner }} | |
| defaults: | |
| run: | |
| shell: bash | |
| permissions: | |
| id-token: write | |
| contents: read | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: | |
| - name: linux | |
| runner: | |
| group: databricks-protected-runner-group-large | |
| labels: linux-ubuntu-latest-large | |
| - name: macos | |
| runner: | |
| labels: macos-latest | |
| # The Windows tests are broken; see https://github.com/databricks/cli/pull/4024. | |
| # - name: windows | |
| # runner: | |
| # group: databricks-protected-runner-group-large | |
| # labels: windows-server-latest-large | |
| steps: | |
| - name: Checkout repository and submodules | |
| uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 | |
| - name: Setup build environment | |
| uses: ./.github/actions/setup-build-environment | |
| with: | |
| cache-key: test-exp-aitools | |
| - name: Run tests | |
| run: | | |
| go tool -modfile=tools/task/go.mod task test-exp-aitools | |
| test-exp-ssh: | |
| needs: | |
| - cleanups | |
| - testmask | |
| # Only run if the target is in the list of targets from testmask | |
| if: ${{ contains(fromJSON(needs.testmask.outputs.targets), 'test-exp-ssh') }} | |
| name: "task test-exp-ssh (${{matrix.os.name}})" | |
| runs-on: ${{ matrix.os.runner }} | |
| defaults: | |
| run: | |
| shell: bash | |
| permissions: | |
| id-token: write | |
| contents: read | |
| env: | |
| TASK_CONCURRENCY: ${{ matrix.os.name == 'windows' && '1' || '' }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: | |
| - name: linux | |
| runner: | |
| group: databricks-protected-runner-group-large | |
| labels: linux-ubuntu-latest-large | |
| - name: windows | |
| runner: | |
| group: databricks-protected-runner-group-large | |
| labels: windows-server-latest-large | |
| - name: macos | |
| runner: | |
| labels: macos-latest | |
| steps: | |
| - name: Checkout repository and submodules | |
| uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 | |
| - name: Setup build environment | |
| uses: ./.github/actions/setup-build-environment | |
| with: | |
| cache-key: test-exp-ssh | |
| - name: Run tests | |
| run: | | |
| go tool -modfile=tools/task/go.mod task test-exp-ssh | |
| test-pipelines: | |
| needs: | |
| - cleanups | |
| - testmask | |
| # Only run if the target is in the list of targets from testmask | |
| if: ${{ contains(fromJSON(needs.testmask.outputs.targets), 'test-pipelines') }} | |
| name: "task test-pipelines (${{matrix.os.name}})" | |
| runs-on: ${{ matrix.os.runner }} | |
| defaults: | |
| run: | |
| shell: bash | |
| permissions: | |
| id-token: write | |
| contents: read | |
| env: | |
| TASK_CONCURRENCY: ${{ matrix.os.name == 'windows' && '1' || '' }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: | |
| - name: linux | |
| runner: | |
| group: databricks-protected-runner-group-large | |
| labels: linux-ubuntu-latest-large | |
| - name: windows | |
| runner: | |
| group: databricks-protected-runner-group-large | |
| labels: windows-server-latest-large | |
| - name: macos | |
| runner: | |
| labels: macos-latest | |
| steps: | |
| - name: Checkout repository and submodules | |
| uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 | |
| - name: Setup build environment | |
| uses: ./.github/actions/setup-build-environment | |
| with: | |
| cache-key: test-pipelines | |
| - name: Run tests | |
| run: | | |
| go tool -modfile=tools/task/go.mod task test-pipelines | |
| # This job groups the result of all the above test jobs. | |
| # It is a required check, so it blocks auto-merge and the merge queue. | |
| # | |
| # We use `if: always()` to ensure this job runs even when dependencies are skipped. | |
| # Without it, GitHub Actions skips jobs whose dependencies are skipped, which would | |
| # incorrectly block the merge queue when optional test jobs don't run. | |
| # | |
| # The step checks `contains(needs.*.result, 'failure')` to fail if any dependency failed. | |
| # Reference: https://github.com/orgs/community/discussions/25970 | |
| test-result: | |
| needs: | |
| - test | |
| - test-exp-aitools | |
| - test-exp-ssh | |
| - test-pipelines | |
| if: ${{ always() }} | |
| name: test-result | |
| runs-on: ubuntu-latest | |
| steps: | |
| - run: | | |
| if [[ "${{ contains(needs.*.result, 'failure') }}" == "true" ]]; then | |
| echo "One or more required jobs failed ❌" | |
| exit 1 | |
| fi | |
| echo "All tests passed ✅" | |
| validate-generated-is-up-to-date: | |
| needs: cleanups | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 | |
| - name: Setup Go | |
| uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0 | |
| with: | |
| go-version-file: go.mod | |
| # Use different schema from regular job, to avoid overwriting the same key | |
| cache-dependency-path: | | |
| go.sum | |
| bundle/internal/schema/*.* | |
| - name: Verify that the schema is up to date | |
| run: | | |
| if ! ( go tool -modfile=tools/task/go.mod task --force generate-schema && git diff --exit-code ); then | |
| echo "The schema is not up to date. Please run './task generate-schema' and commit the changes." | |
| exit 1 | |
| fi | |
| - name: Verify that the generated enum and required fields are up to date | |
| run: | | |
| if ! ( go tool -modfile=tools/task/go.mod task --force generate-validation && git diff --exit-code ); then | |
| echo "The generated enum and required fields are not up to date. Please run './task generate-validation' and commit the changes." | |
| exit 1 | |
| fi | |
| validate-python-codegen: | |
| needs: cleanups | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 | |
| - name: Setup Go | |
| uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0 | |
| with: | |
| go-version-file: go.mod | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0 | |
| with: | |
| version: "0.6.5" | |
| - name: Verify that python/codegen is up to date | |
| run: |- | |
| go tool -modfile=tools/task/go.mod task pydabs-codegen | |
| if ! ( git diff --exit-code ); then | |
| echo "Generated Python code is not up-to-date. Please run './task pydabs-codegen' and commit the changes." | |
| exit 1 | |
| fi | |
| # Trigger integration tests in a separate repository. | |
| # Writes the same-org "Integration Tests" check run for skip/auto-approve | |
| # paths on deco runners. The cross-org `gh workflow run` dispatch is split | |
| # into the sibling `trigger-tests` job so it can run on emu-access runners | |
| # that are allowlisted in the databricks-eng org. | |
| integration-trigger: | |
| needs: | |
| - testmask | |
| if: >- | |
| (github.event_name == 'pull_request' && !github.event.pull_request.head.repo.fork && github.actor != 'dependabot[bot]') || | |
| (github.event_name == 'merge_group') | |
| runs-on: | |
| group: databricks-deco-testing-runner-group | |
| labels: ubuntu-latest-deco | |
| permissions: | |
| checks: write | |
| contents: read | |
| environment: "test-trigger-is" | |
| steps: | |
| - name: Generate GitHub App Token (check runs) | |
| if: >- | |
| (github.event_name == 'merge_group') || | |
| (github.event_name == 'pull_request' && !contains(fromJSON(needs.testmask.outputs.targets), 'test') && !contains(fromJSON(needs.testmask.outputs.targets), 'test-exp-ssh')) | |
| id: generate-check-token | |
| uses: actions/create-github-app-token@f8d387b68d61c58ab83c6c016672934102569859 # v3.0.0 | |
| with: | |
| app-id: ${{ secrets.DECO_TEST_APPROVAL_APP_ID }} | |
| private-key: ${{ secrets.DECO_TEST_APPROVAL_PRIVATE_KEY }} | |
| # DECO_TEST_APPROVAL is installed on the databricks org (not databricks-eng). | |
| owner: databricks | |
| repositories: cli | |
| # Skip integration tests if the primary "test" target is not triggered by this change. | |
| # Use Checks API (not Statuses API) to match the required "Integration Tests" check. | |
| - name: Skip integration tests (pull request) | |
| if: ${{ github.event_name == 'pull_request' && !contains(fromJSON(needs.testmask.outputs.targets), 'test') && !contains(fromJSON(needs.testmask.outputs.targets), 'test-exp-ssh') }} | |
| uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 | |
| with: | |
| github-token: ${{ steps.generate-check-token.outputs.token }} | |
| script: | | |
| await github.rest.checks.create({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| name: 'Integration Tests', | |
| head_sha: '${{ github.event.pull_request.head.sha }}', | |
| status: 'completed', | |
| conclusion: 'success', | |
| output: { | |
| title: 'Integration Tests', | |
| summary: '⏭️ Skipped (changes do not require integration tests)' | |
| } | |
| }); | |
| # Auto-approve for merge group since tests already passed on the PR. | |
| - name: Auto-approve for merge group | |
| if: ${{ github.event_name == 'merge_group' }} | |
| uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 | |
| with: | |
| github-token: ${{ steps.generate-check-token.outputs.token }} | |
| script: | | |
| await github.rest.checks.create({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| name: 'Integration Tests', | |
| head_sha: context.sha, | |
| status: 'completed', | |
| conclusion: 'success', | |
| output: { | |
| title: 'Integration Tests', | |
| summary: '⏭️ Auto-approved for merge queue (tests already passed on PR)' | |
| } | |
| }); | |
| # Cross-org dispatch to databricks-eng/eng-dev-ecosystem. Must run on an | |
| # emu-access runner because the databricks-eng org IP-allowlists only the | |
| # release runner group, not deco. See databricks/databricks-sdk-go#1638. | |
| trigger-tests: | |
| needs: | |
| - testmask | |
| if: >- | |
| (github.event_name == 'pull_request' && !github.event.pull_request.head.repo.fork && github.actor != 'dependabot[bot]' && (contains(fromJSON(needs.testmask.outputs.targets), 'test') || contains(fromJSON(needs.testmask.outputs.targets), 'test-exp-ssh'))) || | |
| (github.event_name == 'push') | |
| runs-on: | |
| group: databricks-release-runner-group-emu-access | |
| labels: linux-ubuntu-latest-emu-access | |
| permissions: | |
| contents: read | |
| environment: "test-trigger-is" | |
| steps: | |
| - name: Generate GitHub App Token | |
| id: generate-token | |
| uses: actions/create-github-app-token@f8d387b68d61c58ab83c6c016672934102569859 # v3.0.0 | |
| with: | |
| app-id: ${{ secrets.DECO_WORKFLOW_TRIGGER_APP_ID }} | |
| private-key: ${{ secrets.DECO_WORKFLOW_TRIGGER_PRIVATE_KEY }} | |
| owner: ${{ secrets.ORG_NAME }} | |
| repositories: ${{ secrets.REPO_NAME }} | |
| - name: Trigger integration tests (pull request) | |
| if: ${{ github.event_name == 'pull_request' }} | |
| env: | |
| GH_TOKEN: ${{ steps.generate-token.outputs.token }} | |
| run: |- | |
| gh workflow run cli-isolated-pr.yml -R ${{ secrets.ORG_NAME }}/${{ secrets.REPO_NAME }} \ | |
| --ref main \ | |
| -f pull_request_number=${{ github.event.pull_request.number }} \ | |
| -f commit_sha=${{ github.event.pull_request.head.sha }} | |
| - name: Trigger integration tests (push to main) | |
| if: ${{ github.event_name == 'push' }} | |
| env: | |
| GH_TOKEN: ${{ steps.generate-token.outputs.token }} | |
| run: |- | |
| gh workflow run cli-isolated-nightly.yml -R ${{ secrets.ORG_NAME }}/${{ secrets.REPO_NAME }} \ | |
| --ref main \ | |
| -f commit_sha=${{ github.event.after }} | |
| # Skip integration tests for dependabot PRs. | |
| # Dependabot has no access to the "test-trigger-is" environment secrets, | |
| # so we use the built-in GITHUB_TOKEN to mark the required "Integration | |
| # Tests" check as passed. | |
| integration-trigger-dependabot: | |
| if: >- | |
| github.event_name == 'pull_request' && | |
| github.actor == 'dependabot[bot]' | |
| runs-on: | |
| group: databricks-deco-testing-runner-group | |
| labels: ubuntu-latest-deco | |
| permissions: | |
| checks: write | |
| steps: | |
| - name: Skip integration tests | |
| uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 | |
| with: | |
| script: |- | |
| await github.rest.checks.create({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| name: 'Integration Tests', | |
| head_sha: '${{ github.event.pull_request.head.sha }}', | |
| status: 'completed', | |
| conclusion: 'success', | |
| output: { | |
| title: 'Integration Tests', | |
| summary: '⏭️ Skipped (dependabot PR)' | |
| } | |
| }); |