data-migration-accelerator/.github/workflows/databricks-asset-bundle-deploy.yml at main · thisisqubika/data-migration-accelerator · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
name: Deploy Migration Accelerator through Databricks Asset Bundles

on:
  push:
    branches: [ main, develop ]
  pull_request:
    branches: [ main, develop ]
    types: [ opened, synchronize, reopened ]
  workflow_dispatch:

env:
  DATABRICKS_HOST: ${{ secrets.DATABRICKS_HOST }}
  DATABRICKS_CLIENT_ID: ${{ secrets.DATABRICKS_CLIENT_ID }}
  DATABRICKS_CLIENT_SECRET: ${{ secrets.DATABRICKS_CLIENT_SECRET }}
  DATABRICKS_AUTH_TYPE: oauth-m2m
  POETRY_VERSION: "2.2.1"
  PYTHON_VERSION: "3.12"

jobs:
  deploy-bundle:
    runs-on: ubuntu-latest

    concurrency:
      group: deploy-app-dbx-job-executor-app-dev
      cancel-in-progress: true

    steps:
    - name: Checkout code
      uses: actions/checkout@v4
      with:
        fetch-depth: 0

    - name: Set up Python
      uses: actions/setup-python@v4
      with:
        python-version: ${{ env.PYTHON_VERSION }}

    - name: Install pipx and Poetry
      run: |
        python -m pip install --upgrade pip pipx
        pipx install poetry==${{ env.POETRY_VERSION }}
        poetry --version

    - name: Install project dependencies
      run: |
        poetry install --no-interaction --no-ansi

    - name: Install Databricks CLI
      uses: databricks/setup-cli@main

    - name: Validate bundle
      run: |
        databricks bundle validate -t dev \
          --var="cluster_id=${{ secrets.DATABRICKS_CLUSTER_ID }}" \
          --var="catalog_name=${{ secrets.UC_CATALOG }}" \
          --var="devs_group=${{ secrets.DEVS_GROUP }}"

    - name: Deploy bundle
      run: |
        databricks bundle deploy -t dev \
          --var="cluster_id=${{ secrets.DATABRICKS_CLUSTER_ID }}" \
          --var="catalog_name=${{ secrets.UC_CATALOG }}" \
          --var="devs_group=${{ secrets.DEVS_GROUP }}"

    - name: Run job executor app
      run: |
        databricks bundle run databricks_job_executor_app -t dev \
          --var="cluster_id=${{ secrets.DATABRICKS_CLUSTER_ID }}" \
          --var="catalog_name=${{ secrets.UC_CATALOG }}" \
          --var="devs_group=${{ secrets.DEVS_GROUP }}"

    - name: Show bundle info
      run: |
        databricks bundle summary -t dev \
          --var="cluster_id=${{ secrets.DATABRICKS_CLUSTER_ID }}" \
          --var="catalog_name=${{ secrets.UC_CATALOG }}" \
          --var="devs_group=${{ secrets.DEVS_GROUP }}"
        databricks bundle resources -t dev \
          --var="cluster_id=${{ secrets.DATABRICKS_CLUSTER_ID }}" \
          --var="catalog_name=${{ secrets.UC_CATALOG }}" \
          --var="devs_group=${{ secrets.DEVS_GROUP }}"