lakeflow_framework/.github/workflows/ci.yml at main · databricks-solutions/lakeflow_framework · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
# Pull-request and branch CI for the Lakeflow Framework repository.
#
# Design: one workflow, parallel jobs. Path-filtered jobs skip when unrelated files
# change. Sample dataflow specs are validated with scripts/validate_dataflows.py
# when files under samples/ change.
#
# | Job               | When it runs                          |
# |-------------------|---------------------------------------|
# | changes           | Always (path filter outputs)          |
# | pytest            | Always on PR / push                   |
# | docs-spelling     | docs/** changed                       |
# | docs-html         | docs/** changed                       |
# | validate-samples  | samples/** changed                    |

name: CI

on:
  pull_request:
    branches:
      - main
  push:
    branches:
      - main

permissions:
  contents: read
  id-token: write

concurrency:
  group: ci-${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: true

jobs:
  changes:
    name: Detect changes
    runs-on:
      group: databricks-solutions-protected-runner-group
      labels: linux-ubuntu-latest
    outputs:
      docs: ${{ steps.filter.outputs.docs }}
      samples: ${{ steps.filter.outputs.samples }}
    steps:
      - name: Checkout
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
        with:
          fetch-depth: 0

      - name: Path filters
        id: filter
        uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2
        with:
          filters: |
            docs:
              - 'docs/**'
              - 'requirements-docs.txt'
              - 'requirements-docs.lock'
            samples:
              - 'samples/**'

  pytest:
    name: Unit tests
    runs-on:
      group: databricks-solutions-protected-runner-group
      labels: linux-ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2

      - name: Set up Python
        uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
        with:
          python-version: '3.12'

      - name: Configure JFrog PyPI registry (OIDC)
        uses: ./.github/actions/setup-jfrog
        with:
          configure-pip: 'true'

      - name: Install dev dependencies
        run: |
          python -m pip install --upgrade pip
          pip install --require-hashes --no-deps -r requirements-dev.lock

      - name: Run pytest
        run: |
          pytest tests/ -m "not integration and not spark" -q

  docs-spelling:
    name: Docs spelling
    needs: changes
    if: needs.changes.outputs.docs == 'true'
    runs-on:
      group: databricks-solutions-protected-runner-group
      labels: linux-ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2

      - name: Set up Python
        uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
        with:
          python-version: '3.12'

      - name: Configure JFrog PyPI registry (OIDC)
        uses: ./.github/actions/setup-jfrog
        with:
          configure-pip: 'true'

      - name: Install spelling dependencies
        run: |
          sudo apt-get update -qq
          sudo apt-get install -y -qq aspell-en libenchant-2-dev
          python -m pip install --upgrade pip
          pip install --require-hashes --no-deps -r requirements-docs.lock

      - name: Docs spelling
        run: bash scripts/ci/docs_spelling_check.sh

  docs-html:
    name: Docs HTML build
    needs: changes
    if: needs.changes.outputs.docs == 'true'
    runs-on:
      group: databricks-solutions-protected-runner-group
      labels: linux-ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2

      - name: Set up Python
        uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
        with:
          python-version: '3.12'

      - name: Configure JFrog PyPI registry (OIDC)
        uses: ./.github/actions/setup-jfrog
        with:
          configure-pip: 'true'

      - name: Install docs dependencies
        run: |
          python -m pip install --upgrade pip
          pip install --require-hashes --no-deps -r requirements-docs.lock

      - name: make html (warning budget)
        run: bash scripts/ci/docs_html_check.sh 19

  validate-samples:
    name: Validate sample dataflows
    needs: changes
    if: needs.changes.outputs.samples == 'true'
    runs-on:
      group: databricks-solutions-protected-runner-group
      labels: linux-ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2

      - name: Set up Python
        uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
        with:
          python-version: '3.12'

      - name: Configure JFrog PyPI registry (OIDC)
        uses: ./.github/actions/setup-jfrog
        with:
          configure-pip: 'true'

      - name: Install validator dependencies
        run: |
          python -m pip install --upgrade pip
          pip install --require-hashes --no-deps -r requirements.lock

      - name: Validate sample dataflows
        run: python scripts/validate_dataflows.py samples/