Skip to content

Commit f96eb23

Browse files
SumanMaharanaCopilotulixius9
authored and
pmbrull
committed
chore(ci): enhance Python E2E and SonarCloud workflows with unit and and integration tests (#26481)
* chore(ci): enhance Python E2E and SonarCloud workflows with unit and integration tests * seperate the unit and integration test * address commensts * Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> * Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> * Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> * Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> * Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> * Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> * Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> * address comments --------- Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> Co-authored-by: ulixius9 <mayursingal9@gmail.com>
1 parent 421c9cd commit f96eb23

2 files changed

Lines changed: 192 additions & 22 deletions

File tree

‎.github/workflows/py-cli-e2e-tests.yml‎

Lines changed: 60 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ on:
1818
e2e-tests:
1919
description: "E2E Tests to run"
2020
required: True
21-
default: '["bigquery", "dbt_redshift", "metabase", "mssql", "mysql", "redash", "snowflake", "tableau", "python", "redshift", "quicksight", "datalake_s3", "postgres", "oracle", "athena", "bigquery_multiple_project"]'
21+
default: '["bigquery", "dbt_redshift", "metabase", "mssql", "mysql", "redash", "snowflake", "tableau", "python-unittests", "python-integration", "redshift", "quicksight", "datalake_s3", "postgres", "oracle", "athena", "bigquery_multiple_project"]'
2222
debug:
2323
description: "If Debugging the Pipeline, Slack and Sonar events won't be triggered [default, true or false]. Default will trigger only on main branch."
2424
required: False
@@ -45,11 +45,12 @@ jobs:
4545
strategy:
4646
fail-fast: false
4747
matrix:
48-
e2e-test: ${{ fromJSON(inputs.e2e-tests || '["bigquery", "dbt_redshift", "metabase", "mssql", "mysql", "redash", "snowflake", "tableau", "python", "redshift", "quicksight", "datalake_s3", "postgres", "oracle", "athena", "bigquery_multiple_project"]') }}
48+
e2e-test: ${{ fromJSON(inputs.e2e-tests || '["bigquery", "dbt_redshift", "metabase", "mssql", "mysql", "redash", "snowflake", "tableau", "python-unittests", "python-integration", "redshift", "quicksight", "datalake_s3", "postgres", "oracle", "athena", "bigquery_multiple_project"]') }}
4949
environment: test
5050

5151
steps:
5252
- name: Free Disk Space (Ubuntu)
53+
if: matrix.e2e-test != 'python-unittests'
5354
uses: jlumbroso/free-disk-space@main
5455
with:
5556
tool-cache: false
@@ -71,21 +72,47 @@ jobs:
7172
role-session-name: github-ci-aws-e2e-tests
7273
aws-region: ${{ secrets.E2E_AWS_REGION }}
7374

75+
- name: Setup Openmetadata Test Environment (without server)
76+
if: matrix.e2e-test == 'python-unittests'
77+
uses: ./.github/actions/setup-openmetadata-test-environment
78+
with:
79+
python-version: '3.10'
80+
install-server: 'false'
81+
7482
- name: Setup Openmetadata Test Environment
83+
if: matrix.e2e-test != 'python-unittests'
7584
uses: ./.github/actions/setup-openmetadata-test-environment
7685
with:
7786
python-version: '3.10'
7887

88+
- name: Run Python Unit Tests
89+
if: matrix.e2e-test == 'python-unittests'
90+
id: python-unittest
91+
continue-on-error: true
92+
run: |
93+
source env/bin/activate
94+
cd ingestion
95+
nox --no-venv -s unit-tests
96+
shell: bash
97+
98+
- name: Rename coverage file for Python unit tests
99+
if: matrix.e2e-test == 'python-unittests' && steps.python-unittest.outcome == 'success' && env.DEBUG == 'false'
100+
run: mv ingestion/.coverage .coverage.python-unittests
79101

80-
- name: Run Python Tests & record coverage
81-
if: matrix.e2e-test == 'python'
82-
id: python-e2e-test
102+
- name: Run Python Integration Tests
103+
if: matrix.e2e-test == 'python-integration'
104+
id: python-integration-test
105+
continue-on-error: true
83106
run: |
84107
source env/bin/activate
85-
make coverage
108+
cd ingestion
109+
nox --no-venv -s integration-tests -- --standalone --durations=5
110+
env:
111+
TESTCONTAINERS_RYUK_DISABLED: true
112+
shell: bash
86113

87114
- name: Run CLI E2E Python Tests & record coverage
88-
if: matrix.e2e-test != 'python'
115+
if: matrix.e2e-test != 'python-unittests' && matrix.e2e-test != 'python-integration'
89116
id: e2e-test
90117
continue-on-error: true
91118
env:
@@ -163,47 +190,62 @@ jobs:
163190
coverage combine --data-file=.coverage.$E2E_TEST --rcfile=ingestion/pyproject.toml --keep -a .coverage*
164191
coverage report --rcfile ingestion/pyproject.toml --data-file .coverage.$E2E_TEST || true
165192
166-
- name: Upload coverage artifact for Python tests
167-
if: matrix.e2e-test == 'python' && steps.python-e2e-test.outcome == 'success' && env.DEBUG == 'false'
193+
- name: Upload coverage artifact for Python unit tests
194+
if: matrix.e2e-test == 'python-unittests' && steps.python-unittest.outcome == 'success' && env.DEBUG == 'false'
195+
uses: actions/upload-artifact@v4
196+
with:
197+
name: coverage-${{ matrix.e2e-test }}
198+
path: .coverage.python-unittests
199+
include-hidden-files: true
200+
201+
- name: Rename coverage file for Python integration tests
202+
if: matrix.e2e-test == 'python-integration' && steps.python-integration-test.outcome == 'success' && env.DEBUG == 'false'
203+
run: mv ingestion/.coverage .coverage.python-integration
204+
205+
- name: Upload coverage artifact for Python integration tests
206+
if: matrix.e2e-test == 'python-integration' && steps.python-integration-test.outcome == 'success' && env.DEBUG == 'false'
168207
uses: actions/upload-artifact@v4
169208
with:
170209
name: coverage-${{ matrix.e2e-test }}
171-
path: .coverage
210+
path: .coverage.python-integration
211+
include-hidden-files: true
172212

173213
- name: Upload coverage artifact for CLI E2E tests
174-
if: matrix.e2e-test != 'python' && steps.e2e-test.outcome == 'success' && env.DEBUG == 'false'
214+
if: matrix.e2e-test != 'python-unittests' && matrix.e2e-test != 'python-integration' && steps.e2e-test.outcome == 'success' && env.DEBUG == 'false'
175215
uses: actions/upload-artifact@v4
176216
with:
177217
name: coverage-${{ matrix.e2e-test }}
178218
path: .coverage.${{ matrix.e2e-test }}
219+
include-hidden-files: true
179220

180221
- name: Upload tests artifact
181-
if: steps.e2e-test.outcome == 'success' || steps.python-e2e-test.outcome == 'success' && env.DEBUG == 'false'
222+
if: (steps.e2e-test.outcome == 'success' || steps.python-unittest.outcome == 'success' || steps.python-integration-test.outcome == 'success') && env.DEBUG == 'false'
182223
uses: actions/upload-artifact@v4
183224
with:
184225
name: tests-${{ matrix.e2e-test }}
185226
path: ingestion/junit/test-results-*.xml
186227

187228
- name: Clean Up
229+
if: matrix.e2e-test != 'python-unittests'
188230
run: |
189231
cd ./docker/development
190232
docker compose down --remove-orphans
191233
sudo rm -rf ${PWD}/docker-volume
192-
234+
193235
- name: Slack on Failure
194-
if: steps.e2e-test.outcome != 'success' && steps.python-e2e-test.outcome != 'success' && env.DEBUG == 'false'
236+
if: (steps.e2e-test.outcome == 'failure' || steps.python-unittest.outcome == 'failure' || steps.python-integration-test.outcome == 'failure') && env.DEBUG == 'false'
195237
uses: slackapi/slack-github-action@v1.23.0
196238
with:
197239
payload: |
198240
{
199-
"text": "🔥 Failed E2E Test for: ${{ matrix.e2e-test }} 🔥\nLogs: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
200-
}
241+
"text": "🔥 Failed E2E Test for: ${{ matrix.e2e-test }} 🔥\nLogs: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
242+
}
201243
env:
202244
SLACK_WEBHOOK_URL: ${{ secrets.E2E_SLACK_WEBHOOK }}
203245
SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK
204246

205247
- name: Force failure
206-
if: steps.e2e-test.outcome != 'success' && steps.python-e2e-test.outcome != 'success'
248+
if: steps.e2e-test.outcome == 'failure' || steps.python-unittest.outcome == 'failure' || steps.python-integration-test.outcome == 'failure'
207249
run: |
208250
exit 1
209251
@@ -235,7 +277,7 @@ jobs:
235277
make install_all install_test
236278
237279
- name: Download all artifacts
238-
uses: actions/download-artifact@v3
280+
uses: actions/download-artifact@v4
239281
with:
240282
path: artifacts
241283

‎.github/workflows/py-sonarcloud-nightly.yml‎

Lines changed: 132 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -29,9 +29,71 @@ env:
2929
PROJECT_BASE_DIR: "ingestion"
3030

3131
jobs:
32-
ingestion-sonarcloud-analysis:
32+
py-unit-tests:
33+
name: Unit Tests
34+
timeout-minutes: 60
35+
runs-on: ubuntu-latest
36+
steps:
37+
- name: Free Disk Space (Ubuntu)
38+
uses: jlumbroso/free-disk-space@main
39+
with:
40+
tool-cache: false
41+
android: true
42+
dotnet: true
43+
haskell: true
44+
large-packages: false
45+
swap-storage: true
46+
docker-images: false
47+
48+
- name: Checkout
49+
uses: actions/checkout@v4
50+
with:
51+
ref: ${{ env.BRANCH_NAME }}
52+
fetch-depth: 0
53+
54+
- name: Setup Openmetadata Test Environment
55+
uses: ./.github/actions/setup-openmetadata-test-environment
56+
with:
57+
python-version: ${{ env.PYTHON_VERSION }}
58+
install-server: 'false'
59+
60+
- name: Run Unit Tests
61+
run: |
62+
source env/bin/activate
63+
cd ingestion
64+
nox --no-venv -s unit-tests
65+
shell: bash
66+
67+
- name: Upload coverage artifact
68+
if: ${{ !cancelled() }}
69+
uses: actions/upload-artifact@v4
70+
with:
71+
name: coverage-unit
72+
path: ingestion/.coverage
73+
include-hidden-files: true
74+
75+
py-integration-tests:
76+
name: "Integration Tests (${{ matrix.shard.name }})"
3377
timeout-minutes: 180
3478
runs-on: ubuntu-latest
79+
strategy:
80+
fail-fast: false
81+
matrix:
82+
shard:
83+
- name: "shard-1"
84+
nox-args: >-
85+
tests/integration/ometa
86+
tests/integration/postgres
87+
tests/integration/mysql
88+
tests/integration/profiler
89+
tests/integration/data_quality
90+
- name: "shard-2"
91+
nox-args: >-
92+
--ignore=tests/integration/ometa
93+
--ignore=tests/integration/postgres
94+
--ignore=tests/integration/mysql
95+
--ignore=tests/integration/profiler
96+
--ignore=tests/integration/data_quality
3597
steps:
3698
- name: Free Disk Space (Ubuntu)
3799
uses: jlumbroso/free-disk-space@main
@@ -57,11 +119,22 @@ jobs:
57119
args: "-m no-ui"
58120
ingestion_dependency: "mysql,elasticsearch,sample-data"
59121

60-
- name: Run Python Tests & Record Coverage
122+
- name: Run Integration Tests
61123
run: |
62124
source env/bin/activate
63-
make coverage
64-
rm pom.xml
125+
cd ingestion
126+
nox --no-venv -s integration-tests -- --standalone --durations=5 ${{ matrix.shard.nox-args }}
127+
env:
128+
TESTCONTAINERS_RYUK_DISABLED: true
129+
shell: bash
130+
131+
- name: Upload coverage artifact
132+
if: ${{ !cancelled() }}
133+
uses: actions/upload-artifact@v4
134+
with:
135+
name: coverage-integration-${{ matrix.shard.name }}
136+
path: ingestion/.coverage
137+
include-hidden-files: true
65138

66139
- name: Clean Up
67140
if: ${{ !cancelled() }}
@@ -70,6 +143,61 @@ jobs:
70143
docker compose down --remove-orphans
71144
sudo rm -rf ${PWD}/docker-volume
72145
146+
py-combine-coverage:
147+
if: ${{ !cancelled() }}
148+
needs: [py-unit-tests, py-integration-tests]
149+
runs-on: ubuntu-latest
150+
timeout-minutes: 10
151+
steps:
152+
- name: Checkout
153+
uses: actions/checkout@v4
154+
with:
155+
ref: ${{ env.BRANCH_NAME }}
156+
fetch-depth: 0
157+
158+
- name: Setup Python
159+
uses: actions/setup-python@v5
160+
with:
161+
python-version: ${{ env.PYTHON_VERSION }}
162+
163+
- name: Install uv
164+
run: pip install uv
165+
shell: bash
166+
167+
- name: Install coverage
168+
run: |
169+
python3 -m venv env
170+
source env/bin/activate
171+
uv pip install "coverage[toml]" nox
172+
shell: bash
173+
174+
- name: Download coverage artifacts
175+
uses: actions/download-artifact@v4
176+
with:
177+
pattern: coverage-*
178+
path: ingestion/coverage-data/
179+
180+
- name: Prepare coverage files
181+
run: |
182+
cd ingestion
183+
[ -f coverage-data/coverage-unit/.coverage ] && mv coverage-data/coverage-unit/.coverage .coverage.unit
184+
for dir in coverage-data/coverage-integration-*/; do
185+
shard=$(basename "$dir" | sed 's/coverage-integration-//')
186+
[ -f "$dir/.coverage" ] && mv "$dir/.coverage" ".coverage.integration-$shard"
187+
done
188+
shell: bash
189+
190+
- name: Combine coverage
191+
run: |
192+
source env/bin/activate
193+
cd ingestion
194+
nox --no-venv -s combine-coverage
195+
shell: bash
196+
197+
- name: Remove pom.xml
198+
run: rm pom.xml
199+
shell: bash
200+
73201
- name: Push Results To Sonar
74202
if: ${{ !cancelled() }}
75203
id: push-to-sonar

0 commit comments

Comments
 (0)