diff --git a/.github/workflows/py-cli-e2e-tests.yml b/.github/workflows/py-cli-e2e-tests.yml index 0f47482c00f7..d41bba0fc6bd 100644 --- a/.github/workflows/py-cli-e2e-tests.yml +++ b/.github/workflows/py-cli-e2e-tests.yml @@ -18,7 +18,7 @@ on: e2e-tests: description: "E2E Tests to run" required: True - default: '["bigquery", "dbt_redshift", "metabase", "mssql", "mysql", "redash", "snowflake", "tableau", "python", "redshift", "quicksight", "datalake_s3", "postgres", "oracle", "athena", "bigquery_multiple_project"]' + default: '["bigquery", "dbt_redshift", "metabase", "mssql", "mysql", "redash", "snowflake", "tableau", "python-unittests", "python-integration", "redshift", "quicksight", "datalake_s3", "postgres", "oracle", "athena", "bigquery_multiple_project"]' debug: description: "If Debugging the Pipeline, Slack and Sonar events won't be triggered [default, true or false]. Default will trigger only on main branch." required: False @@ -45,11 +45,12 @@ jobs: strategy: fail-fast: false matrix: - e2e-test: ${{ fromJSON(inputs.e2e-tests || '["bigquery", "dbt_redshift", "metabase", "mssql", "mysql", "redash", "snowflake", "tableau", "python", "redshift", "quicksight", "datalake_s3", "postgres", "oracle", "athena", "bigquery_multiple_project"]') }} + e2e-test: ${{ fromJSON(inputs.e2e-tests || '["bigquery", "dbt_redshift", "metabase", "mssql", "mysql", "redash", "snowflake", "tableau", "python-unittests", "python-integration", "redshift", "quicksight", "datalake_s3", "postgres", "oracle", "athena", "bigquery_multiple_project"]') }} environment: test steps: - name: Free Disk Space (Ubuntu) + if: matrix.e2e-test != 'python-unittests' uses: jlumbroso/free-disk-space@main with: tool-cache: false @@ -71,21 +72,47 @@ jobs: role-session-name: github-ci-aws-e2e-tests aws-region: ${{ secrets.E2E_AWS_REGION }} + - name: Setup Openmetadata Test Environment (without server) + if: matrix.e2e-test == 'python-unittests' + uses: ./.github/actions/setup-openmetadata-test-environment + with: + python-version: '3.10' + install-server: 'false' + - name: Setup Openmetadata Test Environment + if: matrix.e2e-test != 'python-unittests' uses: ./.github/actions/setup-openmetadata-test-environment with: python-version: '3.10' + - name: Run Python Unit Tests + if: matrix.e2e-test == 'python-unittests' + id: python-unittest + continue-on-error: true + run: | + source env/bin/activate + cd ingestion + nox --no-venv -s unit-tests + shell: bash + + - name: Rename coverage file for Python unit tests + if: matrix.e2e-test == 'python-unittests' && steps.python-unittest.outcome == 'success' && env.DEBUG == 'false' + run: mv ingestion/.coverage .coverage.python-unittests - - name: Run Python Tests & record coverage - if: matrix.e2e-test == 'python' - id: python-e2e-test + - name: Run Python Integration Tests + if: matrix.e2e-test == 'python-integration' + id: python-integration-test + continue-on-error: true run: | source env/bin/activate - make coverage + cd ingestion + nox --no-venv -s integration-tests -- --standalone --durations=5 + env: + TESTCONTAINERS_RYUK_DISABLED: true + shell: bash - name: Run CLI E2E Python Tests & record coverage - if: matrix.e2e-test != 'python' + if: matrix.e2e-test != 'python-unittests' && matrix.e2e-test != 'python-integration' id: e2e-test continue-on-error: true env: @@ -163,47 +190,62 @@ jobs: coverage combine --data-file=.coverage.$E2E_TEST --rcfile=ingestion/pyproject.toml --keep -a .coverage* coverage report --rcfile ingestion/pyproject.toml --data-file .coverage.$E2E_TEST || true - - name: Upload coverage artifact for Python tests - if: matrix.e2e-test == 'python' && steps.python-e2e-test.outcome == 'success' && env.DEBUG == 'false' + - name: Upload coverage artifact for Python unit tests + if: matrix.e2e-test == 'python-unittests' && steps.python-unittest.outcome == 'success' && env.DEBUG == 'false' + uses: actions/upload-artifact@v4 + with: + name: coverage-${{ matrix.e2e-test }} + path: .coverage.python-unittests + include-hidden-files: true + + - name: Rename coverage file for Python integration tests + if: matrix.e2e-test == 'python-integration' && steps.python-integration-test.outcome == 'success' && env.DEBUG == 'false' + run: mv ingestion/.coverage .coverage.python-integration + + - name: Upload coverage artifact for Python integration tests + if: matrix.e2e-test == 'python-integration' && steps.python-integration-test.outcome == 'success' && env.DEBUG == 'false' uses: actions/upload-artifact@v4 with: name: coverage-${{ matrix.e2e-test }} - path: .coverage + path: .coverage.python-integration + include-hidden-files: true - name: Upload coverage artifact for CLI E2E tests - if: matrix.e2e-test != 'python' && steps.e2e-test.outcome == 'success' && env.DEBUG == 'false' + if: matrix.e2e-test != 'python-unittests' && matrix.e2e-test != 'python-integration' && steps.e2e-test.outcome == 'success' && env.DEBUG == 'false' uses: actions/upload-artifact@v4 with: name: coverage-${{ matrix.e2e-test }} path: .coverage.${{ matrix.e2e-test }} + include-hidden-files: true - name: Upload tests artifact - if: steps.e2e-test.outcome == 'success' || steps.python-e2e-test.outcome == 'success' && env.DEBUG == 'false' + if: (steps.e2e-test.outcome == 'success' || steps.python-unittest.outcome == 'success' || steps.python-integration-test.outcome == 'success') && env.DEBUG == 'false' uses: actions/upload-artifact@v4 with: name: tests-${{ matrix.e2e-test }} path: ingestion/junit/test-results-*.xml - name: Clean Up + if: matrix.e2e-test != 'python-unittests' run: | cd ./docker/development docker compose down --remove-orphans sudo rm -rf ${PWD}/docker-volume - + - name: Slack on Failure - if: steps.e2e-test.outcome != 'success' && steps.python-e2e-test.outcome != 'success' && env.DEBUG == 'false' + if: (steps.e2e-test.outcome == 'failure' || steps.python-unittest.outcome == 'failure' || steps.python-integration-test.outcome == 'failure') && env.DEBUG == 'false' uses: slackapi/slack-github-action@v1.23.0 with: payload: | { - "text": "🔥 Failed E2E Test for: ${{ matrix.e2e-test }} 🔥\nLogs: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" - } + "text": "🔥 Failed E2E Test for: ${{ matrix.e2e-test }} 🔥\nLogs: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" + } env: SLACK_WEBHOOK_URL: ${{ secrets.E2E_SLACK_WEBHOOK }} SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK - name: Force failure - if: steps.e2e-test.outcome != 'success' && steps.python-e2e-test.outcome != 'success' + if: steps.e2e-test.outcome == 'failure' || steps.python-unittest.outcome == 'failure' || steps.python-integration-test.outcome == 'failure' run: | exit 1 @@ -235,7 +277,7 @@ jobs: make install_all install_test - name: Download all artifacts - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: path: artifacts diff --git a/.github/workflows/py-sonarcloud-nightly.yml b/.github/workflows/py-sonarcloud-nightly.yml index fe3214d80046..f583bfa0e063 100644 --- a/.github/workflows/py-sonarcloud-nightly.yml +++ b/.github/workflows/py-sonarcloud-nightly.yml @@ -29,9 +29,71 @@ env: PROJECT_BASE_DIR: "ingestion" jobs: - ingestion-sonarcloud-analysis: + py-unit-tests: + name: Unit Tests + timeout-minutes: 60 + runs-on: ubuntu-latest + steps: + - name: Free Disk Space (Ubuntu) + uses: jlumbroso/free-disk-space@main + with: + tool-cache: false + android: true + dotnet: true + haskell: true + large-packages: false + swap-storage: true + docker-images: false + + - name: Checkout + uses: actions/checkout@v4 + with: + ref: ${{ env.BRANCH_NAME }} + fetch-depth: 0 + + - name: Setup Openmetadata Test Environment + uses: ./.github/actions/setup-openmetadata-test-environment + with: + python-version: ${{ env.PYTHON_VERSION }} + install-server: 'false' + + - name: Run Unit Tests + run: | + source env/bin/activate + cd ingestion + nox --no-venv -s unit-tests + shell: bash + + - name: Upload coverage artifact + if: ${{ !cancelled() }} + uses: actions/upload-artifact@v4 + with: + name: coverage-unit + path: ingestion/.coverage + include-hidden-files: true + + py-integration-tests: + name: "Integration Tests (${{ matrix.shard.name }})" timeout-minutes: 180 runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + shard: + - name: "shard-1" + nox-args: >- + tests/integration/ometa + tests/integration/postgres + tests/integration/mysql + tests/integration/profiler + tests/integration/data_quality + - name: "shard-2" + nox-args: >- + --ignore=tests/integration/ometa + --ignore=tests/integration/postgres + --ignore=tests/integration/mysql + --ignore=tests/integration/profiler + --ignore=tests/integration/data_quality steps: - name: Free Disk Space (Ubuntu) uses: jlumbroso/free-disk-space@main @@ -57,11 +119,22 @@ jobs: args: "-m no-ui" ingestion_dependency: "mysql,elasticsearch,sample-data" - - name: Run Python Tests & Record Coverage + - name: Run Integration Tests run: | source env/bin/activate - make coverage - rm pom.xml + cd ingestion + nox --no-venv -s integration-tests -- --standalone --durations=5 ${{ matrix.shard.nox-args }} + env: + TESTCONTAINERS_RYUK_DISABLED: true + shell: bash + + - name: Upload coverage artifact + if: ${{ !cancelled() }} + uses: actions/upload-artifact@v4 + with: + name: coverage-integration-${{ matrix.shard.name }} + path: ingestion/.coverage + include-hidden-files: true - name: Clean Up if: ${{ !cancelled() }} @@ -70,6 +143,61 @@ jobs: docker compose down --remove-orphans sudo rm -rf ${PWD}/docker-volume + py-combine-coverage: + if: ${{ !cancelled() }} + needs: [py-unit-tests, py-integration-tests] + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + ref: ${{ env.BRANCH_NAME }} + fetch-depth: 0 + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + + - name: Install uv + run: pip install uv + shell: bash + + - name: Install coverage + run: | + python3 -m venv env + source env/bin/activate + uv pip install "coverage[toml]" nox + shell: bash + + - name: Download coverage artifacts + uses: actions/download-artifact@v4 + with: + pattern: coverage-* + path: ingestion/coverage-data/ + + - name: Prepare coverage files + run: | + cd ingestion + [ -f coverage-data/coverage-unit/.coverage ] && mv coverage-data/coverage-unit/.coverage .coverage.unit + for dir in coverage-data/coverage-integration-*/; do + shard=$(basename "$dir" | sed 's/coverage-integration-//') + [ -f "$dir/.coverage" ] && mv "$dir/.coverage" ".coverage.integration-$shard" + done + shell: bash + + - name: Combine coverage + run: | + source env/bin/activate + cd ingestion + nox --no-venv -s combine-coverage + shell: bash + + - name: Remove pom.xml + run: rm pom.xml + shell: bash + - name: Push Results To Sonar if: ${{ !cancelled() }} id: push-to-sonar