From 23a1a332a2908a13cb0dc0f5b4b0708209ff4e9a Mon Sep 17 00:00:00 2001 From: Mohamed Awnallah Date: Mon, 25 Aug 2025 20:16:15 +0000 Subject: [PATCH 1/4] workflows: capture DinD tests in PreCommit Py Coverage workflow --- .../beam_PreCommit_Python_Coverage.yml | 31 ++++++++++++++----- 1 file changed, 23 insertions(+), 8 deletions(-) diff --git a/.github/workflows/beam_PreCommit_Python_Coverage.yml b/.github/workflows/beam_PreCommit_Python_Coverage.yml index b21ad50e9da2..95bc78c9dd5f 100644 --- a/.github/workflows/beam_PreCommit_Python_Coverage.yml +++ b/.github/workflows/beam_PreCommit_Python_Coverage.yml @@ -58,35 +58,44 @@ env: jobs: beam_PreCommit_Python_Coverage: - name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) - runs-on: [self-hosted, ubuntu-20.04, highmem] + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }} ${{ matrix.python_version }}) + runs-on: ${{ matrix.os }} strategy: + fail-fast: false matrix: job_name: [beam_PreCommit_Python_Coverage] job_phrase: [Run Python_Coverage PreCommit] + python_version: ['3.9'] + # Run on both self-hosted and GitHub-hosted runners. + # Some tests (marked require_docker_in_docker) can't run on Beam's + # self-hosted runners due to Docker-in-Docker environment constraint. + # These tests will only execute on ubuntu-latest (GitHub-hosted). + # Context: https://github.com/apache/beam/pull/35585 + os: [[self-hosted, ubuntu-20.04, highmem], [ubuntu-latest]] timeout-minutes: 180 if: | github.event_name == 'push' || github.event_name == 'pull_request_target' || (github.event_name == 'schedule' && github.repository == 'apache/beam') || github.event_name == 'workflow_dispatch' || - github.event.comment.body == 'Run Python_Coverage PreCommit' + startswith(github.event.comment.body, 'Run Python_Coverage PreCommit 3.') steps: - uses: actions/checkout@v4 - name: Setup repository uses: ./.github/actions/setup-action with: - comment_phrase: ${{ matrix.job_phrase }} + comment_phrase: ${{ matrix.job_phrase }} ${{ matrix.python_version }} github_token: ${{ secrets.GITHUB_TOKEN }} - github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }} ${{ matrix.python_version }}) - name: Setup environment uses: ./.github/actions/setup-environment-action with: java-version: default - python-version: default + python-version: ${{ matrix.python_version }} - name: Start DinD uses: ./.github/actions/dind-up-action id: dind + if: contains(matrix.os, 'self-hosted') with: # Enable all the new features cleanup-dind-on-start: "true" @@ -97,9 +106,9 @@ jobs: export-gh-env: "true" - name: Run preCommitPyCoverage env: - DOCKER_HOST: ${{ steps.dind.outputs.docker-host }} + DOCKER_HOST: ${{ contains(matrix.os, 'self-hosted') && steps.dind.outputs.docker-host || '' }} TOX_TESTENV_PASSENV: "DOCKER_*,TESTCONTAINERS_*,TC_*,BEAM_*,GRPC_*,OMP_*,OPENBLAS_*,PYTHONHASHSEED,PYTEST_*" - TESTCONTAINERS_HOST_OVERRIDE: ${{ env.DIND_IP }} + TESTCONTAINERS_HOST_OVERRIDE: ${{ contains(matrix.os, 'self-hosted') && env.DIND_IP || '' }} TESTCONTAINERS_DOCKER_SOCKET_OVERRIDE: "/var/run/docker.sock" TESTCONTAINERS_RYUK_DISABLED: "false" TESTCONTAINERS_RYUK_CONTAINER_PRIVILEGED: "true" @@ -110,6 +119,12 @@ jobs: uses: ./.github/actions/gradle-command-self-hosted-action with: gradle-command: :sdks:python:test-suites:tox:py39:preCommitPyCoverage + arguments: | + -Pposargs="${{ + contains(matrix.os, 'self-hosted') && + '-m (not require_docker_in_docker)' || + '-m require_docker_in_docker' + }}" - uses: codecov/codecov-action@v3 with: flags: python From a129b7bd2f2e178f44f4f42fa9fed33efd19ec8a Mon Sep 17 00:00:00 2001 From: Mohamed Awnallah Date: Mon, 25 Aug 2025 22:37:19 +0000 Subject: [PATCH 2/4] workflows: temporarily removing `ubuntu-latest` till resolving deps --- .github/workflows/beam_PreCommit_Python_Coverage.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/beam_PreCommit_Python_Coverage.yml b/.github/workflows/beam_PreCommit_Python_Coverage.yml index 95bc78c9dd5f..018b516b9cfa 100644 --- a/.github/workflows/beam_PreCommit_Python_Coverage.yml +++ b/.github/workflows/beam_PreCommit_Python_Coverage.yml @@ -71,7 +71,8 @@ jobs: # self-hosted runners due to Docker-in-Docker environment constraint. # These tests will only execute on ubuntu-latest (GitHub-hosted). # Context: https://github.com/apache/beam/pull/35585 - os: [[self-hosted, ubuntu-20.04, highmem], [ubuntu-latest]] + # Temporary removed the ubuntu-latest env till resolving deps issues. + os: [[self-hosted, ubuntu-20.04, highmem]] timeout-minutes: 180 if: | github.event_name == 'push' || From 1bf4e08c947d4fb2bf46c721a880368c849a1dab Mon Sep 17 00:00:00 2001 From: Mohamed Awnallah Date: Tue, 26 Aug 2025 17:15:26 +0000 Subject: [PATCH 3/4] workflows: add `matrix.os` label to `beam_PreCommit_Python_Coverage` --- .github/workflows/beam_PreCommit_Python_Coverage.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/beam_PreCommit_Python_Coverage.yml b/.github/workflows/beam_PreCommit_Python_Coverage.yml index 018b516b9cfa..3da51a2eceda 100644 --- a/.github/workflows/beam_PreCommit_Python_Coverage.yml +++ b/.github/workflows/beam_PreCommit_Python_Coverage.yml @@ -58,7 +58,7 @@ env: jobs: beam_PreCommit_Python_Coverage: - name: ${{ matrix.job_name }} (${{ matrix.job_phrase }} ${{ matrix.python_version }}) + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }} ${{ matrix.python_version }}) (${{ join(matrix.os, ', ') }}) runs-on: ${{ matrix.os }} strategy: fail-fast: false @@ -87,7 +87,7 @@ jobs: with: comment_phrase: ${{ matrix.job_phrase }} ${{ matrix.python_version }} github_token: ${{ secrets.GITHUB_TOKEN }} - github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }} ${{ matrix.python_version }}) + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }} ${{ matrix.python_version }}) (${{ join(matrix.os, ', ') }}) - name: Setup environment uses: ./.github/actions/setup-environment-action with: @@ -134,7 +134,7 @@ jobs: uses: actions/upload-artifact@v4 if: failure() with: - name: Python Test Results + name: Python ${{ matrix.python_version }} Test Results (${{ join(matrix.os, ', ') }}) path: '**/pytest*.xml' - name: Publish Python Test Results env: @@ -145,4 +145,5 @@ jobs: commit: '${{ env.prsha || env.GITHUB_SHA }}' comment_mode: ${{ github.event_name == 'issue_comment' && 'always' || 'off' }} files: '**/pytest*.xml' - large_files: true \ No newline at end of file + large_files: true + check_name: "Python ${{ matrix.python_version }} Test Results (${{ join(matrix.os, ', ') }})" From fd2bfc95b9f5a1faead3e110d9dc7313194659a6 Mon Sep 17 00:00:00 2001 From: Mohamed Awnallah Date: Wed, 27 Aug 2025 20:01:59 +0000 Subject: [PATCH 4/4] workflows: remove `Pposargs` --- .github/workflows/beam_PreCommit_Python_Coverage.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.github/workflows/beam_PreCommit_Python_Coverage.yml b/.github/workflows/beam_PreCommit_Python_Coverage.yml index 3da51a2eceda..646114e7c31b 100644 --- a/.github/workflows/beam_PreCommit_Python_Coverage.yml +++ b/.github/workflows/beam_PreCommit_Python_Coverage.yml @@ -120,12 +120,6 @@ jobs: uses: ./.github/actions/gradle-command-self-hosted-action with: gradle-command: :sdks:python:test-suites:tox:py39:preCommitPyCoverage - arguments: | - -Pposargs="${{ - contains(matrix.os, 'self-hosted') && - '-m (not require_docker_in_docker)' || - '-m require_docker_in_docker' - }}" - uses: codecov/codecov-action@v3 with: flags: python