workflows: run ML tests on ubuntu-latest

In this commit, we run ML tests requiring
docker-in-docker environment on `ubuntu-latest`. The
motivation behind this is Milvus itests was requiring
docker-in-docker environment due to setup issues on
beam CI self-hosted environment.

Author: mohamedawnallah

.github/workflows/beam_PostCommit_Python.yml

@@ -54,14 +54,20 @@ env:
 jobs:
   beam_PostCommit_Python:
     name: ${{ matrix.job_name }} (${{ matrix.job_phrase }} ${{ matrix.python_version }})
-    runs-on: [self-hosted, ubuntu-20.04, highmem22]
+    runs-on: ${{ matrix.os }}
     timeout-minutes: 240
     strategy:
       fail-fast: false
       matrix:
         job_name: [beam_PostCommit_Python]
         job_phrase: [Run Python PostCommit]
         python_version: ['3.9', '3.10', '3.11', '3.12']
+        # Run on both self-hosted and GitHub-hosted runners.
+        # Some tests (marked require_docker_in_docker) can't run on Beam's
+        # self-hosted runners due to Docker-in-Docker environment constraint.
+        # These tests will only execute on ubuntu-latest (GitHub-hosted).
+        # Context: https://github.com/apache/beam/pull/35585
+        os: [[self-hosted, ubuntu-20.04, highmem22], [ubuntu-latest]]
     if: |
       github.event_name == 'workflow_dispatch' ||
       github.event_name == 'pull_request_target' ||
@@ -99,6 +105,11 @@ jobs:
           arguments: |
             -Pjava21Home=$JAVA_HOME_21_X64 \
             -PuseWheelDistribution \
+            -Pposargs="${{
+              contains(matrix.os, 'self-hosted') &&
+                '-m ''not require_docker_in_docker''' ||
+                '-m ''require_docker_in_docker'''
+            }}" \
             -PpythonVersion=${{ matrix.python_version }} \
         env:
           CLOUDSDK_CONFIG: ${{ env.KUBELET_GCLOUD_CONFIG_PATH}}

.github/workflows/beam_PreCommit_Python_ML.yml

@@ -58,14 +58,20 @@ env:
 jobs:
   beam_PreCommit_Python_ML:
     name: ${{ matrix.job_name }} (${{ matrix.job_phrase }} ${{ matrix.python_version }})
-    runs-on: [self-hosted, ubuntu-20.04, main]
+    runs-on: ${{ matrix.os }}
     timeout-minutes: 180
     strategy:
       fail-fast: false
       matrix:
         job_name: ['beam_PreCommit_Python_ML']
         job_phrase: ['Run Python_ML PreCommit']
         python_version: ['3.9','3.10','3.11','3.12']
+        # Run on both self-hosted and GitHub-hosted runners.
+        # Some tests (marked require_docker_in_docker) can't run on Beam's
+        # self-hosted runners due to Docker-in-Docker environment constraint.
+        # These tests will only execute on ubuntu-latest (GitHub-hosted).
+        # Context: https://github.com/apache/beam/pull/35585
+        os: [[self-hosted, ubuntu-20.04, main], [ubuntu-latest]]
     if: |
       github.event_name == 'push' ||
       github.event_name == 'pull_request_target' ||
@@ -96,7 +102,11 @@ jobs:
         with:
           gradle-command: :sdks:python:test-suites:tox:py${{steps.set_py_ver_clean.outputs.py_ver_clean}}:testPy${{steps.set_py_ver_clean.outputs.py_ver_clean}}ML
           arguments: |
-            -Pposargs=apache_beam/ml/ \
+            -Pposargs="${{
+              contains(matrix.os, 'self-hosted') &&
+                'apache_beam/ml/ -m ''not require_docker_in_docker''' ||
+                'apache_beam/ml/ -m ''require_docker_in_docker'''
+            }}" \
             -PpythonVersion=${{ matrix.python_version }}
       - name: Archive Python Test Results
         uses: actions/upload-artifact@v4