ci: run Python checks on ARC runners (#234)

Author: sipercai

.github/workflows/generate_workflows.py

@@ -9,8 +9,21 @@
 
 tox_ini_path = Path(__file__).parent.parent.parent.joinpath("tox.ini")
 workflows_directory_path = Path(__file__).parent
+arc_runner_label = "loongsuite-python-arc"
 
-generate_test_workflow(tox_ini_path, workflows_directory_path, "ubuntu-latest")
-generate_lint_workflow(tox_ini_path, workflows_directory_path)
+generate_test_workflow(
+    tox_ini_path,
+    workflows_directory_path,
+    arc_runner_label,
+    batch_size=4,
+    parallelism=4,
+)
+generate_lint_workflow(
+    tox_ini_path,
+    workflows_directory_path,
+    arc_runner_label,
+    batch_size=4,
+    parallelism=4,
+)
 generate_misc_workflow(tox_ini_path, workflows_directory_path)
 generate_contrib_workflow(workflows_directory_path)

.github/workflows/generate_workflows_lib/src/generate_workflows_lib/__init__.py

@@ -16,6 +16,17 @@
 _tox_contrib_env_regex = re_compile(
     r"py39-test-(?P<name>[-\w]+\w)-?(?P<contrib_requirements>\d+)?"
 )
+_OS_ALIASES = {
+    "ubuntu-latest": "Ubuntu",
+    "windows-latest": "Windows",
+    "loongsuite-python-arc": "ARC",
+}
+
+
+def _slug(value: str) -> str:
+    return (
+        str(value).lower().replace(".", "").replace("-", "_").replace(" ", "_")
+    )
 
 
 def _get_job_package_name(name: str, package_names=None) -> str:
@@ -59,8 +70,6 @@ def get_test_job_datas(
     operating_systems: list,
     package_names=None,
 ) -> list:
-    os_alias = {"ubuntu-latest": "Ubuntu", "windows-latest": "Windows"}
-
     python_version_alias = {
         "pypy3": "pypy-3.9",
         "pypy310": "pypy-3.10",
@@ -103,7 +112,7 @@ def get_test_job_datas(
                         f"{groups['name']}"
                         f"{test_requirements}"
                         f"{aliased_python_version} "
-                        f"{os_alias[operating_system]}"
+                        f"{_OS_ALIASES[operating_system]}"
                     ),
                     "package": _get_job_package_name(
                         groups["name"],
@@ -193,9 +202,80 @@ def get_misc_job_datas(tox_envs: list) -> list:
     return misc_job_datas
 
 
+def _batch_job_datas(
+    job_datas: list,
+    batch_size: int,
+    parallelism: int,
+    group_keys=(),
+) -> list:
+    if batch_size < 1:
+        raise ValueError("batch_size must be >= 1")
+
+    if parallelism < 1:
+        raise ValueError("parallelism must be >= 1")
+
+    grouped_job_datas = {}
+    for job_data in job_datas:
+        key = tuple(job_data.get(group_key) for group_key in group_keys)
+        grouped_job_datas.setdefault(key, []).append(job_data)
+
+    batched_job_datas = []
+    batch_index = 0
+
+    for group in grouped_job_datas.values():
+        for chunk_start in range(0, len(group), batch_size):
+            chunk = group[chunk_start : chunk_start + batch_size]
+            first_job_data = chunk[0]
+            tox_envs = [job_data["tox_env"] for job_data in chunk]
+
+            if len(chunk) == 1:
+                batched_job_data = dict(first_job_data)
+                batched_job_data["tox_envs"] = tox_envs
+                batched_job_data["parallelism"] = 1
+                batched_job_data["env_count"] = 1
+                batched_job_datas.append(batched_job_data)
+                continue
+
+            batch_index += 1
+            name_parts = [
+                _slug(first_job_data[group_key])
+                for group_key in group_keys
+                if group_key in first_job_data
+            ]
+            name_suffix = "_".join(name_parts) if name_parts else "tox"
+
+            if "python_version" in first_job_data and "os" in first_job_data:
+                ui_prefix = (
+                    f"{first_job_data['python_version']} "
+                    f"{_OS_ALIASES.get(first_job_data['os'], first_job_data['os'])}"
+                )
+            else:
+                ui_prefix = "tox"
+
+            batched_job_data = dict(first_job_data)
+            batched_job_data["name"] = f"batch_{batch_index:03d}_{name_suffix}"
+            batched_job_data["ui_name"] = (
+                f"{ui_prefix} batch {batch_index:03d} ({len(chunk)} envs)"
+            )
+            batched_job_data["tox_env"] = tox_envs[0]
+            batched_job_data["tox_envs"] = tox_envs
+            batched_job_data["parallelism"] = min(parallelism, len(chunk))
+            batched_job_data["env_count"] = len(chunk)
+            batched_job_datas.append(batched_job_data)
+
+    return batched_job_datas
+
+
 def _generate_workflow(
-    job_datas: list, name: str, workflow_directory_path: Path, max_jobs=250
+    job_datas: list,
+    name: str,
+    workflow_directory_path: Path,
+    max_jobs=250,
+    runner="ubuntu-latest",
 ):
+    for stale_workflow_path in workflow_directory_path.glob(f"{name}_*.yml"):
+        stale_workflow_path.unlink()
+
     # Github seems to limit the amount of jobs in a workflow file, that is why
     # they are split in groups of 250 per workflow file.
     for file_number, job_datas in enumerate(
@@ -210,16 +290,32 @@ def _generate_workflow(
             test_yml_file.write(
                 Environment(loader=FileSystemLoader(Path(__file__).parent))
                 .get_template(f"{name}.yml.j2")
-                .render(job_datas=job_datas, file_number=file_number)
+                .render(
+                    job_datas=job_datas,
+                    file_number=file_number,
+                    runner=runner,
+                )
             )
             test_yml_file.write("\n")
 
 
 def generate_test_workflow(
-    tox_ini_path: Path, workflow_directory_path: Path, *operating_systems
+    tox_ini_path: Path,
+    workflow_directory_path: Path,
+    *operating_systems,
+    batch_size=1,
+    parallelism=1,
 ) -> None:
+    job_datas = get_test_job_datas(
+        get_tox_envs(tox_ini_path), operating_systems
+    )
     _generate_workflow(
-        get_test_job_datas(get_tox_envs(tox_ini_path), operating_systems),
+        _batch_job_datas(
+            job_datas,
+            batch_size=batch_size,
+            parallelism=parallelism,
+            group_keys=("os", "python_version"),
+        ),
         "test",
         workflow_directory_path,
     )
@@ -228,11 +324,20 @@ def generate_test_workflow(
 def generate_lint_workflow(
     tox_ini_path: Path,
     workflow_directory_path: Path,
+    runner="ubuntu-latest",
+    batch_size=1,
+    parallelism=1,
 ) -> None:
+    job_datas = get_lint_job_datas(get_tox_envs(tox_ini_path))
     _generate_workflow(
-        get_lint_job_datas(get_tox_envs(tox_ini_path)),
+        _batch_job_datas(
+            job_datas,
+            batch_size=batch_size,
+            parallelism=parallelism,
+        ),
         "lint",
         workflow_directory_path,
+        runner=runner,
     )
 
 
@@ -293,6 +398,7 @@ def generate_extension_test_workflow(
     workflow_directory_path: Path,
     additional_config_path: Path,
     *operating_systems,
+    control_runner="ubuntu-latest",
 ) -> None:
     loongsuite_envs = get_loongsuite_tox_envs(additional_config_path)
     if not loongsuite_envs:
@@ -310,13 +416,16 @@ def generate_extension_test_workflow(
         "loongsuite_test",
         "loongsuite_test",
         workflow_directory_path,
+        control_runner=control_runner,
     )
 
 
 def generate_extension_lint_workflow(
     tox_ini_path: Path,
     workflow_directory_path: Path,
     additional_config_path: Path,
+    runner="ubuntu-latest",
+    control_runner="ubuntu-latest",
 ) -> None:
     loongsuite_envs = get_loongsuite_tox_envs(additional_config_path)
     if not loongsuite_envs:
@@ -327,6 +436,8 @@ def generate_extension_lint_workflow(
         "loongsuite_lint",
         "loongsuite_lint",
         workflow_directory_path,
+        runner=runner,
+        control_runner=control_runner,
     )
 
 
@@ -353,6 +464,8 @@ def _generate_workflow_with_template(
     template_name: str,
     workflow_directory_path: Path,
     max_jobs=250,
+    runner="ubuntu-latest",
+    control_runner="ubuntu-latest",
 ):
     if (
         name in {"loongsuite_lint", "loongsuite_test"}
@@ -364,6 +477,9 @@ def _generate_workflow_with_template(
             "cross-workflow aggregate before allowing generated chunks."
         )
 
+    for stale_workflow_path in workflow_directory_path.glob(f"{name}_*.yml"):
+        stale_workflow_path.unlink()
+
     # Github seems to limit the amount of jobs in a workflow file, that is why
     # they are split in groups of 250 per workflow file.
     for file_number, job_datas in enumerate(
@@ -378,6 +494,11 @@ def _generate_workflow_with_template(
             test_yml_file.write(
                 Environment(loader=FileSystemLoader(Path(__file__).parent))
                 .get_template(f"{template_name}.yml.j2")
-                .render(job_datas=job_datas, file_number=file_number)
+                .render(
+                    job_datas=job_datas,
+                    file_number=file_number,
+                    runner=runner,
+                    control_runner=control_runner,
+                )
             )
             test_yml_file.write("\n")

.github/workflows/generate_workflows_lib/src/generate_workflows_lib/lint.yml.j2

@@ -5,16 +5,15 @@ name: Lint {{ file_number }}
 
 on:
   push:
-    branches-ignore:
-    - 'release/*'
-    - 'otelbot/*'
+    branches:
+    - main
   pull_request:
 
 permissions:
   contents: read
 
 concurrency:
-  group: ${% raw %}{{ github.workflow }}-${{ github.head_ref || github.run_id }}{% endraw %}
+  group: ${% raw %}{{ github.workflow }}-${{ github.event_name == 'pull_request' && format('pr-{0}', github.event.pull_request.number) || github.ref || github.run_id }}{% endraw %}
   cancel-in-progress: true
 
 env:
@@ -36,7 +35,7 @@ jobs:
 
   {{ job_data.name }}:
     name: {{ job_data.ui_name }}
-    runs-on: ubuntu-latest
+    runs-on: {{ runner }}
     timeout-minutes: 30
     steps:
       - name: Checkout repo @ SHA - ${% raw %}{{ github.sha }}{% endraw %}
@@ -51,5 +50,9 @@ jobs:
         run: pip install tox-uv
 
       - name: Run tests
-        run: tox -e {{ job_data.tox_env }}
+        {%- if job_data.env_count == 1 %}
+        run: tox -e {{ job_data.tox_envs[0] }}
+        {%- else %}
+        run: tox run-parallel -p {{ job_data.parallelism }} -e {{ job_data.tox_envs | join(',') }}
+        {%- endif %}
   {%- endfor %}

.github/workflows/generate_workflows_lib/src/generate_workflows_lib/loongsuite_lint.yml.j2

@@ -15,7 +15,7 @@ permissions:
   contents: read
 
 concurrency:
-  group: ${% raw %}{{ github.workflow }}-${{ github.head_ref || github.run_id }}{% endraw %}
+  group: ${% raw %}{{ github.workflow }}-${{ github.event_name == 'pull_request' && format('pr-{0}', github.event.pull_request.number) || github.ref || github.run_id }}{% endraw %}
   cancel-in-progress: true
 
 env:
@@ -34,7 +34,7 @@ env:
 jobs:
   loongsuite_changes:
     name: LoongSuite changed packages
-    runs-on: ubuntu-latest
+    runs-on: {{ control_runner }}
     outputs:
       full: ${% raw %}{{ steps.detect.outputs.full }}{% endraw %}
       packages: ${% raw %}{{ steps.detect.outputs.packages }}{% endraw %}
@@ -105,7 +105,7 @@ jobs:
     name: LoongSuite ${% raw %}{{ matrix.ui_name }}{% endraw %}
     needs: loongsuite_changes
     if: ${% raw %}{{ needs.loongsuite_changes.outputs.has_jobs == 'true' }}{% endraw %}
-    runs-on: ubuntu-latest
+    runs-on: {{ runner }}
     timeout-minutes: 30
     strategy:
       fail-fast: false
@@ -131,7 +131,7 @@ jobs:
       - loongsuite_changes
       - loongsuite_lint
     if: ${% raw %}{{ always() }}{% endraw %}
-    runs-on: ubuntu-latest
+    runs-on: {{ control_runner }}
     steps:
       - name: Check LoongSuite lint result
         shell: bash

.github/workflows/generate_workflows_lib/src/generate_workflows_lib/loongsuite_misc.yml.j2

@@ -15,7 +15,7 @@ permissions:
   contents: read
 
 concurrency:
-  group: ${% raw %}{{ github.workflow }}-${{ github.head_ref || github.run_id }}{% endraw %}
+  group: ${% raw %}{{ github.workflow }}-${{ github.event_name == 'pull_request' && format('pr-{0}', github.event.pull_request.number) || github.ref || github.run_id }}{% endraw %}
   cancel-in-progress: true
 
 env:

.github/workflows/generate_workflows_lib/src/generate_workflows_lib/loongsuite_test.yml.j2

@@ -15,7 +15,7 @@ permissions:
   contents: read
 
 concurrency:
-  group: ${% raw %}{{ github.workflow }}-${{ github.head_ref || github.run_id }}{% endraw %}
+  group: ${% raw %}{{ github.workflow }}-${{ github.event_name == 'pull_request' && format('pr-{0}', github.event.pull_request.number) || github.ref || github.run_id }}{% endraw %}
   cancel-in-progress: true
 
 env:
@@ -34,7 +34,7 @@ env:
 jobs:
   loongsuite_changes:
     name: LoongSuite changed packages
-    runs-on: ubuntu-latest
+    runs-on: {{ control_runner }}
     outputs:
       full: ${% raw %}{{ steps.detect.outputs.full }}{% endraw %}
       packages: ${% raw %}{{ steps.detect.outputs.packages }}{% endraw %}
@@ -137,7 +137,7 @@ jobs:
       - loongsuite_changes
       - loongsuite_test
     if: ${% raw %}{{ always() }}{% endraw %}
-    runs-on: ubuntu-latest
+    runs-on: {{ control_runner }}
     steps:
       - name: Check LoongSuite test result
         shell: bash

.github/workflows/generate_workflows_lib/src/generate_workflows_lib/misc.yml.j2

@@ -14,7 +14,7 @@ permissions:
   contents: read
 
 concurrency:
-  group: ${% raw %}{{ github.workflow }}-${{ github.head_ref || github.run_id }}{% endraw %}
+  group: ${% raw %}{{ github.workflow }}-${{ github.event_name == 'pull_request' && format('pr-{0}', github.event.pull_request.number) || github.ref || github.run_id }}{% endraw %}
   cancel-in-progress: true
 
 env: