Merge pull request #212 from AlmaLinux/fix-qa

Copy cached QA repos on Nebula VMs if they exist

Author: anfimovdm

alts/shared/models.py

@@ -243,6 +243,13 @@ class ThirdPartyRepoSshHost(BaseModel):
     user: Optional[str] = None
 
 
+class CachedTestRepo(BaseModel):
+    # Baked-in repo cache (`src`) on the VM image, re-seeded into the
+    # host-aware path (`dest`) during provisioning to avoid a full clone.
+    src: str
+    dest: str
+
+
 class CeleryConfig(BaseModel):
     def __init__(self, **data):
         super().__init__(**data)
@@ -353,6 +360,10 @@ def __init__(self, **data):
     # default (current SSH user). Configured per-deployment; empty by
     # default so no `~/.ssh/config` is written unless explicitly set.
     third_party_repo_ssh_hosts: List[ThirdPartyRepoSshHost] = []
+    # Baked-in QA repo caches to re-seed into the host-aware layout during
+    # provisioning (avoids a full network clone on every VM). Empty by
+    # default; configured per-deployment to match the VM image's caches.
+    cached_test_repos: List[CachedTestRepo] = []
     tests_base_dir: str = '/tests'
     package_proxy: str = ''
     disabled_al_repos: List[str] = []

alts/worker/runners/base.py

@@ -751,6 +751,10 @@ def initial_provision(self, verbose=False):
                 entry.model_dump(exclude_none=True)
                 for entry in CONFIG.third_party_repo_ssh_hosts
             ],
+            'cached_test_repos': [
+                entry.model_dump()
+                for entry in CONFIG.cached_test_repos
+            ],
         }
         dist_major_version = self.dist_version[0]
         if self.dist_name in CONFIG.rhel_flavors and dist_major_version in ('6', '7'):

resources/roles/preparation/defaults/main.yml

@@ -6,3 +6,7 @@ centos_repo_baseurl: "https://vault.centos.org"
 # extra-var by the runner. Empty list here is a no-op fallback so the
 # role still works if invoked standalone (e.g. ansible-lint).
 third_party_repo_ssh_hosts: []
+# Source of truth lives in the alts config (`CelerConfig.cached_test_repos`)
+# and is passed in as an extra-var by the runner. Empty list here is a
+# no-op fallback so the role still works if invoked standalone.
+cached_test_repos: []

resources/roles/preparation/tasks/main.yml

@@ -62,6 +62,49 @@
   tags:
     - initial_provision
 
+# Re-seed baked-in QA caches into the host-aware layout so the test
+# runner reuses them (light `git pull`) instead of a full network clone.
+# Docker envs receive repos via `docker cp` from the worker, so skip them.
+- name: Check which baked test repo caches exist
+  ansible.builtin.stat:
+    path: "{{ item.src }}"
+  register: cached_test_repo_stats
+  loop: "{{ cached_test_repos }}"
+  when:
+    - connection_type != 'docker'
+    - cached_test_repos | length > 0
+  tags:
+    - initial_provision
+
+- name: Ensure parent directories for seeded test repos exist
+  ansible.builtin.file:
+    path: "{{ item.item.dest | dirname }}"
+    state: directory
+    mode: '0755'
+  loop: "{{ cached_test_repo_stats.results | default([]) }}"
+  loop_control:
+    label: "{{ item.item.dest }}"
+  when:
+    - not (item.skipped | default(false))
+    - item.stat.exists
+  tags:
+    - initial_provision
+
+- name: Seed host-aware test repo paths from baked caches
+  ansible.builtin.copy:
+    src: "{{ item.item.src }}/"
+    dest: "{{ item.item.dest }}"
+    remote_src: true
+    mode: preserve
+  loop: "{{ cached_test_repo_stats.results | default([]) }}"
+  loop_control:
+    label: "{{ item.item.dest }}"
+  when:
+    - not (item.skipped | default(false))
+    - item.stat.exists
+  tags:
+    - initial_provision
+
 - name: Copy tests to test environment
   copy:
     src: "{{ integrity_tests_dir }}"

tests/runners/test_provision_ssh_hosts.py

@@ -14,7 +14,7 @@
 
 from alts.worker import runners
 from alts.worker.runners.base import GenericVMRunner
-from alts.shared.models import ThirdPartyRepoSshHost
+from alts.shared.models import CachedTestRepo, ThirdPartyRepoSshHost
 
 
 def _make_runner(tmp_path):
@@ -97,3 +97,58 @@ def fake_run(cmd_args, timeout=None):
 
         extra_vars = _extract_extra_vars(captured['args'])
         assert extra_vars['third_party_repo_ssh_hosts'] == []
+
+
+class TestInitialProvisionCachedTestRepos:
+    def test_extra_vars_include_configured_caches(
+        self, tmp_path, monkeypatch,
+    ):
+        monkeypatch.setattr(
+            runners.base.CONFIG,
+            'cached_test_repos',
+            [
+                CachedTestRepo(
+                    src='/opt/QA',
+                    dest='/opt/gerrit.cloudlinux.com/QA',
+                ),
+            ],
+            raising=False,
+        )
+        captured = {}
+
+        def fake_run(cmd_args, timeout=None):
+            captured['args'] = cmd_args
+            return 0, '', ''
+
+        runner = _make_runner(tmp_path)
+        runner.run_ansible_command = fake_run
+
+        runner.initial_provision()
+
+        extra_vars = _extract_extra_vars(captured['args'])
+        assert extra_vars['cached_test_repos'] == [
+            {'src': '/opt/QA', 'dest': '/opt/gerrit.cloudlinux.com/QA'},
+        ]
+
+    def test_extra_vars_empty_when_unconfigured(
+        self, tmp_path, monkeypatch,
+    ):
+        monkeypatch.setattr(
+            runners.base.CONFIG,
+            'cached_test_repos',
+            [],
+            raising=False,
+        )
+        captured = {}
+
+        def fake_run(cmd_args, timeout=None):
+            captured['args'] = cmd_args
+            return 0, '', ''
+
+        runner = _make_runner(tmp_path)
+        runner.run_ansible_command = fake_run
+
+        runner.initial_provision()
+
+        extra_vars = _extract_extra_vars(captured['args'])
+        assert extra_vars['cached_test_repos'] == []

tests/shared/test_third_party_ssh_hosts.py

@@ -7,7 +7,11 @@
 """
 import pytest
 
-from alts.shared.models import CeleryConfig, ThirdPartyRepoSshHost
+from alts.shared.models import (
+    CachedTestRepo,
+    CeleryConfig,
+    ThirdPartyRepoSshHost,
+)
 
 
 class TestThirdPartyRepoSshHost:
@@ -35,12 +39,31 @@ def test_host_is_required(self):
             ThirdPartyRepoSshHost()
 
 
+class TestCachedTestRepo:
+    def test_requires_src_and_dest(self):
+        entry = CachedTestRepo(
+            src='/opt/QA',
+            dest='/opt/gerrit.cloudlinux.com/QA',
+        )
+        assert entry.model_dump() == {
+            'src': '/opt/QA',
+            'dest': '/opt/gerrit.cloudlinux.com/QA',
+        }
+
+    @pytest.mark.parametrize('kwargs', [{}, {'src': '/opt/QA'}])
+    def test_missing_field_raises(self, kwargs):
+        with pytest.raises(Exception):
+            CachedTestRepo(**kwargs)
+
+
 class TestCeleryConfigDefault:
-    def test_third_party_repo_ssh_hosts_defaults_to_empty(self):
-        # No built-in hosts: an unconfigured deployment writes no
-        # ~/.ssh/config (the role task is gated on a non-empty list).
-        config = CeleryConfig.__new__(CeleryConfig)
-        field = CeleryConfig.model_fields['third_party_repo_ssh_hosts']
-        # Pydantic stores the default factory / default value.
+    @pytest.mark.parametrize(
+        'field_name',
+        ['third_party_repo_ssh_hosts', 'cached_test_repos'],
+    )
+    def test_field_defaults_to_empty(self, field_name):
+        # No built-in entries: an unconfigured deployment is a no-op
+        # (the role tasks are gated on non-empty lists).
+        field = CeleryConfig.model_fields[field_name]
         default = field.get_default(call_default_factory=True)
         assert default == []