RHOAIENG-69301: fix disconnected env job submission: tier1 (3.4)

Author: pawelpaszki

tests/e2e/mnist_raycluster_sdk_oauth_test.py

@@ -142,13 +142,10 @@ def assert_jobsubmit_withoutLogin(self, cluster):
             # API endpoint is directly under the hostname
             api_url = dashboard_url + "/api/jobs/"
 
+        job_spec = get_mnist_job_submission_spec()
         jobdata = {
-            "entrypoint": "python mnist.py",
-            "runtime_env": {
-                "working_dir": "./tests/e2e/",
-                "pip": "./tests/e2e/mnist_pip_requirements.txt",
-                "env_vars": get_setup_env_variables(),
-            },
+            "entrypoint": job_spec["entrypoint"],
+            "runtime_env": job_spec["runtime_env"],
         }
 
         # Try to submit a job without authentication
@@ -382,13 +379,11 @@ def assert_jobsubmit_withlogin(self, cluster):
                 "Verified: No jobs exist from the previous unauthenticated submission attempt."
             )
 
+        job_spec = get_mnist_job_submission_spec()
+        print(f"Submitting job: {job_spec['entrypoint']}")
         submission_id = client.submit_job(
-            entrypoint="python mnist.py",
-            runtime_env={
-                "working_dir": "./tests/e2e/",
-                "pip": "./tests/e2e/mnist_pip_requirements.txt",
-                "env_vars": get_setup_env_variables(),
-            },
+            entrypoint=job_spec["entrypoint"],
+            runtime_env=job_spec["runtime_env"],
             entrypoint_num_cpus=1,
         )
         print(f"Submitted job with ID: {submission_id}")
@@ -414,9 +409,10 @@ def assert_jobsubmit_withlogin(self, cluster):
         client.delete_job(submission_id)
 
     def assert_job_completion(self, status):
-        if status == "SUCCEEDED":
-            print(f"Job has completed: '{status}'")
+        status_value = getattr(status, "value", status)
+        if status_value == "SUCCEEDED":
+            print(f"Job has completed: '{status_value}'")
             assert True
         else:
-            print(f"Job has completed: '{status}'")
+            print(f"Job has completed: '{status_value}'")
             assert False

tests/e2e/support.py

@@ -165,44 +165,12 @@ def _env_flag_disabled(name):
     return os.environ.get(name, "").strip().lower() in ("0", "false", "no", "off")
 
 
-def _has_registry_mirror_configured():
-    """
-    Detect mirror-only registry layout typical of disconnected OpenShift installs.
-
-    Checks ImageDigestMirrorSet (OCP 4.14+) and ImageContentSourcePolicy (legacy).
-    """
-    try:
-        from kubernetes import client as k8s_client
-
-        custom_api = k8s_client.CustomObjectsApi()
-        mirror_types = [
-            ("config.openshift.io", "v1", "imagedigestmirrorsets"),
-            ("operator.openshift.io", "v1alpha1", "imagecontentsourcepolicies"),
-        ]
-        for group, version, plural in mirror_types:
-            try:
-                result = custom_api.list_cluster_custom_object(
-                    group=group,
-                    version=version,
-                    plural=plural,
-                )
-                if result.get("items"):
-                    return True
-            except Exception:
-                continue
-    except Exception:
-        pass
-    return False
-
-
 def _disconnected_cluster_signals():
     """Return True when the cluster is likely disconnected / air-gapped."""
     if _env_flag_enabled("DISCONNECTED_CLUSTER") or _env_flag_enabled(
         "IS_DISCONNECTED_CLUSTER"
     ):
         return True
-    if _has_registry_mirror_configured():
-        return True
     try:
         server = (run_oc_command(["whoami", "--show-server=true"]) or "").lower()
         if "-dis-" in server or "disconnected" in server:
@@ -212,7 +180,7 @@ def _disconnected_cluster_signals():
     return False
 
 
-def _upgrade_mnist_prerequisites_met():
+def _mnist_prerequisites_met():
     """
     Return True when full MNIST can run (pip packages + dataset reachable).
 
@@ -229,49 +197,42 @@ def _upgrade_mnist_prerequisites_met():
     if pip_ok and aws_ok:
         print(
             "Disconnected cluster with PIP mirror and S3 endpoint configured; "
-            "using full MNIST upgrade job"
+            "using full MNIST job"
         )
         return True
     return False
 
 
-def use_upgrade_smoke_job():
+def use_smoke_job():
     """
-    Use a lightweight Ray job for upgrade tests when full MNIST is not viable.
-
-    Full MNIST pulls torch/pytorch via pip and may download datasets from the
-    public internet — both fail on disconnected clusters without mirrors.
+    Use a lightweight Ray job when full MNIST is not viable.
 
     Detection order (first match wins):
-      1. UPGRADE_USE_SMOKE_JOB=true|false (explicit override)
+      1. USE_SMOKE_JOB / UPGRADE_USE_SMOKE_JOB=true|false (explicit override)
       2. Full MNIST prerequisites met (connected, or disconnected with mirrors)
       3. DISCONNECTED_CLUSTER / IS_DISCONNECTED_CLUSTER env (Jenkins)
-      4. ImageDigestMirrorSet / ImageContentSourcePolicy on cluster
-      5. API server URL heuristic (-dis- / disconnected), last resort
+      4. API server URL heuristic (-dis- / disconnected), last resort
+
+    ImageDigestMirrorSet / ICSP are intentionally not used: many connected
+    OpenShift clusters mirror container registries without blocking pip/PyPI.
     """
-    if _env_flag_enabled("UPGRADE_USE_SMOKE_JOB"):
-        print("UPGRADE_USE_SMOKE_JOB enabled; using upgrade smoke job")
-        return True
-    if _env_flag_disabled("UPGRADE_USE_SMOKE_JOB"):
-        print("UPGRADE_USE_SMOKE_JOB disabled; using full MNIST upgrade job")
-        return False
+    for name in ("USE_SMOKE_JOB", "UPGRADE_USE_SMOKE_JOB"):
+        if _env_flag_enabled(name):
+            print(f"{name} enabled; using smoke job (no pip install)")
+            return True
+        if _env_flag_disabled(name):
+            print(f"{name} disabled; using full MNIST job")
+            return False
 
-    if _upgrade_mnist_prerequisites_met():
+    if _mnist_prerequisites_met():
         return False
 
     if _env_flag_enabled("DISCONNECTED_CLUSTER") or _env_flag_enabled(
         "IS_DISCONNECTED_CLUSTER"
     ):
         print(
             "Disconnected cluster env set without PIP/S3 mirrors; "
-            "using upgrade smoke job (no pip install)"
-        )
-        return True
-
-    if _has_registry_mirror_configured():
-        print(
-            "Registry mirror detected (ImageDigestMirrorSet/ICSP) without "
-            "PIP/S3 mirrors; using upgrade smoke job (no pip install)"
+            "using smoke job (no pip install)"
         )
         return True
 
@@ -280,7 +241,7 @@ def use_upgrade_smoke_job():
         if "-dis-" in server or "disconnected" in server:
             print(
                 "Detected disconnected cluster from API server URL; "
-                "using upgrade smoke job (no pip install)"
+                "using smoke job (no pip install)"
             )
             return True
     except Exception:
@@ -289,28 +250,37 @@ def use_upgrade_smoke_job():
     return False
 
 
-def get_upgrade_job_submission_spec():
-    """
-    Return entrypoint and runtime_env for post-upgrade job submission tests.
-    """
-    if use_upgrade_smoke_job():
+def use_upgrade_smoke_job():
+    """Backward-compatible alias for upgrade tests."""
+    return use_smoke_job()
+
+
+def get_mnist_job_submission_spec(**kwargs):
+    """Return entrypoint and runtime_env for tier1 / upgrade MNIST job submission tests."""
+    env_vars = get_setup_env_variables(**kwargs)
+    if use_smoke_job():
         return {
             "entrypoint": "python upgrade_job_smoke.py",
             "runtime_env": {
                 "working_dir": "./tests/e2e/",
-                "env_vars": get_setup_env_variables(),
+                "env_vars": env_vars,
             },
         }
     return {
         "entrypoint": "python mnist.py",
         "runtime_env": {
             "working_dir": "./tests/e2e/",
             "pip": "./tests/e2e/mnist_pip_requirements.txt",
-            "env_vars": get_setup_env_variables(),
+            "env_vars": env_vars,
         },
     }
 
 
+def get_upgrade_job_submission_spec(**kwargs):
+    """Backward-compatible alias for post-upgrade job submission tests."""
+    return get_mnist_job_submission_spec(**kwargs)
+
+
 def random_choice():
     alphabet = string.ascii_lowercase + string.digits
     return "".join(random.choices(alphabet, k=5))
@@ -831,18 +801,12 @@ def assert_get_cluster_and_jobsubmit(
     client = cluster.job_client
 
     # Submit a job and get the submission ID
-    env_vars = (
-        get_setup_env_variables(ACCELERATOR=accelerator)
-        if accelerator
-        else get_setup_env_variables()
-    )
+    spec_kwargs = {"ACCELERATOR": accelerator} if accelerator else {}
+    job_spec = get_mnist_job_submission_spec(**spec_kwargs)
+    print(f"Submitting job: {job_spec['entrypoint']}")
     submission_id = client.submit_job(
-        entrypoint="python mnist.py",
-        runtime_env={
-            "working_dir": "./tests/e2e/",
-            "pip": "./tests/e2e/mnist_pip_requirements.txt",
-            "env_vars": env_vars,
-        },
+        entrypoint=job_spec["entrypoint"],
+        runtime_env=job_spec["runtime_env"],
         entrypoint_num_cpus=1 if number_of_gpus is None else None,
         entrypoint_num_gpus=number_of_gpus,
     )

tests/upgrade/01_raycluster_sdk_upgrade_test.py

@@ -249,7 +249,7 @@ def assert_jobsubmit_withlogin(self, cluster):
         client = RayJobClient(address=ray_dashboard, headers=header, verify=False)
 
         job_spec = get_upgrade_job_submission_spec()
-        print(f"Submitting upgrade job: {job_spec['entrypoint']}")
+        print(f"Submitting job: {job_spec['entrypoint']}")
 
         # Submit the job
         submission_id = client.submit_job(

tests/upgrade/README.md

@@ -49,22 +49,22 @@ Post-upgrade job tests use `is_byoidc_cluster_detected()` from `tests/e2e/suppor
 
 On **htpasswd/LDAP** QE clusters (e.g. ods-qe-psi-17), job tests use `oc whoami --show-token=true` with `OCP_ADMIN_USER_*` credentials — not Keycloak password grant.
 
-## Disconnected clusters (post-upgrade job submission)
+## Disconnected clusters (post-upgrade and tier1 job submission)
 
 Full MNIST installs torch via pip and may fetch datasets from the public internet. On disconnected clusters that fails unless mirrors are configured.
 
-Post-upgrade job tests use `get_upgrade_job_submission_spec()` from `tests/e2e/support.py`:
+Post-upgrade and tier1 MNIST job tests use `get_mnist_job_submission_spec()` from `tests/e2e/support.py`:
 
 | Job | When |
 |-----|------|
 | **`upgrade_job_smoke.py`** (no pip) | Disconnected / mirror-only cluster without PyPI + S3 env |
 | **`mnist.py`** (full pip + training) | Connected cluster, or disconnected with `PIP_INDEX_URL` (non-pypi.org) **and** `AWS_DEFAULT_ENDPOINT` set |
 
-**Detection order:** `UPGRADE_USE_SMOKE_JOB` override → MNIST prerequisites → `DISCONNECTED_CLUSTER` / `IS_DISCONNECTED_CLUSTER` (Jenkins) → `ImageDigestMirrorSet` / `ImageContentSourcePolicy` → API URL `-dis-` heuristic (last resort).
+**Detection order:** `USE_SMOKE_JOB` / `UPGRADE_USE_SMOKE_JOB` override → MNIST prerequisites → `DISCONNECTED_CLUSTER` / `IS_DISCONNECTED_CLUSTER` (Jenkins) → API URL `-dis-` heuristic (last resort).
 
-Set `UPGRADE_USE_SMOKE_JOB=false` to force full MNIST when mirrors are configured. Set `DISCONNECTED_CLUSTER=true` in the test env file for explicit lab config (aligns with Jenkins `IS_DISCONNECTED_CLUSTER`).
+Registry image mirrors (`ImageDigestMirrorSet` / `ICSP`) are **not** used for detection — connected OpenShift clusters often have them without blocking pip.
 
-**Porting:** apply the same helpers (`use_upgrade_smoke_job`, `upgrade_job_smoke.py`, `get_upgrade_job_submission_spec` usage) to **`main`** (3.5 / `2.x-3.x-upgrade-tests-post-upgrade`) and **`3.3`** upgrade branches.
+Set `USE_SMOKE_JOB=false` or `UPGRADE_USE_SMOKE_JOB=false` to force full MNIST when mirrors are configured. Set `DISCONNECTED_CLUSTER=true` in the test env file for explicit lab config (aligns with Jenkins `IS_DISCONNECTED_CLUSTER`).
 
 ## Migration script