fix(kubeflow): tail only first and last node logs

KubeflowExecutor.fetch_logs streamed every replica pod
(--max-log-requests num_nodes). Each pod's stdout already includes all of that
node's local ranks, so following all replicas multiplies aggregate log volume
by num_nodes and overruns CI/runner job-log size limits at scale (e.g. a 16-node
job exceeded GitLab's 128MB cap). Restrict the selector to completion indices 0
and num_nodes-1 (first + last node) — enough for rank-0 driver output and the
far end of the world for spotting straggler/per-rank failures.

Signed-off-by: oliver könig <okoenig@nvidia.com>

Author: ko3n1g

nemo_run/core/execution/kubeflow.py

@@ -331,7 +331,18 @@ def fetch_logs(
         until pods are running (up to 10 minutes).  Otherwise it returns the last
         *lines* lines from a single ``kubectl logs`` call.
         """
-        label_selector = f"jobset.sigs.k8s.io/jobset-name={job_name}"
+        # Tail only the first and last node of the job. Each pod's stdout
+        # already carries all of that node's local ranks, so following every
+        # replica multiplies the aggregate output by num_nodes and can blow
+        # past CI / runner job-log size limits at scale. The first and last
+        # completion indices are enough to surface rank-0 driver output and the
+        # opposite end of the world for spotting straggler / per-rank failures.
+        last_index = max(self.num_nodes - 1, 0)
+        node_indices = "0" if last_index == 0 else f"0,{last_index}"
+        label_selector = (
+            f"jobset.sigs.k8s.io/jobset-name={job_name},"
+            f"batch.kubernetes.io/job-completion-index in ({node_indices})"
+        )
         cmd = [
             "kubectl",
             "logs",
@@ -342,7 +353,7 @@ def fetch_logs(
             "--tail",
             str(lines),
             "--max-log-requests",
-            str(self.num_nodes),
+            str(1 if last_index == 0 else 2),
         ]
         if stream:
             cmd.append("-f")