fix(training_report): improve can_parse validation and logs

Reject missing or invalid training artifacts before parsing and log why report generation is skipped.

Signed-off-by: Ben Lugassi <blugassi@nvidia.com>

Author: blugassi

src/cloudai/report_generator/training/parser.py

@@ -72,13 +72,34 @@ def get_model_config(self, tr: TestRun) -> dict:
     def get_model_name(self, tr: TestRun) -> str:
         """CloudAI-sourced model identity for the run."""
 
+    @staticmethod
+    def _has_tb_event_files(tb_dir: Path) -> bool:
+        # TensorBoard documents event files as having "tfevents" in the filename.
+        return any(path.is_file() for path in tb_dir.rglob("*tfevents*"))
+
     def can_parse(self, tr: TestRun) -> bool:
         """Return True when the run produced the TB events and config artifact this parser needs."""
+        name = type(self).__name__
         tb_dir = self.get_tb_dir(tr)
-        if not (tb_dir.is_dir() and any(tb_dir.iterdir())):
+        if not (tb_dir.is_dir() and self._has_tb_event_files(tb_dir)):
+            logging.warning(f"{name}: no TensorBoard events at '{tb_dir}'; skipping training report")
             return False
         config_path = self.get_config_path(tr)
-        return config_path is not None and config_path.is_file()
+        if config_path is None or not config_path.is_file():
+            logging.warning(f"{name}: config artifact not found under '{tr.output_path}'; skipping training report")
+            return False
+        if config_path.suffix.lower() in {".json", ".yaml", ".yml"}:
+            try:
+                config = self.get_model_config(tr)
+            except (json.JSONDecodeError, yaml.YAMLError) as exc:
+                logging.warning(f"{name}: invalid config artifact at '{config_path}' ({exc}); skipping training report")
+                return False
+            if not isinstance(config, dict) or not config:
+                logging.warning(
+                    f"{name}: empty or invalid config artifact at '{config_path}'; skipping training report"
+                )
+                return False
+        return True
 
     def parse(self, tr: TestRun, system: System) -> TrainingResults:
         """Read TB scalars + the config artifact and assemble TrainingResults."""

tests/report_generator/training/test_training_parser.py

@@ -472,13 +472,56 @@ def test_can_parse_requires_nonempty_tb_dir(tmp_path):
 
     tb_dir = tmp_path / "tb_logs" / "default"
     tb_dir.mkdir(parents=True)
-    (tb_dir / "events.out.tfevents").write_text("x")
+    (tb_dir / "placeholder.txt").write_text("x")
+    assert parser.can_parse(_tr(output_path=tmp_path)) is False  # non-empty dir without event files
+
+    (tb_dir / "custom.tfevents.log").write_text("x")
     assert parser.can_parse(_tr(output_path=tmp_path)) is False  # file-backed parsers also need config
 
-    (tmp_path / "nemo_config.json").write_text("{}")
+    (tmp_path / "nemo_config.json").write_text('{"model": {}}')
     assert parser.can_parse(_tr(output_path=tmp_path)) is True
 
 
+@pytest.mark.parametrize(
+    ("parser", "tb_path", "config_path", "config_contents"),
+    (
+        pytest.param(NeMoRunParser(), "tb_logs/default", "nemo_config.json", "", id="nemo-empty-file"),
+        pytest.param(NeMoRunParser(), "tb_logs/default", "nemo_config.json", "{}", id="nemo-empty-object"),
+        pytest.param(NeMoRunParser(), "tb_logs/default", "nemo_config.json", "{", id="nemo-malformed"),
+        pytest.param(
+            MegatronBridgeParser(),
+            "experiment/tb_logs",
+            "experiment/configs/ConfigContainer.yaml",
+            "",
+            id="bridge-empty-file",
+        ),
+        pytest.param(
+            MegatronBridgeParser(),
+            "experiment/tb_logs",
+            "experiment/configs/ConfigContainer.yaml",
+            "{}",
+            id="bridge-empty-object",
+        ),
+        pytest.param(
+            MegatronBridgeParser(),
+            "experiment/tb_logs",
+            "experiment/configs/ConfigContainer.yaml",
+            "key: [",
+            id="bridge-malformed",
+        ),
+    ),
+)
+def test_can_parse_rejects_invalid_file_config(tmp_path, parser, tb_path, config_path, config_contents):
+    tb_dir = tmp_path / tb_path
+    tb_dir.mkdir(parents=True)
+    (tb_dir / "events.out.tfevents.1").write_text("x")
+    artifact = tmp_path / config_path
+    artifact.parent.mkdir(parents=True, exist_ok=True)
+    artifact.write_text(config_contents)
+
+    assert parser.can_parse(_tr(output_path=tmp_path)) is False
+
+
 def test_megatron_config_path_points_to_tb_event_file(tmp_path):
     tb_dir = tmp_path / "tensorboard"
     tb_dir.mkdir()