initial commit for diff only

Author: cquil11

perf-changelog.yaml

@@ -0,0 +1,3 @@
+changelog:
+- config-key: dsr1-fp8-h200-trt
+  description: Test change

utils/matrix_logic/generate_sweep_configs.py

@@ -1,8 +1,12 @@
 import json
-import yaml
 import argparse
 
-from validation import validate_master_config, validate_matrix_entry, validate_runner_config, Fields
+from validation import (
+    validate_matrix_entry,
+    load_config_files,
+    load_runner_file,
+    Fields
+)
 
 seq_len_stoi = {
     "1k1k": (1024, 1024),
@@ -366,44 +370,6 @@ def get_lowest_conc(search_space_entry):
     return matrix_values
 
 
-def load_config_files(config_files):
-    """Load and merge configuration files."""
-    all_config_data = {}
-    for config_file in config_files:
-        try:
-            with open(config_file, 'r') as f:
-                config_data = yaml.safe_load(f)
-                assert isinstance(
-                    config_data, dict), f"Config file '{config_file}' must contain a dictionary"
-
-                # Check for duplicate keys, this is only in place to prevent against the very unlikely
-                # case where an entry in one config accidentally/purposefully tries to override an entry in another config
-                duplicate_keys = set(all_config_data.keys()) & set(
-                    config_data.keys())
-                if duplicate_keys:
-                    raise ValueError(
-                        f"Duplicate configuration keys found in '{config_file}': {', '.join(sorted(duplicate_keys))}"
-                    )
-
-                all_config_data.update(config_data)
-        except FileNotFoundError:
-            raise ValueError(f"Input file '{config_file}' does not exist.")
-
-    return all_config_data
-
-
-def load_runner_file(runner_file):
-    """Load runner configuration file."""
-    try:
-        with open(runner_file, 'r') as f:
-            runner_config = yaml.safe_load(f)
-    except FileNotFoundError as e:
-        raise ValueError(
-            f"Runner config file '{runner_file}' does not exist.")
-
-    return runner_config
-
-
 def main():
     # Create parent parser with common arguments
     parent_parser = argparse.ArgumentParser(add_help=False)
@@ -547,11 +513,9 @@ def main():
 
     args = parser.parse_args()
 
-    # Load and validate configuration files
+    # Load and validate configuration files (validation happens by default in load functions)
     all_config_data = load_config_files(args.config_files)
     runner_data = load_runner_file(args.runner_config)
-    validate_master_config(all_config_data)
-    validate_runner_config(runner_data)
 
     # Route to appropriate function based on subcommand
     if args.command == 'full-sweep':

utils/matrix_logic/test_generate_sweep_configs.py

@@ -7,8 +7,6 @@
     seq_len_to_str,
     generate_full_sweep,
     generate_runner_model_sweep_config,
-    load_config_files,
-    load_runner_file,
 )
 
 
@@ -583,90 +581,6 @@ def test_uses_lowest_conc(self, sample_single_node_config, sample_runner_config,
         assert all(entry["conc"] == 4 for entry in result)
 
 
-# =============================================================================
-# Test load_config_files
-# =============================================================================
-
-class TestLoadConfigFiles:
-    """Tests for load_config_files function."""
-
-    def test_load_single_file(self, tmp_path):
-        """Should load a single config file."""
-        config_file = tmp_path / "config.yaml"
-        config_file.write_text("""
-test-config:
-  image: test-image
-  model: test-model
-""")
-        result = load_config_files([str(config_file)])
-        assert "test-config" in result
-        assert result["test-config"]["image"] == "test-image"
-
-    def test_load_multiple_files(self, tmp_path):
-        """Should merge multiple config files."""
-        config1 = tmp_path / "config1.yaml"
-        config1.write_text("""
-config-one:
-  value: 1
-""")
-        config2 = tmp_path / "config2.yaml"
-        config2.write_text("""
-config-two:
-  value: 2
-""")
-        result = load_config_files([str(config1), str(config2)])
-        assert "config-one" in result
-        assert "config-two" in result
-
-    def test_duplicate_keys_raise_error(self, tmp_path):
-        """Duplicate keys across files should raise error."""
-        config1 = tmp_path / "config1.yaml"
-        config1.write_text("""
-duplicate-key:
-  value: 1
-""")
-        config2 = tmp_path / "config2.yaml"
-        config2.write_text("""
-duplicate-key:
-  value: 2
-""")
-        with pytest.raises(ValueError) as exc_info:
-            load_config_files([str(config1), str(config2)])
-        assert "Duplicate configuration keys" in str(exc_info.value)
-
-    def test_nonexistent_file_raises_error(self):
-        """Nonexistent file should raise error."""
-        with pytest.raises(ValueError) as exc_info:
-            load_config_files(["nonexistent.yaml"])
-        assert "does not exist" in str(exc_info.value)
-
-
-# =============================================================================
-# Test load_runner_file
-# =============================================================================
-
-class TestLoadRunnerFile:
-    """Tests for load_runner_file function."""
-
-    def test_load_runner_file(self, tmp_path):
-        """Should load runner config file."""
-        runner_file = tmp_path / "runners.yaml"
-        runner_file.write_text("""
-h100:
-- h100-node-0
-- h100-node-1
-""")
-        result = load_runner_file(str(runner_file))
-        assert "h100" in result
-        assert len(result["h100"]) == 2
-
-    def test_nonexistent_runner_file(self):
-        """Nonexistent runner file should raise error."""
-        with pytest.raises(ValueError) as exc_info:
-            load_runner_file("nonexistent.yaml")
-        assert "does not exist" in str(exc_info.value)
-
-
 # =============================================================================
 # Test edge cases and special configurations
 # =============================================================================

utils/matrix_logic/test_validation.py

@@ -14,6 +14,8 @@
     validate_matrix_entry,
     validate_master_config,
     validate_runner_config,
+    load_config_files,
+    load_runner_file,
 )
 
 
@@ -738,3 +740,130 @@ def test_multiple_runner_types(self, valid_runner_config):
         assert "h200" in result
         assert "mi300x" in result
         assert "gb200" in result
+
+
+# =============================================================================
+# Test load_config_files
+# =============================================================================
+
+class TestLoadConfigFiles:
+    """Tests for load_config_files function."""
+
+    def test_load_single_file_with_validation(self, tmp_path, valid_single_node_master_config):
+        """Should load and validate a single config file."""
+        config_file = tmp_path / "config.yaml"
+        import yaml
+        config_file.write_text(yaml.dump({"test-config": valid_single_node_master_config}))
+        result = load_config_files([str(config_file)])
+        assert "test-config" in result
+        assert result["test-config"]["image"] == valid_single_node_master_config["image"]
+
+    def test_load_single_file_without_validation(self, tmp_path):
+        """Should load a single config file without validation when validate=False."""
+        config_file = tmp_path / "config.yaml"
+        config_file.write_text("""
+test-config:
+  image: test-image
+  model: test-model
+""")
+        result = load_config_files([str(config_file)], validate=False)
+        assert "test-config" in result
+        assert result["test-config"]["image"] == "test-image"
+
+    def test_load_multiple_files(self, tmp_path):
+        """Should merge multiple config files."""
+        config1 = tmp_path / "config1.yaml"
+        config1.write_text("""
+config-one:
+  value: 1
+""")
+        config2 = tmp_path / "config2.yaml"
+        config2.write_text("""
+config-two:
+  value: 2
+""")
+        result = load_config_files([str(config1), str(config2)], validate=False)
+        assert "config-one" in result
+        assert "config-two" in result
+
+    def test_duplicate_keys_raise_error(self, tmp_path):
+        """Duplicate keys across files should raise error."""
+        config1 = tmp_path / "config1.yaml"
+        config1.write_text("""
+duplicate-key:
+  value: 1
+""")
+        config2 = tmp_path / "config2.yaml"
+        config2.write_text("""
+duplicate-key:
+  value: 2
+""")
+        with pytest.raises(ValueError) as exc_info:
+            load_config_files([str(config1), str(config2)], validate=False)
+        assert "Duplicate configuration keys" in str(exc_info.value)
+
+    def test_nonexistent_file_raises_error(self):
+        """Nonexistent file should raise error."""
+        with pytest.raises(ValueError) as exc_info:
+            load_config_files(["nonexistent.yaml"])
+        assert "does not exist" in str(exc_info.value)
+
+    def test_validation_runs_by_default(self, tmp_path):
+        """Validation should run by default and catch invalid configs."""
+        config_file = tmp_path / "config.yaml"
+        config_file.write_text("""
+invalid-config:
+  image: test-image
+  # Missing required fields like model, model-prefix, precision, etc.
+""")
+        with pytest.raises(ValueError) as exc_info:
+            load_config_files([str(config_file)])
+        assert "failed validation" in str(exc_info.value)
+
+
+# =============================================================================
+# Test load_runner_file
+# =============================================================================
+
+class TestLoadRunnerFile:
+    """Tests for load_runner_file function."""
+
+    def test_load_runner_file_with_validation(self, tmp_path):
+        """Should load and validate runner config file."""
+        runner_file = tmp_path / "runners.yaml"
+        runner_file.write_text("""
+h100:
+- h100-node-0
+- h100-node-1
+""")
+        result = load_runner_file(str(runner_file))
+        assert "h100" in result
+        assert len(result["h100"]) == 2
+
+    def test_load_runner_file_without_validation(self, tmp_path):
+        """Should load runner config file without validation when validate=False."""
+        runner_file = tmp_path / "runners.yaml"
+        runner_file.write_text("""
+h100:
+- h100-node-0
+- h100-node-1
+""")
+        result = load_runner_file(str(runner_file), validate=False)
+        assert "h100" in result
+        assert len(result["h100"]) == 2
+
+    def test_nonexistent_runner_file(self):
+        """Nonexistent runner file should raise error."""
+        with pytest.raises(ValueError) as exc_info:
+            load_runner_file("nonexistent.yaml")
+        assert "does not exist" in str(exc_info.value)
+
+    def test_validation_runs_by_default(self, tmp_path):
+        """Validation should run by default and catch invalid configs."""
+        runner_file = tmp_path / "runners.yaml"
+        runner_file.write_text("""
+h100: not-a-list
+""")
+        with pytest.raises(ValueError) as exc_info:
+            load_runner_file(str(runner_file))
+        assert "must be a list" in str(exc_info.value)

utils/matrix_logic/validation.py

@@ -3,6 +3,7 @@
 from enum import Enum
 
 import pprint
+import yaml
 
 """
     The below class defines the field names expected to be present in the JSON entries
@@ -315,3 +316,73 @@ def validate_runner_config(runner_configs: dict) -> List[dict]:
                 f"Runner config entry '{key}' cannot be an empty list")
 
     return runner_configs
+
+
+# =============================================================================
+# File Loading Functions
+# =============================================================================
+
+
+def load_config_files(config_files: List[str], validate: bool = True) -> dict:
+    """Load and merge configuration files.
+
+    Args:
+        config_files: List of paths to YAML configuration files.
+        validate: If True, run validate_master_config on loaded data. Defaults to True.
+
+    Returns:
+        Merged configuration dictionary.
+
+    Raises:
+        ValueError: If file doesn't exist, isn't a dict, or has duplicate keys.
+    """
+    all_config_data = {}
+    for config_file in config_files:
+        try:
+            with open(config_file, 'r') as f:
+                config_data = yaml.safe_load(f)
+                assert isinstance(
+                    config_data, dict), f"Config file '{config_file}' must contain a dictionary"
+
+                # Check for duplicate keys
+                duplicate_keys = set(all_config_data.keys()) & set(
+                    config_data.keys())
+                if duplicate_keys:
+                    raise ValueError(
+                        f"Duplicate configuration keys found in '{config_file}': {', '.join(sorted(duplicate_keys))}"
+                    )
+
+                all_config_data.update(config_data)
+        except FileNotFoundError:
+            raise ValueError(f"Input file '{config_file}' does not exist.")
+
+    if validate:
+        validate_master_config(all_config_data)
+
+    return all_config_data
+
+
+def load_runner_file(runner_file: str, validate: bool = True) -> dict:
+    """Load runner configuration file.
+
+    Args:
+        runner_file: Path to the runner YAML configuration file.
+        validate: If True, run validate_runner_config on loaded data. Defaults to True.
+
+    Returns:
+        Runner configuration dictionary.
+
+    Raises:
+        ValueError: If file doesn't exist or fails validation.
+    """
+    try:
+        with open(runner_file, 'r') as f:
+            runner_config = yaml.safe_load(f)
+    except FileNotFoundError:
+        raise ValueError(
+            f"Runner config file '{runner_file}' does not exist.")
+
+    if validate:
+        validate_runner_config(runner_config)
+
+    return runner_config