Support mixed-precision per layer quant config in config.json (#929)

## What does this PR do?

**Type of change:** ? <!-- Use one of the following: Bug fix, new
feature, new example, new tests, documentation. -->

**Overview:** Support mixed-precision per layer quant config in
config.json, since it's the first-class source of truth in deployment
FWs.


## Usage
<!-- You can potentially add a usage example below. -->

```python
python3 -c "
from modelopt.torch.export.convert_hf_config import convert_hf_quant_config_format
import json

# Test 1: Existing FP8 case still works
fp8_config = {
    'producer': {'name': 'modelopt', 'version': '0.29.0'},
    'quantization': {
        'quant_algo': 'FP8',
        'kv_cache_quant_algo': 'FP8',
        'exclude_modules': ['lm_head'],
    },
}
result = convert_hf_quant_config_format(fp8_config)
print('=== FP8 (existing) ===')
print(json.dumps(result, indent=2))
assert result['quant_algo'] == 'FP8'
assert 'group_0' in result['config_groups']
assert result['ignore'] == ['lm_head']

# Test 2: Mixed precision
mixed_config = {
    'producer': {'name': 'modelopt', 'version': '0.29.0'},
    'quantization': {
        'quant_algo': 'MIXED_PRECISION',
        'kv_cache_quant_algo': 'FP8',
        'quantized_layers': {
            'model.layers.0.self_attn.q_proj': {'quant_algo': 'FP8'},
            'model.layers.0.self_attn.k_proj': {'quant_algo': 'FP8'},
            'model.layers.0.mlp.gate_proj': {'quant_algo': 'NVFP4', 'group_size': 16},
            'model.layers.0.mlp.up_proj': {'quant_algo': 'NVFP4', 'group_size': 16},
            'model.layers.1.self_attn.q_proj': {'quant_algo': 'FP8'},
        },
    },
}
result = convert_hf_quant_config_format(mixed_config)
print()
print('=== MIXED_PRECISION ===')
print(json.dumps(result, indent=2))

assert result['quant_algo'] == 'MIXED_PRECISION'
assert 'config_groups' in result
assert 'quantized_layers' in result
# Should have 2 groups: one for FP8 layers, one for NVFP4 layers
assert len(result['config_groups']) == 2

# Verify per-layer detail is preserved
assert 'model.layers.0.self_attn.q_proj' in result['quantized_layers']
assert result['quantized_layers']['model.layers.0.mlp.gate_proj']['quant_algo'] == 'NVFP4'

# Check that FP8 group has correct targets
for gname, gcfg in result['config_groups'].items():
    if gcfg.get('weights', {}).get('num_bits') == 8:
        assert len(gcfg['targets']) == 3  # 3 FP8 layers
    elif gcfg.get('weights', {}).get('num_bits') == 4:
        assert len(gcfg['targets']) == 2  # 2 NVFP4 layers

print()
print('All tests passed!')
"

```

## Testing
<!-- Mention how have you tested your change if applicable. -->

## Before your PR is "*Ready for review*"
<!-- If you haven't finished some of the above items you can still open
`Draft` PR. -->

- **Make sure you read and follow [Contributor
guidelines](https://github.com/NVIDIA/Model-Optimizer/blob/main/CONTRIBUTING.md)**
and your commits are signed.
- **Is this change backward compatible?**: Yes <!--- If No, explain why.
-->
- **Did you write any new necessary tests?**: Yes/No
- **Did you add or update any necessary documentation?**: Yes/No
- **Did you update
[Changelog](https://github.com/NVIDIA/Model-Optimizer/blob/main/CHANGELOG.rst)?**:
Yes/No <!--- Only for new features, API changes, critical bug fixes or
bw breaking changes. -->

## Additional Information
<!-- E.g. related issue. -->


<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->

## Summary by CodeRabbit

## Release Notes

* **New Features**
* Added support for MIXED_PRECISION quantization configurations in model
export, enabling automatic aggregation of layers by their individual
quantization settings.
* Enhanced quantization config handling to dynamically manage multiple
configuration groups for complex quantization scenarios.
* Maintained backward compatibility with existing quantization
algorithms.

<!-- end of auto-generated comment: release notes by coderabbit.ai -->

---------

Signed-off-by: Zhiyu Cheng <zhiyuc@nvidia.com>

Author: Edwardf0t1

modelopt/torch/export/convert_hf_config.py

@@ -15,9 +15,100 @@
 
 """Convert modelopt quantization export config to align with llm-compressor config format."""
 
+import warnings
+from collections import defaultdict
 from typing import Any
 
 
+def _quant_algo_to_group_config(quant_algo: str, group_size: int | None = None) -> dict[str, Any]:
+    """Map a per-layer quant_algo string to compressed-tensors config group details.
+
+    Args:
+        quant_algo: The quantization algorithm name (e.g. "FP8", "NVFP4").
+        group_size: Optional group size for block-wise quantization algorithms.
+
+    Returns:
+        Dictionary with ``input_activations`` and ``weights`` entries suitable for
+        a compressed-tensors ``config_groups`` entry.
+    """
+    if quant_algo == "FP8":
+        return {
+            "input_activations": {"dynamic": False, "num_bits": 8, "type": "float"},
+            "weights": {"dynamic": False, "num_bits": 8, "type": "float"},
+        }
+    elif quant_algo == "FP8_PER_CHANNEL_PER_TOKEN":
+        return {
+            "input_activations": {"dynamic": False, "num_bits": 8, "type": "float"},
+            "weights": {"dynamic": False, "num_bits": 8, "type": "float", "strategy": "channel"},
+        }
+    elif quant_algo == "NVFP4":
+        gs = group_size or 16
+        return {
+            "input_activations": {
+                "dynamic": False,
+                "num_bits": 4,
+                "type": "float",
+                "group_size": gs,
+            },
+            "weights": {"dynamic": False, "num_bits": 4, "type": "float", "group_size": gs},
+        }
+    elif quant_algo == "W4A16_AWQ":
+        gs = group_size or 128
+        return {
+            "weights": {"dynamic": False, "num_bits": 4, "type": "int", "group_size": gs},
+        }
+    elif quant_algo in ("NVFP4_AWQ", "W4A8_AWQ"):
+        gs = group_size or 128
+        return {
+            "input_activations": {
+                "dynamic": False,
+                "num_bits": 8,
+                "type": "float",
+                "group_size": gs,
+            },
+            "weights": {"dynamic": False, "num_bits": 4, "type": "float", "group_size": gs},
+        }
+    elif quant_algo == "W8A16":
+        return {
+            "weights": {"dynamic": False, "num_bits": 8, "type": "int"},
+        }
+    elif quant_algo == "W8A8_SQ_PER_CHANNEL":
+        return {
+            "input_activations": {"dynamic": False, "num_bits": 8, "type": "int"},
+            "weights": {
+                "dynamic": False,
+                "num_bits": 8,
+                "type": "int",
+                "strategy": "channel",
+            },
+        }
+    elif quant_algo in ("W4A8_NVFP4_FP8", "W4A8_MXFP4_FP8"):
+        gs = group_size or 16
+        return {
+            "input_activations": {"dynamic": False, "num_bits": 8, "type": "float"},
+            "weights": {"dynamic": False, "num_bits": 4, "type": "float", "group_size": gs},
+        }
+    elif quant_algo == "MXFP8":
+        gs = group_size or 32
+        return {
+            "input_activations": {
+                "dynamic": False,
+                "num_bits": 8,
+                "type": "float",
+                "group_size": gs,
+            },
+            "weights": {"dynamic": False, "num_bits": 8, "type": "float", "group_size": gs},
+        }
+    else:
+        warnings.warn(
+            f"Unsupported quantization algorithm '{quant_algo}' in "
+            f"_quant_algo_to_group_config. The resulting config group will not contain "
+            f"'input_activations' or 'weights' keys and may not be compatible with "
+            f"compressed-tensors consumers. Please add explicit support for this algorithm."
+        )
+        return {"quant_algo": quant_algo}
+
+
 def convert_hf_quant_config_format(input_config: dict[str, Any]) -> dict[str, Any]:
     """Converts modelopt quantization config dictionary to align with llm-compressor config format.
 
@@ -92,6 +183,30 @@ def convert_hf_quant_config_format(input_config: dict[str, Any]) -> dict[str, An
             "targets": ["Linear"],
         }
         new_config["config_groups"] = {"group_0": config_group_details}
+    elif quant_algo_value == "MIXED_PRECISION":
+        quantized_layers = original_quantization_details.get("quantized_layers", {})
+
+        # Group layers by their unique quantization config so each distinct
+        # (quant_algo, group_size, ...) combination becomes one config_group.
+        algo_to_layers: dict[tuple, list[str]] = defaultdict(list)
+        for layer_name, layer_cfg in quantized_layers.items():
+            # Create a hashable key from the layer config
+            key = tuple(sorted(layer_cfg.items()))
+            algo_to_layers[key].append(layer_name)
+
+        config_groups: dict[str, Any] = {}
+        for idx, (config_key, layer_names) in enumerate(algo_to_layers.items()):
+            layer_cfg = dict(config_key)
+            algo = layer_cfg.get("quant_algo", "")
+            layer_group_size = layer_cfg.get("group_size")
+
+            group_config = _quant_algo_to_group_config(algo, layer_group_size)
+            group_config["targets"] = sorted(layer_names)
+            config_groups[f"group_{idx}"] = group_config
+
+        new_config["config_groups"] = config_groups
+        # Preserve the full per-layer detail for consumers that need it.
+        new_config["quantized_layers"] = quantized_layers
 
     exclude_modules = original_quantization_details.get("exclude_modules")