refactor: add validate_coreml_palettization_compatibility for combo checks

Vector palettization, LUT quantization, and per-channel scale are each
independently CoreML-exportable, but combining any two breaks CoreML/MIL's
constexpr_lut_to_dense op with a rank mismatch, a missing vector_axis, or an
indices/LUT divisibility error depending on the pair. Add a dedicated
validator (composing the existing LUT dtype check) so the kmeans export path
has one entry point to reject these combinations with a clear
CoreMLExportError, instead of relying on a test-side skip to hide them.

Author: guru-desh

src/coreai_opt/_utils/export_utils.py

@@ -11,6 +11,7 @@
 from coreai_opt._utils.torch_utils import is_tensor_on_cpu
 from coreai_opt.common import CoreMLExportError, ExportBackend
 from coreai_opt.config.spec import CompressionTargetTensor
+from coreai_opt.quantization.spec import QuantizationSpec
 from coreai_opt.quantization.spec.granularity import PerTensorGranularity, QuantizationGranularity
 
 COREML_SUPPORTED_WEIGHT_DTYPES: frozenset[torch.dtype] = frozenset(
@@ -78,6 +79,52 @@ def validate_coreml_compatibility(
         raise CoreMLExportError.from_config(granularity, context)
 
 
+def validate_coreml_palettization_compatibility(
+    cluster_dim: int,
+    lut_qspec: QuantizationSpec | None,
+    enable_per_channel_scale: bool,
+    context: str,
+) -> None:
+    """Raise CoreMLExportError if this palettization config isn't CoreML-exportable.
+
+    Checks the LUT dtype (delegating to validate_coreml_compatibility) and
+    whether the config combines multiple features CoreML/MIL export cannot yet
+    fuse into a single compatible op chain: CoreML export supports at most one
+    of {vector palettization, LUT quantization, per-channel scale} at a time;
+    combining any two hits an unsupported CoreML/MIL op configuration
+    (mismatched tensor ranks, or `lut_to_dense` divisibility errors).
+
+    Args:
+        cluster_dim (int): Palettization cluster dimension; > 1 indicates
+            vector palettization.
+        lut_qspec (QuantizationSpec | None): LUT quantization spec, or None if
+            the LUT is not quantized.
+        enable_per_channel_scale (bool): Whether per-channel scaling is enabled.
+        context (str): Human-readable description of what's being checked.
+
+    Raises:
+        CoreMLExportError: If the LUT dtype isn't supported, or if two or more
+            of the three features above are combined.
+    """
+    if lut_qspec is not None:
+        validate_coreml_compatibility(
+            CompressionTargetTensor.LUT, lut_qspec.dtype, f"LUT of {context}"
+        )
+
+    active_features = []
+    if cluster_dim > 1:
+        active_features.append("vector palettization")
+    if lut_qspec is not None:
+        active_features.append("LUT quantization")
+    if enable_per_channel_scale:
+        active_features.append("per-channel scale")
+
+    if len(active_features) >= 2:
+        raise CoreMLExportError(
+            f"CoreML export does not support {' + '.join(active_features)} on {context}."
+        )
+
+
 def validate_mmap_backend_and_device(
     model: torch.nn.Module,
     backend: ExportBackend,