Merge branch 'feature/custom-models-region-visibility' into 'develop'

feat(ui): hide Custom Models nav when deployed outside us-east-1

See merge request genaiic-reusable-assets/engagement-artifacts/genaiic-idp-accelerator!619

Author: rstrahan

CHANGELOG.md

@@ -22,6 +22,9 @@ SPDX-License-Identifier: MIT-0
 
 ### Fixed
 
+- **Fixed** Nova 2.0 fine-tuning failure (`Invalid input error: Nova 2.0 doesn't support validation set`) — Nova 2.x models do not support validation datasets in the Bedrock `CreateModelCustomizationJob` API. The `finetuning_job_creator` Lambda and `ModelFinetuningService` now detect Nova 2.x models and skip the `validationDataConfig` parameter. Validation data is still generated and stored in S3 for internal metrics, but is no longer passed to the Bedrock API for these models. Added `model_capabilities` section to `finetuning_models.yaml` for config-driven control.
+- **Fixed** agentic extraction crash (`TypeError: unsupported format string passed to NoneType.__format__`) when table parsing stats contain `None` values for `avg_confidence` or `parse_success_rate`.
+- **Fixed** agentic extraction `map_table_to_schema` producing phantom empty rows from non-matching tables (e.g. account_summary rows prepended to transaction_details), causing list item ordering to be shifted by several positions.
 - **Error Analyzer model selection** — The agent was using the Chat Companion's model instead of its own configured model.
 - **Error Analyzer log processing** — Fixed early termination that stopped searching after the first Lambda function with errors; now searches all relevant log groups.
 - **Error Analyzer log truncation** — Fixed handling of long log messages to trim them rather than skip them entirely.

config_library/finetuning_models.yaml

@@ -54,6 +54,27 @@ model_mappings:
   "amazon.nova-lite-v1:0": "amazon.nova-lite-v1:0:300k"
   "amazon.nova-pro-v1:0": "amazon.nova-pro-v1:0:300k"
 
+# ---------------------------------------------------------------------------
+# Model Capabilities
+# ---------------------------------------------------------------------------
+# Some models have restrictions on what features they support during
+# fine-tuning. This section documents those restrictions.
+#
+# supports_validation: Whether the model supports a validation dataset
+#   during fine-tuning. Nova 2.x models do NOT support validation sets.
+#   When false, the validation data will still be generated for internal
+#   metrics but will NOT be passed to the Bedrock CreateModelCustomizationJob API.
+model_capabilities:
+  "amazon.nova-2-lite-v1:0":
+    supports_validation: false
+  "amazon.nova-2-pro-v1:0":
+    supports_validation: false
+  # Nova 1.x models support validation sets
+  "amazon.nova-lite-v1:0":
+    supports_validation: true
+  "amazon.nova-pro-v1:0":
+    supports_validation: true
+
 # ---------------------------------------------------------------------------
 # Default Hyperparameters
 # ---------------------------------------------------------------------------

docs/custom-model-finetuning.md

@@ -38,6 +38,7 @@ Custom Model Fine-tuning is available to **Admin** and **Author** roles. The Gra
 
 ## Prerequisites
 
+- **Deployed in `us-east-1` region**: Custom Model Fine-Tuning is only available when the IDP Accelerator is deployed to the `us-east-1` AWS region. The Custom Models navigation link is automatically hidden in deployments to other regions.
 - An existing Test Set with:
   - At least 10 documents
   - At least 2 document classes

docs/nova-finetuning.md

@@ -24,6 +24,8 @@ This guide provides comprehensive step-by-step instructions for fine-tuning Amaz
 
 ## Prerequisites
 
+> **Region Availability**: Nova fine-tuning is only available in the `us-east-1` AWS region. Ensure your IDP Accelerator stack is deployed to `us-east-1` to use this feature. The Custom Models UI is automatically hidden in deployments to other regions.
+
 ### AWS Setup
 Set up AWS CLI and credentials:
 ```bash

lib/idp_common_pkg/idp_common/model_finetuning/service.py

@@ -34,6 +34,11 @@
     {"id": "us.amazon.nova-2-pro-v1:0", "name": "Nova 2 Pro", "provider": "Amazon"},
 ]
 
+# Nova 2.x models do NOT support validation datasets during fine-tuning.
+# Passing a validation set to the Bedrock API for these models results in:
+# "Invalid input error: Nova 2.0 doesn't support validation set"
+_NOVA_2_MODEL_PREFIXES = ("amazon.nova-2-",)
+
 # Module-level cache for supported models
 _supported_models_cache: Optional[List[Dict[str, str]]] = None
 
@@ -355,6 +360,29 @@ def create_finetuning_job(
 
         return result
 
+    @staticmethod
+    def _is_nova_2_model(model_id: str) -> bool:
+        """
+        Check if a model identifier refers to a Nova 2.x model.
+
+        Nova 2.x models have specific restrictions, such as not supporting
+        validation datasets during fine-tuning.
+
+        Args:
+            model_id: Model identifier (may include cross-region prefix or context window suffix)
+
+        Returns:
+            True if the model is a Nova 2.x model
+        """
+        # Strip cross-region prefix (e.g., "us.amazon.nova-2-lite-v1:0" -> "amazon.nova-2-lite-v1:0")
+        normalized = model_id
+        if "." in normalized and not normalized.startswith("arn:"):
+            parts = normalized.split(".", 1)
+            if len(parts[0]) <= 3 and parts[0].isalpha():
+                normalized = parts[1]
+
+        return any(normalized.startswith(prefix) for prefix in _NOVA_2_MODEL_PREFIXES)
+
     def _create_nova_job_params(
         self, config: FinetuningJobConfig, data_uris: Dict[str, str]
     ) -> Dict[str, Any]:
@@ -389,10 +417,19 @@ def _create_nova_job_params(
         if config.output_uri:
             job_params["outputDataConfig"] = {"s3Uri": config.output_uri}
 
+        # Add validation data only if the model supports it.
+        # Nova 2.x models do NOT support validation sets — passing one causes:
+        # "Invalid input error: Nova 2.0 doesn't support validation set"
         if "validation_data_uri" in data_uris:
-            job_params["validationDataConfig"] = {
-                "validators": [{"s3Uri": data_uris["validation_data_uri"]}]
-            }
+            if self._is_nova_2_model(config.base_model):
+                logger.info(
+                    f"Skipping validation data for Nova 2.x model '{config.base_model}' "
+                    "— model does not support validation sets"
+                )
+            else:
+                job_params["validationDataConfig"] = {
+                    "validators": [{"s3Uri": data_uris["validation_data_uri"]}]
+                }
 
         if config.client_request_token:
             job_params["clientRequestToken"] = config.client_request_token

lib/idp_common_pkg/tests/unit/test_model_finetuning.py

@@ -372,7 +372,7 @@ def test_validate_nova_hyperparameters_invalid_learning_rate(self, service):
 
     @pytest.mark.unit
     def test_create_nova_job_params(self, service):
-        """Test Nova job parameters creation."""
+        """Test Nova v1 job parameters creation includes validation data."""
         config = FinetuningJobConfig(
             base_model="us.amazon.nova-lite-v1:0",
             training_data_uri="s3://bucket/training.jsonl",
@@ -401,6 +401,72 @@ def test_create_nova_job_params(self, service):
             == "s3://bucket/validation.jsonl"
         )
 
+    @pytest.mark.unit
+    def test_create_nova_2_job_params_skips_validation(self, service):
+        """Test Nova 2.x job parameters skip validation data.
+
+        Nova 2.0 does not support validation sets. Passing one causes:
+        'Invalid input error: Nova 2.0 doesn't support validation set'
+        """
+        config = FinetuningJobConfig(
+            base_model="us.amazon.nova-2-lite-v1:0",
+            training_data_uri="s3://bucket/training.jsonl",
+            role_arn="arn:aws:iam::123456789012:role/BedrockRole",
+            job_name="test-job",
+            model_name="test-model",
+            hyperparameters={"epochCount": "2"},
+        )
+
+        data_uris = {
+            "training_data_uri": "s3://bucket/training.jsonl",
+            "validation_data_uri": "s3://bucket/validation.jsonl",
+        }
+
+        params = service._create_nova_job_params(config, data_uris)
+
+        assert params["customizationType"] == "FINE_TUNING"
+        assert params["baseModelIdentifier"] == "us.amazon.nova-2-lite-v1:0"
+        assert params["trainingDataConfig"]["s3Uri"] == "s3://bucket/training.jsonl"
+        # Validation data should NOT be included for Nova 2.x models
+        assert "validationDataConfig" not in params
+
+    @pytest.mark.unit
+    def test_create_nova_2_pro_job_params_skips_validation(self, service):
+        """Test Nova 2 Pro job parameters also skip validation data."""
+        config = FinetuningJobConfig(
+            base_model="amazon.nova-2-pro-v1:0",
+            training_data_uri="s3://bucket/training.jsonl",
+            role_arn="arn:aws:iam::123456789012:role/BedrockRole",
+            job_name="test-job",
+            model_name="test-model",
+        )
+
+        data_uris = {
+            "training_data_uri": "s3://bucket/training.jsonl",
+            "validation_data_uri": "s3://bucket/validation.jsonl",
+        }
+
+        params = service._create_nova_job_params(config, data_uris)
+
+        # Validation data should NOT be included for Nova 2.x models
+        assert "validationDataConfig" not in params
+
+    @pytest.mark.unit
+    def test_is_nova_2_model(self, service):
+        """Test _is_nova_2_model correctly identifies Nova 2.x models."""
+        # Nova 2.x models (should return True)
+        assert service._is_nova_2_model("amazon.nova-2-lite-v1:0") is True
+        assert service._is_nova_2_model("amazon.nova-2-pro-v1:0") is True
+        assert service._is_nova_2_model("us.amazon.nova-2-lite-v1:0") is True
+        assert service._is_nova_2_model("us.amazon.nova-2-pro-v1:0") is True
+        assert service._is_nova_2_model("amazon.nova-2-lite-v1:0:256k") is True
+
+        # Nova 1.x models (should return False)
+        assert service._is_nova_2_model("amazon.nova-lite-v1:0") is False
+        assert service._is_nova_2_model("amazon.nova-pro-v1:0") is False
+        assert service._is_nova_2_model("us.amazon.nova-lite-v1:0") is False
+        assert service._is_nova_2_model("us.amazon.nova-pro-v1:0") is False
+
     @pytest.mark.unit
     def test_create_finetuning_job(self, service, mock_bedrock_client):
         """Test creating a fine-tuning job."""

src/lambda/finetuning_job_creator/index.py

@@ -48,7 +48,7 @@ def _load_finetuning_config() -> Dict[str, Any]:
     The configuration is cached after first load to avoid repeated S3 calls.
     
     Returns:
-        Dictionary containing model_mappings and default_hyperparameters
+        Dictionary containing model_mappings, default_hyperparameters, and model_capabilities
     """
     global _config_cache
 
@@ -69,6 +69,12 @@ def _load_finetuning_config() -> Dict[str, Any]:
             "learningRate": "0.00001",
             "batchSize": "1",
         },
+        "model_capabilities": {
+            "amazon.nova-2-lite-v1:0": {"supports_validation": False},
+            "amazon.nova-2-pro-v1:0": {"supports_validation": False},
+            "amazon.nova-lite-v1:0": {"supports_validation": True},
+            "amazon.nova-pro-v1:0": {"supports_validation": True},
+        },
     }
 
     # Try to load from S3
@@ -89,6 +95,9 @@ def _load_finetuning_config() -> Dict[str, Any]:
                 "default_hyperparameters": config.get(
                     "default_hyperparameters", default_config["default_hyperparameters"]
                 ),
+                "model_capabilities": config.get(
+                    "model_capabilities", default_config["model_capabilities"]
+                ),
             }
             logger.info(f"Loaded fine-tuning config with {len(_config_cache['model_mappings'])} model mappings")
             return _config_cache
@@ -116,6 +125,51 @@ def _get_default_hyperparameters() -> Dict[str, str]:
     return config.get("default_hyperparameters", {})
 
 
+def _model_supports_validation(model_id: str) -> bool:
+    """
+    Check if a model supports validation datasets during fine-tuning.
+    
+    Nova 2.x models do NOT support validation sets. Passing a validation
+    dataset to the Bedrock API for these models results in:
+    "Invalid input error: Nova 2.0 doesn't support validation set"
+    
+    Args:
+        model_id: The model identifier (can be cross-region profile, standard ID, or mapped ID)
+        
+    Returns:
+        True if the model supports validation datasets, False otherwise.
+        Defaults to True for unknown models (backward compatible).
+    """
+    config = _load_finetuning_config()
+    model_capabilities = config.get("model_capabilities", {})
+    
+    # Strip cross-region prefix for lookup (e.g., "us.amazon.nova-2-lite-v1:0" -> "amazon.nova-2-lite-v1:0")
+    normalized = model_id
+    if "." in normalized and not normalized.startswith("arn:"):
+        parts = normalized.split(".", 1)
+        if len(parts[0]) <= 3 and parts[0].isalpha():
+            normalized = parts[1]
+    
+    # Strip context window suffix for lookup (e.g., "amazon.nova-2-lite-v1:0:256k" -> "amazon.nova-2-lite-v1:0")
+    # Model capabilities are keyed by base model ID without context window suffix
+    base_id = normalized
+    # Count colons - base IDs have format "amazon.nova-2-lite-v1:0", mapped IDs have "amazon.nova-2-lite-v1:0:256k"
+    colon_parts = base_id.split(":")
+    if len(colon_parts) > 2:
+        # Has context window suffix, strip it (keep only first two parts: "amazon.nova-2-lite-v1:0")
+        base_id = ":".join(colon_parts[:2])
+    
+    # Look up capabilities
+    capabilities = model_capabilities.get(base_id, {})
+    supports_validation = capabilities.get("supports_validation", True)
+    
+    logger.info(
+        f"Model '{model_id}' (base_id='{base_id}') supports_validation={supports_validation}"
+    )
+    
+    return supports_validation
+
+
 def _normalize_model_identifier(model_id: str) -> str:
     """
     Normalize model identifier for Bedrock fine-tuning API.
@@ -212,13 +266,20 @@ def handler(event: Dict[str, Any], context: Any) -> Dict[str, Any]:
             "hyperParameters": _get_default_hyperparameters(),
         }
 
-        # Add validation data if provided
-        if validation_data_uri:
+        # Add validation data if provided AND the model supports it.
+        # Nova 2.x models do NOT support validation sets — passing one causes:
+        # "Invalid input error: Nova 2.0 doesn't support validation set"
+        if validation_data_uri and _model_supports_validation(base_model):
             job_params["validationDataConfig"] = {
                 "validators": [
                     {"s3Uri": validation_data_uri}
                 ]
             }
+        elif validation_data_uri:
+            logger.info(
+                f"Skipping validation data for model '{base_model}' — "
+                "model does not support validation sets"
+            )
 
         # Add output data config
         output_uri = f"s3://{FINETUNING_BUCKET}/finetuning/{job_id}/output/"

src/ui/src/components/genaiidp-layout/navigation.tsx

@@ -187,7 +187,9 @@ const Navigation = ({
     return viewerNavItems;
   }, [items, isAdmin, isAuthor, isViewerOnly, isReviewerOnly]);
 
-  // Filter out Capacity Planning link if pattern is not Pattern-2
+  // Filter out navigation items based on deployment context:
+  // - Capacity Planning: hidden if pattern is not Pattern-2 or Unified
+  // - Custom Models: hidden if deployed region is not us-east-1
   const filteredItems = useMemo(() => {
     const pattern = (settings?.IDPPattern as string | undefined)?.toLowerCase();
 
@@ -201,29 +203,42 @@ const Navigation = ({
       pattern.includes('unified') ||
       /pattern[\s\-_]?2/.test(pattern); // Regex: "pattern" followed by optional separator, then "2"
 
+    // Custom Models is only available in us-east-1 (Nova fine-tuning region requirement)
+    const deployedRegion = import.meta.env.VITE_AWS_REGION as string | undefined;
+    const isCustomModelsSupported = deployedRegion === 'us-east-1';
+
     // Debug logging (remove after testing)
     if (pattern) {
       console.log('[Navigation] IDPPattern detected:', settings.IDPPattern, '| Capacity Planning supported:', isCapacityPlanningSupported);
     }
+    console.log('[Navigation] Region:', deployedRegion, '| Custom Models supported:', isCustomModelsSupported);
+
+    // Build list of items to hide from the Configuration section
+    const hiddenConfigItems = new Set<string>();
+    if (!isCapacityPlanningSupported) {
+      hiddenConfigItems.add('Capacity Planning');
+    }
+    if (!isCustomModelsSupported) {
+      hiddenConfigItems.add('Custom Models');
+    }
 
-    if (isCapacityPlanningSupported) {
-      // Show Capacity Planning for Pattern-2, Unified, or if pattern is unknown
+    if (hiddenConfigItems.size === 0) {
       return baseItems;
     }
 
-    // Filter out Capacity Planning for Pattern 1 and Pattern 3
+    // Filter out hidden items from Configuration section and top-level
     return baseItems
       .map((item) => {
         if (item.type === 'section' && item.text === 'Configuration') {
           const section = item as SideNavigationProps.Section;
           return {
             ...item,
-            items: section.items.filter((subItem) => (subItem as { text?: string }).text !== 'Capacity Planning'),
+            items: section.items.filter((subItem) => !hiddenConfigItems.has((subItem as { text?: string }).text ?? '')),
           };
         }
         return item;
       })
-      .filter((item) => (item as { text?: string }).text !== 'Capacity Planning'); // Also filter top-level if it exists
+      .filter((item) => !hiddenConfigItems.has((item as { text?: string }).text ?? ''));
   }, [baseItems, settings?.IDPPattern]);
 
   // Determine active link based on current path