refactor(scoring): structural completeness fallback instead of N/A

Per feedback: Completed runs must always show a meaningful number; Failed runs and genuine zeros stay at 0%.

- save_handler._derive_aggregate_scores picks the best available signal: (1) probabilistic confidence when logprobs available; (2) structural completeness (filled fields / total) when no logprobs (reasoning models, image-only flow); (3) 0.0 when no extraction data at all.

- _is_filled_value heuristic: None/empty/whitespace count as not filled; descends into nested dicts/lists.

- Reverted models from Optional[float]=None back to default 0.0.

- Reverted frontend: no N/A path; renders 0% for null/missing scores.

- 15 new tests covering all 3 paths + _is_filled_value heuristic.

Author: Prachig-Microsoft

src/ContentProcessor/src/libs/models/content_process.py

@@ -67,9 +67,9 @@ class ContentProcess(BaseModel):
     last_modified_time: datetime.datetime = datetime.datetime.now(datetime.UTC)
     last_modified_by: Optional[str] = None
     status: str
-    entity_score: Optional[float] = None
-    min_extracted_entity_score: Optional[float] = None
-    schema_score: Optional[float] = None
+    entity_score: Optional[float] = 0.0
+    min_extracted_entity_score: Optional[float] = 0.0
+    schema_score: Optional[float] = 0.0
     result: Optional[dict] = None
     confidence: Optional[dict] = None
     target_schema: Optional[Schema] = None

src/ContentProcessor/src/libs/pipeline/handlers/save_handler.py

@@ -112,13 +112,12 @@ def find_process_result(step_name: str):
             )
         )
 
-        # Determine whether per-field confidence could actually be computed.
-        # When `total_evaluated_fields_count == 0`, no field-level confidence
-        # signal was produced (e.g. logprobs unavailable on reasoning models, or
-        # an image flow with no Content Understanding signal). In that case the
-        # entity/schema scores are *unavailable* rather than genuinely zero, and
-        # we propagate ``None`` so downstream consumers (API + UI) can render
-        # an explicit "N/A" instead of a misleading "0%".
+        # Compute the aggregate scores. Successful (Completed) processing
+        # always yields numeric scores: when probabilistic confidence is
+        # available (logprobs from non-reasoning models / Content Understanding
+        # signal) we use it; otherwise we fall back to a structural
+        # completeness score (fraction of expected fields actually filled).
+        # Failed runs and genuinely empty extractions remain at ``0.0``.
         entity_score, schema_score, min_extracted_entity_score = (
             self._derive_aggregate_scores(evaluated_result)
         )
@@ -235,22 +234,51 @@ def _summarize_processed_time(self, step_results: list[StepResult]) -> str:
         formatted_elapsed_time = f"{total_hours:02}:{total_minutes:02}:{total_seconds:02}.{total_milliseconds:03}"
         return formatted_elapsed_time
 
+    @staticmethod
+    def _is_filled_value(value: object) -> bool:
+        """Heuristic: does an extracted value count as "actually filled"?
+
+        Treats ``None``, empty strings, whitespace-only strings, and empty
+        containers as *not* filled. Recursively descends into dicts/lists so a
+        nested object that contains only nulls is still counted as empty.
+        """
+        if value is None:
+            return False
+        if isinstance(value, bool):
+            return True
+        if isinstance(value, str):
+            return value.strip() != ""
+        if isinstance(value, dict):
+            return any(SaveHandler._is_filled_value(v) for v in value.values())
+        if isinstance(value, (list, tuple, set)):
+            return any(SaveHandler._is_filled_value(v) for v in value)
+        return True
+
     @staticmethod
     def _derive_aggregate_scores(
         evaluated_result: DataExtractionResult,
-    ) -> tuple[float | None, float | None, float | None]:
+    ) -> tuple[float, float, float]:
         """Compute ``(entity_score, schema_score, min_extracted_entity_score)``.
 
-        Returns ``(None, None, None)`` when no per-field confidence signal was
-        produced (i.e. ``total_evaluated_fields_count == 0`` or there are no
-        comparison items). This happens, for example, when the LLM call could
-        not return logprobs (reasoning models) and there is no Content
-        Understanding signal to fall back on. Treating that case as "unknown"
-        rather than ``0.0`` lets the API and UI render "N/A" instead of a
-        misleading "0%".
+        Score selection order:
+
+        1. **Probabilistic confidence** — when the evaluate step produced
+           per-field confidence (``total_evaluated_fields_count > 0``), use the
+           probabilistic ``overall_confidence`` plus the ratio of
+           above-threshold fields. This is the highest-fidelity signal.
+
+        2. **Structural completeness fallback** — when no probabilistic
+           signal was produced (e.g. reasoning models like ``gpt-5``/``o1``/``o3``
+           don't return logprobs, and image-only flow has no Content
+           Understanding signal), but extraction still produced a comparison
+           table, score by *how much of the schema was actually filled*. This
+           replaces the old behaviour of falsely emitting ``0%`` for completed
+           runs that simply lacked logprobs.
 
-        A genuine zero confidence (e.g. a model that emitted fields but
-        every token had ``logprob == -inf``) is preserved verbatim.
+        3. **Zero** — only when there is literally no extraction data
+           (failed pipeline / genuinely empty result). Failed processing
+           continues to surface as ``0`` so the UI consistently renders
+           ``0%`` for failures and genuine zeros.
         """
         confidence = evaluated_result.confidence or {}
         total_evaluated_fields_count = confidence.get(
@@ -261,14 +289,29 @@ def _derive_aggregate_scores(
             if evaluated_result.comparison_result is not None
             else []
         )
-        if total_evaluated_fields_count == 0 or not comparison_items:
-            return (None, None, None)
 
-        zero_count = confidence.get("zero_confidence_fields_count", 0)
-        schema_score = round(
-            (len(comparison_items) - zero_count) / len(comparison_items),
-            3,
-        )
-        entity_score = confidence.get("overall_confidence")
-        min_extracted_entity_score = confidence.get("min_extracted_field_confidence")
-        return (entity_score, schema_score, min_extracted_entity_score)
+        # Path 1: probabilistic confidence
+        if total_evaluated_fields_count > 0 and comparison_items:
+            zero_count = confidence.get("zero_confidence_fields_count", 0)
+            schema_score = round(
+                (len(comparison_items) - zero_count) / len(comparison_items),
+                3,
+            )
+            entity_score = float(confidence.get("overall_confidence") or 0.0)
+            min_extracted_entity_score = float(
+                confidence.get("min_extracted_field_confidence") or 0.0
+            )
+            return (entity_score, schema_score, min_extracted_entity_score)
+
+        # Path 2: structural completeness fallback
+        if comparison_items:
+            filled = sum(
+                1
+                for item in comparison_items
+                if SaveHandler._is_filled_value(item.Extracted)
+            )
+            ratio = round(filled / len(comparison_items), 3)
+            return (ratio, ratio, ratio)
+
+        # Path 3: nothing to score on
+        return (0.0, 0.0, 0.0)

src/ContentProcessor/tests/unit/pipeline/test_save_handler_scores.py

@@ -3,11 +3,12 @@
 
 """Tests for ``SaveHandler._derive_aggregate_scores``.
 
-Covers the score-availability semantics:
-- valid scores flow through verbatim
-- missing per-field signal yields ``None`` (rendered as "N/A" in the UI)
-- a genuine zero is preserved as ``0`` (rendered as "0%")
-- failed processing (no comparison items) yields ``None``
+Covers the score-derivation contract:
+- probabilistic confidence flows through verbatim when available
+- structural completeness fallback fires for Completed runs without logprobs
+  (e.g. reasoning models / image-only flow) instead of emitting a misleading 0%
+- a genuine zero is preserved as ``0.0``
+- failed/empty runs return ``0.0``
 """
 
 from __future__ import annotations
@@ -35,7 +36,7 @@ def _make_result(
     )
 
 
-class TestDeriveAggregateScores:
+class TestProbabilisticPath:
     def test_valid_scores_flow_through(self):
         """A normal evaluate-step result must produce numeric scores."""
         items = [
@@ -63,75 +64,174 @@ def test_valid_scores_flow_through(self):
         assert schema == round(2 / 3, 3)
         assert min_score == 0.0
 
-    def test_missing_per_field_signal_returns_none(self):
-        """Reasoning-model / image-only flow: no signal → ``None`` everywhere."""
-        items: list[ExtractionComparisonItem] = []
+    def test_all_fields_above_threshold(self):
+        items = [
+            ExtractionComparisonItem(
+                Field="a", Extracted="x", Confidence="95.00%", IsAboveThreshold="True"
+            ),
+            ExtractionComparisonItem(
+                Field="b", Extracted="y", Confidence="90.00%", IsAboveThreshold="True"
+            ),
+        ]
         confidence = {
-            "total_evaluated_fields_count": 0,
-            "overall_confidence": 0.0,
-            "min_extracted_field_confidence": 0.0,
+            "total_evaluated_fields_count": 2,
+            "overall_confidence": 0.925,
+            "min_extracted_field_confidence": 0.9,
             "zero_confidence_fields_count": 0,
         }
         entity, schema, min_score = SaveHandler._derive_aggregate_scores(
             _make_result(items=items, confidence=confidence)
         )
-        assert entity is None
-        assert schema is None
-        assert min_score is None
+        assert entity == 0.925
+        assert schema == 1.0
+        assert min_score == 0.9
+
 
-    def test_no_comparison_items_returns_none(self):
-        """Even if confidence claims fields exist, an empty comparison list is unknown."""
+class TestStructuralFallback:
+    """When logprobs are unavailable (reasoning model / image-only) but
+    extraction succeeded, the Completed file must still get a meaningful
+    numeric score based on schema completeness."""
+
+    def test_all_fields_filled_yields_one(self):
+        items = [
+            ExtractionComparisonItem(
+                Field="a", Extracted="x", Confidence="0.00%", IsAboveThreshold="False"
+            ),
+            ExtractionComparisonItem(
+                Field="b", Extracted="y", Confidence="0.00%", IsAboveThreshold="False"
+            ),
+            ExtractionComparisonItem(
+                Field="c", Extracted=42, Confidence="0.00%", IsAboveThreshold="False"
+            ),
+        ]
+        # No probabilistic signal: total_evaluated_fields_count == 0
         confidence = {
-            "total_evaluated_fields_count": 5,
-            "overall_confidence": 0.9,
-            "min_extracted_field_confidence": 0.5,
+            "total_evaluated_fields_count": 0,
+            "overall_confidence": 0.0,
+            "min_extracted_field_confidence": 0.0,
             "zero_confidence_fields_count": 0,
         }
         entity, schema, min_score = SaveHandler._derive_aggregate_scores(
-            _make_result(items=[], confidence=confidence)
+            _make_result(items=items, confidence=confidence)
         )
-        assert entity is None
-        assert schema is None
-        assert min_score is None
+        assert entity == 1.0
+        assert schema == 1.0
+        assert min_score == 1.0
 
-    def test_genuine_zero_score_preserved(self):
-        """A real ``0`` confidence (e.g. all fields below threshold) must NOT become ``None``."""
+    def test_partial_fill_yields_ratio(self):
         items = [
             ExtractionComparisonItem(
                 Field="a", Extracted="x", Confidence="0.00%", IsAboveThreshold="False"
             ),
+            ExtractionComparisonItem(
+                Field="b", Extracted=None, Confidence="0.00%", IsAboveThreshold="False"
+            ),
+            ExtractionComparisonItem(
+                Field="c", Extracted="", Confidence="0.00%", IsAboveThreshold="False"
+            ),
+            ExtractionComparisonItem(
+                Field="d", Extracted="z", Confidence="0.00%", IsAboveThreshold="False"
+            ),
+        ]
+        confidence = {"total_evaluated_fields_count": 0}
+        entity, schema, min_score = SaveHandler._derive_aggregate_scores(
+            _make_result(items=items, confidence=confidence)
+        )
+        # 2 of 4 fields actually filled → 0.5
+        assert entity == 0.5
+        assert schema == 0.5
+        assert min_score == 0.5
+
+    def test_all_fields_empty_yields_zero(self):
+        """Genuine-empty extraction: structural fallback collapses to ``0.0``."""
+        items = [
+            ExtractionComparisonItem(
+                Field="a", Extracted=None, Confidence="0.00%", IsAboveThreshold="False"
+            ),
+            ExtractionComparisonItem(
+                Field="b", Extracted="", Confidence="0.00%", IsAboveThreshold="False"
+            ),
+            ExtractionComparisonItem(
+                Field="c", Extracted="   ", Confidence="0.00%", IsAboveThreshold="False"
+            ),
         ]
+        confidence = {"total_evaluated_fields_count": 0}
+        entity, schema, min_score = SaveHandler._derive_aggregate_scores(
+            _make_result(items=items, confidence=confidence)
+        )
+        assert entity == 0.0
+        assert schema == 0.0
+        assert min_score == 0.0
+
+
+class TestZeroPath:
+    def test_no_comparison_items_returns_zero(self):
+        """No extraction data at all (failed pipeline) → ``0.0``."""
         confidence = {
-            "total_evaluated_fields_count": 1,
+            "total_evaluated_fields_count": 0,
             "overall_confidence": 0.0,
             "min_extracted_field_confidence": 0.0,
-            "zero_confidence_fields_count": 1,
+            "zero_confidence_fields_count": 0,
         }
         entity, schema, min_score = SaveHandler._derive_aggregate_scores(
-            _make_result(items=items, confidence=confidence)
+            _make_result(items=[], confidence=confidence)
         )
         assert entity == 0.0
         assert schema == 0.0
         assert min_score == 0.0
 
-    def test_all_fields_above_threshold(self):
+    def test_genuine_zero_probabilistic_score_preserved(self):
+        """A real ``0`` confidence (every field below threshold) must NOT be
+        replaced by the structural fallback — it's genuinely 0%."""
         items = [
             ExtractionComparisonItem(
-                Field="a", Extracted="x", Confidence="95.00%", IsAboveThreshold="True"
-            ),
-            ExtractionComparisonItem(
-                Field="b", Extracted="y", Confidence="90.00%", IsAboveThreshold="True"
+                Field="a", Extracted="x", Confidence="0.00%", IsAboveThreshold="False"
             ),
         ]
         confidence = {
-            "total_evaluated_fields_count": 2,
-            "overall_confidence": 0.925,
-            "min_extracted_field_confidence": 0.9,
-            "zero_confidence_fields_count": 0,
+            "total_evaluated_fields_count": 1,
+            "overall_confidence": 0.0,
+            "min_extracted_field_confidence": 0.0,
+            "zero_confidence_fields_count": 1,
         }
         entity, schema, min_score = SaveHandler._derive_aggregate_scores(
             _make_result(items=items, confidence=confidence)
         )
-        assert entity == 0.925
-        assert schema == 1.0
-        assert min_score == 0.9
+        assert entity == 0.0
+        assert schema == 0.0
+        assert min_score == 0.0
+
+
+class TestIsFilledValue:
+    """Coverage for the ``_is_filled_value`` helper used by the structural fallback."""
+
+    def test_none_is_empty(self):
+        assert SaveHandler._is_filled_value(None) is False
+
+    def test_empty_string_is_empty(self):
+        assert SaveHandler._is_filled_value("") is False
+        assert SaveHandler._is_filled_value("   ") is False
+
+    def test_non_empty_string_is_filled(self):
+        assert SaveHandler._is_filled_value("x") is True
+
+    def test_zero_int_is_filled(self):
+        # A literal ``0`` is a valid extracted value (e.g. count fields).
+        assert SaveHandler._is_filled_value(0) is True
+
+    def test_bool_is_filled(self):
+        assert SaveHandler._is_filled_value(False) is True
+        assert SaveHandler._is_filled_value(True) is True
+
+    def test_empty_container_is_empty(self):
+        assert SaveHandler._is_filled_value([]) is False
+        assert SaveHandler._is_filled_value({}) is False
+
+    def test_nested_all_null_is_empty(self):
+        assert SaveHandler._is_filled_value({"a": None, "b": ""}) is False
+        assert SaveHandler._is_filled_value([None, "", {"c": None}]) is False
+
+    def test_nested_with_value_is_filled(self):
+        assert SaveHandler._is_filled_value({"a": None, "b": "x"}) is True
+        assert SaveHandler._is_filled_value([None, "x"]) is True
+

src/ContentProcessorAPI/app/routers/models/contentprocessor/claim_process.py

@@ -53,13 +53,13 @@ class Content_Process(EntityBase):
     mime_type: Optional[str] = Field(
         description="MIME type of the processed content file", default=None
     )
-    entity_score: Optional[float] = Field(
-        description="Score indicating the quality of entity extraction from the content. ``None`` means the score was not available (e.g. logprobs unavailable on reasoning models).",
-        default=None,
+    entity_score: float = Field(
+        description="Score indicating the quality of entity extraction from the content. For Completed runs this is either the probabilistic confidence (when logprobs are available) or a structural completeness fallback (fraction of expected fields actually filled). Failed runs and genuinely empty extractions remain at ``0.0``.",
+        default=0.0,
     )
-    schema_score: Optional[float] = Field(
-        description="Score indicating the quality of schema matching for the content. ``None`` means the score was not available.",
-        default=None,
+    schema_score: float = Field(
+        description="Score indicating the quality of schema matching for the content. For Completed runs this is either the probabilistic above-threshold ratio or a structural completeness fallback. Failed runs remain at ``0.0``.",
+        default=0.0,
     )
     status: Optional[str] = Field(
         description="Indicates the current status in the content processing pipeline",