From b3dbe558e0f0eec1ba300f7237a9bae59eef94ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Boschi?= Date: Wed, 27 May 2026 10:21:17 +0200 Subject: [PATCH 1/2] test: stabilize two LLM-flake tests surfaced after PR #1469 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 1. test_high_skepticism_response_is_more_hedged_than_low (hs_llm_core): The source claim was "Sam is *supposedly* the most productive engineer ...". The built-in hedge ("supposedly") primes both low- and high-skepticism reflects to echo it, shrinking the gap the judge has to detect. Rephrasing the claim as a direct assertion gives the disposition room to matter — high-skepticism should now hedge while low-skepticism states it directly. 2. test_comprehensive_multi_dimension (was hs_llm_mat): Module-level marker is hs_llm_core; this method was overriding to hs_llm_mat, which sent it through the bedrock/nova-2-lite weak model. That model consistently drops one of the two required dimensions (emotional or preferential) and fails the judge. This is a quality assertion, not a provider-compatibility check, so it belongs in the single-strong-provider tier (matching the pattern PR #1469 used). --- .../tests/test_fact_extraction_quality.py | 11 +++++++++-- hindsight-api-slim/tests/test_quality_integration.py | 12 ++++++++---- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/hindsight-api-slim/tests/test_fact_extraction_quality.py b/hindsight-api-slim/tests/test_fact_extraction_quality.py index 8c207ae78b..3303dddc45 100644 --- a/hindsight-api-slim/tests/test_fact_extraction_quality.py +++ b/hindsight-api-slim/tests/test_fact_extraction_quality.py @@ -365,10 +365,17 @@ async def test_evaluative_preferential_dimension(self): msg=f"Evaluative/preferential dimension should be preserved. Facts: {[f.fact for f in facts]}", ) - @pytest.mark.hs_llm_mat @pytest.mark.asyncio async def test_comprehensive_multi_dimension(self): - """Test a realistic scenario with multiple dimensions in one fact.""" + """Test a realistic scenario with multiple dimensions in one fact. + + Inherits the module-level `hs_llm_core` marker rather than running on + the full `hs_llm_mat` matrix — the weakest matrix providers (notably + bedrock/nova-2-lite) drop one of the two required dimensions (emotional + or preferential) and fail the judge. This is a quality assertion, not + a provider-compatibility check, so single-strong-provider is the right + tier. + """ text = """ I was thrilled to receive such positive feedback on my presentation yesterday! I wasn't sure if my approach would resonate, but the audience seemed enthusiastic. diff --git a/hindsight-api-slim/tests/test_quality_integration.py b/hindsight-api-slim/tests/test_quality_integration.py index 7c5c6e50ae..51c65ef94a 100644 --- a/hindsight-api-slim/tests/test_quality_integration.py +++ b/hindsight-api-slim/tests/test_quality_integration.py @@ -160,7 +160,11 @@ async def test_high_skepticism_response_is_more_hedged_than_low(self, memory: Me This catches the case where the disposition trait isn't wired into the prompt at all — both responses would then look equally confident. """ - claim = "Sam is supposedly the most productive engineer on the team by a wide margin." + # The claim is stated as a direct assertion (no built-in hedges like + # "supposedly" / "reportedly"). If the source text already hedges, both + # low- and high-skepticism reflects tend to echo the hedge, shrinking + # the gap the judge has to detect. + claim = "Sam is the most productive engineer on the team by a wide margin." query = "What can you tell me about Sam's productivity?" bank_low = f"test-disposition-low-{uuid.uuid4().hex[:8]}" @@ -199,9 +203,9 @@ async def test_high_skepticism_response_is_more_hedged_than_low(self, memory: Me "show the same level of skepticism, the criterion is NOT met." ), context=( - "Both banks stored the same claim ('Sam is supposedly the most productive " - "engineer...') and were asked the same query. The only difference is " - "their skepticism disposition trait. A: skepticism=5, B: skepticism=1." + "Both banks stored the same claim ('Sam is the most productive engineer " + "on the team by a wide margin.') and were asked the same query. The only " + "difference is their skepticism disposition trait. A: skepticism=5, B: skepticism=1." ), msg=( f"Disposition should make response A more skeptical than B.\n" From 021bc4a40306ea9a1014d8c279747e4208a43f19 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Boschi?= Date: Wed, 27 May 2026 10:50:51 +0200 Subject: [PATCH 2/2] test: give skepticism test something to actually be skeptical of MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CI on the first fix attempt still failed identically — both low- and high-skepticism reflects produced "Sam is considered the most productive engineer..." on gemini-2.5-flash-lite. Root cause: with a single assertive claim and no contradicting signal, skepticism has nothing to express. The disposition trait can only show up when there's tension between facts to weigh differently. Add one piece of contradicting evidence ("Sam's manager noted Sam had missed two deadlines last quarter."). Now skepticism=5 should acknowledge the tension while skepticism=1 should defer to the headline claim. Updated the judge criteria and context accordingly. --- .../tests/test_quality_integration.py | 32 ++++++++++++------- 1 file changed, 21 insertions(+), 11 deletions(-) diff --git a/hindsight-api-slim/tests/test_quality_integration.py b/hindsight-api-slim/tests/test_quality_integration.py index 51c65ef94a..5c8ef6c758 100644 --- a/hindsight-api-slim/tests/test_quality_integration.py +++ b/hindsight-api-slim/tests/test_quality_integration.py @@ -160,11 +160,15 @@ async def test_high_skepticism_response_is_more_hedged_than_low(self, memory: Me This catches the case where the disposition trait isn't wired into the prompt at all — both responses would then look equally confident. """ - # The claim is stated as a direct assertion (no built-in hedges like - # "supposedly" / "reportedly"). If the source text already hedges, both - # low- and high-skepticism reflects tend to echo the hedge, shrinking - # the gap the judge has to detect. + # Skepticism only has something to express when there's *something to + # be skeptical of*. With a single assertive claim and no contradicting + # signal, both low- and high-skepticism reflects converge on the same + # restatement (verified on gemini-2.5-flash-lite). We store one + # assertive claim plus one piece of contradicting evidence so the + # high-skepticism bank can flag the tension while the low-skepticism + # bank still defers to the headline claim. claim = "Sam is the most productive engineer on the team by a wide margin." + contradicting = "Sam's manager noted Sam had missed two deadlines last quarter." query = "What can you tell me about Sam's productivity?" bank_low = f"test-disposition-low-{uuid.uuid4().hex[:8]}" @@ -178,6 +182,7 @@ async def test_high_skepticism_response_is_more_hedged_than_low(self, memory: Me request_context=request_context, ) await memory.retain_async(bank_id=bank_id, content=claim, request_context=request_context) + await memory.retain_async(bank_id=bank_id, content=contradicting, request_context=request_context) low_result = await memory.reflect_async(bank_id=bank_low, query=query, request_context=request_context) high_result = await memory.reflect_async(bank_id=bank_high, query=query, request_context=request_context) @@ -197,15 +202,20 @@ async def test_high_skepticism_response_is_more_hedged_than_low(self, memory: Me response=comparison, criteria=( "Response A shows more skepticism than Response B about the productivity " - "claim — A uses more hedging language ('apparently', 'reportedly', 'might', " - "'allegedly', etc.) or more explicitly flags the claim as unverified, while " - "B states it more directly or with less qualification. If the two responses " - "show the same level of skepticism, the criterion is NOT met." + "claim — A acknowledges the tension between 'most productive' and the missed " + "deadlines, uses hedging language ('apparently', 'mixed signals', 'might', etc.), " + "or more explicitly flags the headline claim as unverified or qualified. " + "B states the 'most productive' claim more directly, downplays the missed " + "deadlines, or treats the headline claim as authoritative. If both responses " + "show the same level of skepticism — e.g. both restate the claim with similar " + "qualification — the criterion is NOT met." ), context=( - "Both banks stored the same claim ('Sam is the most productive engineer " - "on the team by a wide margin.') and were asked the same query. The only " - "difference is their skepticism disposition trait. A: skepticism=5, B: skepticism=1." + "Both banks stored TWO facts: the headline claim 'Sam is the most productive " + "engineer on the team by a wide margin.' AND a contradicting signal 'Sam's " + "manager noted Sam had missed two deadlines last quarter.' They were asked " + "the same query. The only configuration difference is the skepticism " + "disposition trait. A: skepticism=5, B: skepticism=1." ), msg=( f"Disposition should make response A more skeptical than B.\n"