diff --git a/hindsight-api-slim/tests/test_fact_extraction_quality.py b/hindsight-api-slim/tests/test_fact_extraction_quality.py index 3303dddc45..6e7fcd4c3c 100644 --- a/hindsight-api-slim/tests/test_fact_extraction_quality.py +++ b/hindsight-api-slim/tests/test_fact_extraction_quality.py @@ -152,10 +152,14 @@ async def test_cognitive_epistemic_dimension(self): response=all_facts_text, criteria=( "The extracted facts preserve cognitive or epistemic states from the input — " - "at least two of: the realisation that the approach wasn't working, her uncertainty " - "about whether the meeting would happen, his conviction that AI will transform " + "at least two of: the realisation that the approach wasn't working, uncertainty " + "about whether the meeting would happen, conviction that AI will transform " "healthcare, or the suggestion to reconsider the timeline. Equivalent phrasing " - "(e.g. 'came to understand' for 'realised') is acceptable." + "(e.g. 'came to understand' for 'realised') is acceptable. Evaluate ONLY whether " + "these cognitive/epistemic states are represented; ignore who each state is " + "attributed to (entity/speaker attribution) and ignore exact wording — a state " + "counts as preserved even if the fact attributes it to the wrong person or omits " + "the person entirely." ), context=( "Input: realising an approach wasn't working; uncertainty about a meeting; " @@ -580,22 +584,25 @@ async def test_date_field_calculation_yesterday(self): assert len(facts) > 0, "Should extract at least one fact" - # Find a fact with occurred_start + # Date correctness is asserted deterministically on the structured + # occurred_start field — that is where the resolved date actually lives, + # so the free-text phrasing ("Yesterday" vs "November 12") is irrelevant + # and must NOT gate the test. Calculating the date for a "yesterday" + # event is the whole point here, so require at least one dated fact. facts_with_date = [f for f in facts if f.occurred_start] + assert facts_with_date, f"Expected a fact with a resolved occurred_start. Facts: {[f.fact for f in facts]}" - # If we got a fact with temporal data, verify the date is reasonable - if facts_with_date: - jogging_fact = facts_with_date[0] - fact_date_str = jogging_fact.occurred_start - if "T" in fact_date_str: - fact_date = datetime.fromisoformat(fact_date_str.replace("Z", "+00:00")) - else: - fact_date = datetime.fromisoformat(fact_date_str) + jogging_fact = facts_with_date[0] + fact_date_str = jogging_fact.occurred_start + if "T" in fact_date_str: + fact_date = datetime.fromisoformat(fact_date_str.replace("Z", "+00:00")) + else: + fact_date = datetime.fromisoformat(fact_date_str) - assert fact_date.year == 2024, "Year should be 2024" - assert fact_date.month == 11, "Month should be November" - # Accept day 12 (ideal: yesterday) or 13 (conversation date) as valid - assert fact_date.day in (12, 13), f"Day should be 12 or 13 (around Nov 13 event), but got {fact_date.day}." + assert fact_date.year == 2024, "Year should be 2024" + assert fact_date.month == 11, "Month should be November" + # Accept day 12 (ideal: yesterday) or 13 (conversation date) as valid + assert fact_date.day in (12, 13), f"Day should be 12 or 13 (around Nov 13 event), but got {fact_date.day}." all_facts_text_lower = " ".join(f.fact.lower() for f in facts) all_facts_text = " ".join(f.fact for f in facts) @@ -604,21 +611,24 @@ async def test_date_field_calculation_yesterday(self): # "yesterday" to a concrete date, not paraphrase it as something equally vague. assert "recently" not in all_facts_text_lower, "Should NOT convert 'yesterday' to 'recently'" - # Semantic: content preservation AND date conversion go through the judge so - # paraphrases ("ran" for "jog", "Nov 12 2024" for "November 12") still satisfy. + # Semantic: only the fuzzy content-preservation claim goes through the + # judge (so paraphrases like "ran" for "jog" still satisfy). The date is + # already verified above via occurred_start, so it is deliberately not + # part of the judged criteria — requiring the absolute date to also + # appear in the fact prose tested phrasing, not capability, and flaked. await assert_meets_criteria( response=all_facts_text, criteria=( - "The extracted facts (1) preserve the activity content — that the speaker went for " - "a morning jog/run for the first time in a nearby park — and (2) reflect that the " - "event happened on November 12, 2024 (the day before the conversation), either by " - "stating the absolute date in the fact text or by using an unambiguous reference." + "The extracted facts preserve the activity content — that the speaker went for a " + "morning jog/run for the first time in a nearby park. Equivalent phrasing is " + "acceptable. Ignore how any dates or times are worded; date correctness is checked " + "separately." ), context=( "Conversation date: 2024-11-13. Input: 'Yesterday I went for a morning jog for the " "first time in a nearby park. It was a beautiful day...'" ), - msg=f"Yesterday content and date conversion should be preserved. Facts: {[f.fact for f in facts]}", + msg=f"Yesterday activity content should be preserved. Facts: {[f.fact for f in facts]}", ) @pytest.mark.asyncio