From 1766dcc4fbf1a3eef54b127d8e62ccb7f57f361e Mon Sep 17 00:00:00 2001 From: douxiao398 Date: Thu, 18 Jun 2026 22:16:03 +0800 Subject: [PATCH] fix(corrective-rag): make relevancy grading robust so the web search actually triggers The corrective-rag and firecrawl-agent workflows decide whether to run a corrective web search with `if "no" in relevancy_results`. Each element of that list is the grader LLM's full response (lower-cased/stripped), so this only matches when a response is exactly "no". Real outputs like "no.", "No, the document is not relevant", etc. never match, so the corrective web search is silently skipped and the core Corrective-RAG mechanism never fires. corrective-rag also filtered relevant docs with `result == "yes"`, which drops any doc whose grade isn't exactly "yes" (e.g. "yes, relevant"). Make the parsing robust: - corrective-rag: treat a doc as relevant when its grade starts with "yes", and trigger the web search when any doc is not a clear "yes". - firecrawl-agent: it already treats a doc as relevant when the grade contains "yes", so trigger the web search when any doc does not contain "yes" (consistent with that definition). --- corrective-rag/workflow.py | 12 ++++++++++-- firecrawl-agent/workflow.py | 7 ++++++- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/corrective-rag/workflow.py b/corrective-rag/workflow.py index a167e4930..ca4821d69 100644 --- a/corrective-rag/workflow.py +++ b/corrective-rag/workflow.py @@ -127,13 +127,21 @@ async def eval_relevance( relevancy = self.llm.complete(prompt) relevancy_results.append(relevancy.text.lower().strip()) + # The grader is asked for a binary 'yes'/'no', but LLMs often add + # punctuation or a short justification (e.g. "No, ..."). Match on the + # leading token so the decision is robust instead of requiring the result + # to be exactly "yes"/"no". relevant_texts = [ retrieved_nodes[i].text for i, result in enumerate(relevancy_results) - if result == "yes" + if result.startswith("yes") ] relevant_text = "\n".join(relevant_texts) - if "no" in relevancy_results: + # Trigger a corrective web search whenever at least one retrieved document + # was judged not relevant (anything that isn't a clear "yes"). The previous + # `"no" in relevancy_results` only matched a result that was exactly "no", + # so answers like "No, ..." or "no." silently skipped the web search. + if any(not result.startswith("yes") for result in relevancy_results): return WebSearchEvent(relevant_text=relevant_text) else: return QueryEvent(relevant_text=relevant_text, search_text="") diff --git a/firecrawl-agent/workflow.py b/firecrawl-agent/workflow.py index 5c85ae6ac..7d0a93d4e 100644 --- a/firecrawl-agent/workflow.py +++ b/firecrawl-agent/workflow.py @@ -180,7 +180,12 @@ async def eval_relevance( print(f"DEBUG: Relevant texts count: {len(relevant_texts)}") print(f"DEBUG: Relevant text preview: {relevant_text[:200]}...") - if "no" in relevancy_results_striped: + # A document is treated as relevant when its grade contains "yes" (see the + # filter above), so trigger a corrective web search whenever any document is + # *not* relevant. The previous `"no" in relevancy_results_striped` only + # matched a grade equal to exactly "no", so answers like "No, ..." or "no." + # silently skipped the web search. + if any("yes" not in result.lower() for result in relevancy_results_striped): print("DEBUG: Some documents irrelevant, returning WebSearchEvent") return WebSearchEvent(relevant_text=relevant_text) else: