fix: remove answer_relevance* intrinsics; fix other intrinsics issues (#690)

* fix: add support back for older models and requirement_check adapter

* fix: fix model used in test_rag for intrinsics

* fix: remove answer_relevance_classifier and answer_relevance_rewriter

Author: jakelorocco

docs/examples/intrinsics/README.md

@@ -13,9 +13,6 @@ Core example showing how to directly use intrinsics with adapters.
 - Working with Granite Common adapters (aLoRA-based)
 - Understanding adapter output formats
 
-### answer_relevance.py
-Evaluates whether an answer is relevant to a question.
-
 ### answerability.py
 Checks if a question can be answered given the context.
 
@@ -71,7 +68,6 @@ out, new_ctx = mfuncs.act(
 ## Available Intrinsics
 
 - **requirement_check**: Validate requirements (used by ALoraRequirement)
-- **answer_relevance**: Check answer-question relevance
 - **answerability**: Determine if question is answerable
 - **citations**: Extract and validate citations
 - **context_relevance**: Assess context-query relevance

docs/examples/intrinsics/answer_relevance.py

@@ -65,16 +65,13 @@ class IntriniscsCatalogEntry(pydantic.BaseModel):
     # Core Intrinsics
     ############################################
     IntriniscsCatalogEntry(name="requirement-check", repo_id=_CORE_R1_REPO),
+    IntriniscsCatalogEntry(
+        name="requirement_check", repo_id=_CORE_REPO
+    ),  # Necessary to support granite 3.2 and 3.3.
     IntriniscsCatalogEntry(name="uncertainty", repo_id=_CORE_R1_REPO),
     ############################################
     # RAG Intrinsics
     ############################################
-    IntriniscsCatalogEntry(
-        name="answer_relevance_classifier",
-        repo_id=_RAG_REPO,
-        adapter_types=(AdapterType.LORA,),
-    ),
-    IntriniscsCatalogEntry(name="answer_relevance_rewriter", repo_id=_RAG_REPO),
     IntriniscsCatalogEntry(name="answerability", repo_id=_RAG_REPO),
     IntriniscsCatalogEntry(name="citations", repo_id=_RAG_REPO),
     IntriniscsCatalogEntry(name="context_relevance", repo_id=_RAG_REPO),

mellea/backends/adapters/catalog.py

@@ -8,24 +8,6 @@
 from ..chat import Message
 from ._util import call_intrinsic
 
-_ANSWER_RELEVANCE_CORRECTION_METHODS = {
-    "Excessive unnecessary information": "removing the excessive information from the "
-    "draft response",
-    "Unduly restrictive": "providing answer without the unwarranted restriction, or "
-    "indicating that the desired answer is not available",
-    "Too vague or generic": "providing more crisp and to-the-point answer, or "
-    "indicating that the desired answer is not available",
-    "Contextual misalignment": "providing a response that answers the last user "
-    "inquiry, taking into account the context of the conversation",
-    "Misinterpreted inquiry": "providing answer only to the correct interpretation of "
-    "the inquiry, or attempting clarification if the inquiry is ambiguous or otherwise "
-    "confusing, or indicating that the desired answer is not available",
-    "No attempt": "providing a relevant response if an inquiry should be answered, or "
-    "providing a short response if the last user utterance contains no inquiry",
-}
-"""Prompting strings for the answer relevance rewriter. This model is a (a)LoRA adapter,
-so it's important to stick to in-domain prompts."""
-
 
 def check_answerability(
     question: str,
@@ -207,62 +189,3 @@ def flag_hallucinated_content(
         backend,
     )
     return result_json
-
-
-def rewrite_answer_for_relevance(
-    response: str,
-    documents: collections.abc.Iterable[Document],
-    context: ChatContext,
-    backend: AdapterMixin,
-    /,
-    rewrite_threshold: float = 0.5,
-) -> str:
-    """Rewrite an assistant answer to improve relevance to the user's question.
-
-    Args:
-        response: The assistant's response to the user's question in the last turn
-            of ``context``.
-        documents: Document snippets that were used to generate ``response``.
-        context: A chat log that ends with a user asking a question.
-        backend: Backend instance that supports the adapters that implement this
-            intrinsic.
-        rewrite_threshold: Number between 0.0 and 1.0 that determines how eagerly
-            to skip rewriting the assistant's answer for relevance. 0.0 means never
-            rewrite and 1.0 means always rewrite.
-
-    Returns:
-        Either the original response, or a rewritten version of the original response.
-    """
-    # First run the classifier to determine the likelihood of a relevant answer
-    # Output will have three fields:
-    # * answer_relevance_analysis
-    # * answer_relevance_category
-    # * answer_relevance_likelihood
-    result_json = call_intrinsic(
-        "answer_relevance_classifier",
-        context.add(Message("assistant", response, documents=list(documents))),
-        backend,
-    )
-    if result_json["answer_relevance_likelihood"] >= rewrite_threshold:
-        return response
-
-    # If we get here, the classifier indicated a likely irrelevant response. Trigger
-    # rewrite.
-    # Rewrite needs a prompt string that is an expanded version of the classifier's
-    # short output.
-    correction_method = _ANSWER_RELEVANCE_CORRECTION_METHODS[
-        result_json["answer_relevance_category"]
-    ]
-
-    result_json = call_intrinsic(
-        "answer_relevance_rewriter",
-        context.add(Message("assistant", response, documents=list(documents))),
-        backend,
-        kwargs={
-            "answer_relevance_category": result_json["answer_relevance_category"],
-            "answer_relevance_analysis": result_json["answer_relevance_analysis"],
-            "correction_method": correction_method,
-        },
-    )
-    # Unpack boxed string
-    return result_json["answer_relevance_rewrite"]

mellea/stdlib/components/intrinsic/rag.py

@@ -193,32 +193,6 @@ def _maybe_download_yaml(self):
         task="context_relevance",
         is_alora=True,
     ),
-    YamlJsonCombo(
-        short_name="answer_relevance_classifier",
-        inputs_file=_INPUT_JSON_DIR / "answer_relevance_classifier.json",
-        task="answer_relevance_classifier",
-    ),
-    # aLoRA adapter for this intrinsic not currently available
-    # YamlJsonCombo(
-    #     short_name="answer_relevance_classifier_alora",
-    #     inputs_file=_INPUT_JSON_DIR / "answer_relevance_classifier.json",
-    #     task="answer_relevance_classifier",
-    #     is_alora=True,
-    # ),
-    YamlJsonCombo(
-        short_name="answer_relevance_rewriter",
-        inputs_file=_INPUT_JSON_DIR / "answer_relevance_rewriter.json",
-        arguments_file=_INPUT_ARGS_DIR / "answer_relevance_rewriter.json",
-        task="answer_relevance_rewriter",
-    ),
-    # aLoRA adapter for this intrinsic not currently available
-    # YamlJsonCombo(
-    #     short_name="answer_relevance_rewriter_alora",
-    #     inputs_file=_INPUT_JSON_DIR / "answer_relevance_rewriter.json",
-    #     arguments_file=_INPUT_ARGS_DIR / "answer_relevance_rewriter.json",
-    #     task="answer_relevance_rewriter",
-    #     is_alora=True,
-    # ),
     YamlJsonCombo(
         short_name="citations",
         inputs_file=_INPUT_JSON_DIR / "citations.json",