Merge pull request #44 from sciknoworg/dev

add dimension - gap pairwise rubric

Author: HamedBabaei

README.md

@@ -124,6 +124,8 @@ from yescieval.rubric.pairwise.depth import MechanisticUnderstanding, CausalReas
 from yescieval.rubric.pairwise.breadth import ContextCoverage, MethodCoverage, DimensionCoverage, ScaleCoverage, ScopeCoverage
 # Rigor rubrics
 from yescieval.rubric.pairiwise.rigor import EpistemicCalibration, QuantitativeEvidenceAndUncertainty, ExplicitUncertainty
+# Gap rubrics
+from yescieval.rubric.pairwise.gap import GapIdentification
 ```
 
 A complete list of rubrics are available at YESciEval [📚 Rubrics](https://yescieval.readthedocs.io/rubrics.html) page.

docs/source/rubrics.rst

@@ -45,6 +45,8 @@ Each rubric can be used in two ways:
         from yescieval.rubric.pairwise.breadth import ContextCoverage, MethodCoverage, DimensionCoverage, ScaleCoverage, ScopeCoverage
         # Pairwise rigor rubrics
         from yescieval.rubric.pairwise.rigor import EpistemicCalibration, QuantitativeEvidenceAndUncertainty, ExplicitUncertainty
+        # Pairwise Gap rubric
+        from yescieval.rubric.pairwise.gap import GapIdentification
 
 
 Pairwise Evaluation
@@ -471,9 +473,7 @@ Detects explicit acknowledgment of unanswered questions or understudied areas.
        terms like "research gap" or "understudied"?
 
 
-
-
-.. tab:: Basic Usage
+.. tab:: Pointwise Usage
 
   .. code-block:: python
 
@@ -485,6 +485,19 @@ Detects explicit acknowledgment of unanswered questions or understudied areas.
      print(instruction)
      print(rubric.name)
 
+.. tab:: Pairwise Usage
+
+  .. code-block:: python
+
+     from yescieval.rubric.pairwise.gap import GapIdentification
+
+     rubric      = GapIdentification(papers=papers, question=question, answer_a=answer_a, answer_b=answer_b)
+     instruction = rubric.instruct()
+
+     print(instruction)
+     print(rubric.name)
+  
+
 .. tab:: Usage with Example and Vocabulary Injectors
 
   .. code-block:: python

yescieval/injector/domains/ecology.py

@@ -487,6 +487,30 @@
                     }
                 ]
             }
+        },
+        "Gap": {
+            "GapIdentification": {
+                "ResponseA": [
+                    {
+                        "rating": "1",
+                        "rationale": "The response only describes known ecological patterns and findings. It does not point out any missing data, unanswered questions, or limitations. It also does not mention underrepresented species, regions, or conflicting results."
+                    },
+                    {
+                        "rating": "4",
+                        "rationale": "The response clearly identifies important gaps, such as missing long-term data for soil invertebrates in semi-arid regions, limited studies in tropical and boreal systems, and conflicting results on predator reintroduction. It explains why these gaps matter, though one point slightly mixes data gaps with methodological issues."
+                    }
+                ],
+                "ResponseB": [
+                    {
+                        "rating": "1",
+                        "rationale": "The response gives general observations about deforestation and habitat fragmentation but does not identify any missing information, open questions, or conflicting evidence. It assumes the existing knowledge is complete."
+                    },
+                    {
+                        "rating": "4",
+                        "rationale": "The response identifies clear gaps, such as lack of controlled studies on edge effects in freshwater areas, missing long-term data for migratory invertebrates, and differences between remote sensing and field data. It explains how these gaps affect conclusions, though it could be more specific about affected species or regions."
+                    }
+                ]
+            }
         }    
     }
 }

yescieval/injector/domains/nlp.py

@@ -493,6 +493,30 @@
                     }
                 ]
             }
+        },
+        "Gap": {
+            "GapIdentification": {
+                "ResponseA": [
+                    {
+                        "rating": "1",
+                        "rationale": "The response is purely descriptive, summarizing reported benchmark scores and model architecture details across several tasks such as named entity recognition and machine translation. It does not identify any missing evaluations, underexplored domains, or unresolved questions relevant to the research question. No mention is made of dataset biases, lack of ablation studies, or conflicting results across benchmarks."
+                    },
+                    {
+                        "rating": "4",
+                        "rationale": "The response clearly identifies several meaningful gaps in the evidence base: it notes that most evaluations of large language models are conducted on high-resource languages leaving low-resource and morphologically complex languages largely underrepresented, that existing benchmarks for dialogue systems lack coverage of multi-turn reasoning under ambiguity, and that conflicting results across GLUE and SuperGLUE variants suggest evaluation inconsistencies that remain unresolved. The response explains why these gaps limit claims about generalizability, though it slightly overgeneralizes when attributing all performance variance to dataset bias without considering architectural differences."
+                    }
+                ],
+                "ResponseB": [
+                    {
+                        "rating": "1",
+                        "rationale": "The response describes existing approaches to coreference resolution and summarization, reporting model performance figures and dataset characteristics without identifying any limitations, missing analyses, or unresolved aspects of the research question. Findings are presented as comprehensive and conclusive, with no acknowledgment of underexplored tasks, missing robustness evaluations, or gaps in the comparative analysis across model families."
+                    },
+                    {
+                        "rating": "4",
+                        "rationale": "The response identifies concrete gaps in the evidence base, such as the absence of out-of-domain generalization evaluations for transformer-based models fine-tuned on narrow corpora, the lack of ablation studies isolating the contribution of pretraining data size versus architectural choices, and the underrepresentation of spoken or informal language registers in standard NLP benchmarks. It explains that these gaps undermine the reliability of conclusions drawn from current leaderboard comparisons, though it could have been more specific about which model families or task categories are most critically affected."
+                    }
+                ]
+            }
         }           
     }
 }

yescieval/rubric/pairwise/gap/__init__.py

@@ -0,0 +1,3 @@
+from .gap_identification import GapIdentification
+
+__all__ = ["GapIdentification"]

yescieval/rubric/pairwise/gap/gap_identification.py

@@ -0,0 +1,85 @@
+from ....base import PairwiseRubric
+
+gap_identification_pairwise_prompt = """<Context>
+Scientific question answering and synthesis often require more than listing findings: high-quality scientific writing identifies what remains unknown, insufficiently addressed, or unresolved in existing research. This is commonly expressed through gap identification, where the text specifies limitations, missing knowledge, unresolved inconsistencies, or missing connections in prior work and explains why they matter for the research question.
+
+Gap identification can refer to (a) field-level gaps across the literature (e.g., missing populations/settings, inconsistent measures, lack of comparative studies, limited external validity), and/or (b) recurring study-level limitations that materially constrain conclusions when framed as evidence-base limitations. High-quality gap statements are specific and scoped (what is missing, where, and why), rather than generic (“more research is needed”). If gap emphasis is not central to the user's question, it should not be forced; however, when included, it should remain relevant and well-defined.
+
+The responses may be short paragraphs or long-form reports. Gap identification should be evaluated independently of presentation style or length.
+
+This rubric focuses exclusively on the presence and quality of gap identification within the provided text of two responses that are compared, emphasizing explicit and relevant statements of limitations, unanswered questions, or missing connections in prior work rather than summaries of what is already known. Other aspects of scientific quality (such as factual accuracy, evidential grounding, completeness, or innovation) are intentionally outside its scope and are assessed by separate evaluation criteria.
+</Context>
+
+<Role>
+You are tasked as a scientific writing quality evaluator performing a pairwise comparison between two texts.
+</Role>
+
+<Task-Description>
+A user will provide:
+1) a research question, and
+2) two written responses (Response A and Response B) intended to address that question.
+
+Your task is to:
+- First, independently evaluate each response using the evaluation characteristic below.
+- Then perform a pairwise comparison of the two responses using the evaluation characteristic below.
+- Then grade each response with a comparative rating from 1 (very bad) to 5 (very good) compared to the other response and a subsequent rationale for each comparative response rating.
+- Note that it is possible for both responses to receive the same rating, if they are equally comparably clear with systematic gap identification and provide complementary or identical insights.
+
+Your judgment must be based solely on the provided question and comparing the two responses w.r.t. addressing the question and w.r.t. each other.
+</Task-Description>
+
+<Evaluation-Characteristic>
+GapIdentification: Does the response identify gaps relevant to the research question by specifying limitations, missing knowledge, unresolved issues, or missing connections in prior work (preferably at the evidence-base/field level), rather than only describing existing findings?
+</Evaluation-Characteristic>
+
+<Domain-Vocabulary-Examples>
+Below are terms and phrases that often signal gap identification. They are examples only: their presence is not required, and their presence alone is not sufficient for a high score.
+{GAP_IDENTIFICATION_VOCAB}
+</Domain-Vocabulary-Examples>
+
+<Rating-Scale>
+For the characteristic above, rate the quality from 1 (very bad) to 5 (very good). Follow the guidelines specified below.
+
+GapIdentification
+Rating 1. Very bad: The response is purely descriptive, summarizing existing findings or observations with no identification of missing, unknown, inconsistent, or unresolved aspects relevant to the research question.
+Rating 2. Bad: The response refers to gaps only in a vague or generic manner (e.g., “more research is needed”) without clearly specifying what is missing or unresolved in the context of the research question.
+Rating 3. Moderate: The response identifies one or more potential gaps with partial clarity, but the nature, scope, location (where in the literature), or relevance of the gaps is incomplete, unclear, or inconsistently articulated.
+Rating 4. Good: The response clearly identifies specific gaps or limitations in the evidence base that are relevant to the research question (e.g., missing comparisons, populations, settings, measures, time horizons, or conflicting results) and provides some explanation of why they matter; minor ambiguity or imprecision may remain.
+Rating 5. Very good: The response explicitly and clearly identifies well-defined gaps or unanswered questions, specifying what is missing, where it occurs in the evidence base (or across studies), and why it matters for the research question, without relying on vague statements. Gap statements are appropriately scoped (avoiding absolute claims like “no research exists” unless clearly justified within the response).
+</Rating-Scale>
+
+<Response-Format>
+Return your evaluation strictly in JSON format:
+
+{
+  "GapIdentification": {
+    "ResponseA": {
+      "rating": "",
+      "rationale": ""
+    },
+    "ResponseB": {
+      "rating": "",
+      "rationale": ""
+    }
+  }
+}
+
+where:
+- "rating" is a number from 1 to 5.
+- "rationale" is the comparative evaluation rating justification.
+
+</Response-Format>
+
+<Example-Responses>
+{EXAMPLE_RESPONSES}
+</Example-Responses>
+
+
+<Note>
+Your evaluation must be based solely on the provided research question and responses. Do not reward keyword mentions alone. Reward distinct, clearly described gaps in existing knowledge that are relevant to the research question and appropriately scoped. This rubric does not assess factual correctness, evidential grounding, completeness, or innovation.
+</Note>
+"""
+
+class GapIdentification(PairwiseRubric):
+    name: str = "GapIdentification"
+    system_prompt_template: str = gap_identification_pairwise_prompt