Add AA-LCR dataset integration

dataset-index.yml

@@ -40,6 +40,12 @@
     paper: https://arxiv.org/pdf/2406.10149
     configpath: opencompass/configs/datasets/babilong
     configpath_llmjudge: ''
+- aa_lcr:
+    name: AA-LCR
+    category: Long Context
+    paper: https://huggingface.co/datasets/ArtificialAnalysis/AA-LCR
+    configpath: opencompass/configs/datasets/aa_lcr/aa_lcr_gen.py
+    configpath_llmjudge: opencompass/configs/datasets/aa_lcr/aa_lcr_gen.py
 - bigcodebench:
     name: BigCodeBench
     category: Code

examples/eval_aa_lcr_gpt55_xhigh.py

@@ -0,0 +1,84 @@
+from copy import deepcopy
+
+from mmengine.config import read_base
+
+from opencompass.models import OpenAISDK, OpenAISDKResponse
+from opencompass.partitioners import NaivePartitioner
+from opencompass.runners import LocalRunner
+from opencompass.tasks import OpenICLEvalTask, OpenICLInferTask
+from opencompass.utils.text_postprocessors import extract_non_reasoning_content
+
+with read_base():
+    from opencompass.configs.datasets.aa_lcr.aa_lcr_gen import \
+        aa_lcr_datasets
+
+
+AA_LCR_NUM_RUNS = 3
+AA_LCR_MAX_OUT_LEN = 128000
+AA_LCR_MAX_SEQ_LEN = 1050000
+AA_LCR_TEST_RANGE = None
+
+work_dir = './output/aa_lcr_gpt55_xhigh'
+
+datasets = deepcopy(aa_lcr_datasets)
+for dataset in datasets:
+    dataset['n'] = AA_LCR_NUM_RUNS
+    dataset['eval_cfg']['evaluator']['dataset_cfg']['n'] = 1
+
+    if AA_LCR_TEST_RANGE:
+        dataset['reader_cfg']['test_range'] = AA_LCR_TEST_RANGE
+        dataset['eval_cfg']['evaluator']['dataset_cfg']['reader_cfg'][
+            'test_range'] = AA_LCR_TEST_RANGE
+
+    judge_cfg = dict(
+        abbr='qwen3-235b-a22b-instruct-2507',
+        type=OpenAISDK,
+        path='qwen3-235b-a22b-instruct-2507',
+        key='ENV',
+        query_per_second=1,
+        batch_size=10,
+        temperature=0,
+        tokenizer_path='gpt-4o-2024-05-13',
+        max_out_len=1024,
+        max_seq_len=131072,
+        retry=5,
+        timeout=3600,
+    )
+    dataset['eval_cfg']['evaluator']['judge_cfg'] = judge_cfg
+
+models = [
+    dict(
+        abbr='gpt-5.5-xhigh',
+        type=OpenAISDKResponse,
+        path='gpt-5.5',
+        key='ENV',
+        query_per_second=1,
+        batch_size=100,
+        tokenizer_path='gpt-4o-2024-05-13',
+        max_out_len=AA_LCR_MAX_OUT_LEN,
+        max_seq_len=AA_LCR_MAX_SEQ_LEN,
+        max_workers=20,
+        retry=5,
+        timeout=3600,
+        openai_extra_kwargs=dict(reasoning=dict(effort='xhigh')),
+        pred_postprocessor=dict(type=extract_non_reasoning_content),
+    )
+]
+
+infer = dict(
+    partitioner=dict(type=NaivePartitioner),
+    runner=dict(
+        type=LocalRunner,
+        max_num_workers=10,
+        task=dict(type=OpenICLInferTask),
+    ),
+)
+
+eval = dict(
+    partitioner=dict(type=NaivePartitioner),
+    runner=dict(
+        type=LocalRunner,
+        max_num_workers=10,
+        task=dict(type=OpenICLEvalTask),
+    ),
+)

opencompass/configs/datasets/aa_lcr/__init__.py

@@ -0,0 +1 @@
+from .aa_lcr_gen import aa_lcr_datasets  # noqa: F401, F403

opencompass/configs/datasets/aa_lcr/aa_lcr_gen.py

@@ -0,0 +1,58 @@
+from opencompass.datasets import AALCRDataset, aa_lcr_llmjudge_postprocess
+from opencompass.evaluator import GenericLLMEvaluator
+from opencompass.openicl.icl_inferencer import GenInferencer
+from opencompass.openicl.icl_raw_prompt_template import RawPromptTemplate
+from opencompass.openicl.icl_retriever import ZeroRetriever
+
+
+AA_LCR_JUDGE_TEMPLATE = """Assess whether the following CANDIDATE ANSWER is \
+CORRECT or INCORRECT.
+For the CANDIDATE ANSWER to be correct, it must be consistent with the \
+OFFICIAL ANSWER.
+
+The question, for reference only: {question}
+The OFFICIAL ANSWER: {answer}
+CANDIDATE ANSWER TO ASSESS: {prediction}
+
+Reply only with CORRECT or INCORRECT."""
+
+
+aa_lcr_reader_cfg = dict(input_columns=['prompt'], output_column='answer')
+
+aa_lcr_infer_cfg = dict(
+    prompt_template=dict(
+        type=RawPromptTemplate,
+        messages=[dict(role='user', content='{prompt}')],
+    ),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=GenInferencer),
+)
+
+aa_lcr_eval_cfg = dict(
+    evaluator=dict(
+        type=GenericLLMEvaluator,
+        prompt_template=dict(
+            type=RawPromptTemplate,
+            messages=[dict(role='user', content=AA_LCR_JUDGE_TEMPLATE)],
+        ),
+        dataset_cfg=dict(
+            type=AALCRDataset,
+            path='ArtificialAnalysis/AA-LCR',
+            reader_cfg=aa_lcr_reader_cfg,
+        ),
+        judge_cfg=dict(),
+        dict_postprocessor=dict(type=aa_lcr_llmjudge_postprocess),
+    ),
+    pred_role='BOT',
+)
+
+aa_lcr_datasets = [
+    dict(
+        abbr='AA-LCR',
+        type=AALCRDataset,
+        path='ArtificialAnalysis/AA-LCR',
+        reader_cfg=aa_lcr_reader_cfg,
+        infer_cfg=aa_lcr_infer_cfg,
+        eval_cfg=aa_lcr_eval_cfg,
+    )
+]

opencompass/datasets/__init__.py

@@ -1,3 +1,4 @@
+from .aa_lcr import AALCRDataset, aa_lcr_llmjudge_postprocess  # noqa: F401
 from .advancedIF import AdvancedIFDataset  # noqa: F401
 from .advancedIF import advancedif_rubric_postprocess  # noqa: F401
 from .advglue import *  # noqa: F401, F403