Feature/jschen 20260702 aalcr and ifbench

ais_bench/benchmark/configs/datasets/aa_lcr/aa_lcr_llmjudge.py

@@ -0,0 +1,102 @@
+from ais_bench.benchmark.datasets.aa_lcr import (
+    AALCRDataset,
+    AALCRJGDataset,
+    AALCRJudgeEvaluator,
+    JUDGE_PROMPT,
+)
+from ais_bench.benchmark.models import VLLMCustomAPIChat
+from ais_bench.benchmark.openicl.icl_inferencer import GenInferencer
+from ais_bench.benchmark.openicl.icl_prompt_template import PromptTemplate
+from ais_bench.benchmark.openicl.icl_retriever import ZeroRetriever
+
+# ---------------------------------------------------------------------------
+# Model inference configuration
+# ---------------------------------------------------------------------------
+# The prompt is fully built by AALCRDataset.load() – the template simply
+# passes through the pre-formatted ``input`` field.
+# ---------------------------------------------------------------------------
+
+aa_lcr_reader_cfg = dict(
+    input_columns=['input'],
+    output_column='answers',
+)
+
+aa_lcr_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template='{input}',
+    ),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=GenInferencer),
+)
+
+# ---------------------------------------------------------------------------
+# Judge model configuration (LLM Judge – mirrors HLE pattern)
+# ---------------------------------------------------------------------------
+
+aa_lcr_judge_infer_cfg = dict(
+    judge_reader_cfg=dict(
+        input_columns=['question', 'answers', 'model_answer'],
+        output_column='model_pred_uuid',
+    ),
+    judge_model=dict(
+        attr='service',
+        type=VLLMCustomAPIChat,
+        abbr='judge',
+        path='',
+        model='',
+        stream=False,
+        request_rate=0,
+        use_timestamp=False,
+        retry=2,
+        api_key='',
+        host_ip='localhost',
+        host_port=8005,
+        url='',
+        max_out_len=512,
+        batch_size=100,
+        trust_remote_code=False,
+        generation_kwargs=dict(
+            temperature=0.01,
+            seed=0,
+            chat_template_kwargs=dict(
+                enable_thinking=False,
+            ),
+        ),
+    ),
+    judge_dataset_type=AALCRJGDataset,
+    prompt_template=dict(
+        type=PromptTemplate,
+        template=dict(
+            round=[
+                dict(role='HUMAN', prompt=JUDGE_PROMPT),
+            ],
+        ),
+    ),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=GenInferencer),
+)
+
+# ---------------------------------------------------------------------------
+# Evaluation configuration
+# ---------------------------------------------------------------------------
+
+aa_lcr_eval_cfg = dict(
+    evaluator=dict(type=AALCRJudgeEvaluator),
+)
+
+# ---------------------------------------------------------------------------
+# Dataset definitions
+# ---------------------------------------------------------------------------
+
+aa_lcr_datasets = [
+    dict(
+        abbr='aa_lcr',
+        type=AALCRDataset,
+        path='benchmark/ais_bench/datasets/aa_lcr/',
+        reader_cfg=aa_lcr_reader_cfg,
+        infer_cfg=aa_lcr_infer_cfg,
+        judge_infer_cfg=aa_lcr_judge_infer_cfg,
+        eval_cfg=aa_lcr_eval_cfg,
+    )
+]

ais_bench/benchmark/configs/datasets/ifbench/ifbench_0_shot_gen_str.py

@@ -0,0 +1,33 @@
+from ais_bench.benchmark.openicl.icl_prompt_template import PromptTemplate
+from ais_bench.benchmark.openicl.icl_retriever import ZeroRetriever
+from ais_bench.benchmark.openicl.icl_inferencer import GenInferencer
+from ais_bench.benchmark.datasets.ifbench import IFBenchDataset, IFBenchEvaluator
+
+ifbench_reader_cfg = dict(
+    input_columns=['prompt'],
+    output_column='reference',
+)
+
+ifbench_infer_cfg = dict(
+    prompt_template=dict(
+        type=PromptTemplate,
+        template='{prompt}',
+    ),
+    retriever=dict(type=ZeroRetriever),
+    inferencer=dict(type=GenInferencer),
+)
+
+ifbench_eval_cfg = dict(
+    evaluator=dict(type=IFBenchEvaluator),
+)
+
+ifbench_datasets = [
+    dict(
+        abbr='ifbench',
+        type=IFBenchDataset,
+        path='ais_bench/datasets/ifbench/data/train-00000-of-00001.parquet',
+        reader_cfg=ifbench_reader_cfg,
+        infer_cfg=ifbench_infer_cfg,
+        eval_cfg=ifbench_eval_cfg,
+    )
+]

ais_bench/benchmark/datasets/__init__.py

@@ -64,6 +64,8 @@
 from ais_bench.benchmark.datasets.swebench_pro import *  # noqa: F401, F403
 from ais_bench.benchmark.datasets.refcoco import *  # noqa: F401, F403
 from ais_bench.benchmark.datasets.hle import * # noqa: F401, F403
+from ais_bench.benchmark.datasets.aa_lcr import *  # noqa: F401, F403
 from ais_bench.benchmark.datasets.realworldqa import * # noqa: F401, F403
+from ais_bench.benchmark.datasets.ifbench import *  # noqa: F401, F403
 from ais_bench.benchmark.datasets.oneig import *  # noqa: F401, F403