Feat/ocrbench omnidocbench lang subset (#1250)

* feat(metrics): add Chinese/English subset scores for OCRBench_v2 and OmniDocBench

* fix(ocrbench_v2): guard spotting_evaluation against missing hmean key

Author: MaxwellJryao

lmms_eval/tasks/ocrbench_v2/ocrbench_v2.yaml

@@ -20,5 +20,11 @@ metric_list:
   - metric: ocrbench_v2_accuracy
     aggregation: !function utils.ocrbench_v2_aggregate_accuracy
     higher_is_better: true
+  - metric: ocrbench_v2_accuracy_en
+    aggregation: !function utils.ocrbench_v2_aggregate_accuracy_en
+    higher_is_better: true
+  - metric: ocrbench_v2_accuracy_cn
+    aggregation: !function utils.ocrbench_v2_aggregate_accuracy_cn
+    higher_is_better: true
 metadata:
   - version: 0.0

lmms_eval/tasks/ocrbench_v2/spotting_metric.py

@@ -173,5 +173,9 @@ def spotting_evaluation(prediction_list, img_metas):
         command = {"g": gt_zip_path, "s": submit_zip_path, "o": temp_dir, "p": '{"IOU_CONSTRAINT":0.5}'}
 
         result = rrc_evaluation_funcs.main_evaluation(command, default_evaluation_params, validate_data, evaluate_method)
-        score = result["method"]["hmean"]
+        method = result.get("method", {})
+        if isinstance(method, dict):
+            score = method.get("hmean", 0)
+        else:
+            score = 0
     return score

lmms_eval/tasks/ocrbench_v2/utils.py

@@ -368,16 +368,30 @@ def ocrbench_v2_process_results(doc, results):
             else:
                 score = spotting_evaluation(predict_bbox, doc)
 
+    payload = {"question_type": data_type, "score": score, "prediction": pred, "ground_truth": gt_ans}
     return {
-        "ocrbench_v2_accuracy": {"question_type": data_type, "score": score, "prediction": pred, "ground_truth": gt_ans},
+        "ocrbench_v2_accuracy": payload,
+        "ocrbench_v2_accuracy_en": payload,
+        "ocrbench_v2_accuracy_cn": payload,
     }
 
 
 def calculate_average_score(categories, score_buckets):
-    return sum(sum(score_buckets[cat]) / len(score_buckets[cat]) if len(score_buckets[cat]) > 0 else 0 for cat in categories) / len(categories)
+    """Weighted average across categories by sample count."""
+    total_score = sum(sum(score_buckets[cat]) for cat in categories)
+    total_count = sum(len(score_buckets[cat]) for cat in categories)
+    if total_count == 0:
+        return 0.0
+    return total_score / total_count
 
 
-def ocrbench_v2_aggregate_accuracy(results, args):
+ENGLISH_TASKS = ["text_recognition_en", "text_detection_en", "text_spotting_en", "relationship_extraction_en", "element_parsing_en", "mathematical_calculation_en", "visual_text_understanding_en", "knowledge_reasoning_en"]
+
+CHINESE_TASKS = ["text_recognition_cn", "relationship_extraction_cn", "element_parsing_cn", "visual_text_understanding_cn", "knowledge_reasoning_cn"]
+
+
+def _fill_score_buckets(results):
+    """Shared logic: populate score buckets and per-question-type scores from results."""
     question_type_scores = {}
     score_buckets = _make_score_buckets()
 
@@ -435,14 +449,19 @@ def ocrbench_v2_aggregate_accuracy(results, args):
             print("No such task!")
             raise TypeError
 
-    english_tasks = ["text_recognition_en", "text_detection_en", "text_spotting_en", "relationship_extraction_en", "element_parsing_en", "mathematical_calculation_en", "visual_text_understanding_en", "knowledge_reasoning_en"]
+    return question_type_scores, score_buckets
+
 
-    chinese_tasks = ["text_recognition_cn", "relationship_extraction_cn", "element_parsing_cn", "visual_text_understanding_cn", "knowledge_reasoning_cn"]
+def ocrbench_v2_aggregate_accuracy(results, args):
+    question_type_scores, score_buckets = _fill_score_buckets(results)
 
-    OCRBench_v2_English_subset_score = calculate_average_score(english_tasks, score_buckets)
-    OCRBench_v2_Chinese_subset_score = calculate_average_score(chinese_tasks, score_buckets)
+    OCRBench_v2_English_subset_score = calculate_average_score(ENGLISH_TASKS, score_buckets)
+    OCRBench_v2_Chinese_subset_score = calculate_average_score(CHINESE_TASKS, score_buckets)
 
-    Final_score = (OCRBench_v2_English_subset_score + OCRBench_v2_Chinese_subset_score) / 2
+    en_count = sum(len(score_buckets[t]) for t in ENGLISH_TASKS)
+    cn_count = sum(len(score_buckets[t]) for t in CHINESE_TASKS)
+    total = en_count + cn_count
+    Final_score = (OCRBench_v2_English_subset_score * en_count + OCRBench_v2_Chinese_subset_score * cn_count) / total if total > 0 else 0.0
     file_name = generate_submission_file("ocrbench_v2_results.txt", args, subpath="results")
     with open(file_name, "w") as f:
         print("######################### OCRBench v2 ##########################", file=f)
@@ -451,13 +470,13 @@ def ocrbench_v2_aggregate_accuracy(results, args):
             avg_score = sum(scores) / len(scores) if len(scores) > 0 else 0
             print(f"{q_type} (sample number: {len(scores)}): {avg_score:.2f}", file=f)
         print("######################### English Subsets ######################", file=f)
-        for task in english_tasks:
+        for task in ENGLISH_TASKS:
             num_samples = len(score_buckets[task])
             avg_score = sum(score_buckets[task]) / num_samples if num_samples > 0 else 0
             print(f"{task.replace('_', ' ').title()} (Total {num_samples}): {avg_score:.2f}", file=f)
         print(f"Overall English Score: {OCRBench_v2_English_subset_score:.2f}", file=f)
         print("######################### Chinese Subsets ######################", file=f)
-        for task in chinese_tasks:
+        for task in CHINESE_TASKS:
             num_samples = len(score_buckets[task])
             avg_score = sum(score_buckets[task]) / num_samples if num_samples > 0 else 0
             print(f"{task.replace('_', ' ').title()} (Total {num_samples}): {avg_score:.2f}", file=f)
@@ -467,3 +486,13 @@ def ocrbench_v2_aggregate_accuracy(results, args):
     logger.info(f"OCRBench v2 results saved to {file_name}")
 
     return Final_score  # return the final score as accuracy
+
+
+def ocrbench_v2_aggregate_accuracy_en(results, args):
+    _, score_buckets = _fill_score_buckets(results)
+    return calculate_average_score(ENGLISH_TASKS, score_buckets)
+
+
+def ocrbench_v2_aggregate_accuracy_cn(results, args):
+    _, score_buckets = _fill_score_buckets(results)
+    return calculate_average_score(CHINESE_TASKS, score_buckets)

lmms_eval/tasks/omnidocbench/omnidocbench.yaml

@@ -23,6 +23,24 @@ metric_list:
   - metric: omnidocbench_nld_score
     aggregation: mean
     higher_is_better: true
+  - metric: omnidocbench_exact_match_en
+    aggregation: !function utils.omnidocbench_aggregate_exact_match_en
+    higher_is_better: true
+  - metric: omnidocbench_exact_match_cn
+    aggregation: !function utils.omnidocbench_aggregate_exact_match_cn
+    higher_is_better: true
+  - metric: omnidocbench_nld_score_en
+    aggregation: !function utils.omnidocbench_aggregate_nld_score_en
+    higher_is_better: true
+  - metric: omnidocbench_nld_score_cn
+    aggregation: !function utils.omnidocbench_aggregate_nld_score_cn
+    higher_is_better: true
+  - metric: omnidocbench_exact_match_mixed
+    aggregation: !function utils.omnidocbench_aggregate_exact_match_mixed
+    higher_is_better: true
+  - metric: omnidocbench_nld_score_mixed
+    aggregation: !function utils.omnidocbench_aggregate_nld_score_mixed
+    higher_is_better: true
 lmms_eval_specific_kwargs:
   default:
     pre_prompt: ""

lmms_eval/tasks/omnidocbench/utils.py

@@ -1,5 +1,7 @@
 import io
+import json
 import re
+from collections import Counter
 from typing import Any
 
 import Levenshtein
@@ -57,6 +59,60 @@ def _to_rgb(image_obj: Any):
     return None
 
 
+def _detect_document_language(doc) -> str:
+    """Detect whether a document is 'en', 'cn', or 'mixed' from its answer JSON.
+
+    Inspects ``page_info.page_attribute.language`` first (most reliable).
+    Falls back to majority vote over ``layout_dets[*].attribute.text_language``
+    and ``layout_dets[*].attribute.language`` (for tables).
+    """
+    answer_raw = doc.get("answer") or doc.get("answers") or doc.get("target")
+    if not answer_raw:
+        return "mixed"
+
+    if isinstance(answer_raw, list):
+        answer_raw = answer_raw[0]
+    if not isinstance(answer_raw, str):
+        return "mixed"
+
+    try:
+        answer = json.loads(answer_raw)
+    except (json.JSONDecodeError, TypeError):
+        return "mixed"
+
+    # Try page-level language first
+    page_lang = None
+    page_info = answer.get("page_info") or {}
+    page_attr = page_info.get("page_attribute") or {}
+    page_lang_raw = page_attr.get("language", "")
+    if "chinese" in page_lang_raw:
+        page_lang = "cn"
+    elif "english" in page_lang_raw:
+        page_lang = "en"
+
+    if page_lang:
+        return page_lang
+
+    # Fall back to element-level majority vote
+    lang_counts = Counter()
+    for det in answer.get("layout_dets", []):
+        attr = det.get("attribute") or {}
+        # Text elements use "text_language", tables use "language"
+        lang_val = attr.get("text_language", "") or attr.get("language", "")
+        if "chinese" in lang_val:
+            lang_counts["cn"] += 1
+        elif "english" in lang_val or lang_val.endswith("_en"):
+            lang_counts["en"] += 1
+        elif "mixed" in lang_val:
+            lang_counts["mixed"] += 1
+
+    if not lang_counts:
+        return "mixed"
+
+    top = lang_counts.most_common(1)[0][0]
+    return top
+
+
 def omnidocbench_doc_to_visual(doc):
     visuals = []
 
@@ -114,22 +170,69 @@ def _normalized_levenshtein_score(pred: str, ref: str) -> float:
 def omnidocbench_process_results(doc, results):
     prediction = _normalize_text(results[0])
     answers = _extract_answers(doc)
+    lang = _detect_document_language(doc)
+
     if not answers:
-        return {"omnidocbench_exact_match": 0.0, "omnidocbench_nld_score": 0.0}
+        em_score = 0.0
+        nld_score = 0.0
+    else:
+        # Exact match
+        answer_set = {_normalize_text(answer) for answer in answers}
+        em_score = float(prediction in answer_set)
 
-    # Exact match
-    answer_set = {_normalize_text(answer) for answer in answers}
-    em_score = float(prediction in answer_set)
+        options = _extract_options(doc)
+        if options:
+            pred_letter = _extract_option_letter(str(results[0]))
+            if pred_letter:
+                for answer in answers:
+                    if pred_letter == answer.strip().upper()[:1]:
+                        em_score = max(em_score, 1.0)
 
-    options = _extract_options(doc)
-    if options:
-        pred_letter = _extract_option_letter(str(results[0]))
-        if pred_letter:
-            for answer in answers:
-                if pred_letter == answer.strip().upper()[:1]:
-                    em_score = max(em_score, 1.0)
+        # Normalized Levenshtein score: (1 - NLD) * 100, take best across answers
+        nld_score = max(_normalized_levenshtein_score(prediction, _normalize_text(answer)) for answer in answers)
+
+    lang_payload_em = {"score": em_score, "lang": lang}
+    lang_payload_nld = {"score": nld_score, "lang": lang}
+
+    return {
+        "omnidocbench_exact_match": em_score,
+        "omnidocbench_nld_score": nld_score,
+        "omnidocbench_exact_match_en": lang_payload_em,
+        "omnidocbench_exact_match_cn": lang_payload_em,
+        "omnidocbench_exact_match_mixed": lang_payload_em,
+        "omnidocbench_nld_score_en": lang_payload_nld,
+        "omnidocbench_nld_score_cn": lang_payload_nld,
+        "omnidocbench_nld_score_mixed": lang_payload_nld,
+    }
+
+
+def _aggregate_by_lang(results, target_langs):
+    """Filter results by language and compute mean score."""
+    filtered = [r["score"] for r in results if r["lang"] in target_langs]
+    if not filtered:
+        return 0.0
+    return sum(filtered) / len(filtered)
+
+
+def omnidocbench_aggregate_exact_match_en(results, args):
+    return _aggregate_by_lang(results, {"en"})
+
+
+def omnidocbench_aggregate_exact_match_cn(results, args):
+    return _aggregate_by_lang(results, {"cn"})
+
+
+def omnidocbench_aggregate_exact_match_mixed(results, args):
+    return _aggregate_by_lang(results, {"mixed"})
+
+
+def omnidocbench_aggregate_nld_score_en(results, args):
+    return _aggregate_by_lang(results, {"en"})
+
+
+def omnidocbench_aggregate_nld_score_cn(results, args):
+    return _aggregate_by_lang(results, {"cn"})
 
-    # Normalized Levenshtein score: (1 - NLD) * 100, take best across answers
-    nld_score = max(_normalized_levenshtein_score(prediction, _normalize_text(answer)) for answer in answers)
 
-    return {"omnidocbench_exact_match": em_score, "omnidocbench_nld_score": nld_score}
+def omnidocbench_aggregate_nld_score_mixed(results, args):
+    return _aggregate_by_lang(results, {"mixed"})