Skip to content

Commit a2742ca

Browse files
jiafatomCopilot
andauthored
Revert "Fix input validation issues and update multiple-choice options handling (microsoft#2501)" (microsoft#2509)
## Describe your changes Reverts PR microsoft#2501 which broke vision VQA accuracy by changing option labels from numbers (1/2/3/4) to letters (A/B/C/D) without updating the ground truth conversion. ### Problem PR microsoft#2501 changed option formatting to A/B/C/D but the ground truth was still converted to 1-based numbers. The evaluator digit extraction took priority over letter extraction, so predictions never matched ground truth (~11% accuracy instead of ~66-75%). ### What this reverts - `pre_process_data.py`: Restores 1-based numeric option labeling and removes `extract_option_letter` field - `olive_evaluator.py`: Removes the dead letter extraction branch, restores digit-only extraction ### What is kept The `graph_surgeries.py` change from microsoft#2501 (supporting `inputs_embeds`) is intentionally **not** reverted as it is unrelated and correct. ### Testing Validated on AI2D with Qwen3-VL-2B-Instruct — accuracy restored from ~11% to expected ~66%. ## Checklist - [x] My code follows the style guidelines of this project - [x] I have performed a self-review of my code Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
1 parent da09b4f commit a2742ca

2 files changed

Lines changed: 5 additions & 16 deletions

File tree

olive/data/component/pre_process_data.py

Lines changed: 2 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -478,16 +478,11 @@ def __getitem__(self, idx):
478478
# Use 1-based numbering (1, 2, 3, 4) which aligns with how VLMs are
479479
# typically prompted and avoids confusion with diagram region labels.
480480
num_choices = 0
481-
has_options = False
482481
if self.options_column and self.options_column in item:
483482
options = item[self.options_column]
484-
if isinstance(options, (list, tuple)):
485-
has_options = True
483+
if isinstance(options, (list, tuple)) and len(options) > 0:
486484
num_choices = len(options)
487-
letters = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
488-
options_text = "\n".join(
489-
f"{letters[i] if i < len(letters) else str(i)}. {opt}" for i, opt in enumerate(options)
490-
)
485+
options_text = "\n".join(f"{i + 1}. {opt}" for i, opt in enumerate(options))
491486
question = f"{question}\n{options_text}"
492487

493488
# Handle list/tuple answers (some datasets have multiple valid answers)
@@ -507,7 +502,6 @@ def __getitem__(self, idx):
507502
"image": image,
508503
"question": question,
509504
"system_prompt": self.system_prompt,
510-
"extract_option_letter": has_options,
511505
"num_choices": num_choices,
512506
"max_length": self.max_length,
513507
}

olive/evaluator/olive_evaluator.py

Lines changed: 3 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -881,7 +881,6 @@ def _inference_vision_genai(
881881
pil_image = item.get("image")
882882
question = item.get("question", "")
883883
sys_prompt = item.get("system_prompt", "")
884-
extract_option_letter = item.get("extract_option_letter", False)
885884
num_choices = item.get("num_choices", 0)
886885
max_length = item.get("max_length", default_max_length)
887886

@@ -938,9 +937,9 @@ def _inference_vision_genai(
938937

939938
sample_idx += 1
940939

941-
# For multiple-choice tasks, extract the answer from responses.
942-
# First try digit extraction (for 1-based numbered options),
943-
# then fall back to letter extraction (for A/B/C/D options).
940+
# For multiple-choice tasks, extract the answer digit from responses
941+
# like "2", "The answer is 3", or "1. D" to match the expected answer format.
942+
# Only enabled when num_choices is between 1 and 9 (single-digit options).
944943
if 1 <= num_choices <= 9 and pred:
945944
pattern = rf"\b([1-{num_choices}])\b"
946945
num_match = re.search(pattern, pred)
@@ -953,10 +952,6 @@ def _inference_vision_genai(
953952
if ch in valid_digits:
954953
pred = ch
955954
break
956-
elif extract_option_letter and pred:
957-
letter_match = re.search(r"\b([A-Z])\b", pred)
958-
if letter_match:
959-
pred = letter_match.group(1)
960955
all_preds.append(pred)
961956

962957
# Collect reference texts (aligned with preds including empty ones for None images)

0 commit comments

Comments
 (0)