Fix input validation issues and update multiple-choice options handling (microsoft#2501)

This pull request introduces improvements to the handling of
multiple-choice questions and options extraction in the data
preprocessing and evaluation pipeline, as well as a minor enhancement to
ONNX model input handling. The main changes include switching from
numeric to letter-based multiple-choice options, extracting the selected
letter during evaluation, and supporting an additional input name in
ONNX graph surgeries.

**Multiple-choice options handling:**

* Updated `pre_process_data.py` to format multiple-choice options using
letters (A, B, C, ...) instead of numbers, and added a new field
`extract_option_letter` to indicate when letter extraction is needed.
[[1]](diffhunk://#diff-df8a7d990b3be7d29f4b7a919903dc72b9e8ff15d13d9282639a3e281bc3f23aR481-R490)
[[2]](diffhunk://#diff-df8a7d990b3be7d29f4b7a919903dc72b9e8ff15d13d9282639a3e281bc3f23aR510)
* Modified `olive_evaluator.py` to extract the selected letter from
model responses for multiple-choice questions, in addition to the
previous digit extraction. This ensures compatibility with the new
letter-based formatting.
[[1]](diffhunk://#diff-816af95cf16e31610395bb20947f1d73cc1ca67355964e074a8f55375c8139a7R884)
[[2]](diffhunk://#diff-816af95cf16e31610395bb20947f1d73cc1ca67355964e074a8f55375c8139a7L940-R943)
[[3]](diffhunk://#diff-816af95cf16e31610395bb20947f1d73cc1ca67355964e074a8f55375c8139a7R956-R959)

**ONNX model input handling:**

* Enhanced the `TieWordEmbeddings` class in `graph_surgeries.py` to
support both `input_embeds` and `inputs_embeds` as valid input names,
improving compatibility with different model variants.

Author: apsonawane

olive/data/component/pre_process_data.py

@@ -478,11 +478,16 @@ def __getitem__(self, idx):
             # Use 1-based numbering (1, 2, 3, 4) which aligns with how VLMs are
             # typically prompted and avoids confusion with diagram region labels.
             num_choices = 0
+            has_options = False
             if self.options_column and self.options_column in item:
                 options = item[self.options_column]
-                if isinstance(options, (list, tuple)) and len(options) > 0:
+                if isinstance(options, (list, tuple)):
+                    has_options = True
                     num_choices = len(options)
-                    options_text = "\n".join(f"{i + 1}. {opt}" for i, opt in enumerate(options))
+                    letters = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
+                    options_text = "\n".join(
+                        f"{letters[i] if i < len(letters) else str(i)}. {opt}" for i, opt in enumerate(options)
+                    )
                     question = f"{question}\n{options_text}"
 
             # Handle list/tuple answers (some datasets have multiple valid answers)
@@ -502,6 +507,7 @@ def __getitem__(self, idx):
                 "image": image,
                 "question": question,
                 "system_prompt": self.system_prompt,
+                "extract_option_letter": has_options,
                 "num_choices": num_choices,
                 "max_length": self.max_length,
             }

olive/evaluator/olive_evaluator.py

@@ -881,6 +881,7 @@ def _inference_vision_genai(
                     pil_image = item.get("image")
                     question = item.get("question", "")
                     sys_prompt = item.get("system_prompt", "")
+                    extract_option_letter = item.get("extract_option_letter", False)
                     num_choices = item.get("num_choices", 0)
                     max_length = item.get("max_length", default_max_length)
 
@@ -937,9 +938,9 @@ def _inference_vision_genai(
 
                     sample_idx += 1
 
-                    # For multiple-choice tasks, extract the answer digit from responses
-                    # like "2", "The answer is 3", or "1. D" to match the expected answer format.
-                    # Only enabled when num_choices is between 1 and 9 (single-digit options).
+                    # For multiple-choice tasks, extract the answer from responses.
+                    # First try digit extraction (for 1-based numbered options),
+                    # then fall back to letter extraction (for A/B/C/D options).
                     if 1 <= num_choices <= 9 and pred:
                         pattern = rf"\b([1-{num_choices}])\b"
                         num_match = re.search(pattern, pred)
@@ -952,6 +953,10 @@ def _inference_vision_genai(
                                 if ch in valid_digits:
                                     pred = ch
                                     break
+                    elif extract_option_letter and pred:
+                        letter_match = re.search(r"\b([A-Z])\b", pred)
+                        if letter_match:
+                            pred = letter_match.group(1)
                     all_preds.append(pred)
 
                 # Collect reference texts (aligned with preds including empty ones for None images)

olive/passes/onnx/graph_surgeries.py

@@ -2151,9 +2151,9 @@ class TieWordEmbeddings(ProtoSurgeon):
     def __call__(self, model: onnx.ModelProto):
         dag = OnnxDAG(model)
 
-        # support both "input_ids" and "input_embeds" as input names
+        # support both "input_ids" and "input_embeds"/"inputs_embeds" as input names
         input_name = None
-        for candidate in ("input_ids", "input_embeds"):
+        for candidate in ("input_ids", "input_embeds", "inputs_embeds"):
             if candidate in dag.ios and dag.is_input(candidate):
                 input_name = candidate
                 break