Skip to content
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 8 additions & 3 deletions vlmeval/dataset/image_mcq.py
Original file line number Diff line number Diff line change
Expand Up @@ -633,15 +633,20 @@ def cot_postproc(self, response):
lines = response.strip().split('\n')
lines = [x.strip() for x in lines]
cands = [x for x in lines if x.startswith('Answer:')]
if len(cands) == 1:
if len(cands) >= 1:
# Inspect only the text AFTER the last "Answer:" marker. Counting over the whole
# line included the capital "A" in "Answer", so the single-letter fast-path almost
# never fired; models that echo the "$LETTER" template literally (e.g. "Answer: $B")
# then fell through to returning raw text that the option matcher fails to parse.
tail = cands[-1][len('Answer:'):]
counter = defaultdict(lambda: 0)
for ch in cands[0]:
for ch in tail:
if ch in string.ascii_uppercase:
counter[ch] += 1
if len(counter) == 1:
return list(counter.keys())[0]
else:
return cands[0][7:]
return tail
return response

def evaluate(self, eval_file, **judge_kwargs):
Expand Down
Loading