Skip to content

Commit 3594ff1

Browse files
committed
interception: lean token-id parse (match #1029), drop the dead token_id-key path
1 parent 21cb55a commit 3594ff1

1 file changed

Lines changed: 3 additions & 8 deletions

File tree

src/openenv/core/sandbox/interception.py

Lines changed: 3 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -462,14 +462,9 @@ def _build_turn_record(
462462
for entry in content_lp:
463463
token = entry.get("token", "")
464464
tokens.append(token)
465-
# OpenAI returns no raw token ids. vLLM with ``--return-tokens-as-token-ids``
466-
# encodes the id in the ``token`` field as ``"token_id:<int>"``; some builds
467-
# instead expose a separate ``token_id`` key. Accept both.
468-
token_id = entry.get("token_id")
469-
if token_id is None and token.startswith("token_id:"):
470-
token_id = token.split(":", 1)[1]
471-
if token_id is not None:
472-
token_ids.append(int(token_id))
465+
# vLLM (--return-tokens-as-token-ids) returns the id in the token field as "token_id:<int>".
466+
if token.startswith("token_id:"):
467+
token_ids.append(int(token.split(":", 1)[1]))
473468
lp = entry.get("logprob")
474469
if lp is not None:
475470
per_token_logps.append(float(lp))

0 commit comments

Comments
 (0)