File tree Expand file tree Collapse file tree
Expand file tree Collapse file tree Original file line number Diff line number Diff line change @@ -462,14 +462,9 @@ def _build_turn_record(
462462 for entry in content_lp :
463463 token = entry .get ("token" , "" )
464464 tokens .append (token )
465- # OpenAI returns no raw token ids. vLLM with ``--return-tokens-as-token-ids``
466- # encodes the id in the ``token`` field as ``"token_id:<int>"``; some builds
467- # instead expose a separate ``token_id`` key. Accept both.
468- token_id = entry .get ("token_id" )
469- if token_id is None and token .startswith ("token_id:" ):
470- token_id = token .split (":" , 1 )[1 ]
471- if token_id is not None :
472- token_ids .append (int (token_id ))
465+ # vLLM (--return-tokens-as-token-ids) returns the id in the token field as "token_id:<int>".
466+ if token .startswith ("token_id:" ):
467+ token_ids .append (int (token .split (":" , 1 )[1 ]))
473468 lp = entry .get ("logprob" )
474469 if lp is not None :
475470 per_token_logps .append (float (lp ))
You can’t perform that action at this time.
0 commit comments