@@ -161,65 +161,53 @@ def _step_in_call(self, ch: str) -> None:
161161 self ._in_string = False
162162 self ._escape = False
163163
164- def _step_in_special (self , ch : str , out : list [str ]) -> None :
165- """Advance the ``<|...|>`` state machine; recurse on tail after ``|>``."""
164+ def _step_in_special (self , ch : str , _out : list [str ]) -> None :
165+ """Advance the ``<|...|>`` state machine; close when ``|>`` arrives.
166+
167+ Because ``feed()`` drives one character at a time and ``_in_special``
168+ is entered with an empty buffer, the close marker is always at the
169+ tail of the buffer — there is no trailing text to recurse on.
170+ """
166171 self ._buffer += ch
167- close_idx = self ._buffer .find (self ._SPECIAL_CLOSE )
168- if close_idx < 0 :
172+ if self ._SPECIAL_CLOSE not in self ._buffer :
169173 return
170- rest = self ._buffer [close_idx + len (self ._SPECIAL_CLOSE ) :]
171174 self ._buffer = ""
172175 self ._in_special = False
173- if rest :
174- out .append (self .feed (rest ))
175176
176- def _try_open_call (self , out : list [str ]) -> bool :
177+ def _try_open_call (self , _out : list [str ]) -> bool :
177178 """If a complete ``functions.<name>...{`` opener sits in buffer, enter call mode.
178179
179180 Returns True if the buffer was consumed (caller skips other checks);
180- False if the marker isn't fully present yet — caller must NOT keep
181- scanning the buffer for ``<|`` (the ``functions.`` prefix already
182- committed us to wait).
181+ False if the marker isn't fully present yet. ``_flush_safe_prefix``
182+ guarantees ``functions.`` always sits at the buffer head when it's
183+ present, and char-by-char feeding means ``{`` is always the tail —
184+ no leading prefix to emit and no trailing text to recurse on.
183185 """
184- call_idx = self ._buffer .find (self ._CALL_START )
185- if call_idx < 0 :
186+ if self ._CALL_START not in self ._buffer :
186187 return False
187- brace_idx = self ._buffer .find ("{" , call_idx + len (self ._CALL_START ))
188+ brace_idx = self ._buffer .find ("{" , len (self ._CALL_START ))
188189 if brace_idx < 0 :
189190 # Marker present but no ``{`` yet — keep buffering, do not
190191 # fall through to the ``<|`` check (it would never match
191192 # ``functions.`` and we'd over-emit).
192193 return True
193- if call_idx > 0 :
194- out .append (self ._buffer [:call_idx ])
195- rest = self ._buffer [brace_idx + 1 :]
196194 self ._buffer = ""
197195 self ._in_call = True
198196 self ._depth = 1
199197 self ._in_string = False
200198 self ._escape = False
201- if rest :
202- out .append (self .feed (rest ))
203199 return True
204200
205- def _try_open_special (self , out : list [str ]) -> bool :
206- """If a ``<|...|>`` token (or its open) is in buffer, drop it; return True."""
207- special_idx = self ._buffer .find (self ._SPECIAL_OPEN )
208- if special_idx < 0 :
201+ def _try_open_special (self , _out : list [str ]) -> bool :
202+ """If a ``<|`` opener sits in buffer, drop it and enter skip mode.
203+
204+ ``_flush_safe_prefix`` guarantees only ``<|`` itself (no trailing
205+ text) ever reaches us, and the closing ``|>`` is consumed later
206+ by ``_step_in_special`` — so we only need to handle the "open
207+ seen, no close yet" case.
208+ """
209+ if self ._SPECIAL_OPEN not in self ._buffer :
209210 return False
210- close_idx = self ._buffer .find (self ._SPECIAL_CLOSE , special_idx + len (self ._SPECIAL_OPEN ))
211- if close_idx >= 0 :
212- if special_idx > 0 :
213- out .append (self ._buffer [:special_idx ])
214- rest = self ._buffer [close_idx + len (self ._SPECIAL_CLOSE ) :]
215- self ._buffer = ""
216- if rest :
217- out .append (self .feed (rest ))
218- return True
219- # Open seen but no close yet — drop everything from ``<|`` on,
220- # emit the prefix, enter token-skip mode.
221- if special_idx > 0 :
222- out .append (self ._buffer [:special_idx ])
223211 self ._buffer = ""
224212 self ._in_special = True
225213 return True
0 commit comments