Skip to content

Commit 3711c0d

Browse files
edelaunaedelauna
committed
test(tag-matcher): consolidate reasoning tag tests into tag-matcher.spec.ts
1 parent c7e3a5a commit 3711c0d

4 files changed

Lines changed: 87 additions & 244 deletions

File tree

‎src/api/providers/__tests__/base-openai-compatible-provider.spec.ts‎

Lines changed: 2 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -222,13 +222,8 @@ describe("BaseOpenAiCompatibleProvider", () => {
222222
chunks.push(chunk)
223223
}
224224

225-
// TagMatcher should handle incomplete tags and flush remaining content
226-
expect(chunks.length).toBeGreaterThan(0)
227-
expect(
228-
chunks.some(
229-
(c) => (c.type === "text" || c.type === "reasoning") && c.text.includes("Incomplete thought"),
230-
),
231-
).toBe(true)
225+
// TagMatcher should flush incomplete reasoning content on stream end
226+
expect(chunks).toContainEqual({ type: "reasoning", text: "Incomplete thought" })
232227
})
233228

234229
it("should handle text without any <think> tags", async () => {

‎src/api/providers/__tests__/openai.spec.ts‎

Lines changed: 0 additions & 234 deletions
Original file line numberDiff line numberDiff line change
@@ -638,145 +638,6 @@ describe("OpenAiHandler", () => {
638638
})
639639

640640
describe("TagMatcher reasoning tags", () => {
641-
it("should handle <think> tags from stream", async () => {
642-
mockCreate.mockImplementationOnce(() => ({
643-
[Symbol.asyncIterator]: () => ({
644-
next: vi
645-
.fn()
646-
.mockResolvedValueOnce({
647-
done: false,
648-
value: { choices: [{ delta: { content: "<think>Let me think" } }] },
649-
})
650-
.mockResolvedValueOnce({
651-
done: false,
652-
value: { choices: [{ delta: { content: " about this</think>" } }] },
653-
})
654-
.mockResolvedValueOnce({
655-
done: false,
656-
value: { choices: [{ delta: { content: "The answer is 42" } }] },
657-
})
658-
.mockResolvedValueOnce({ done: true }),
659-
}),
660-
}))
661-
662-
const stream = handler.createMessage(systemPrompt, messages)
663-
const chunks: any[] = []
664-
for await (const chunk of stream) {
665-
chunks.push(chunk)
666-
}
667-
668-
expect(chunks).toEqual([
669-
{ type: "reasoning", text: "Let me think" },
670-
{ type: "reasoning", text: " about this" },
671-
{ type: "text", text: "The answer is 42" },
672-
])
673-
})
674-
675-
it("should handle <thought> tags from stream", async () => {
676-
mockCreate.mockImplementationOnce(() => ({
677-
[Symbol.asyncIterator]: () => ({
678-
next: vi
679-
.fn()
680-
.mockResolvedValueOnce({
681-
done: false,
682-
value: { choices: [{ delta: { content: "<thought>Deep thought" } }] },
683-
})
684-
.mockResolvedValueOnce({
685-
done: false,
686-
value: { choices: [{ delta: { content: " here</thought>" } }] },
687-
})
688-
.mockResolvedValueOnce({
689-
done: false,
690-
value: { choices: [{ delta: { content: "Result: 42" } }] },
691-
})
692-
.mockResolvedValueOnce({ done: true }),
693-
}),
694-
}))
695-
696-
const stream = handler.createMessage(systemPrompt, messages)
697-
const chunks: any[] = []
698-
for await (const chunk of stream) {
699-
chunks.push(chunk)
700-
}
701-
702-
expect(chunks).toEqual([
703-
{ type: "reasoning", text: "Deep thought" },
704-
{ type: "reasoning", text: " here" },
705-
{ type: "text", text: "Result: 42" },
706-
])
707-
})
708-
709-
it("should not close <think> tag with </thought> tag", async () => {
710-
mockCreate.mockImplementationOnce(() => ({
711-
[Symbol.asyncIterator]: () => ({
712-
next: vi
713-
.fn()
714-
.mockResolvedValueOnce({
715-
done: false,
716-
value: { choices: [{ delta: { content: "<think>Thinking" } }] },
717-
})
718-
.mockResolvedValueOnce({
719-
done: false,
720-
value: { choices: [{ delta: { content: " but closing with wrong tag</thought>" } }] },
721-
})
722-
.mockResolvedValueOnce({
723-
done: false,
724-
value: { choices: [{ delta: { content: " still thinking</think>" } }] },
725-
})
726-
.mockResolvedValueOnce({
727-
done: false,
728-
value: { choices: [{ delta: { content: "final text" } }] },
729-
})
730-
.mockResolvedValueOnce({ done: true }),
731-
}),
732-
}))
733-
734-
const stream = handler.createMessage(systemPrompt, messages)
735-
const chunks: any[] = []
736-
for await (const chunk of stream) {
737-
chunks.push(chunk)
738-
}
739-
740-
// The </thought> tag should not match the active <think> tag, so the closing
741-
// tag is treated as text. The " still thinking" stays reasoning since <think>
742-
// was never closed with </think>.
743-
expect(chunks).toEqual([
744-
{ type: "reasoning", text: "Thinking" },
745-
{ type: "reasoning", text: " but closing with wrong tag</thought>" },
746-
{ type: "reasoning", text: " still thinking" },
747-
{ type: "text", text: "final text" },
748-
])
749-
})
750-
751-
it("should handle text without any tags", async () => {
752-
mockCreate.mockImplementationOnce(() => ({
753-
[Symbol.asyncIterator]: () => ({
754-
next: vi
755-
.fn()
756-
.mockResolvedValueOnce({
757-
done: false,
758-
value: { choices: [{ delta: { content: "Just regular text" } }] },
759-
})
760-
.mockResolvedValueOnce({
761-
done: false,
762-
value: { choices: [{ delta: { content: " without reasoning" } }] },
763-
})
764-
.mockResolvedValueOnce({ done: true }),
765-
}),
766-
}))
767-
768-
const stream = handler.createMessage(systemPrompt, messages)
769-
const chunks: any[] = []
770-
for await (const chunk of stream) {
771-
chunks.push(chunk)
772-
}
773-
774-
expect(chunks).toEqual([
775-
{ type: "text", text: "Just regular text" },
776-
{ type: "text", text: " without reasoning" },
777-
])
778-
})
779-
780641
it("should treat stray closing tag as plain text when no tag is open", async () => {
781642
mockCreate.mockImplementationOnce(() => ({
782643
[Symbol.asyncIterator]: () => ({
@@ -826,101 +687,6 @@ describe("OpenAiHandler", () => {
826687
])
827688
})
828689

829-
it("should handle <think> tags that start at beginning of stream", async () => {
830-
mockCreate.mockImplementationOnce(() => ({
831-
[Symbol.asyncIterator]: () => ({
832-
next: vi
833-
.fn()
834-
.mockResolvedValueOnce({
835-
done: false,
836-
value: { choices: [{ delta: { content: "<think>reasoning" } }] },
837-
})
838-
.mockResolvedValueOnce({
839-
done: false,
840-
value: { choices: [{ delta: { content: " content</think>" } }] },
841-
})
842-
.mockResolvedValueOnce({
843-
done: false,
844-
value: { choices: [{ delta: { content: " normal text" } }] },
845-
})
846-
.mockResolvedValueOnce({ done: true }),
847-
}),
848-
}))
849-
850-
const stream = handler.createMessage(systemPrompt, messages)
851-
const chunks: any[] = []
852-
for await (const chunk of stream) {
853-
chunks.push(chunk)
854-
}
855-
856-
expect(chunks).toEqual([
857-
{ type: "reasoning", text: "reasoning" },
858-
{ type: "reasoning", text: " content" },
859-
{ type: "text", text: " normal text" },
860-
])
861-
})
862-
863-
it("should handle incomplete <think> tag at end of stream", async () => {
864-
mockCreate.mockImplementationOnce(() => ({
865-
[Symbol.asyncIterator]: () => ({
866-
next: vi
867-
.fn()
868-
.mockResolvedValueOnce({
869-
done: false,
870-
value: { choices: [{ delta: { content: "<think>Incomplete thought" } }] },
871-
})
872-
.mockResolvedValueOnce({ done: true }),
873-
}),
874-
}))
875-
876-
const stream = handler.createMessage(systemPrompt, messages)
877-
const chunks: any[] = []
878-
for await (const chunk of stream) {
879-
chunks.push(chunk)
880-
}
881-
882-
// TagMatcher should flush remaining reasoning content on final()
883-
expect(chunks.length).toBeGreaterThan(0)
884-
expect(
885-
chunks.some(
886-
(c) => (c.type === "text" || c.type === "reasoning") && c.text.includes("Incomplete thought"),
887-
),
888-
).toBe(true)
889-
})
890-
891-
it("should handle complete <think> tag in a single chunk", async () => {
892-
mockCreate.mockImplementationOnce(() => ({
893-
[Symbol.asyncIterator]: () => ({
894-
next: vi
895-
.fn()
896-
.mockResolvedValueOnce({
897-
done: false,
898-
value: { choices: [{ delta: { content: "text before " } }] },
899-
})
900-
.mockResolvedValueOnce({
901-
done: false,
902-
value: { choices: [{ delta: { content: "<think>Complete thought</think>" } }] },
903-
})
904-
.mockResolvedValueOnce({
905-
done: false,
906-
value: { choices: [{ delta: { content: " text after" } }] },
907-
})
908-
.mockResolvedValueOnce({ done: true }),
909-
}),
910-
}))
911-
912-
const stream = handler.createMessage(systemPrompt, messages)
913-
const chunks: any[] = []
914-
for await (const chunk of stream) {
915-
chunks.push(chunk)
916-
}
917-
918-
// The TagMatcher processes the whole chunk character by character,
919-
// so the complete tag is detected and yields reasoning text
920-
expect(chunks.length).toBeGreaterThan(0)
921-
expect(chunks[0]).toEqual({ type: "text", text: "text before " })
922-
})
923-
924690
it("should handle nested mixed tags with correct closure matching", async () => {
925691
mockCreate.mockImplementationOnce(() => ({
926692
[Symbol.asyncIterator]: () => ({

‎src/utils/__tests__/tag-matcher.spec.ts‎

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,86 @@ describe("TagMatcher", () => {
5454
})
5555
})
5656

57+
describe("multi-tag constructor (string[])", () => {
58+
it("opens and closes <thought> when constructed with array", () => {
59+
const matcher = new TagMatcher(["think", "thought"])
60+
const result = matcher.final("<thought>deep reasoning</thought>done")
61+
expect(result.some((r) => r.matched && r.data === "deep reasoning")).toBe(true)
62+
expect(result.some((r) => !r.matched && r.data === "done")).toBe(true)
63+
})
64+
65+
it("opens and closes <think> when constructed with array", () => {
66+
const matcher = new TagMatcher(["think", "thought"])
67+
const result = matcher.final("<think>thinking</think>done")
68+
expect(result.some((r) => r.matched && r.data === "thinking")).toBe(true)
69+
expect(result.some((r) => !r.matched && r.data === "done")).toBe(true)
70+
})
71+
72+
it("<think> open is not closed by </thought> (cross-tag isolation)", () => {
73+
const matcher = new TagMatcher(["think", "thought"])
74+
const result = matcher.final("<think>reasoning</thought>still reasoning</think>done")
75+
// </thought> must be treated as text since active tag is <think>
76+
expect(result.some((r) => r.matched && r.data.includes("</thought>"))).toBe(true)
77+
expect(result.some((r) => !r.matched && r.data === "done")).toBe(true)
78+
})
79+
80+
it("<thought> open is not closed by </think> (inverse cross-tag isolation)", () => {
81+
const matcher = new TagMatcher(["think", "thought"])
82+
const result = matcher.final("<thought>reasoning</think>still reasoning</thought>done")
83+
// </think> must be treated as text since active tag is <thought>
84+
expect(result.some((r) => r.matched && r.data.includes("</think>"))).toBe(true)
85+
expect(result.some((r) => !r.matched && r.data === "done")).toBe(true)
86+
})
87+
})
88+
89+
describe("chunk split at mid-tag-name boundary", () => {
90+
it("correctly opens tag split across two update() calls", () => {
91+
const matcher = new TagMatcher("think")
92+
const first = matcher.update("<thi")
93+
// Tag not yet complete — no chunks emitted yet
94+
expect(first).toEqual([])
95+
const second = matcher.update("nk>content</think>")
96+
expect(second.some((r) => r.matched && r.data === "content")).toBe(true)
97+
})
98+
})
99+
100+
describe("unmatched > in TAG_OPEN falls back to TEXT", () => {
101+
it("treats <xyz> as plain text when xyz is not a configured tag name", () => {
102+
const matcher = new TagMatcher("think")
103+
const result = matcher.final("<xyz>content")
104+
expect(result.every((r) => !r.matched)).toBe(true)
105+
})
106+
107+
it("treats stray closing tag as plain text when no tag is open", () => {
108+
const matcher = new TagMatcher(["think", "thought"])
109+
const result = matcher.final("final</think>text")
110+
expect(result).toEqual([{ matched: false, data: "final</think>text" }])
111+
})
112+
113+
it("treats extra closing tag after a closed block as plain text", () => {
114+
const matcher = new TagMatcher(["think", "thought"])
115+
const result = matcher.final("<think>thinking</think>final</think>text")
116+
expect(result.some((r) => r.matched && r.data === "thinking")).toBe(true)
117+
expect(result.some((r) => !r.matched && r.data === "final</think>text")).toBe(true)
118+
})
119+
})
120+
121+
describe("nested tags", () => {
122+
it("treats inner <thought> as text when outer <think> is active", () => {
123+
const matcher = new TagMatcher(["think", "thought"])
124+
const result = matcher.final("<think>outer<thought>inner</thought> middle</think>final")
125+
expect(result.some((r) => r.matched && r.data.includes("<thought>inner</thought>"))).toBe(true)
126+
expect(result.some((r) => !r.matched && r.data === "final")).toBe(true)
127+
})
128+
129+
it("correctly unwinds nested same-name tags", () => {
130+
const matcher = new TagMatcher(["think", "thought"])
131+
const result = matcher.final("<think>outer<think>inner</think> middle</think>final")
132+
expect(result.some((r) => r.matched && r.data.includes("<think>inner</think>"))).toBe(true)
133+
expect(result.some((r) => !r.matched && r.data === "final")).toBe(true)
134+
})
135+
})
136+
57137
describe("space handling in TAG_CLOSE (line 119)", () => {
58138
it("tolerates a trailing space before > in closing tag (</think >)", () => {
59139
// space at index === tagName.length hits line 119 (continue)

‎src/utils/tag-matcher.ts‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ export class TagMatcher<Result = TagMatcherResult> {
2222
private candidates: { name: string; index: number }[] = []
2323

2424
constructor(
25-
tagName: string | string[],
25+
tagName: string | [string, ...string[]],
2626
readonly transform?: (chunks: TagMatcherResult) => Result,
2727
readonly position = 0,
2828
) {
@@ -62,7 +62,6 @@ export class TagMatcher<Result = TagMatcherResult> {
6262
if (this.state === "TEXT") {
6363
if (char === "<" && (this.pointer <= this.position + 1 || this.matched)) {
6464
this.state = "TAG_OPEN"
65-
this.index = 0
6665
if (this.depth === 0) {
6766
this.candidates = this.tagNames.map((name) => ({ name, index: 0 }))
6867
} else {
@@ -84,6 +83,9 @@ export class TagMatcher<Result = TagMatcherResult> {
8483
this.depth++
8584
this.matched = true
8685
continue
86+
} else {
87+
this.state = "TEXT"
88+
this.collect()
8789
}
8890
} else if (this.candidates.every((c) => c.index === 0) && char === "/") {
8991
this.state = "TAG_CLOSE"
@@ -106,7 +108,7 @@ export class TagMatcher<Result = TagMatcherResult> {
106108
}
107109
}
108110
} else if (this.state === "TAG_CLOSE") {
109-
const tagName = this.activeTagNames.at(-1) || this.tagNames[0]
111+
const tagName = this.activeTagNames.at(-1) ?? this.tagNames[0]
110112
if (char === ">" && this.index === tagName.length) {
111113
this.state = "TEXT"
112114
this.depth--

0 commit comments

Comments
 (0)