|
| 1 | +package openai |
| 2 | + |
| 3 | +import ( |
| 4 | + "strings" |
| 5 | + "unicode" |
| 6 | + "unicode/utf8" |
| 7 | + |
| 8 | + "github.com/rivo/uniseg" |
| 9 | +) |
| 10 | + |
| 11 | +// Default clause-chunker bounds (in runes). minRunes gates only sub-sentence |
| 12 | +// (clause-mark / Thai-space) cuts so we don't synthesize tiny choppy fragments; |
| 13 | +// full sentences always flush regardless of length. maxRunes caps an |
| 14 | +// unterminated run so a long punctuation-less span doesn't buffer unbounded. |
| 15 | +const ( |
| 16 | + defaultClauseMinRunes = 12 |
| 17 | + defaultClauseMaxRunes = 200 |
| 18 | +) |
| 19 | + |
| 20 | +// clauseChunker splits streamed LLM content into speakable clauses for |
| 21 | +// incremental TTS, in a SCRIPT-AWARE way so it works for languages without |
| 22 | +// whitespace word boundaries. It leans on UAX #29 sentence segmentation (which |
| 23 | +// natively terminates on CJK 。！？ as well as Latin .!?), adds CJK clause |
| 24 | +// punctuation (，、；：) and Thai/Lao spaces as finer boundaries, and caps an |
| 25 | +// over-long unterminated run via UAX #14 line-break opportunities. |
| 26 | +// |
| 27 | +// Unlike the old ASCII .!?/newline segmenter (dropped in 076dcdbe), it does not |
| 28 | +// degrade to whole-message buffering for CJK (handled natively) or Thai/Lao |
| 29 | +// (handled via spaces, which Thai uses at clause/sentence boundaries). Scripts |
| 30 | +// that genuinely need a dictionary (Khmer/Myanmar) simply stay buffered until a |
| 31 | +// space or end-of-message — no worse than the buffered default. |
| 32 | +// |
| 33 | +// It is not safe for concurrent use; callers feed it from a single goroutine |
| 34 | +// (the LLM token callback). |
| 35 | +type clauseChunker struct { |
| 36 | + buf strings.Builder |
| 37 | + minRunes int |
| 38 | + maxRunes int |
| 39 | +} |
| 40 | + |
| 41 | +func newClauseChunker(minRunes, maxRunes int) *clauseChunker { |
| 42 | + return &clauseChunker{minRunes: minRunes, maxRunes: maxRunes} |
| 43 | +} |
| 44 | + |
| 45 | +// push appends streamed content and returns any clauses that are now complete — |
| 46 | +// "complete" meaning confirmed by following content, so we never speak a clause |
| 47 | +// that the next token might extend. Incomplete trailing text stays buffered. |
| 48 | +func (c *clauseChunker) push(text string) []string { |
| 49 | + c.buf.WriteString(text) |
| 50 | + return c.drain(false) |
| 51 | +} |
| 52 | + |
| 53 | +// flush returns the remaining buffered clauses, treating end-of-input as a hard |
| 54 | +// boundary, and clears the buffer. |
| 55 | +func (c *clauseChunker) flush() []string { |
| 56 | + return c.drain(true) |
| 57 | +} |
| 58 | + |
| 59 | +func (c *clauseChunker) drain(final bool) []string { |
| 60 | + s := c.buf.String() |
| 61 | + rest := s |
| 62 | + var out []string |
| 63 | + for rest != "" { |
| 64 | + end, ok := c.nextBoundary(rest, final) |
| 65 | + if !ok { |
| 66 | + break |
| 67 | + } |
| 68 | + if seg := strings.TrimSpace(rest[:end]); seg != "" { |
| 69 | + out = append(out, seg) |
| 70 | + } |
| 71 | + rest = rest[end:] |
| 72 | + } |
| 73 | + // Rewriting the builder reallocates and copies the whole buffer; skip it on |
| 74 | + // the common per-token call where no boundary was confirmed. |
| 75 | + if len(rest) != len(s) { |
| 76 | + c.buf.Reset() |
| 77 | + c.buf.WriteString(rest) |
| 78 | + } |
| 79 | + return out |
| 80 | +} |
| 81 | + |
| 82 | +// nextBoundary returns the byte offset just past the first emittable clause in |
| 83 | +// s, or ok=false when more input is needed (final=false) and no boundary is |
| 84 | +// confirmed yet. |
| 85 | +func (c *clauseChunker) nextBoundary(s string, final bool) (int, bool) { |
| 86 | + if s == "" { |
| 87 | + return 0, false |
| 88 | + } |
| 89 | + |
| 90 | + // 1) UAX #29 sentence boundary. When the first sentence is followed by more |
| 91 | + // text it is a confirmed complete sentence (handles Latin .!? with |
| 92 | + // abbreviation/decimal guards, and CJK 。！？ with no whitespace). |
| 93 | + sentence, rest, _ := uniseg.FirstSentenceInString(s, -1) |
| 94 | + if rest != "" { |
| 95 | + // Optionally cut finer inside the sentence at a clause boundary. |
| 96 | + if cut, ok := c.firstClauseCut(sentence); ok { |
| 97 | + return cut, true |
| 98 | + } |
| 99 | + return len(sentence), true |
| 100 | + } |
| 101 | + |
| 102 | + // 2) Unterminated tail: look for a sub-sentence clause boundary (CJK |
| 103 | + // punctuation or a Thai/Lao space) confirmed by following content. |
| 104 | + if cut, ok := c.firstClauseCut(s); ok { |
| 105 | + return cut, true |
| 106 | + } |
| 107 | + |
| 108 | + // 3) Over-long punctuation-less run: force a typographically legal break so |
| 109 | + // we don't buffer unbounded (e.g. a long CJK run with no punctuation). |
| 110 | + if !final && c.maxRunes > 0 && utf8.RuneCountInString(s) > c.maxRunes { |
| 111 | + if cut, ok := lineBreakCut(s, c.maxRunes); ok { |
| 112 | + return cut, true |
| 113 | + } |
| 114 | + } |
| 115 | + |
| 116 | + // 4) End of input: emit whatever remains as the final clause. |
| 117 | + if final { |
| 118 | + return len(s), true |
| 119 | + } |
| 120 | + return 0, false |
| 121 | +} |
| 122 | + |
| 123 | +// firstClauseCut returns the byte offset just past the first sub-sentence clause |
| 124 | +// boundary in s — a CJK clause punctuation mark, or a space following a Thai/Lao |
| 125 | +// letter — provided the prefix is at least minRunes long and non-space content |
| 126 | +// follows. The boundary mark (and any trailing spaces) stay with the left clause. |
| 127 | +func (c *clauseChunker) firstClauseCut(s string) (int, bool) { |
| 128 | + var prev rune |
| 129 | + runes := 0 |
| 130 | + for i, r := range s { |
| 131 | + boundary := isCJKClausePunct(r) || (unicode.IsSpace(r) && isThaiLao(prev)) |
| 132 | + if boundary && runes+1 >= c.minRunes { |
| 133 | + end := i + utf8.RuneLen(r) |
| 134 | + for end < len(s) { |
| 135 | + nr, sz := utf8.DecodeRuneInString(s[end:]) |
| 136 | + if !unicode.IsSpace(nr) { |
| 137 | + break |
| 138 | + } |
| 139 | + end += sz |
| 140 | + } |
| 141 | + if end < len(s) { // confirmed: real content follows the boundary |
| 142 | + return end, true |
| 143 | + } |
| 144 | + // Boundary sits at the end of the buffer with nothing after it yet — |
| 145 | + // wait for the next token to confirm it rather than emit early. |
| 146 | + return 0, false |
| 147 | + } |
| 148 | + prev = r |
| 149 | + runes++ |
| 150 | + } |
| 151 | + return 0, false |
| 152 | +} |
| 153 | + |
| 154 | +// lineBreakCut walks UAX #14 line segments and returns the byte offset of the |
| 155 | +// last legal break opportunity at or before maxRunes. Returns ok=false when the |
| 156 | +// run has no internal break opportunity (e.g. a space-less Thai run), leaving it |
| 157 | +// buffered. |
| 158 | +func lineBreakCut(s string, maxRunes int) (int, bool) { |
| 159 | + state := -1 |
| 160 | + rest := s |
| 161 | + consumed := 0 |
| 162 | + runes := 0 |
| 163 | + for rest != "" { |
| 164 | + seg, rem, _, st := uniseg.FirstLineSegmentInString(rest, state) |
| 165 | + state = st |
| 166 | + runes += utf8.RuneCountInString(seg) |
| 167 | + consumed += len(seg) |
| 168 | + rest = rem |
| 169 | + if runes >= maxRunes { |
| 170 | + if consumed < len(s) { |
| 171 | + return consumed, true |
| 172 | + } |
| 173 | + return 0, false |
| 174 | + } |
| 175 | + } |
| 176 | + return 0, false |
| 177 | +} |
| 178 | + |
| 179 | +// isCJKClausePunct reports whether r is a CJK clause-level separator worth a |
| 180 | +// soft TTS break. Sentence terminators (。！？) are intentionally excluded — UAX |
| 181 | +// #29 sentence segmentation already handles those. |
| 182 | +func isCJKClausePunct(r rune) bool { |
| 183 | + switch r { |
| 184 | + case '，', // ， fullwidth comma |
| 185 | + '、', // 、 ideographic comma |
| 186 | + '；', // ； fullwidth semicolon |
| 187 | + '：', // ： fullwidth colon |
| 188 | + '・', // ・ katakana middle dot |
| 189 | + '･': // ・ halfwidth katakana middle dot |
| 190 | + return true |
| 191 | + } |
| 192 | + return false |
| 193 | +} |
| 194 | + |
| 195 | +// isThaiLao reports whether r is a Thai or Lao letter. Those scripts have no |
| 196 | +// inter-word spaces; an ASCII space inside such a run marks a clause/sentence |
| 197 | +// boundary, which is the only no-dictionary segmentation signal available. |
| 198 | +func isThaiLao(r rune) bool { |
| 199 | + return unicode.Is(unicode.Thai, r) || unicode.Is(unicode.Lao, r) |
| 200 | +} |
0 commit comments