Skip to content

Commit 31f2d8b

Browse files
fqscfqjfqscfqj
committed
feat(crispasr): add word-level timestamp support
Add word-level timestamp extraction to the crispasr backend by calling the CrispASR C library's word accessor functions that are already exported by libgocraspasr but were not previously bound by the Go wrapper. Two families of word functions are supported: 1. Session-based (get_word_count/text/t0/t1) — works per-segment for whisper-like backends. 2. Parakeet-specific (get_parakeet_word_count/text/t0/t1) — returns a global word list for TDT/CTC/RNNT parakeet models where the session API does not expose per-segment word data. The Go code tries session-based first and falls back to parakeet-specific when the session word count is zero. Depends on #10402 (grpc server Words forwarding) for the words to reach the HTTP response.
1 parent 29dbba7 commit 31f2d8b

4 files changed

Lines changed: 128 additions & 0 deletions

File tree

‎backend/go/crispasr/cpp/crispasr_shim.cpp‎

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,74 @@ extern "C" void set_abort(int v) {
4747
g_abort.store(v, std::memory_order_relaxed);
4848
}
4949

50+
// --- word-level timestamp accessors ---
51+
extern "C" {
52+
int crispasr_session_result_n_words(crispasr_session_result *r, int seg_i);
53+
const char *crispasr_session_result_word_text(crispasr_session_result *r,
54+
int seg_i, int word_i);
55+
int64_t crispasr_session_result_word_t0(crispasr_session_result *r, int seg_i,
56+
int word_i);
57+
int64_t crispasr_session_result_word_t1(crispasr_session_result *r, int seg_i,
58+
int word_i);
59+
60+
// Parakeet-specific word accessors
61+
int crispasr_parakeet_result_n_words(void *r);
62+
const char *crispasr_parakeet_result_word_text(void *r, int word_i);
63+
int64_t crispasr_parakeet_result_word_t0(void *r, int word_i);
64+
int64_t crispasr_parakeet_result_word_t1(void *r, int word_i);
65+
}
66+
67+
void *get_result(void) { return g_result; }
68+
69+
int get_word_count(int seg_i) {
70+
if (!g_result)
71+
return 0;
72+
return crispasr_session_result_n_words(g_result, seg_i);
73+
}
74+
75+
const char *get_word_text(int seg_i, int word_i) {
76+
if (!g_result)
77+
return "";
78+
return crispasr_session_result_word_text(g_result, seg_i, word_i);
79+
}
80+
81+
int64_t get_word_t0(int seg_i, int word_i) {
82+
if (!g_result)
83+
return 0;
84+
return crispasr_session_result_word_t0(g_result, seg_i, word_i);
85+
}
86+
87+
int64_t get_word_t1(int seg_i, int word_i) {
88+
if (!g_result)
89+
return 0;
90+
return crispasr_session_result_word_t1(g_result, seg_i, word_i);
91+
}
92+
93+
// Parakeet-specific word accessors
94+
int get_parakeet_word_count(void) {
95+
if (!g_result)
96+
return 0;
97+
return crispasr_parakeet_result_n_words(g_result);
98+
}
99+
100+
const char *get_parakeet_word_text(int word_i) {
101+
if (!g_result)
102+
return "";
103+
return crispasr_parakeet_result_word_text(g_result, word_i);
104+
}
105+
106+
int64_t get_parakeet_word_t0(int word_i) {
107+
if (!g_result)
108+
return 0;
109+
return crispasr_parakeet_result_word_t0(g_result, word_i);
110+
}
111+
112+
int64_t get_parakeet_word_t1(int word_i) {
113+
if (!g_result)
114+
return 0;
115+
return crispasr_parakeet_result_word_t1(g_result, word_i);
116+
}
117+
50118
static void ggml_log_cb(enum ggml_log_level level, const char *log,
51119
void *data) {
52120
const char *level_str;

‎backend/go/crispasr/cpp/crispasr_shim.h‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,4 +20,18 @@ float *tts_synthesize(const char *text, int *out_n_samples); // 24kHz mono float
2020
void tts_free(float *pcm);
2121
int tts_set_voice(const char *name); // best-effort speaker selection; 0 ok
2222
int tts_set_voice_file(const char *path, const char *ref_text); // load voice pack (.gguf) or zero-shot clone (.wav + ref_text)
23+
24+
// --- word-level timestamp accessors ---
25+
// Session-based (works for whisper-like backends)
26+
void *get_result(void);
27+
int get_word_count(int seg_i);
28+
const char *get_word_text(int seg_i, int word_i);
29+
int64_t get_word_t0(int seg_i, int word_i);
30+
int64_t get_word_t1(int seg_i, int word_i);
31+
32+
// Parakeet-specific (global word list, no segment index)
33+
int get_parakeet_word_count(void);
34+
const char *get_parakeet_word_text(int word_i);
35+
int64_t get_parakeet_word_t0(int word_i);
36+
int64_t get_parakeet_word_t1(int word_i);
2337
}

‎backend/go/crispasr/gocrispasr.go‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,18 @@ var (
3434
CppTTSFree func(ptr uintptr)
3535
CppTTSSetVoice func(name string) int
3636
CppTTSSetVoiceFile func(path string, refText string) int
37+
38+
// Word-level timestamp accessors (session-based, per-segment)
39+
CppGetWordCount func(segI int) int
40+
CppGetWordText func(segI int, wordI int) string
41+
CppGetWordT0 func(segI int, wordI int) int64
42+
CppGetWordT1 func(segI int, wordI int) int64
43+
44+
// Parakeet-specific word accessors (global, no segment index)
45+
CppGetParakeetWordCount func() int
46+
CppGetParakeetWordText func(wordI int) string
47+
CppGetParakeetWordT0 func(wordI int) int64
48+
CppGetParakeetWordT1 func(wordI int) int64
3749
)
3850

3951
type CrispASR struct {
@@ -290,10 +302,36 @@ func (w *CrispASR) AudioTranscription(ctx context.Context, opts *pb.TranscriptRe
290302
// IDs, so Tokens is left empty.
291303
txt := strings.ToValidUTF8(strings.Clone(CppGetSegmentText(i)), "�")
292304

305+
// Populate word-level timestamps. Try session-based functions first
306+
// (per-segment); fall back to parakeet-specific functions (global word
307+
// list with no segment index — only populated on the first segment to
308+
// avoid duplication).
309+
words := []*pb.TranscriptWord{}
310+
wordCount := CppGetWordCount(i)
311+
if wordCount == 0 && i == 0 {
312+
wordCount = CppGetParakeetWordCount()
313+
for j := 0; j < wordCount; j++ {
314+
words = append(words, &pb.TranscriptWord{
315+
Start: CppGetParakeetWordT0(j) * (10000000),
316+
End: CppGetParakeetWordT1(j) * (10000000),
317+
Text: strings.ToValidUTF8(strings.Clone(CppGetParakeetWordText(j)), "�"),
318+
})
319+
}
320+
} else {
321+
for j := 0; j < wordCount; j++ {
322+
words = append(words, &pb.TranscriptWord{
323+
Start: CppGetWordT0(i, j) * (10000000),
324+
End: CppGetWordT1(i, j) * (10000000),
325+
Text: strings.ToValidUTF8(strings.Clone(CppGetWordText(i, j)), "�"),
326+
})
327+
}
328+
}
329+
293330
segment := &pb.TranscriptSegment{
294331
Id: int32(i),
295332
Text: txt,
296333
Start: s, End: t,
334+
Words: words,
297335
}
298336

299337
segments = append(segments, segment)

‎backend/go/crispasr/main.go‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,14 @@ func main() {
4444
{&CppTTSFree, "tts_free"},
4545
{&CppTTSSetVoice, "tts_set_voice"},
4646
{&CppTTSSetVoiceFile, "tts_set_voice_file"},
47+
{&CppGetWordCount, "get_word_count"},
48+
{&CppGetWordText, "get_word_text"},
49+
{&CppGetWordT0, "get_word_t0"},
50+
{&CppGetWordT1, "get_word_t1"},
51+
{&CppGetParakeetWordCount, "get_parakeet_word_count"},
52+
{&CppGetParakeetWordText, "get_parakeet_word_text"},
53+
{&CppGetParakeetWordT0, "get_parakeet_word_t0"},
54+
{&CppGetParakeetWordT1, "get_parakeet_word_t1"},
4755
}
4856

4957
for _, lf := range libFuncs {

0 commit comments

Comments
 (0)