Skip to content

Commit af83518

Browse files
egliamudler
eglia
and
mudler
authored
feat: support word-level timestamps for faster-whisper (#9621)
Signed-off-by: Andreas Egli <github@kharan.ch> Signed-off-by: Ettore Di Giacinto <mudler@users.noreply.github.com> Co-authored-by: Ettore Di Giacinto <mudler@users.noreply.github.com>
1 parent a315c32 commit af83518

6 files changed

Lines changed: 146 additions & 11 deletions

File tree

‎backend/backend.proto‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -355,13 +355,20 @@ message TranscriptStreamResponse {
355355
TranscriptResult final_result = 2;
356356
}
357357

358+
message TranscriptWord {
359+
int64 start = 1;
360+
int64 end = 2;
361+
string text = 3;
362+
}
363+
358364
message TranscriptSegment {
359365
int32 id = 1;
360366
int64 start = 2;
361367
int64 end = 3;
362368
string text = 4;
363369
repeated int32 tokens = 5;
364370
string speaker = 6;
371+
repeated TranscriptWord words = 7;
365372
}
366373

367374
message GenerateImageRequest {

‎backend/python/faster-whisper/backend.py‎

Lines changed: 18 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -55,11 +55,27 @@ def AudioTranscription(self, request, context):
5555
resultSegments = []
5656
text = ""
5757
try:
58-
segments, info = self.model.transcribe(request.dst, beam_size=5, condition_on_previous_text=False)
58+
word_timestamps = "word" in request.timestamp_granularities
59+
segments, info = self.model.transcribe(request.dst, beam_size=5, condition_on_previous_text=False, word_timestamps=word_timestamps)
5960
id = 0
6061
for segment in segments:
6162
print("[%.2fs -> %.2fs] %s" % (segment.start, segment.end, segment.text))
62-
resultSegments.append(backend_pb2.TranscriptSegment(id=id, start=int(segment.start*1e9), end=int(segment.end*1e9), text=segment.text))
63+
words = []
64+
if word_timestamps and hasattr(segment, 'words'):
65+
for word in segment.words:
66+
words.append(backend_pb2.TranscriptWord(
67+
start=int(word.start * 1e9),
68+
end=int(word.end * 1e9),
69+
text=word.word
70+
))
71+
72+
resultSegments.append(backend_pb2.TranscriptSegment(
73+
id=id,
74+
start=int(segment.start * 1e9),
75+
end=int(segment.end * 1e9),
76+
text=segment.text,
77+
words=words
78+
))
6379
text += segment.text
6480
id += 1
6581
except Exception as err:

‎core/backend/transcript.go‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -179,11 +179,22 @@ func transcriptResultFromProto(r *proto.TranscriptResult) *schema.TranscriptionR
179179
Language: r.Language,
180180
Duration: float64(r.Duration),
181181
}
182+
182183
for _, s := range r.Segments {
183184
var tks []int
184185
for _, t := range s.Tokens {
185186
tks = append(tks, int(t))
186187
}
188+
var words []schema.TranscriptionWord
189+
for _, w := range s.Words {
190+
var word = schema.TranscriptionWord {
191+
Start: time.Duration(w.Start),
192+
End: time.Duration(w.End),
193+
Text: w.Text,
194+
}
195+
words = append(words, word)
196+
tr.Words = append(tr.Words, word)
197+
}
187198
tr.Segments = append(tr.Segments,
188199
schema.TranscriptionSegment{
189200
Text: s.Text,
@@ -192,6 +203,7 @@ func transcriptResultFromProto(r *proto.TranscriptResult) *schema.TranscriptionR
192203
End: time.Duration(s.End),
193204
Tokens: tks,
194205
Speaker: s.Speaker,
206+
Words: words,
195207
})
196208
}
197209
return tr

‎core/cli/transcript.go‎

Lines changed: 36 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -81,14 +81,48 @@ func (t *TranscriptCMD) Run(ctx *cliContext.Context) error {
8181
fmt.Println(schema.TranscriptionResponse(tr, t.ResponseFormat))
8282
case schema.TranscriptionResponseFormatJson:
8383
tr.Segments = nil
84+
tr.Words = nil
8485
fallthrough
8586
case schema.TranscriptionResponseFormatJsonVerbose:
87+
trs := schema.TranscriptionResultSeconds{
88+
Text: tr.Text,
89+
Language: tr.Language,
90+
Duration: tr.Duration,
91+
Words: []schema.TranscriptionWordSeconds{},
92+
Segments: []schema.TranscriptionSegmentSeconds{},
93+
}
94+
for _, word := range(tr.Words) {
95+
trs.Words = append(trs.Words, schema.TranscriptionWordSeconds{
96+
Start: word.Start.Seconds(),
97+
End: word.End.Seconds(),
98+
Text: word.Text,
99+
})
100+
}
101+
for _, seg := range(tr.Segments) {
102+
segWords := []schema.TranscriptionWordSeconds{}
103+
for _, word := range(seg.Words) {
104+
segWords = append(segWords, schema.TranscriptionWordSeconds{
105+
Start: word.Start.Seconds(),
106+
End: word.End.Seconds(),
107+
Text: word.Text,
108+
})
109+
}
110+
trs.Segments = append(trs.Segments, schema.TranscriptionSegmentSeconds{
111+
Id: seg.Id,
112+
Start: seg.Start.Seconds(),
113+
End: seg.End.Seconds(),
114+
Text: seg.Text,
115+
Tokens: seg.Tokens,
116+
Speaker: seg.Speaker,
117+
Words: segWords,
118+
})
119+
}
86120
var mtr []byte
87121
var err error
88122
if t.PrettyPrint {
89-
mtr, err = json.MarshalIndent(tr, "", " ")
123+
mtr, err = json.MarshalIndent(trs, "", " ")
90124
} else {
91-
mtr, err = json.Marshal(tr)
125+
mtr, err = json.Marshal(trs)
92126
}
93127
if err != nil {
94128
return err

‎core/http/endpoints/openai/transcription.go‎

Lines changed: 35 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -138,9 +138,43 @@ func TranscriptEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, app
138138
return c.String(http.StatusOK, schema.TranscriptionResponse(tr, responseFormat))
139139
case schema.TranscriptionResponseFormatJson:
140140
tr.Segments = nil
141+
tr.Words = nil
141142
fallthrough
142143
case schema.TranscriptionResponseFormatJsonVerbose, "": // maintain backwards compatibility
143-
return c.JSON(http.StatusOK, tr)
144+
trs := schema.TranscriptionResultSeconds{
145+
Text: tr.Text,
146+
Language: tr.Language,
147+
Duration: tr.Duration,
148+
Words: []schema.TranscriptionWordSeconds{},
149+
Segments: []schema.TranscriptionSegmentSeconds{},
150+
}
151+
for _, word := range(tr.Words) {
152+
trs.Words = append(trs.Words, schema.TranscriptionWordSeconds{
153+
Start: word.Start.Seconds(),
154+
End: word.End.Seconds(),
155+
Text: word.Text,
156+
})
157+
}
158+
for _, seg := range(tr.Segments) {
159+
segWords := []schema.TranscriptionWordSeconds{}
160+
for _, word := range(seg.Words) {
161+
segWords = append(segWords, schema.TranscriptionWordSeconds{
162+
Start: word.Start.Seconds(),
163+
End: word.End.Seconds(),
164+
Text: word.Text,
165+
})
166+
}
167+
trs.Segments = append(trs.Segments, schema.TranscriptionSegmentSeconds{
168+
Id: seg.Id,
169+
Start: seg.Start.Seconds(),
170+
End: seg.End.Seconds(),
171+
Text: seg.Text,
172+
Tokens: seg.Tokens,
173+
Speaker: seg.Speaker,
174+
Words: segWords,
175+
})
176+
}
177+
return c.JSON(http.StatusOK, trs)
144178
default:
145179
return errors.New("invalid response_format")
146180
}

‎core/schema/transcription.go‎

Lines changed: 38 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -3,17 +3,49 @@ package schema
33
import "time"
44

55
type TranscriptionSegment struct {
6-
Id int `json:"id"`
7-
Start time.Duration `json:"start"`
8-
End time.Duration `json:"end"`
9-
Text string `json:"text"`
10-
Tokens []int `json:"tokens"`
11-
Speaker string `json:"speaker,omitempty"`
6+
Id int `json:"id"`
7+
Start time.Duration `json:"start"`
8+
End time.Duration `json:"end"`
9+
Text string `json:"text"`
10+
Tokens []int `json:"tokens"`
11+
Speaker string `json:"speaker,omitempty"`
12+
Words []TranscriptionWord `json:"words,omitempty"`
13+
}
14+
15+
type TranscriptionWord struct {
16+
Start time.Duration `json:"start"`
17+
End time.Duration `json:"end"`
18+
Text string `json:"text"`
1219
}
1320

1421
type TranscriptionResult struct {
1522
Segments []TranscriptionSegment `json:"segments,omitempty"`
23+
Words []TranscriptionWord `json:"words,omitempty"`
1624
Text string `json:"text"`
1725
Language string `json:"language,omitempty"`
1826
Duration float64 `json:"duration,omitempty"`
1927
}
28+
29+
type TranscriptionSegmentSeconds struct {
30+
Id int `json:"id"`
31+
Start float64 `json:"start"`
32+
End float64 `json:"end"`
33+
Text string `json:"text"`
34+
Tokens []int `json:"tokens"`
35+
Speaker string `json:"speaker,omitempty"`
36+
Words []TranscriptionWordSeconds `json:"words,omitempty"`
37+
}
38+
39+
type TranscriptionWordSeconds struct {
40+
Start float64 `json:"start"`
41+
End float64 `json:"end"`
42+
Text string `json:"text"`
43+
}
44+
45+
type TranscriptionResultSeconds struct {
46+
Segments []TranscriptionSegmentSeconds `json:"segments,omitempty"`
47+
Words []TranscriptionWordSeconds `json:"words,omitempty"`
48+
Text string `json:"text"`
49+
Language string `json:"language,omitempty"`
50+
Duration float64 `json:"duration,omitempty"`
51+
}

0 commit comments

Comments
 (0)