fix: improve heuristic for not combining paragraphs

this changes the detection heuristic for overly long sentences from the previous "longer than 30s" to a word based approach of "more than 161 words".
moreover it gets rid of the stateful approach that we would disable all recombination after the first overly long paragraph.

overall this is not a perfect approach but produces saner results than before.

this should temporarily fix the situation in which transcribee generates only one word per paragraph.
the proper fix is resetting the whisper state from time to time and will land in the future.

Author: anuejn

worker/tests/data/test_strict_sentence_paragraphs-doest_combine_long_para.json

@@ -4,7 +4,7 @@
 
 import pytest
 from pydantic import BaseModel
-from transcribee_proto.document import Paragraph
+from transcribee_proto.document import Atom, Paragraph
 from transcribee_worker.whisper_transcribe import (
     move_space_to_prev_token,
     strict_sentence_paragraphs,
@@ -43,6 +43,28 @@ def test_strict_sentence_paragraphs(data_file):
     assert output == test_data.expected
 
 
+def test_strict_sentence_paragraphs_dont_combine_overly_long_paras():
+    test_data = [
+        Paragraph(
+            lang="en",
+            children=[
+                Atom(text=f"{word} ", start=0, end=0, conf=1, conf_ts=0)
+                for word in f"this is long sentence {i} without a sentence ending".split()
+            ],
+        )
+        for i in range(19)
+    ]
+
+    output = list(doc_chain_func_to_list(strict_sentence_paragraphs)(test_data))
+
+    for para in output:
+        for atom in para.children:
+            print(atom.text, end="")
+        print()
+
+    assert output == test_data
+
+
 @pytest.mark.parametrize(
     "data_file",
     glob.glob(

worker/tests/test_transcribe.py

@@ -90,29 +90,18 @@ def strict_sentence_paragraphs(
 ) -> Iterator[Paragraph]:
     acc_paragraph = None
     acc_used_paras = []
-    combination_active = True
     for paragraph in iter:
-        if not combination_active:
-            yield paragraph
-            continue
-        elif acc_paragraph is None:
+        if acc_paragraph is not None and len(acc_paragraph.children) > 161:
+            # it seems like whisper is on some path that does not involve sentence breaks
+            # as a workaround we just emit the raw whisper bars
+            yield from acc_used_paras
+            acc_paragraph = None
+
+        if acc_paragraph is None:
             acc_paragraph = Paragraph(
                 lang=paragraph.lang, speaker=paragraph.speaker, children=[]
             )
             acc_used_paras = []
-        elif (
-            (start := acc_paragraph.start()) is not None
-            and (end := paragraph.end()) is not None
-            and end - start > 30
-        ):
-            # It seems like whisper doesn't produce sentence breaks. Ignore the
-            # current `acc_paragraph` and yield the original paras instead,
-            # disable this step until the end of the document
-            combination_active = False
-            for para in acc_used_paras:
-                yield para
-            yield paragraph
-            continue
         elif (
             acc_paragraph.lang != paragraph.lang
             or acc_paragraph.speaker != paragraph.speaker
@@ -163,7 +152,7 @@ def strict_sentence_paragraphs(
                 children=paragraph.children[acc_yield_offset:],
             )
         )
-    if acc_paragraph is not None and acc_paragraph.children and combination_active:
+    if acc_paragraph is not None and acc_paragraph.children:
         yield acc_paragraph