ICU-23440 Merge the end-of-text and Sentence_Break=Sep symbols in the sentence breaking state machine

Author: eggrobin

icu4c/source/data/brkitr/rules/sent.txt

@@ -76,7 +76,12 @@ $ATermEx $CloseEx* $SpEx* $NotLettersEx* $Lower;
 ($STermEx | $ATermEx) $CloseEx* $SpEx* ($SContinueEx | $STermEx | $ATermEx);
 
 #Rule 9, 10, 11
-($STermEx | $ATermEx) $CloseEx* $SpEx* ($Sep | $CR | $LF)?;
+# Including {eof} here does not change the observable behaviour (if {eof} were not here, on STerm at
+# the end of text, we would reach an accepting state which would have no transition on the {eof}
+# symbol, and break at the end of text with status 0), but it allows us to merge the {eof} and Sep
+# symbols; on STerm at the end of text or before Sep, we transition on {eof} or Sep to the same
+# state and accept that one.
+($STermEx | $ATermEx) $CloseEx* $SpEx* [$Sep $LF $CR {eof}]?;
 
 #Rule 998
 [^$STerm $ATerm $Close $Sp $Sep $LF $CR $Format $Extend] ($Extend | $Format | $Close | $Sp)* .;

icu4c/source/data/brkitr/rules/sent_el.txt

@@ -76,7 +76,12 @@ $ATermEx $CloseEx* $SpEx* $NotLettersEx* $Lower;
 ($STermEx | $ATermEx) $CloseEx* $SpEx* ($SContinueEx | $STermEx | $ATermEx);
 
 #Rule 9, 10, 11
-($STermEx | $ATermEx) $CloseEx* $SpEx* ($Sep | $CR | $LF)?;
+# Including {eof} here does not change the observable behaviour (if {eof} were not here, on STerm at
+# the end of text, we would reach an accepting state which would have no transition on the {eof}
+# symbol, and break at the end of text with status 0), but it allows us to merge the {eof} and Sep
+# symbols; on STerm at the end of text or before Sep, we transition on {eof} or Sep to the same
+# state and accept that one.
+($STermEx | $ATermEx) $CloseEx* $SpEx* [$Sep $LF $CR {eof}]?;
 
 #Rule 998
 [^$STerm $ATerm $Close $Sp $Sep $LF $CR $Format $Extend] ($Extend | $Format | $Close | $Sp)* .;