RDBC-1068 Fix OverlapTokens supported-methods set to match server/C#

Overlap tokens are only consumed by the two paragraph chunking methods
(PlainTextSplitParagraphs, MarkDownSplitParagraphs) on the server. The
Python set wrongly allowed PlainTextSplit/PlainTextSplitLines (silently
ignored by the server) and omitted MarkDownSplitParagraphs (spurious
client-side rejection of a valid config). Also makes the validation
message derive from the set so it can't drift again.

Author: poissoncorp

ravendb/documents/operations/ai/chunking_options.py

@@ -11,11 +11,11 @@ class ChunkingMethod(Enum):
     HTML_STRIP = "HtmlStrip"
 
 
-# Methods that support overlap tokens
+# Methods that support overlap tokens. Mirrors the server (TextChunker.cs) and the C#/Node clients:
+# only the two *paragraph* methods consume OverlapTokens; every other method ignores it.
 METHODS_SUPPORTING_OVERLAP_TOKENS = {
-    ChunkingMethod.PLAIN_TEXT_SPLIT,
-    ChunkingMethod.PLAIN_TEXT_SPLIT_LINES,
     ChunkingMethod.PLAIN_TEXT_SPLIT_PARAGRAPHS,
+    ChunkingMethod.MARK_DOWN_SPLIT_PARAGRAPHS,
 }
 
 
@@ -84,10 +84,8 @@ def validate(self, source: str, errors: List[str]) -> None:
             errors.append(f"{source}: OverlapTokens cannot be greater than MaxTokensPerChunk.")
 
         if self.overlap_tokens > 0 and self.chunking_method not in METHODS_SUPPORTING_OVERLAP_TOKENS:
-            errors.append(
-                f"{source}: OverlapTokens is only supported for PlainTextSplit, "
-                f"PlainTextSplitLines, and PlainTextSplitParagraphs chunking methods."
-            )
+            supported = ", ".join(sorted(method.value for method in METHODS_SUPPORTING_OVERLAP_TOKENS))
+            errors.append(f"{source}: OverlapTokens is only supported for the following chunking methods: {supported}.")
 
     @staticmethod
     def are_equal(left: Optional["ChunkingOptions"], right: Optional["ChunkingOptions"]) -> bool:

ravendb/tests/embeddings_generation_tests/test_chunking_options.py

@@ -45,6 +45,25 @@ def test_no_chunking_marker_bypasses_budget_validation(self):
         options.validate("source", errors)
         self.assertEqual([], errors)
 
+    def test_overlap_allowed_only_on_paragraph_methods(self):
+        # Matches the server / C# / Node clients: overlap is consumed only by the two paragraph methods.
+        for method in (ChunkingMethod.PLAIN_TEXT_SPLIT_PARAGRAPHS, ChunkingMethod.MARK_DOWN_SPLIT_PARAGRAPHS):
+            errors = []
+            ChunkingOptions(method, 100, 10).validate("source", errors)
+            self.assertEqual([], errors, method)
+
+    def test_overlap_rejected_on_non_paragraph_methods(self):
+        for method in (
+            ChunkingMethod.PLAIN_TEXT_SPLIT,
+            ChunkingMethod.PLAIN_TEXT_SPLIT_LINES,
+            ChunkingMethod.MARK_DOWN_SPLIT_LINES,
+            ChunkingMethod.HTML_STRIP,
+        ):
+            errors = []
+            ChunkingOptions(method, 100, 10).validate("source", errors)
+            self.assertEqual(1, len(errors), method)
+            self.assertIn("OverlapTokens", errors[0])
+
 
 if __name__ == "__main__":
     unittest.main()