@@ -891,7 +891,7 @@ Example:
891891from haystack import Document
892892from haystack.components.preprocessors import RecursiveDocumentSplitter
893893
894- chunker = RecursiveDocumentSplitter(split_length = 260 , split_overlap = 0 , separators = [" \n\n " , " \n " , " ." , " " ])
894+ chunker = RecursiveDocumentSplitter(split_length = 15 , split_overlap = 0 , separators = [" \n\n " , " \n " , " ." , " " ])
895895text = (''' Artificial intelligence (AI) - Introduction
896896
897897AI, in its broadest sense, is intelligence exhibited by machines, particularly computer systems.
@@ -900,10 +900,11 @@ doc = Document(content=text)
900900doc_chunks = chunker.run([doc])
901901print (doc_chunks[" documents" ])
902902# [
903- # Document(id=..., content: 'Artificial intelligence (AI) - Introduction\n\n', meta: {'original_id': '...', 'split_id': 0, 'split_idx_start': 0, '_split_overlap': []})
904- # Document(id=..., content: 'AI, in its broadest sense, is intelligence exhibited by machines, particularly computer systems.\n', meta: {'original_id': '...', 'split_id': 1, 'split_idx_start': 45, '_split_overlap': []})
905- # Document(id=..., content: 'AI technology is widely used throughout industry, government, and science.', meta: {'original_id': '...', 'split_id': 2, 'split_idx_start': 142, '_split_overlap': []})
906- # Document(id=..., content: ' Some high-profile applications include advanced web search engines; recommendation systems; interac...', meta: {'original_id': '...', 'split_id': 3, 'split_idx_start': 216, '_split_overlap': []})
903+ # Document(id=..., content: 'Artificial intelligence (AI) - Introduction\n\n', meta: {'source_id': '...', 'parent_id': '...', 'split_id': 0, 'split_idx_start': 0, '_split_overlap': None, 'page_number': 1})
904+ # Document(id=..., content: 'AI, in its broadest sense, is intelligence exhibited by machines, particularly computer systems.\n', meta: {'source_id': '...', 'parent_id': '...', 'split_id': 1, 'split_idx_start': 45, '_split_overlap': None, 'page_number': 1})
905+ # Document(id=..., content: 'AI technology is widely used throughout industry, government, and science.', meta: {'source_id': '...', 'parent_id': '...', 'split_id': 2, 'split_idx_start': 142, '_split_overlap': None, 'page_number': 1})
906+ # Document(id=..., content: ' Some high-profile applications include advanced web search engines; recommendation systems; interac...', meta: {'source_id': '...', 'parent_id': '...', 'split_id': 3, 'split_idx_start': 216, '_split_overlap': None, 'page_number': 1})
907+ # Document(id=..., content: 'vehicles; generative and creative tools; and superhuman play and analysis in strategy games.', meta: {'source_id': '...', 'parent_id': '...', 'split_id': 4, 'split_idx_start': 350, '_split_overlap': None, 'page_number': 1})
907908# ]
908909```
909910
0 commit comments