@@ -25,24 +25,36 @@ def __init__(
2525 chunking_method : Optional [ChunkingMethod ] = None ,
2626 max_tokens_per_chunk : int = 512 ,
2727 overlap_tokens : int = 0 ,
28+ context_prefix : Optional [str ] = None ,
2829 ):
2930 self .chunking_method = chunking_method
3031 self .max_tokens_per_chunk = max_tokens_per_chunk
3132 self .overlap_tokens = overlap_tokens
3233
34+ # Optional constant text prepended to every produced chunk before it is sent to the embedding model.
35+ # Useful for adding broader document context (e.g. title) to isolated chunks. The prefix's tokens count
36+ # against max_tokens_per_chunk - the effective chunking budget is reduced accordingly.
37+ self .context_prefix = context_prefix
38+
39+ # Internal-only marker: when set, the value is emitted unchunked with context_prefix prepended, and
40+ # max_tokens_per_chunk / overlap_tokens are ignored. Never set by user-constructed config; not serialized.
41+ self .no_chunking = False
42+
3343 @classmethod
3444 def from_json (cls , json_dict : Dict [str , Any ]) -> "ChunkingOptions" :
3545 return cls (
3646 chunking_method = ChunkingMethod (json_dict ["ChunkingMethod" ]),
3747 max_tokens_per_chunk = json_dict .get ("MaxTokensPerChunk" , None ),
3848 overlap_tokens = json_dict .get ("OverlapTokens" , None ),
49+ context_prefix = json_dict .get ("ContextPrefix" , None ),
3950 )
4051
4152 def to_json (self ) -> Dict [str , Any ]:
4253 return {
4354 "ChunkingMethod" : self .chunking_method .value if self .chunking_method else None ,
4455 "MaxTokensPerChunk" : self .max_tokens_per_chunk ,
4556 "OverlapTokens" : self .overlap_tokens ,
57+ "ContextPrefix" : self .context_prefix ,
4658 }
4759
4860 def validate (self , source : str , errors : List [str ]) -> None :
@@ -52,6 +64,16 @@ def validate(self, source: str, errors: List[str]) -> None:
5264 source: The source context for error messages (e.g., 'embeddings.generate').
5365 errors: List to append validation errors to.
5466 """
67+ if self .context_prefix is not None and not self .context_prefix .strip ():
68+ errors .append (
69+ f"{ source } : ContextPrefix cannot be empty or whitespace-only. "
70+ f"Either provide a non-empty value or omit it."
71+ )
72+
73+ # no_chunking is set only by the with_context_prefix handler on raw strings/arrays and bypasses budget rules.
74+ if self .no_chunking :
75+ return
76+
5577 if self .max_tokens_per_chunk <= 0 :
5678 errors .append (f"{ source } : MaxTokensPerChunk must be greater than 0." )
5779
@@ -87,7 +109,17 @@ def __eq__(self, other: object) -> bool:
87109 self .chunking_method == other .chunking_method
88110 and self .max_tokens_per_chunk == other .max_tokens_per_chunk
89111 and self .overlap_tokens == other .overlap_tokens
112+ and self .context_prefix == other .context_prefix
113+ and self .no_chunking == other .no_chunking
90114 )
91115
92116 def __hash__ (self ) -> int :
93- return hash ((self .chunking_method , self .max_tokens_per_chunk , self .overlap_tokens ))
117+ return hash (
118+ (
119+ self .chunking_method ,
120+ self .max_tokens_per_chunk ,
121+ self .overlap_tokens ,
122+ self .context_prefix ,
123+ self .no_chunking ,
124+ )
125+ )
0 commit comments