feat(oss-opensearch): Add KNN derived source configuration option

Author: Akhil-Pathivada

vectordb_bench/backend/clients/oss_opensearch/config.py

@@ -77,6 +77,7 @@ class OSSOpenSearchIndexConfig(BaseModel, DBCaseConfig):
     oversample_factor: float = 1.0
     quantization_type: OSSOpenSearchQuantization = OSSOpenSearchQuantization.fp32
     replication_type: str | None = "DOCUMENT"
+    knn_derived_source_enabled: str | None = None
 
     @root_validator
     def validate_engine_name(cls, values: dict):
@@ -103,6 +104,7 @@ def __eq__(self, obj: any):
             and self.use_routing == obj.use_routing
             and self.quantization_type == obj.quantization_type
             and self.replication_type == obj.replication_type
+            and self.knn_derived_source_enabled == obj.knn_derived_source_enabled
         )
 
     def __hash__(self) -> int:
@@ -117,6 +119,7 @@ def __hash__(self) -> int:
                 self.use_routing,
                 self.quantization_type,
                 self.replication_type,
+                self.knn_derived_source_enabled,
             )
         )
 

vectordb_bench/backend/clients/oss_opensearch/oss_opensearch.py

@@ -270,6 +270,7 @@ def _create_index(self, client: OpenSearch) -> None:
         log.info(f"Creating index with ef_search: {ef_search_value}")
         log.info(f"Creating index with number_of_replicas: {self.case_config.number_of_replicas}")
         log.info(f"Creating index with replication_type: {self.case_config.replication_type}")
+        log.info(f"Creating index with knn_derived_source_enabled: {self.case_config.knn_derived_source_enabled}")
         log.info(f"Creating index with engine: {self.case_config.engine}")
         log.info(f"Creating index with metric type: {self.case_config.metric_type_name}")
         log.info(f"All case_config parameters: {self.case_config.__dict__}")
@@ -293,6 +294,12 @@ def _create_index(self, client: OpenSearch) -> None:
             },
             "refresh_interval": self.case_config.refresh_interval,
         }
+        # Only add knn.derived_source.enabled if explicitly set (None = skip for versions < 3.x compatibility)
+        if self.case_config.knn_derived_source_enabled and self.case_config.knn_derived_source_enabled != "None":
+            is_knn_derived_source_enabled = self.case_config.knn_derived_source_enabled == "True"
+            log.info(f"Adding knn.derived_source.enabled={is_knn_derived_source_enabled} to index settings")
+            settings["index"]["knn.derived_source.enabled"] = is_knn_derived_source_enabled
+
         settings["index"]["knn.algo_param.ef_search"] = ef_search_value
 
         version_specific_settings = self._get_version_specific_settings(self._get_cluster_version(client))

vectordb_bench/frontend/config/dbCaseConfigs.py

@@ -1802,6 +1802,17 @@ class CaseConfigInput(BaseModel):
     },
 )
 
+CaseConfigParamInput_KNN_DERIVED_SOURCE_ENABLED_AWSOpensearch = CaseConfigInput(
+    label=CaseConfigParamType.knn_derived_source_enabled,
+    displayLabel="KNN Derived Source Enabled",
+    inputHelp="OpenSearch 3.x+ set this to 'True' by default. Use 'False' to disable it. Use 'None' for <3.x versions.",
+    inputType=InputType.Option,
+    inputConfig={
+        "options": ["None", "True", "False"],
+        "default": "None",
+    },
+)
+
 MilvusLoadConfig = [
     CaseConfigParamInput_IndexType,
     CaseConfigParamInput_M,
@@ -1883,12 +1894,14 @@ class CaseConfigInput(BaseModel):
     CaseConfigParamInput_EFConstruction_AWSOpensearch,
     CaseConfigParamInput_M_AWSOpensearch,
     CaseConfigParamInput_REPLICATION_TYPE_AWSOpensearch,
+    CaseConfigParamInput_KNN_DERIVED_SOURCE_ENABLED_AWSOpensearch,
 ]
 AWSOpenSearchPerformanceConfig = [
     CaseConfigParamInput_EFConstruction_AWSOpensearch,
     CaseConfigParamInput_M_AWSOpensearch,
     CaseConfigParamInput_EF_SEARCH_AWSOpensearch,
     CaseConfigParamInput_REPLICATION_TYPE_AWSOpensearch,
+    CaseConfigParamInput_KNN_DERIVED_SOURCE_ENABLED_AWSOpensearch,
 ]
 
 AliyunOpensearchLoadingConfig = []
@@ -2215,6 +2228,7 @@ class CaseConfigInput(BaseModel):
     CaseConfigParamInput_EFConstruction_AWSOpensearch,
     CaseConfigParamInput_NUMBER_OF_SHARDS_AWSOpensearch,
     CaseConfigParamInput_NUMBER_OF_REPLICAS_AWSOpensearch,
+    CaseConfigParamInput_KNN_DERIVED_SOURCE_ENABLED_AWSOpensearch,
     CaseConfigParamInput_NUMBER_OF_INDEXING_CLIENTS_AWSOpensearch,
     CaseConfigParamInput_INDEX_THREAD_QTY_AWSOpensearch,
     CaseConfigParamInput_REPLICATION_TYPE_AWSOpensearch,
@@ -2230,6 +2244,7 @@ class CaseConfigInput(BaseModel):
     CaseConfigParamInput_EFConstruction_AWSOpensearch,
     CaseConfigParamInput_NUMBER_OF_SHARDS_AWSOpensearch,
     CaseConfigParamInput_NUMBER_OF_REPLICAS_AWSOpensearch,
+    CaseConfigParamInput_KNN_DERIVED_SOURCE_ENABLED_AWSOpensearch,
     CaseConfigParamInput_NUMBER_OF_INDEXING_CLIENTS_AWSOpensearch,
     CaseConfigParamInput_INDEX_THREAD_QTY_AWSOpensearch,
     CaseConfigParamInput_REPLICATION_TYPE_AWSOpensearch,

vectordb_bench/models.py

@@ -128,6 +128,7 @@ class CaseConfigParamType(Enum):
     use_routing = "use_routing"
     replication_type = "replication_type"
     cache_size = "cache_size"
+    knn_derived_source_enabled = "knn_derived_source_enabled"
 
     dataset_with_size_type = "dataset_with_size_type"
     filter_rate = "filter_rate"