feature: add agentic search

Author: densumesh

clients/ts-sdk/openapi.json

@@ -11670,11 +11670,6 @@
             "description": "No result message for when there are no chunks found above the score threshold.",
             "nullable": true
           },
-          "only_include_docs_used": {
-            "type": "boolean",
-            "description": "Only include docs used in the completion. If not specified, this defaults to false.",
-            "nullable": true
-          },
           "page_size": {
             "type": "integer",
             "format": "int64",
@@ -11732,6 +11727,11 @@
             ],
             "nullable": true
           },
+          "use_agentic_search": {
+            "type": "boolean",
+            "description": "If true, the search will be conducted using llm tool calling. If not specified, this defaults to false.",
+            "nullable": true
+          },
           "use_group_search": {
             "type": "boolean",
             "description": "If use_group_search is set to true, the search will be conducted using the `search_over_groups` api. If not specified, this defaults to false.",
@@ -12766,11 +12766,6 @@
             "description": "No result message for when there are no chunks found above the score threshold.",
             "nullable": true
           },
-          "only_include_docs_used": {
-            "type": "boolean",
-            "description": "Only include docs used in the completion. If not specified, this defaults to false.",
-            "nullable": true
-          },
           "page_size": {
             "type": "integer",
             "format": "int64",
@@ -12828,6 +12823,11 @@
             ],
             "nullable": true
           },
+          "use_agentic_search": {
+            "type": "boolean",
+            "description": "If true, the search will be conducted using llm tool calling. If not specified, this defaults to false.",
+            "nullable": true
+          },
           "use_group_search": {
             "type": "boolean",
             "nullable": true
@@ -19323,11 +19323,6 @@
             "description": "No result message for when there are no chunks found above the score threshold.",
             "nullable": true
           },
-          "only_include_docs_used": {
-            "type": "boolean",
-            "description": "Only include docs used in the completion. If not specified, this defaults to false.",
-            "nullable": true
-          },
           "page_size": {
             "type": "integer",
             "format": "int64",
@@ -19385,6 +19380,11 @@
             ],
             "nullable": true
           },
+          "use_agentic_search": {
+            "type": "boolean",
+            "description": "If true, the search will be conducted using llm tool calling. If not specified, this defaults to false.",
+            "nullable": true
+          },
           "use_group_search": {
             "type": "boolean",
             "description": "If use_group_search is set to true, the search will be conducted using the `search_over_groups` api. If not specified, this defaults to false.",

clients/ts-sdk/src/types.gen.ts

@@ -1086,10 +1086,6 @@ export type CreateMessageReqPayload = {
      * No result message for when there are no chunks found above the score threshold.
      */
     no_result_message?: (string) | null;
-    /**
-     * Only include docs used in the completion. If not specified, this defaults to false.
-     */
-    only_include_docs_used?: (boolean) | null;
     /**
      * Page size is the number of chunks to fetch during RAG. If 0, then no search will be performed. If specified, this will override the N retrievals to include in the dataset configuration. Default is None.
      */
@@ -1117,6 +1113,10 @@ export type CreateMessageReqPayload = {
      */
     topic_id: string;
     typo_options?: ((TypoOptions) | null);
+    /**
+     * If true, the search will be conducted using llm tool calling. If not specified, this defaults to false.
+     */
+    use_agentic_search?: (boolean) | null;
     /**
      * If use_group_search is set to true, the search will be conducted using the `search_over_groups` api. If not specified, this defaults to false.
      */
@@ -1594,10 +1594,6 @@ export type EditMessageReqPayload = {
      * No result message for when there are no chunks found above the score threshold.
      */
     no_result_message?: (string) | null;
-    /**
-     * Only include docs used in the completion. If not specified, this defaults to false.
-     */
-    only_include_docs_used?: (boolean) | null;
     /**
      * Page size is the number of chunks to fetch during RAG. If 0, then no search will be performed. If specified, this will override the N retrievals to include in the dataset configuration. Default is None.
      */
@@ -1625,6 +1621,10 @@ export type EditMessageReqPayload = {
      */
     topic_id: string;
     typo_options?: ((TypoOptions) | null);
+    /**
+     * If true, the search will be conducted using llm tool calling. If not specified, this defaults to false.
+     */
+    use_agentic_search?: (boolean) | null;
     use_group_search?: (boolean) | null;
     /**
      * If true, quoted and - prefixed words will be parsed from the queries and used as required and negated words respectively. Default is false.
@@ -3695,10 +3695,6 @@ export type RegenerateMessageReqPayload = {
      * No result message for when there are no chunks found above the score threshold.
      */
     no_result_message?: (string) | null;
-    /**
-     * Only include docs used in the completion. If not specified, this defaults to false.
-     */
-    only_include_docs_used?: (boolean) | null;
     /**
      * Page size is the number of chunks to fetch during RAG. If 0, then no search will be performed. If specified, this will override the N retrievals to include in the dataset configuration. Default is None.
      */
@@ -3726,6 +3722,10 @@ export type RegenerateMessageReqPayload = {
      */
     topic_id: string;
     typo_options?: ((TypoOptions) | null);
+    /**
+     * If true, the search will be conducted using llm tool calling. If not specified, this defaults to false.
+     */
+    use_agentic_search?: (boolean) | null;
     /**
      * If use_group_search is set to true, the search will be conducted using the `search_over_groups` api. If not specified, this defaults to false.
      */

server/src/bin/csv-jsonl-worker.rs

@@ -228,7 +228,7 @@ async fn file_worker(
             Err(err) => {
                 log::error!(
                     "File {} has not yet been uploaded to the signed put url: {:?}",
-                    csv_jsonl_worker_message.file_id.to_string(),
+                    csv_jsonl_worker_message.file_id,
                     err.to_string(),
                 );
 
@@ -619,10 +619,7 @@ fn convert_value_to_chunkreqpayload(
         }
         _ => value.clone(),
     };
-    let chunk_html = match serde_json::to_string(&cleaned_value) {
-        Ok(chunk_html) => Some(chunk_html),
-        Err(_) => None,
-    };
+    let chunk_html = serde_json::to_string(&cleaned_value).ok();
     let mut chunk_req_payload = ChunkReqPayload {
         chunk_html,
         semantic_content: None,

server/src/bin/ingestion-worker.rs

@@ -648,9 +648,7 @@ pub async fn bulk_upload_chunks(
     } else {
         let content_size = fulltext_content_and_boosts.len();
 
-        Ok(std::iter::repeat(vec![(0, 0.0)])
-            .take(content_size)
-            .collect())
+        Ok(std::iter::repeat_n(vec![(0, 0.0)], content_size).collect())
     }?;
 
     let bm25_vectors = if dataset_config.BM25_ENABLED

server/src/data/models.rs

@@ -3667,7 +3667,6 @@ impl DatasetConfigurationDTO {
     "DOCUMENT_UPLOAD_FEATURE": true,
     "FILE_NAME_KEY": "file_name_key",
 }))]
-
 pub struct DatasetAndOrgWithSubAndPlan {
     pub dataset: Dataset,
     pub organization: OrganizationWithSubAndPlan,
@@ -4541,13 +4540,13 @@ impl ApiKeyRequestParams {
             currency: payload.currency,
             context_options: payload.context_options,
             no_result_message: self.no_result_message.or(payload.no_result_message),
-            only_include_docs_used: payload.only_include_docs_used,
             use_quote_negated_terms: self
                 .use_quote_negated_terms
                 .or(payload.use_quote_negated_terms),
             remove_stop_words: self.remove_stop_words.or(payload.remove_stop_words),
             typo_options: self.typo_options.or(payload.typo_options),
             metadata: payload.metadata,
+            use_agentic_search: payload.use_agentic_search,
         }
     }
 
@@ -5149,7 +5148,7 @@ pub enum RangeCondition {
     Int(i64),
 }
 
-#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)]
+#[derive(Serialize, Deserialize, Debug, Clone, ToSchema, Default)]
 #[schema(example = json!({
     "gte": 0.0,
     "lte": 1.0,
@@ -5259,7 +5258,7 @@ pub struct HasChunkIDCondition {
     pub tracking_ids: Option<Vec<String>>,
 }
 
-#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)]
+#[derive(Serialize, Deserialize, Debug, Clone, ToSchema, Default)]
 #[schema(example = json!({
     "field": "metadata.key1",
     "match": ["value1", "value2"],
@@ -8263,7 +8262,7 @@ pub enum SearchAnalyticsResponse {
     #[schema(title = "CountQueries")]
     CountQueries(QueryCountResponse),
     #[schema(title = "QueryDetails")]
-    QueryDetails(SearchQueryEvent),
+    QueryDetails(Box<SearchQueryEvent>),
     #[schema(title = "PopularFilters")]
     PopularFilters(PopularFiltersResponse),
     #[schema(title = "CTRMetricsOverTime")]
@@ -8297,7 +8296,7 @@ pub enum RecommendationAnalyticsResponse {
     #[schema(title = "RecommendationQueries")]
     RecommendationQueries(RecommendationsEventResponse),
     #[schema(title = "QueryDetails")]
-    QueryDetails(RecommendationEvent),
+    QueryDetails(Box<RecommendationEvent>),
     #[schema(title = "RecommendationUsageGraph")]
     RecommendationUsageGraph(RecommendationUsageGraphResponse),
     #[schema(title = "RecommendationsPerUser")]
@@ -9424,10 +9423,10 @@ impl<'de> Deserialize<'de> for CreateMessageReqPayload {
             pub use_group_search: Option<bool>,
             pub context_options: Option<ContextOptions>,
             pub no_result_message: Option<String>,
-            pub only_include_docs_used: Option<bool>,
             pub currency: Option<String>,
             metadata: Option<serde_json::Value>,
             pub rag_context: Option<String>,
+            pub use_agentic_search: Option<bool>,
             #[serde(flatten)]
             other: std::collections::HashMap<String, serde_json::Value>,
             use_quote_negated_terms: Option<bool>,
@@ -9469,10 +9468,10 @@ impl<'de> Deserialize<'de> for CreateMessageReqPayload {
             no_result_message: helper.no_result_message,
             metadata: helper.metadata,
             rag_context: helper.rag_context,
-            only_include_docs_used: helper.only_include_docs_used,
             use_quote_negated_terms: helper.use_quote_negated_terms,
             remove_stop_words: helper.remove_stop_words,
             typo_options: helper.typo_options,
+            use_agentic_search: helper.use_agentic_search,
         })
     }
 }
@@ -9498,7 +9497,6 @@ impl<'de> Deserialize<'de> for RegenerateMessageReqPayload {
             pub use_group_search: Option<bool>,
             pub context_options: Option<ContextOptions>,
             pub no_result_message: Option<String>,
-            pub only_include_docs_used: Option<bool>,
             pub currency: Option<String>,
             metadata: Option<serde_json::Value>,
             #[serde(flatten)]
@@ -9507,6 +9505,7 @@ impl<'de> Deserialize<'de> for RegenerateMessageReqPayload {
             pub remove_stop_words: Option<bool>,
             pub typo_options: Option<TypoOptions>,
             pub rag_context: Option<String>,
+            pub use_agentic_search: Option<bool>,
         }
 
         let mut helper = Helper::deserialize(deserializer)?;
@@ -9539,11 +9538,11 @@ impl<'de> Deserialize<'de> for RegenerateMessageReqPayload {
             context_options,
             metadata: helper.metadata,
             no_result_message: helper.no_result_message,
-            only_include_docs_used: helper.only_include_docs_used,
             use_quote_negated_terms: helper.use_quote_negated_terms,
             remove_stop_words: helper.remove_stop_words,
             typo_options: helper.typo_options,
             rag_context: helper.rag_context,
+            use_agentic_search: helper.use_agentic_search,
         })
     }
 }
@@ -9573,7 +9572,6 @@ impl<'de> Deserialize<'de> for EditMessageReqPayload {
             pub user_id: Option<String>,
             pub context_options: Option<ContextOptions>,
             pub no_result_message: Option<String>,
-            pub only_include_docs_used: Option<bool>,
             pub currency: Option<String>,
             metadata: Option<serde_json::Value>,
             #[serde(flatten)]
@@ -9582,6 +9580,7 @@ impl<'de> Deserialize<'de> for EditMessageReqPayload {
             pub remove_stop_words: Option<bool>,
             pub typo_options: Option<TypoOptions>,
             pub rag_context: Option<String>,
+            pub use_agentic_search: Option<bool>,
         }
 
         let mut helper = Helper::deserialize(deserializer)?;
@@ -9618,11 +9617,11 @@ impl<'de> Deserialize<'de> for EditMessageReqPayload {
             context_options,
             metadata: helper.metadata,
             no_result_message: helper.no_result_message,
-            only_include_docs_used: helper.only_include_docs_used,
             use_quote_negated_terms: helper.use_quote_negated_terms,
             remove_stop_words: helper.remove_stop_words,
             typo_options: helper.typo_options,
             rag_context: helper.rag_context,
+            use_agentic_search: helper.use_agentic_search,
         })
     }
 }

server/src/errors.rs

@@ -126,6 +126,12 @@ impl From<DBError> for ServiceError {
     }
 }
 
+impl From<actix_web::Error> for ServiceError {
+    fn from(error: actix_web::Error) -> Self {
+        ServiceError::InternalServerError(error.to_string())
+    }
+}
+
 pub fn custom_json_error_handler(
     err: JsonPayloadError,
     _req: &actix_web::HttpRequest,

server/src/handlers/analytics_handler.rs

@@ -283,7 +283,7 @@ pub async fn get_search_analytics(
             )
             .await?;
 
-            SearchAnalyticsResponse::QueryDetails(query)
+            SearchAnalyticsResponse::QueryDetails(Box::new(query))
         }
         SearchAnalytics::CountQueries {
             filter,
@@ -745,7 +745,7 @@ pub async fn get_recommendation_analytics(
                 clickhouse_client.get_ref(),
             )
             .await?;
-            RecommendationAnalyticsResponse::QueryDetails(recommendation_query)
+            RecommendationAnalyticsResponse::QueryDetails(Box::new(recommendation_query))
         }
         RecommendationAnalytics::RecommendationUsageGraph {
             filter,