refactor(settings): update embedding model handling and clean up dataset settings form

Author: skeptrunedev

frontends/dashboard/src/components/dataset-settings/GeneralSettings.tsx

@@ -14,8 +14,6 @@ export const GeneralServerSettings = (props: {
 }) => {
   return (
     <form class="flex flex-col gap-3">
-      {/* General LLM Settings */}
-
       {/* Embedding Settings */}
       <div
         class="rounded-md border shadow sm:overflow-hidden"
@@ -35,7 +33,7 @@ export const GeneralServerSettings = (props: {
             <div class="col-span-4 space-y-1 sm:col-span-2">
               <div class="flex items-center">
                 <label
-                  for="embeddingSize"
+                  for="embeddingModel"
                   class="mr-2 block text-sm font-medium leading-6"
                 >
                   Embedding Model
@@ -46,11 +44,11 @@ export const GeneralServerSettings = (props: {
                 />
               </div>
               <select
-                id="embeddingSize"
+                id="embeddingModel"
                 aria-readonly
                 title="Embedding Model is only editable on creation"
                 disabled
-                name="embeddingSize"
+                name="embeddingModel"
                 class="col-span-2 block w-full cursor-not-allowed rounded-md border-[0.5px] border-neutral-300 bg-white px-3 py-1.5 shadow-sm placeholder:text-neutral-400 focus:outline-magenta-500 sm:text-sm sm:leading-6"
                 value={
                   availableEmbeddingModels.find(
@@ -68,7 +66,7 @@ export const GeneralServerSettings = (props: {
             <div class="col-span-4 space-y-1 sm:col-span-2">
               <div class="flex items-center">
                 <label
-                  for="embeddingSize"
+                  for="embeddingQueryPrefix"
                   class="mr-2 block text-sm font-medium leading-6"
                 >
                   Embedding Query Prefix
@@ -98,7 +96,7 @@ export const GeneralServerSettings = (props: {
             <div class="col-span-4 space-y-1 sm:col-span-2">
               <div class="flex items-center">
                 <label
-                  for="embeddingSize"
+                  for="rerankerModel"
                   class="mr-2 block text-sm font-medium leading-6"
                 >
                   Reranker Model
@@ -176,7 +174,7 @@ export const GeneralServerSettings = (props: {
             <div class="col-span-4 space-y-1 sm:col-span-2">
               <div class="flex items-center">
                 <label
-                  for="embeddingSize"
+                  for="distanceMetric"
                   class="mr-2 block text-sm font-medium leading-6"
                 >
                   Distance Metric
@@ -187,11 +185,11 @@ export const GeneralServerSettings = (props: {
                 />
               </div>
               <select
-                id="embeddingSize"
+                id="distanceMetric"
                 aria-readonly
                 title="Embedding Model is only editable on creation"
                 disabled
-                name="embeddingSize"
+                name="distanceMetric"
                 class="col-span-2 block w-full cursor-not-allowed rounded-md border-[0.5px] border-neutral-300 bg-white px-3 py-1.5 shadow-sm placeholder:text-neutral-400 focus:outline-magenta-500 sm:text-sm sm:leading-6"
                 value={
                   availableDistanceMetrics.find(
@@ -398,7 +396,7 @@ export const GeneralServerSettings = (props: {
               />
               <div class="flex items-center">
                 <label
-                  for="embeddingSize"
+                  for="indexedOnly"
                   class="mr-2 block text-sm font-medium leading-6"
                 >
                   Indexed Only
@@ -427,7 +425,7 @@ export const GeneralServerSettings = (props: {
               />
               <div class="flex items-center">
                 <label
-                  for="embeddingSize"
+                  for="qdrantOnly"
                   class="mr-2 block text-sm font-medium leading-6"
                 >
                   Qdrant Only

server/src/data/models.rs

@@ -2808,12 +2808,25 @@ impl From<DatasetConfigurationDTO> for DatasetConfiguration {
             RERANKER_API_KEY: dto.RERANKER_API_KEY.unwrap_or("".to_string()),
             RERANKER_MODEL_NAME: dto.RERANKER_MODEL_NAME.unwrap_or("bge-reranker-large".to_string()),
             EMBEDDING_BASE_URL: dto.EMBEDDING_BASE_URL.unwrap_or("https://embedding.trieve.ai".to_string()),
-            EMBEDDING_MODEL_NAME: dto.EMBEDDING_MODEL_NAME.unwrap_or("jina-base-en".to_string()),
+            EMBEDDING_MODEL_NAME: dto.EMBEDDING_MODEL_NAME.clone().unwrap_or("jina-base-en".to_string()),
             RERANKER_BASE_URL: dto.RERANKER_BASE_URL.unwrap_or("".to_string()),
             MESSAGE_TO_QUERY_PROMPT: dto.MESSAGE_TO_QUERY_PROMPT.unwrap_or("Write a 1-2 sentence semantic search query along the lines of a hypothetical response to: \n\n".to_string()),
             RAG_PROMPT: dto.RAG_PROMPT.unwrap_or("Use the following retrieved documents to respond briefly and accurately:".to_string()),
             N_RETRIEVALS_TO_INCLUDE: dto.N_RETRIEVALS_TO_INCLUDE.unwrap_or(8),
-            EMBEDDING_SIZE: dto.EMBEDDING_SIZE.unwrap_or(768),
+            EMBEDDING_SIZE: if let Some(embedding_size) = dto.EMBEDDING_SIZE {
+                embedding_size
+            } else if let Some(embedding_model_name) = dto.EMBEDDING_MODEL_NAME {
+                match embedding_model_name.as_str() {
+                    "text-embedding-3-small" => 1536,
+                    "text-embedding-3-large" => 3072,
+                    "bge-m3" => 1024,
+                    "jina-embeddings-v2-base-code" => 768,
+                    "jina-base-en" => 768,
+                    _ => 768,
+                }
+            } else {
+                768
+            },
             DISTANCE_METRIC: dto.DISTANCE_METRIC.unwrap_or(DistanceMetric::Cosine),
             LLM_DEFAULT_MODEL: dto.LLM_DEFAULT_MODEL.unwrap_or("gpt-4o".to_string()),
             BM25_ENABLED: dto.BM25_ENABLED.unwrap_or(true),

server/src/operators/dataset_operator.rs

@@ -488,6 +488,7 @@ pub async fn clear_dataset_query(
 
     let chunk_groups = chunk_group::chunk_group
         .filter(chunk_group::dataset_id.eq(id))
+        .filter(chunk_group::created_at.le(deleted_at))
         .select(chunk_group::id)
         .load::<uuid::Uuid>(&mut conn)
         .await
@@ -585,6 +586,10 @@ pub async fn clear_dataset_query(
         delete_points_from_qdrant(qdrant_point_ids, qdrant_collection.clone())
             .await
             .map_err(|err| {
+                log::error!(
+                    "Could not delete points in current batch from qdrant: {}",
+                    err
+                );
                 ServiceError::BadRequest(format!(
                     "Could not delete points in current batch from qdrant: {}",
                     err