bugfix(server): enhance error logging and improve handling of chunk metadata in processing functions

Author: skeptrunedev

server/src/operators/chunk_operator.rs

@@ -273,7 +273,8 @@ pub async fn get_content_chunk_from_point_ids_query(
                 .filter(chunk_metadata_columns::qdrant_point_id.eq_any(&point_ids))
                 .load(&mut conn)
                 .await
-                .map_err(|_| {
+                .map_err(|e| {
+                    log::error!("Failed to load content chunk metadatas: {:?}", e);
                     ServiceError::BadRequest("Failed to load content chunk metadatas".to_string())
                 })?;
 

server/src/operators/model_operator.rs

@@ -934,14 +934,15 @@ pub async fn cross_encoder(
     let common_request_docs = results
         .clone()
         .into_iter()
-        .map(|x| {
-            let chunk = match x.metadata.get(0).cloned() {
-                Some(ChunkMetadataTypes::Metadata(metadata)) => Ok(metadata),
+        .map(|score_chunk_dto| {
+            let chunk_html = match score_chunk_dto.metadata.get(0).cloned() {
+                Some(ChunkMetadataTypes::Metadata(chunk_metadata_string_tag_set)) => Ok(chunk_metadata_string_tag_set.chunk_html),
+                Some(ChunkMetadataTypes::Content(content_chunk_metadata)) => Ok(content_chunk_metadata.chunk_html),
                 _ => Err(ServiceError::BadRequest(
-                    "ChunkMetadataStringTagSet not found for chunk in results".to_string(),
+                    "Cannot rerank SlimChunkMetadata ScoreChunks since they do not have any text in their chunk_html".to_string(),
                 )),
             }?;
-            Ok(convert_html_to_text(&chunk.chunk_html.unwrap_or_default())
+            Ok(convert_html_to_text(&chunk_html.unwrap_or_default())
                 .split_whitespace()
                 .take(max_words_to_rerank)
                 .join(" "))