feat: offload document processing to an Arbiter for improved async handling

Author: skeptrunedev

server/src/handlers/file_handler.rs

@@ -18,6 +18,7 @@ use crate::{
         organization_operator::{get_file_size_sum_org, hash_function},
     },
 };
+use actix::Arbiter;
 use actix_web::{web, HttpResponse};
 use base64::{
     alphabet,
@@ -720,14 +721,16 @@ pub async fn upload_html_page(
         return Err(ServiceError::BadRequest("Webhook secret does not match.".to_string()).into());
     }
 
-    process_crawl_doc(
-        dataset_id,
-        req_payload.scrape_id,
-        req_payload.data,
-        broccoli_queue,
-        pool,
-    )
-    .await?;
+    Arbiter::new().spawn(async move {
+        let _ = process_crawl_doc(
+            dataset_id,
+            req_payload.scrape_id,
+            req_payload.data,
+            broccoli_queue,
+            pool,
+        )
+        .await;
+    });
 
     Ok(HttpResponse::NoContent().finish())
 }