fix

Author: Cifko

atoma-bin/atoma_node.rs

@@ -376,6 +376,8 @@ async fn main() -> Result<()> {
         too_many_requests: Arc::new(DashSet::new()),
         too_many_requests_timeout_ms: u128::from(config.service.too_many_requests_timeout_ms),
         running_num_requests: Arc::new(RequestCounter::new()),
+        lower_memory_threshold: config.service.lower_memory_threshold,
+        upper_memory_threshold: config.service.upper_memory_threshold,
     };
 
     let chat_completions_service_urls = app_state

atoma-service/src/config.rs

@@ -60,6 +60,11 @@ pub struct AtomaServiceConfig {
 
     /// The timeout for the too many requests error in milliseconds.
     pub too_many_requests_timeout_ms: u64,
+
+    ///Lower threshold for memory usage, if the memory usage goes below this value, the service will not be considered overloaded
+    pub lower_memory_threshold: f64,
+    /// Upper threshold for memory usage, if the memory usage goes above this value, the service will be considered overloaded
+    pub upper_memory_threshold: f64,
 }
 
 impl AtomaServiceConfig {