feat: add mem exceeded exception

Author: yuanchaoa

agent/src/exception.rs

@@ -49,6 +49,10 @@ impl ExceptionHandler {
         }
     }
 
+    pub fn peek(&self) -> u64 {
+        self.exception.load(Ordering::Relaxed)
+    }
+
     pub fn has(&self, e: Exception) -> bool {
         let e = e as u64;
         self.exception.load(Ordering::Relaxed) & e == e

agent/src/rpc/synchronizer.rs

@@ -2064,7 +2064,14 @@ impl Synchronizer {
                     info!("sync interval set to {:?}", sync_interval);
                 }
 
-                time::sleep(sync_interval).await;
+                let last_exception = exception_handler.peek();
+                let count = sync_interval.as_secs().max(1);
+                for _ in 0..count {
+                    time::sleep(Duration::from_secs(1)).await;
+                    if last_exception != exception_handler.peek() {
+                        break;
+                    }
+                }
             }
         }));
     }

agent/src/utils/guard.rs

@@ -698,6 +698,7 @@ impl Guard {
                         match get_memory_rss() {
                             Ok(memory_usage) => {
                                 if memory_usage >= memory_limit {
+                                    exception_handler.set(Exception::MemExceeded, None);
                                     if over_memory_limit {
                                         error!(
                                     "memory usage over memory limit twice, current={}, memory_limit={}, deepflow-agent restart...",
@@ -712,12 +713,18 @@ impl Guard {
                                     );
                                         over_memory_limit = true;
                                     }
+                                } else {
+                                    over_memory_limit = false;
+                                    exception_handler.clear(Exception::MemExceeded);
                                 }
                             }
                             Err(e) => {
                                 warn!("{}", e);
                             }
                         }
+                    } else {
+                        over_memory_limit = false;
+                        exception_handler.clear(Exception::MemExceeded);
                     }
                 }
 

message/agent.proto

@@ -78,6 +78,7 @@ enum Exception {
     DATA_BPS_THRESHOLD_EXCEEDED = 2097152;
     FREE_DISK_CIRCUIT_BREAKER = 4194304;
     KERNEL_VERSION_CIRCUIT_BREAKER = 8388608;
+    MEM_EXCEEDED = 16777216;
     // 2^31及以下由 agent ，agent 最大可用异常是2^31，顺序从前往后
     // 2^32及以上由控制器使用，顺序从后往前
 }