feat: Implement automatic log rotation

Author: michaelmior

ROADMAP.md

@@ -21,7 +21,7 @@ This stage will implement logging of all operations in ArgusDB to allow for faul
 - [x] Define a log file format in specs/logging.md
 - [x] Add log entries whenever new documents are inserted
 - [x] Implement log recovery to read the log on restart and recover from a crash
-- [] Implement log rotation and deletion of old log files
+- [x] Implement log rotation and deletion of old log files
 
 # Stage 4 - Disk storage
 

src/log.rs

@@ -1,9 +1,9 @@
 use chrono::{DateTime, Utc};
 use serde::{Deserialize, Serialize};
 use serde_json::Value;
-use std::fs::OpenOptions;
+use std::fs::{self, OpenOptions};
 use std::io::Write;
-use std::path::Path;
+use std::path::{Path, PathBuf};
 
 #[derive(Serialize, Deserialize, Debug)]
 pub enum Operation {
@@ -28,24 +28,72 @@ pub struct LogEntry {
 
 pub struct Logger {
     file: std::fs::File,
+    path: PathBuf,
+    rotation_threshold: u64,
 }
 
 impl Logger {
-    pub fn new<P: AsRef<Path>>(path: P) -> std::io::Result<Self> {
+    pub fn new<P: AsRef<Path>>(path: P, rotation_threshold: u64) -> std::io::Result<Self> {
+        let path = path.as_ref().to_path_buf();
         let file = OpenOptions::new()
             .create(true)
             .write(true)
             .append(true)
-            .open(path)?;
-        Ok(Logger { file })
+            .open(&path)?;
+        Ok(Logger {
+            file,
+            path,
+            rotation_threshold,
+        })
     }
 
     pub fn log(&mut self, op: Operation) -> std::io::Result<()> {
+        if self.file.metadata()?.len() > self.rotation_threshold {
+            self.rotate()?;
+        }
         let entry = LogEntry {
             ts: Utc::now(),
             op,
         };
         let json = serde_json::to_string(&entry)?;
         writeln!(self.file, "{}", json)
     }
+
+    pub fn rotate(&mut self) -> std::io::Result<()> {
+        let new_path = self.path.with_extension("log.1");
+        fs::rename(&self.path, new_path)?;
+        self.file = OpenOptions::new()
+            .create(true)
+            .write(true)
+            .append(true)
+            .open(&self.path)?;
+        Ok(())
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use serde_json::json;
+    use tempfile::NamedTempFile;
+
+    #[test]
+    fn test_log_rotate() {
+        let log_file = NamedTempFile::new().unwrap();
+        let mut logger = Logger::new(log_file.path(), 1024 * 1024).unwrap();
+        let op = Operation::Insert {
+            id: "test-id".to_string(),
+            doc: json!({"a": 1}),
+        };
+        logger.log(op).unwrap();
+
+        logger.rotate().unwrap();
+
+        let log_content = std::fs::read_to_string(log_file.path()).unwrap();
+        assert!(log_content.is_empty());
+
+        let rotated_log_path = log_file.path().with_extension("log.1");
+        let rotated_log_content = std::fs::read_to_string(rotated_log_path).unwrap();
+        assert!(!rotated_log_content.is_empty());
+    }
 }

src/storage.rs

@@ -11,8 +11,8 @@ pub struct MemTable {
 }
 
 impl MemTable {
-    pub fn new(log_path: &str) -> Self {
-        let logger = Logger::new(log_path).unwrap();
+    pub fn new(log_path: &str, rotation_threshold: u64) -> Self {
+        let logger = Logger::new(log_path, rotation_threshold).unwrap();
         let mut memtable = MemTable {
             documents: BTreeMap::new(),
             schema: Schema {
@@ -171,7 +171,7 @@ mod tests {
 
     fn create_test_memtable() -> (NamedTempFile, MemTable) {
         let log_file = NamedTempFile::new().unwrap();
-        let memtable = MemTable::new(log_file.path().to_str().unwrap());
+        let memtable = MemTable::new(log_file.path().to_str().unwrap(), 1024 * 1024);
         (log_file, memtable)
     }
 
@@ -266,8 +266,31 @@ mod tests {
 
         memtable.delete(&id1);
 
-        let memtable2 = MemTable::new(log_file.path().to_str().unwrap());
+        let memtable2 = MemTable::new(log_file.path().to_str().unwrap(), 1024 * 1024);
         assert_eq!(memtable2.documents.len(), 1);
         assert_eq!(*memtable2.documents.get(&id2).unwrap(), doc2);
     }
+
+    #[test]
+    fn test_automatic_log_rotation() {
+        let log_file = NamedTempFile::new().unwrap();
+        let mut memtable = MemTable::new(log_file.path().to_str().unwrap(), 100);
+        let doc1 = json!({"a": 1});
+        memtable.insert(doc1);
+
+        let log_content = std::fs::read_to_string(log_file.path()).unwrap();
+        assert!(!log_content.is_empty());
+
+        let doc2 = json!({"b": "a long string to make the log entry bigger than the threshold"});
+        memtable.insert(doc2);
+
+        let log_content_after_rotation = std::fs::read_to_string(log_file.path()).unwrap();
+        let rotated_log_path = log_file.path().with_extension("log.1");
+        let rotated_log_content = std::fs::read_to_string(rotated_log_path).unwrap();
+
+        assert!(!rotated_log_content.is_empty());
+        assert!(rotated_log_content.contains("{\"a\":1}"));
+        assert!(!log_content_after_rotation.contains("{\"a\":1}"));
+        assert!(log_content_after_rotation.contains("a long string"));
+    }
 }