api: implement API for dirty memory

Implement API /memory/dirty which returns a bitmap tracking dirty guest
memory. The bitmap is structured as a vector of u64, so its length is:
total_number_of_pages.div_ceil(64).

Pages are ordered in the order of pages as reported by /memory/mappings.

Signed-off-by: Babis Chalios <babis.chalios@e2b.dev>

Author: bchalios

resources/seccomp/x86_64-unknown-linux-musl.json

@@ -216,6 +216,10 @@
                 "syscall": "mincore",
                 "comment": "Used by get_memory_dirty_bitmap to check if memory pages are resident"
             },
+            {
+                "syscall": "pread64",
+                "comment": "Used by get_dirty_memory to read /proc/self/pagemap entries"
+            },
             {
                 "syscall": "mmap",
                 "comment": "Used by the VirtIO balloon device",

src/firecracker/src/api_server/parsed_request.rs

@@ -21,7 +21,7 @@ use super::request::logger::parse_put_logger;
 use super::request::machine_configuration::{
     parse_get_machine_config, parse_patch_machine_config, parse_put_machine_config,
 };
-use super::request::memory::{parse_get_memory, parse_get_memory_mappings};
+use super::request::memory::{parse_get_memory, parse_get_memory_dirty, parse_get_memory_mappings};
 use super::request::metrics::parse_put_metrics;
 use super::request::mmds::{parse_get_mmds, parse_patch_mmds, parse_put_mmds};
 use super::request::net::{parse_patch_net, parse_put_net};
@@ -85,6 +85,7 @@ impl TryFrom<&Request> for ParsedRequest {
             (Method::Get, "machine-config", None) => parse_get_machine_config(),
             (Method::Get, "memory", None) => match path_tokens.next() {
                 Some("mappings") => parse_get_memory_mappings(),
+                Some("dirty") => parse_get_memory_dirty(),
                 None => parse_get_memory(),
                 _ => Err(RequestError::InvalidPathMethod(
                     request_uri.to_string(),
@@ -183,6 +184,7 @@ impl ParsedRequest {
                 VmmData::InstanceInformation(info) => Self::success_response_with_data(info),
                 VmmData::MemoryMappings(mappings) => Self::success_response_with_data(mappings),
                 VmmData::Memory(memory) => Self::success_response_with_data(memory),
+                VmmData::MemoryDirty(dirty) => Self::success_response_with_data(dirty),
                 VmmData::VmmVersion(version) => Self::success_response_with_data(
                     &serde_json::json!({ "firecracker_version": version.as_str() }),
                 ),
@@ -585,6 +587,9 @@ pub mod tests {
                 VmmData::Memory(memory) => {
                     http_response(&serde_json::to_string(memory).unwrap(), 200)
                 }
+                VmmData::MemoryDirty(dirty) => {
+                    http_response(&serde_json::to_string(dirty).unwrap(), 200)
+                }
                 VmmData::VmmVersion(version) => http_response(
                     &serde_json::json!({ "firecracker_version": version.as_str() }).to_string(),
                     200,
@@ -615,6 +620,9 @@ pub mod tests {
                 empty: vec![],
             },
         ));
+        verify_ok_response_with(VmmData::MemoryDirty(
+            vmm::vmm_config::instance_info::MemoryDirty { bitmap: vec![] },
+        ));
         verify_ok_response_with(VmmData::VmmVersion(String::default()));
 
         // Error.
@@ -712,6 +720,18 @@ pub mod tests {
         ParsedRequest::try_from(&req).unwrap();
     }
 
+    #[test]
+    fn test_try_from_get_memory_dirty() {
+        let (mut sender, receiver) = UnixStream::pair().unwrap();
+        let mut connection = HttpConnection::new(receiver);
+        sender
+            .write_all(http_request("GET", "/memory/dirty", None).as_bytes())
+            .unwrap();
+        connection.try_read().unwrap();
+        let req = connection.pop_parsed_request().unwrap();
+        ParsedRequest::try_from(&req).unwrap();
+    }
+
     #[test]
     fn test_try_from_get_version() {
         let (mut sender, receiver) = UnixStream::pair().unwrap();

src/firecracker/src/api_server/request/memory.rs

@@ -16,6 +16,11 @@ pub(crate) fn parse_get_memory() -> Result<ParsedRequest, RequestError> {
     Ok(ParsedRequest::new_sync(VmmAction::GetMemory))
 }
 
+pub(crate) fn parse_get_memory_dirty() -> Result<ParsedRequest, RequestError> {
+    METRICS.get_api_requests.instance_info_count.inc();
+    Ok(ParsedRequest::new_sync(VmmAction::GetMemoryDirty))
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -36,4 +41,12 @@ mod tests {
             _ => panic!("Test failed."),
         }
     }
+
+    #[test]
+    fn test_parse_get_memory_dirty_request() {
+        match parse_get_memory_dirty().unwrap().into_parts() {
+            (RequestAction::Sync(action), _) if *action == VmmAction::GetMemoryDirty => {}
+            _ => panic!("Test failed."),
+        }
+    }
 }

src/vmm/src/lib.rs

@@ -249,6 +249,8 @@ pub enum VmmError {
     VcpuMessage,
     /// Cannot spawn Vcpu thread: {0}
     VcpuSpawn(io::Error),
+    /// Pagemap error: {0}
+    Pagemap(#[from] utils::pagemap::PagemapError),
     /// Vm error: {0}
     Vm(#[from] vstate::vm::VmError),
     /// Kvm error: {0}
@@ -758,6 +760,44 @@ impl Vmm {
         }
     }
 
+    /// Get dirty pages bitmap for guest memory.
+    ///
+    /// Returns a bitmap (Vec<u64>) where each bit corresponds to a guest page of `page_size`.
+    /// A set bit indicates the page is present in RAM and has been written to (not write-protected).
+    /// Pages are ordered in the same order as reported by `/memory/mappings`.
+    pub fn get_dirty_memory(&self, page_size: usize) -> Result<Vec<u64>, VmmError> {
+        let pagemap = utils::pagemap::PagemapReader::new(page_size)?;
+        let mut dirty_bitmap = vec![];
+
+        for region in self.vm.guest_memory().iter() {
+            let base_addr = region.as_ptr() as usize;
+            let len = region.size() as usize;
+            let nr_pages = len / page_size;
+
+            // Use mincore to get resident pages bitmap at guest page size granularity
+            let resident_bitmap =
+                vstate::vm::mincore_bitmap(base_addr as *mut u8, len, page_size)?;
+
+            // Build dirty bitmap: check pagemap only for pages that mincore reports resident.
+            // This reduces the number of /proc/self/pagemap reads.
+            let mut slot_bitmap = vec![0u64; nr_pages.div_ceil(64)];
+            for page_idx in 0..nr_pages {
+                let is_resident =
+                    (resident_bitmap[page_idx / 64] & (1u64 << (page_idx % 64))) != 0;
+                if is_resident {
+                    let virt_addr = base_addr + (page_idx * page_size);
+                    if pagemap.is_page_dirty(virt_addr)? {
+                        slot_bitmap[page_idx / 64] |= 1u64 << (page_idx % 64);
+                    }
+                }
+            }
+
+            dirty_bitmap.extend_from_slice(&slot_bitmap);
+        }
+
+        Ok(dirty_bitmap)
+    }
+
     /// Signals Vmm to stop and exit.
     pub fn stop(&mut self, exit_code: FcExitCode) {
         // To avoid cycles, all teardown paths take the following route:

src/vmm/src/rpc_interface.rs

@@ -26,7 +26,9 @@ use crate::vmm_config::balloon::{
 use crate::vmm_config::boot_source::{BootSourceConfig, BootSourceConfigError};
 use crate::vmm_config::drive::{BlockDeviceConfig, BlockDeviceUpdateConfig, DriveError};
 use crate::vmm_config::entropy::{EntropyDeviceConfig, EntropyDeviceError};
-use crate::vmm_config::instance_info::{InstanceInfo, MemoryMappingsResponse, MemoryResponse};
+use crate::vmm_config::instance_info::{
+    InstanceInfo, MemoryDirty, MemoryMappingsResponse, MemoryResponse, VmState,
+};
 use crate::vmm_config::machine_config::{MachineConfig, MachineConfigError, MachineConfigUpdate};
 use crate::vmm_config::metrics::{MetricsConfig, MetricsConfigError};
 use crate::vmm_config::mmds::{MmdsConfig, MmdsConfigError};
@@ -69,6 +71,8 @@ pub enum VmmAction {
     GetMemoryMappings,
     /// Get memory info (resident and empty pages).
     GetMemory,
+    /// Get guest memory dirty pages information.
+    GetMemoryDirty,
     /// Get microVM version.
     GetVmmVersion,
     /// Flush the metrics. This action can only be called after the logger has been configured.
@@ -168,6 +172,8 @@ pub enum VmmActionError {
     OperationNotSupportedPostBoot,
     /// The requested operation is not supported before starting the microVM.
     OperationNotSupportedPreBoot,
+    /// The requested operation is not supported while the microVM is running.
+    OperationNotSupportedWhileRunning,
     /// Start microvm error: {0}
     StartMicrovm(#[from] StartMicrovmError),
     /// Vsock config error: {0}
@@ -197,6 +203,8 @@ pub enum VmmData {
     MemoryMappings(MemoryMappingsResponse),
     /// Memory info (resident and empty pages).
     Memory(MemoryResponse),
+    /// The guest memory dirty pages information.
+    MemoryDirty(MemoryDirty),
     /// The microVM version.
     VmmVersion(String),
 }
@@ -427,7 +435,9 @@ impl<'a> PrebootApiController<'a> {
                 self.vm_resources.machine_config.clone(),
             )),
             GetVmInstanceInfo => Ok(VmmData::InstanceInformation(self.instance_info.clone())),
-            GetMemoryMappings | GetMemory => Err(VmmActionError::OperationNotSupportedPreBoot),
+            GetMemoryMappings | GetMemory | GetMemoryDirty => {
+                Err(VmmActionError::OperationNotSupportedPreBoot)
+            }
             GetVmmVersion => Ok(VmmData::VmmVersion(self.instance_info.vmm_version.clone())),
             InsertBlockDevice(config) => self.insert_block_device(config),
             InsertNetworkDevice(config) => self.insert_net_device(config),
@@ -680,6 +690,7 @@ impl RuntimeApiController {
                     empty: empty_bitmap,
                 }))
             }
+            GetMemoryDirty => self.get_dirty_memory_info(),
             GetVmmVersion => Ok(VmmData::VmmVersion(
                 self.vmm.lock().expect("Poisoned lock").version(),
             )),
@@ -825,6 +836,27 @@ impl RuntimeApiController {
         Ok(VmmData::Empty)
     }
 
+    /// Get dirty pages information for guest memory.
+    fn get_dirty_memory_info(&self) -> Result<VmmData, VmmActionError> {
+        let start_us = get_time_us(ClockType::Monotonic);
+        let vmm = self.vmm.lock().expect("Poisoned lock");
+
+        // Dirty memory can only be queried while the VM is paused
+        if vmm.instance_info.state != VmState::Paused {
+            return Err(VmmActionError::OperationNotSupportedWhileRunning);
+        }
+
+        let page_size = self.vm_resources.machine_config.huge_pages.page_size();
+        let bitmap = vmm
+            .get_dirty_memory(page_size)
+            .map_err(VmmActionError::InternalVmm)?;
+
+        let elapsed_time_us = get_time_us(ClockType::Monotonic) - start_us;
+        info!("'get dirty memory' VMM action took {elapsed_time_us} us.");
+
+        Ok(VmmData::MemoryDirty(MemoryDirty { bitmap }))
+    }
+
     /// Updates block device properties:
     ///  - path of the host file backing the emulated block device, update the disk image on the
     ///    device and its virtio configuration

src/vmm/src/utils/mod.rs

@@ -9,6 +9,8 @@ pub mod net;
 pub mod signal;
 /// Module with state machine
 pub mod sm;
+/// Module with pagemap utilities
+pub mod pagemap;
 
 use std::num::Wrapping;
 use std::result::Result;

src/vmm/src/utils/pagemap.rs

@@ -0,0 +1,115 @@
+//! Utilities for reading /proc/self/pagemap to track dirty pages.
+
+#![allow(clippy::cast_possible_wrap)]
+
+use std::fs::File;
+use std::os::unix::io::AsRawFd;
+
+use crate::arch::host_page_size;
+
+const PAGEMAP_ENTRY_SIZE: usize = 8;
+
+/// Errors related to pagemap operations
+#[derive(Debug, thiserror::Error, displaydoc::Display)]
+pub enum PagemapError {
+    /// Failed to open /proc/self/pagemap: {0}
+    OpenPagemap(#[source] std::io::Error),
+    /// Failed to read pagemap entry: {0}
+    ReadEntry(#[source] std::io::Error),
+    /// Failed to open /proc/self/clear_refs: {0}
+    OpenClearRefs(#[source] std::io::Error),
+    /// Failed to clear soft-dirty bits: {0}
+    ClearSoftDirty(#[source] std::io::Error),
+}
+
+/// Represents a single entry in /proc/pid/pagemap.
+///
+/// Each virtual page has an 8-byte entry with the following layout:
+/// - Bits 0-54:  Page frame number (PFN) if present
+/// - Bit 55:     Page is soft-dirty (written to since last clear)
+/// - Bit 56:     Page is exclusively mapped
+/// - Bit 57:     Page is write-protected via userfaultfd
+/// - Bit 58:     Unused
+/// - Bit 59-60:  Unused
+/// - Bit 61:     Page is file-page or shared-anon
+/// - Bit 62:     Page is swapped
+/// - Bit 63:     Page is present in RAM
+#[derive(Debug, Clone, Copy)]
+pub struct PagemapEntry {
+    raw: u64,
+}
+
+impl PagemapEntry {
+    /// Create a PagemapEntry from bytes (little-endian)
+    pub fn from_bytes(bytes: [u8; 8]) -> Self {
+        Self {
+            raw: u64::from_ne_bytes(bytes),
+        }
+    }
+
+    /// Check if page is write-protected via userfaultfd
+    pub fn is_write_protected(&self) -> bool {
+        (self.raw & (1u64 << 57)) != 0
+    }
+
+    /// Check if page is present in RAM (bit 63)
+    pub fn is_present(&self) -> bool {
+        (self.raw & (1u64 << 63)) != 0
+    }
+}
+
+/// Reader for /proc/self/pagemap
+#[derive(Debug)]
+pub struct PagemapReader {
+    pagemap_fd: File,
+}
+
+impl PagemapReader {
+    /// Create a new PagemapReader
+    pub fn new(_page_size: usize) -> Result<Self, PagemapError> {
+        let pagemap_fd = File::open("/proc/self/pagemap").map_err(PagemapError::OpenPagemap)?;
+
+        Ok(Self { pagemap_fd })
+    }
+
+    /// Check if a single page is dirty (write-protected bit cleared).
+    ///
+    /// Checks the first host page (4K) of the guest page at the given address.
+    /// For huge pages, all host pages within the huge page typically have the same
+    /// dirty status, so sampling the first is sufficient.
+    ///
+    /// # Arguments
+    /// * `virt_addr` - Virtual address of the page to check
+    ///
+    /// # Returns
+    /// True if the page is present and write-protected bit is cleared (dirty).
+    pub fn is_page_dirty(&self, virt_addr: usize) -> Result<bool, PagemapError> {
+        // Pagemap always uses host (4K) page size
+        let host_page_size = host_page_size();
+
+        // Calculate offset for this virtual page (using host page size)
+        let host_vpn = virt_addr / host_page_size;
+        let offset = (host_vpn * PAGEMAP_ENTRY_SIZE) as i64;
+
+        let mut entry_bytes = [0u8; 8];
+
+        // SAFETY: pread is safe as long as the fd is valid and the buffer is properly sized
+        let ret = unsafe {
+            libc::pread(
+                self.pagemap_fd.as_raw_fd(),
+                entry_bytes.as_mut_ptr().cast(),
+                PAGEMAP_ENTRY_SIZE,
+                offset,
+            )
+        };
+
+        if ret != PAGEMAP_ENTRY_SIZE as isize {
+            return Err(PagemapError::ReadEntry(std::io::Error::last_os_error()));
+        }
+
+        let entry = PagemapEntry::from_bytes(entry_bytes);
+
+        // Page must be present and the write_protected bit cleared (indicating it was written to)
+        Ok(entry.is_present() && !entry.is_write_protected())
+    }
+}

src/vmm/src/vmm_config/instance_info.rs

@@ -67,3 +67,11 @@ pub struct MemoryResponse {
     /// This is a subset of the resident pages.
     pub empty: Vec<u64>,
 }
+
+/// Information about dirty guest memory pages
+#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize)]
+pub struct MemoryDirty {
+    /// Bitmap for dirty pages. The bitmap is encoded as a vector of u64 values.
+    /// Each bit represents whether a page has been written since the last snapshot.
+    pub bitmap: Vec<u64>,
+}