api: implement API for dirty memory

Implement API /memory/dirty which returns a bitmap tracking dirty guest
memory. The bitmap is structured as a vector of u64, so its length is:
total_number_of_pages.div_ceil(64).

Pages are ordered in the order of pages as reported by /memory/mappings.

Signed-off-by: Babis Chalios <babis.chalios@e2b.dev>

Author: bchalios

resources/seccomp/x86_64-unknown-linux-musl.json

@@ -220,6 +220,10 @@
                 "syscall": "mincore",
                 "comment": "Used by get_memory_dirty_bitmap to check if memory pages are resident"
             },
+            {
+                "syscall": "pread64",
+                "comment": "Used by get_dirty_memory to read pagemap entries"
+            },
             {
                 "syscall": "mmap",
                 "comment": "Used by the VirtIO balloon device",

src/firecracker/src/api_server/parsed_request.rs

@@ -20,7 +20,7 @@ use super::request::logger::parse_put_logger;
 use super::request::machine_configuration::{
     parse_get_machine_config, parse_patch_machine_config, parse_put_machine_config,
 };
-use super::request::memory::{parse_get_memory, parse_get_memory_mappings};
+use super::request::memory::{parse_get_memory, parse_get_memory_dirty, parse_get_memory_mappings};
 use super::request::metrics::parse_put_metrics;
 use super::request::mmds::{parse_get_mmds, parse_patch_mmds, parse_put_mmds};
 use super::request::net::{parse_patch_net, parse_put_net};
@@ -85,6 +85,7 @@ impl TryFrom<&Request> for ParsedRequest {
             (Method::Get, "machine-config", None) => parse_get_machine_config(),
             (Method::Get, "memory", None) => match path_tokens.next() {
                 Some("mappings") => parse_get_memory_mappings(),
+                Some("dirty") => parse_get_memory_dirty(),
                 None => parse_get_memory(),
                 _ => Err(RequestError::InvalidPathMethod(
                     request_uri.to_string(),
@@ -183,6 +184,7 @@ impl ParsedRequest {
                 VmmData::InstanceInformation(info) => Self::success_response_with_data(info),
                 VmmData::MemoryMappings(mappings) => Self::success_response_with_data(mappings),
                 VmmData::Memory(memory) => Self::success_response_with_data(memory),
+                VmmData::MemoryDirty(dirty) => Self::success_response_with_data(dirty),
                 VmmData::VmmVersion(version) => Self::success_response_with_data(
                     &serde_json::json!({ "firecracker_version": version.as_str() }),
                 ),
@@ -585,6 +587,9 @@ pub mod tests {
                 VmmData::Memory(memory) => {
                     http_response(&serde_json::to_string(memory).unwrap(), 200)
                 }
+                VmmData::MemoryDirty(dirty) => {
+                    http_response(&serde_json::to_string(dirty).unwrap(), 200)
+                }
                 VmmData::VmmVersion(version) => http_response(
                     &serde_json::json!({ "firecracker_version": version.as_str() }).to_string(),
                     200,
@@ -615,6 +620,9 @@ pub mod tests {
                 empty: vec![],
             },
         ));
+        verify_ok_response_with(VmmData::MemoryDirty(
+            vmm::vmm_config::instance_info::MemoryDirty { bitmap: vec![] },
+        ));
         verify_ok_response_with(VmmData::VmmVersion(String::default()));
 
         // Error.

src/firecracker/src/api_server/request/memory.rs

@@ -16,6 +16,11 @@ pub(crate) fn parse_get_memory() -> Result<ParsedRequest, RequestError> {
     Ok(ParsedRequest::new_sync(VmmAction::GetMemory))
 }
 
+pub(crate) fn parse_get_memory_dirty() -> Result<ParsedRequest, RequestError> {
+    METRICS.get_api_requests.instance_info_count.inc();
+    Ok(ParsedRequest::new_sync(VmmAction::GetMemoryDirty))
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -36,4 +41,12 @@ mod tests {
             _ => panic!("Test failed."),
         }
     }
+
+    #[test]
+    fn test_parse_get_memory_dirty_request() {
+        match parse_get_memory_dirty().unwrap().into_parts() {
+            (RequestAction::Sync(action), _) if *action == VmmAction::GetMemoryDirty => {}
+            _ => panic!("Test failed."),
+        }
+    }
 }

src/vmm/src/lib.rs

@@ -276,6 +276,8 @@ pub enum VmmError {
     VmmObserverTeardown(vmm_sys_util::errno::Error),
     /// VMGenID error: {0}
     VMGenID(#[from] VmGenIdError),
+    /// Pagemap error: {0}
+    Pagemap(#[from] utils::pagemap::PagemapError),
 }
 
 /// Shorthand type for KVM dirty page bitmap.
@@ -482,6 +484,69 @@ impl Vmm {
         mappings
     }
 
+    /// Get dirty pages bitmap for guest memory.
+    ///
+    /// Returns a bitmap where each bit represents whether a guest page has been written to
+    /// (i.e., present in RAM and not write-protected via userfaultfd). Pages are ordered
+    /// following the order of memory regions as returned by `guest_memory_mappings`.
+    pub fn get_dirty_memory(&self, page_size: usize) -> Result<Vec<u64>, VmmError> {
+        let pagemap = utils::pagemap::PagemapReader::new(page_size)?;
+        let mut dirty_bitmap = vec![];
+
+        let sys_page_size = utils::get_page_size().expect("Failed to get system page size");
+
+        for region in self.guest_memory().iter() {
+            let base_addr = region.as_ptr() as usize;
+            let len = region.size();
+            let nr_pages = len / page_size;
+
+            // Use mincore to get resident pages at guest page size granularity
+            let mincore_n = len.div_ceil(sys_page_size);
+            let mut mincore_vec = vec![0u8; mincore_n];
+
+            // SAFETY: base_addr points to a valid guest memory region we own.
+            let mincore_result = unsafe {
+                libc::mincore(
+                    base_addr as *mut libc::c_void,
+                    len,
+                    mincore_vec.as_mut_ptr(),
+                )
+            };
+
+            // Build dirty bitmap: check pagemap only for pages that mincore reports resident.
+            let mut slot_bitmap = vec![0u64; nr_pages.div_ceil(64)];
+            for page_idx in 0..nr_pages {
+                let page_offset = page_idx * page_size;
+
+                let is_resident = if mincore_result == 0 {
+                    let start = page_offset / sys_page_size;
+                    let count = page_size.div_ceil(sys_page_size);
+                    if start + count <= mincore_vec.len() {
+                        mincore_vec[start..start + count]
+                            .iter()
+                            .any(|&v| (v & 0x1) != 0)
+                    } else {
+                        false
+                    }
+                } else {
+                    // If mincore failed, assume resident (conservative)
+                    true
+                };
+
+                if is_resident {
+                    let virt_addr = base_addr + page_offset;
+                    if pagemap.is_page_dirty(virt_addr)? {
+                        slot_bitmap[page_idx / 64] |= 1u64 << (page_idx % 64);
+                    }
+                }
+            }
+
+            dirty_bitmap.extend_from_slice(&slot_bitmap);
+        }
+
+        Ok(dirty_bitmap)
+    }
+
     /// Sets RDA bit in serial console
     pub fn emulate_serial_init(&self) -> Result<(), EmulateSerialInitError> {
         // When restoring from a previously saved state, there is no serial

src/vmm/src/rpc_interface.rs

@@ -25,7 +25,9 @@ use crate::vmm_config::balloon::{
 use crate::vmm_config::boot_source::{BootSourceConfig, BootSourceConfigError};
 use crate::vmm_config::drive::{BlockDeviceConfig, BlockDeviceUpdateConfig, DriveError};
 use crate::vmm_config::entropy::{EntropyDeviceConfig, EntropyDeviceError};
-use crate::vmm_config::instance_info::{InstanceInfo, MemoryMappingsResponse, MemoryResponse};
+use crate::vmm_config::instance_info::{
+    InstanceInfo, MemoryDirty, MemoryMappingsResponse, MemoryResponse, VmState,
+};
 use crate::vmm_config::machine_config::{MachineConfig, MachineConfigUpdate, VmConfigError};
 use crate::vmm_config::metrics::{MetricsConfig, MetricsConfigError};
 use crate::vmm_config::mmds::{MmdsConfig, MmdsConfigError};
@@ -69,6 +71,8 @@ pub enum VmmAction {
     GetMemoryMappings,
     /// Get memory info (resident and empty pages).
     GetMemory,
+    /// Get guest memory dirty pages information
+    GetMemoryDirty,
     /// Get microVM version.
     GetVmmVersion,
     /// Flush the metrics. This action can only be called after the logger has been configured.
@@ -168,6 +172,8 @@ pub enum VmmActionError {
     OperationNotSupportedPostBoot,
     /// The requested operation is not supported before starting the microVM.
     OperationNotSupportedPreBoot,
+    /// The requested operation is not supported while the microVM is running.
+    OperationNotSupportedWhileRunning,
     /// Start microvm error: {0}
     StartMicrovm(#[from] StartMicrovmError),
     /// Vsock config error: {0}
@@ -197,6 +203,8 @@ pub enum VmmData {
     MemoryMappings(MemoryMappingsResponse),
     /// Memory info (resident and empty pages).
     Memory(MemoryResponse),
+    /// The guest memory dirty pages information
+    MemoryDirty(MemoryDirty),
     /// The microVM version.
     VmmVersion(String),
 }
@@ -427,7 +435,9 @@ impl<'a> PrebootApiController<'a> {
                 &self.vm_resources.vm_config,
             ))),
             GetVmInstanceInfo => Ok(VmmData::InstanceInformation(self.instance_info.clone())),
-            GetMemoryMappings | GetMemory => Err(VmmActionError::OperationNotSupportedPreBoot),
+            GetMemoryMappings | GetMemory | GetMemoryDirty => {
+                Err(VmmActionError::OperationNotSupportedPreBoot)
+            }
             GetVmmVersion => Ok(VmmData::VmmVersion(self.instance_info.vmm_version.clone())),
             InsertBlockDevice(config) => self.insert_block_device(config),
             InsertNetworkDevice(config) => self.insert_net_device(config),
@@ -680,6 +690,7 @@ impl RuntimeApiController {
                     empty: empty_bitmap,
                 }))
             }
+            GetMemoryDirty => self.get_dirty_memory_info(),
             GetVmmVersion => Ok(VmmData::VmmVersion(
                 self.vmm.lock().expect("Poisoned lock").version(),
             )),
@@ -779,6 +790,28 @@ impl RuntimeApiController {
             .map_err(VmmActionError::InternalVmm)
     }
 
+    /// Get dirty pages information for guest memory
+    fn get_dirty_memory_info(&self) -> Result<VmmData, VmmActionError> {
+        let start_us = get_time_us(ClockType::Monotonic);
+        let vmm = self.vmm.lock().expect("Poisoned lock");
+
+        // Dirty page tracking via pagemap requires the VM to be paused so that guest
+        // pages are not modified while we are reading the pagemap.
+        if vmm.instance_info.state != VmState::Paused {
+            return Err(VmmActionError::OperationNotSupportedWhileRunning);
+        }
+
+        let page_size = self.vm_resources.vm_config.huge_pages.page_size_kib();
+        let bitmap = vmm
+            .get_dirty_memory(page_size)
+            .map_err(VmmActionError::InternalVmm)?;
+
+        let elapsed_time_us = get_time_us(ClockType::Monotonic) - start_us;
+        info!("'get dirty memory' VMM action took {elapsed_time_us} us.");
+
+        Ok(VmmData::MemoryDirty(MemoryDirty { bitmap }))
+    }
+
     fn create_snapshot(
         &mut self,
         create_params: &CreateSnapshotParams,

src/vmm/src/utils/mod.rs

@@ -9,6 +9,8 @@ pub mod net;
 pub mod signal;
 /// Module with state machine
 pub mod sm;
+/// Module with pagemap utilities
+pub mod pagemap;
 
 use std::num::Wrapping;
 use std::result::Result;

src/vmm/src/utils/pagemap.rs

@@ -0,0 +1,111 @@
+//! Utilities for reading /proc/self/pagemap to track dirty pages.
+
+#![allow(clippy::cast_possible_wrap)]
+
+use std::fs::File;
+use std::os::unix::io::AsRawFd;
+
+use crate::utils::get_page_size;
+
+const PAGEMAP_ENTRY_SIZE: usize = 8;
+
+/// Errors related to pagemap operations
+#[derive(Debug, thiserror::Error, displaydoc::Display)]
+pub enum PagemapError {
+    /// Failed to open /proc/self/pagemap: {0}
+    OpenPagemap(#[source] std::io::Error),
+    /// Failed to read pagemap entry: {0}
+    ReadEntry(#[source] std::io::Error),
+}
+
+/// Represents a single entry in /proc/pid/pagemap.
+///
+/// Each virtual page has an 8-byte entry with the following layout:
+/// - Bits 0-54:  Page frame number (PFN) if present
+/// - Bit 55:     Page is soft-dirty (written to since last clear)
+/// - Bit 56:     Page is exclusively mapped
+/// - Bit 57:     Page is write-protected via userfaultfd
+/// - Bit 58:     Unused
+/// - Bit 59-60:  Unused
+/// - Bit 61:     Page is file-page or shared-anon
+/// - Bit 62:     Page is swapped
+/// - Bit 63:     Page is present in RAM
+#[derive(Debug, Clone, Copy)]
+pub struct PagemapEntry {
+    raw: u64,
+}
+
+impl PagemapEntry {
+    /// Create a PagemapEntry from bytes (little-endian)
+    pub fn from_bytes(bytes: [u8; 8]) -> Self {
+        Self {
+            raw: u64::from_ne_bytes(bytes),
+        }
+    }
+
+    /// Check if page is write-protected via userfaultfd
+    pub fn is_write_protected(&self) -> bool {
+        (self.raw & (1u64 << 57)) != 0
+    }
+
+    /// Check if page is present in RAM (bit 63)
+    pub fn is_present(&self) -> bool {
+        (self.raw & (1u64 << 63)) != 0
+    }
+}
+
+/// Reader for /proc/self/pagemap
+#[derive(Debug)]
+pub struct PagemapReader {
+    pagemap_fd: File,
+}
+
+impl PagemapReader {
+    /// Create a new PagemapReader
+    pub fn new(_page_size: usize) -> Result<Self, PagemapError> {
+        let pagemap_fd = File::open("/proc/self/pagemap").map_err(PagemapError::OpenPagemap)?;
+
+        Ok(Self { pagemap_fd })
+    }
+
+    /// Check if a single page is dirty (write-protected bit cleared).
+    ///
+    /// Checks the first host page (4K) of the guest page at the given address.
+    /// For huge pages, all host pages within the huge page typically have the same
+    /// dirty status, so sampling the first is sufficient.
+    ///
+    /// # Arguments
+    /// * `virt_addr` - Virtual address of the page to check
+    ///
+    /// # Returns
+    /// True if the page is present and write-protected bit is cleared (dirty).
+    pub fn is_page_dirty(&self, virt_addr: usize) -> Result<bool, PagemapError> {
+        // Pagemap always uses host (4K) page size
+        let host_page_size = get_page_size().expect("Failed to get system page size");
+
+        // Calculate offset for this virtual page (using host page size)
+        let host_vpn = virt_addr / host_page_size;
+        let offset = (host_vpn * PAGEMAP_ENTRY_SIZE) as i64;
+
+        let mut entry_bytes = [0u8; 8];
+
+        // SAFETY: pread is safe as long as the fd is valid and the buffer is properly sized
+        let ret = unsafe {
+            libc::pread(
+                self.pagemap_fd.as_raw_fd(),
+                entry_bytes.as_mut_ptr().cast(),
+                PAGEMAP_ENTRY_SIZE,
+                offset,
+            )
+        };
+
+        if ret != PAGEMAP_ENTRY_SIZE as isize {
+            return Err(PagemapError::ReadEntry(std::io::Error::last_os_error()));
+        }
+
+        let entry = PagemapEntry::from_bytes(entry_bytes);
+
+        // Page must be present and the write_protected bit cleared (indicating it was written to)
+        Ok(entry.is_present() && !entry.is_write_protected())
+    }
+}

src/vmm/src/vmm_config/instance_info.rs

@@ -68,3 +68,11 @@ pub struct MemoryResponse {
     /// This is a subset of the resident pages.
     pub empty: Vec<u64>,
 }
+
+/// Information about dirty guest memory pages
+#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize)]
+pub struct MemoryDirty {
+    /// Bitmap for dirty pages. The bitmap is encoded as a vector of u64 values.
+    /// Each bit represents whether a page has been written since the last snapshot.
+    pub bitmap: Vec<u64>,
+}