From bc845d6357e0edce63ccb667ff6bd57121eae60c Mon Sep 17 00:00:00 2001 From: Nikita Utkin Date: Sun, 24 Nov 2024 16:56:23 +0500 Subject: [PATCH 1/4] Add mmap_file --- Cargo.lock | 10 ++++++++++ Cargo.toml | 1 + src/common.rs | 31 +++++++++++++++++++++++++++++++ 3 files changed, 42 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index a4d2af845..62a040f27 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -123,6 +123,7 @@ dependencies = [ "flate2", "log", "md5", + "memmap2", "miniz_oxide 0.8.0", "plotters", "serde", @@ -676,6 +677,15 @@ version = "2.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" +[[package]] +name = "memmap2" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd3f7eed9d3848f8b98834af67102b720745c4ec028fcd0aa0239277e7de374f" +dependencies = [ + "libc", +] + [[package]] name = "miniz_oxide" version = "0.7.4" diff --git a/Cargo.toml b/Cargo.toml index ae2819fee..5b0f30bea 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -33,6 +33,7 @@ aho-corasick = "1.1.3" serde = { version = "1.0", features = ["derive"]} clap = { version = "4.5.16", features = ["derive"] } xxhash-rust = { version = "0.8.12", features = ["xxh32"] } +memmap2 = "0.9.5" [dependencies.uuid] version = "1.10.0" diff --git a/src/common.rs b/src/common.rs index cb1276a4c..a92836109 100644 --- a/src/common.rs +++ b/src/common.rs @@ -1,6 +1,7 @@ //! Common Functions use chrono::prelude::DateTime; use log::{debug, error}; +use memmap2::Mmap; use std::fs::File; use std::io::Read; @@ -39,6 +40,36 @@ pub fn read_file(file: impl Into) -> Result, std::io::Error> { } } +/// Map the file into memory and return a [Mmap] instance +/// that provides access into the file's contents. +/// +/// ## Example +/// +/// ``` +/// use binwalk::common::mmap_file; +/// +/// let file_data = mmap_file("/etc/passwd")?; +/// assert!(file_data.len() > 0); +/// # Ok::<(), std::io::Error>(()) +/// ``` +pub fn mmap_file(file: impl Into) -> Result { + let file_path = file.into(); + + match File::open(&file_path) { + Err(e) => { + error!("Failed to open file {}: {}", file_path, e); + Err(e) + } + Ok(fp) => match unsafe { Mmap::map(&fp) } { + Err(e) => { + error!("Failed to map file {} into memory: {}", file_path, e); + Err(e) + } + Ok(mmap) => Ok(mmap), + }, + } +} + /// Calculates the CRC32 checksum of the given data. /// /// ## Notes From cd0125398e90ccb747f9ff94a1d4542b2f586255 Mon Sep 17 00:00:00 2001 From: Nikita Utkin Date: Sun, 24 Nov 2024 16:56:53 +0500 Subject: [PATCH 2/4] Use mmap_file for analysis --- src/binwalk.rs | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/binwalk.rs b/src/binwalk.rs index 933970ded..482de9b60 100644 --- a/src/binwalk.rs +++ b/src/binwalk.rs @@ -14,7 +14,7 @@ use std::os::windows; #[cfg(unix)] use std::os::unix; -use crate::common::{is_offset_safe, read_file}; +use crate::common::{is_offset_safe, mmap_file}; use crate::extractors; use crate::magic; use crate::signatures; @@ -699,11 +699,10 @@ impl Binwalk { debug!("Analysis start: {}", target_file); - // Read file into memory - if let Ok(file_data) = read_file(target_file) { + if let Ok(file_mmap) = mmap_file(target_file) { // Scan file data for signatures info!("Scanning {}", target_file); - results.file_map = self.scan(&file_data); + results.file_map = self.scan(&file_mmap); // Only extract if told to, and if there were some signatures found in this file if do_extraction && !results.file_map.is_empty() { @@ -712,7 +711,7 @@ impl Binwalk { "Submitting {} signature results to extractor", results.file_map.len() ); - results.extractions = self.extract(&file_data, target_file, &results.file_map); + results.extractions = self.extract(&file_mmap, target_file, &results.file_map); } } From 3b5ff37284d8f32d5da3c1dc2876ea2c9bcd3459 Mon Sep 17 00:00:00 2001 From: Nikita Utkin Date: Sun, 24 Nov 2024 16:58:29 +0500 Subject: [PATCH 3/4] Replace other uses of read_file with mmap_file --- src/entropy.rs | 4 ++-- src/main.rs | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/entropy.rs b/src/entropy.rs index e3455262f..5628ad890 100644 --- a/src/entropy.rs +++ b/src/entropy.rs @@ -1,4 +1,4 @@ -use crate::common::read_file; +use crate::common::mmap_file; use entropy::shannon_entropy; use log::error; use plotters::prelude::*; @@ -87,7 +87,7 @@ pub fn plot(file_path: impl Into) -> Result { } // Read in the target file data - if let Ok(file_data) = read_file(&target_file) { + if let Ok(file_data) = mmap_file(&target_file) { let mut points: Vec<(i32, i32)> = vec![]; // Calculate the entropy for each file block diff --git a/src/main.rs b/src/main.rs index 586d1f968..0e389e25d 100644 --- a/src/main.rs +++ b/src/main.rs @@ -305,7 +305,7 @@ fn carve_file_map(results: &binwalk::AnalysisResults) -> usize { // No results, don't do anything if !results.file_map.is_empty() { // Read in the source file - if let Ok(file_data) = common::read_file(&results.file_path) { + if let Ok(file_data) = common::mmap_file(&results.file_path) { // Loop through all identified signatures in the file for signature_result in &results.file_map { // If there is data between the last signature and this signature, it is some chunk of unknown data From cae18b8cb6c8bb263e62f675add2bce825e821e5 Mon Sep 17 00:00:00 2001 From: Nikita Utkin Date: Sun, 24 Nov 2024 16:59:09 +0500 Subject: [PATCH 4/4] Remove read_file --- src/common.rs | 42 +++--------------------------------------- 1 file changed, 3 insertions(+), 39 deletions(-) diff --git a/src/common.rs b/src/common.rs index a92836109..8e0b59aeb 100644 --- a/src/common.rs +++ b/src/common.rs @@ -1,50 +1,14 @@ //! Common Functions use chrono::prelude::DateTime; -use log::{debug, error}; +use log::error; use memmap2::Mmap; use std::fs::File; -use std::io::Read; - -/// Read a file into memory and return its contents. -/// -/// ## Example -/// -/// ``` -/// # fn main() { #[allow(non_snake_case)] fn _doctest_main_src_common_rs_11_0() -> Result<(), Box> { -/// use binwalk::common::read_file; -/// -/// let file_data = read_file("/etc/passwd")?; -/// assert!(file_data.len() > 0); -/// # Ok(()) -/// # } _doctest_main_src_common_rs_11_0(); } -/// ``` -pub fn read_file(file: impl Into) -> Result, std::io::Error> { - let mut file_data = Vec::new(); - let file_path = file.into(); - - match File::open(&file_path) { - Err(e) => { - error!("Failed to open file {}: {}", file_path, e); - Err(e) - } - Ok(mut fp) => match fp.read_to_end(&mut file_data) { - Err(e) => { - error!("Failed to read file {} into memory: {}", file_path, e); - Err(e) - } - Ok(file_size) => { - debug!("Loaded {} bytes from {}", file_size, file_path); - Ok(file_data) - } - }, - } -} /// Map the file into memory and return a [Mmap] instance /// that provides access into the file's contents. -/// +/// /// ## Example -/// +/// /// ``` /// use binwalk::common::mmap_file; ///