[BNTL] Misc improvements

Author: emesare

Cargo.lock

@@ -23,7 +23,7 @@ walkdir = "2.5"
 dashmap = "6.1"
 
 # For TBD parsing
-serde-saphyr = { version = "0.0.18", default-features = false, features = [] }
+serde-saphyr = { version = "0.0.19", default-features = false, features = [] }
 
 # For reports
 minijinja = "2.10.2"

plugins/bntl_utils/Cargo.toml

@@ -34,6 +34,8 @@ Examples:
     - Downloads and processes all files in the project, placing potentially multiple `.bntl` files in the `output` directory.
 - `./bntl_cli create sqlite3.dll "windows-x86_64" ./winmd/ ./output/`
   - `winmd` files are also supported as input, they will be processed together. You also probably want to provide some apiset schema files as well.
+- `./bntl_cli create sqlite3.dll "windows-x86_64" ./headers/ ./output/ --include-directories ./system_headers/`
+  - You can also specify additional include directories to search for referenced headers.
 
 #### Dump
 
@@ -45,7 +47,7 @@ Examples:
 
 #### Diff
 
-Compare two type libraries and generate a .diff file containing a similarity ratio.
+Compare two type libraries and generate a .diff file.
 
 Examples:
 

plugins/bntl_utils/cli/README.md

@@ -14,6 +14,9 @@ pub struct CreateArgs {
     pub platform: String,
     pub input: Input,
     pub output_directory: Option<PathBuf>,
+    /// A list of directories to use for include paths when parsing C header files.
+    #[clap(long)]
+    pub include_directories: Vec<PathBuf>,
     #[clap(long)]
     pub dry_run: bool,
 }
@@ -37,7 +40,8 @@ impl CreateArgs {
         }
         std::fs::create_dir_all(&output_path).expect("Failed to create output directory");
 
-        let processor = TypeLibProcessor::new(&self.name, &self.platform);
+        let processor = TypeLibProcessor::new(&self.name, &self.platform)
+            .with_include_directories(self.include_directories.clone());
         // TODO: Need progress indicator here, when downloading files.
         let resolved_input = self.input.resolve().expect("Failed to resolve input");
 

plugins/bntl_utils/cli/src/create.rs

@@ -1,7 +1,7 @@
 use crate::command::{InputDirectoryField, OutputDirectoryField};
 use crate::helper::path_to_type_libraries;
 use crate::validate::TypeLibValidater;
-use binaryninja::binary_view::{BinaryView, BinaryViewExt};
+use binaryninja::binary_view::BinaryView;
 use binaryninja::command::Command;
 use binaryninja::interaction::Form;
 use binaryninja::platform::Platform;

plugins/bntl_utils/src/command/validate.rs

@@ -354,12 +354,12 @@ impl ProcessedData {
             (QualifiedName, CoreArchitecture),
             Vec<Ref<TypeLibrary>>,
         > = HashMap::new();
-        for merged_type_library in &self.type_libraries {
-            for named_type in &merged_type_library.named_types() {
+        for tl in &self.type_libraries {
+            for named_type in &tl.named_types() {
                 mapped_named_types
-                    .entry((named_type.name.clone(), merged_type_library.arch()))
+                    .entry((named_type.name.clone(), tl.arch()))
                     .or_default()
-                    .push(merged_type_library.clone());
+                    .push(tl.clone());
             }
         }
 
@@ -553,12 +553,16 @@ impl TypeLibProcessor {
             .filter_map(|res| match res {
                 Ok(result) => Some(Ok(result)),
                 Err(ProcessingError::SkippedFile(path)) => {
-                    tracing::debug!("Skipping project root file: {:?}", path);
+                    tracing::debug!("Skipping project file: {:?}", path);
                     None
                 }
                 Err(ProcessingError::Cancelled) => Some(Err(ProcessingError::Cancelled)),
+                Err(ProcessingError::NoPathToProjectFile(path)) => {
+                    tracing::warn!("Project file not downloaded: {:?}", path);
+                    None
+                }
                 Err(e) => {
-                    tracing::error!("Project root file processing error: {:?}", e);
+                    tracing::error!("Project file processing error: {:?}", e);
                     None
                 }
             })
@@ -588,12 +592,16 @@ impl TypeLibProcessor {
             .filter_map(|res| match res {
                 Ok(result) => Some(Ok(result)),
                 Err(ProcessingError::SkippedFile(path)) => {
-                    tracing::debug!("Skipping project directory file: {:?}", path);
+                    tracing::debug!("Skipping project file: {:?}", path);
                     None
                 }
                 Err(ProcessingError::Cancelled) => Some(Err(ProcessingError::Cancelled)),
+                Err(ProcessingError::NoPathToProjectFile(path)) => {
+                    tracing::warn!("Project file not downloaded: {:?}", path);
+                    None
+                }
                 Err(e) => {
-                    tracing::error!("Project folder file processing error: {:?}", e);
+                    tracing::error!("Project file processing error: {:?}", e);
                     None
                 }
             })
@@ -1093,13 +1101,13 @@ pub fn is_parsable(path: &Path) -> bool {
     if path.extension() == Some(OsStr::new("pdb")) {
         return false;
     }
-    let mut metadata = FileMetadata::with_file_path(path);
-    let Ok(view) = BinaryView::from_path(&metadata, path) else {
+    let metadata = FileMetadata::with_file_path(path);
+    let Ok(view) = BinaryView::from_metadata(&metadata) else {
         return false;
     };
     // If any view type parses this file, consider it for this source.
     // All files will have a "Raw" file type, so we account for that.
-    BinaryViewType::list_valid_types_for(&view).len() > 1
+    BinaryViewType::valid_types_for_data(&view).len() > 1
 }
 
 #[cfg(test)]

plugins/bntl_utils/src/process.rs

@@ -1,3 +1,5 @@
+//! Parse Apples TBD file format, which gives information about where source symbols are located.
+
 use binaryninja::architecture::CoreArchitecture;
 use binaryninja::platform::Platform;
 use binaryninja::rc::Ref;