feat(ast-doc-core): enable hotpath feature for improved performance tracking

Author: Akagi201

Cargo.toml

@@ -24,6 +24,7 @@ cucumber = "0.22.1"
 eyre = "0.6.12"
 git2 = "0.20.4"
 globset = "0.4.18"
+hotpath = { version = "0.5", default-features = false }
 ignore = "0.4.25"
 proptest = "1.11.0"
 rayon = "1.11.0"

bin/ast-doc/Cargo.toml

@@ -11,10 +11,14 @@ categories.workspace = true
 description = "CLI for generating optimized llms.txt documentation from codebases"
 readme = "README.md"
 
+[features]
+hotpath = ["ast-doc-core/hotpath", "dep:hotpath"]
+
 [dependencies]
 ast-doc-core = { workspace = true }
 clap = { workspace = true, features = ["derive"] }
 eyre = { workspace = true }
+hotpath = { workspace = true, optional = true }
 tracing = { workspace = true }
 tracing-subscriber = { workspace = true }
 

bin/ast-doc/src/main.rs

@@ -106,6 +106,9 @@ pub fn build_config(args: &Args) -> ast_doc_core::AstDocConfig {
 }
 
 fn main() -> Result<()> {
+    #[cfg(feature = "hotpath")]
+    let _guard = hotpath::GuardBuilder::new("main").build();
+
     let args = Args::parse();
 
     tracing_subscriber::fmt()

crates/ast-doc-core/Cargo.toml

@@ -20,6 +20,7 @@ all-languages = [
     "lang-c",
 ]
 default = ["lang-rust"]
+hotpath = ["dep:hotpath"]
 lang-c = ["tree-sitter-c"]
 lang-go = ["tree-sitter-go"]
 lang-python = ["tree-sitter-python"]
@@ -30,6 +31,7 @@ lang-typescript = ["tree-sitter-typescript"]
 eyre = { workspace = true }
 git2 = { workspace = true }
 globset = { workspace = true }
+hotpath = { workspace = true, optional = true }
 ignore = { workspace = true }
 rayon = { workspace = true }
 serde_json = { workspace = true }
@@ -48,6 +50,7 @@ tree-sitter-typescript = { workspace = true, optional = true }
 workspace = true
 
 [dev-dependencies]
+ctor = "0.2"
 cucumber = { workspace = true }
 proptest = { workspace = true }
 tempfile = { workspace = true }

crates/ast-doc-core/src/ingestion/mod.rs

@@ -54,6 +54,7 @@ pub struct IngestionResult {
 /// # Errors
 ///
 /// Returns an error if directory walking or git operations fail.
+#[cfg_attr(feature = "hotpath", hotpath::measure)]
 pub fn run_ingestion(config: &AstDocConfig) -> Result<IngestionResult, AstDocError> {
     let root = config
         .path
@@ -123,8 +124,14 @@ pub fn run_ingestion(config: &AstDocConfig) -> Result<IngestionResult, AstDocErr
 }
 
 /// Count tokens in a string using `tiktoken-rs`.
+///
+/// Uses a cached BPE instance to avoid repeated initialization.
 fn count_tokens(text: &str) -> usize {
-    tiktoken_rs::cl100k_base().map_or(0, |bpe| bpe.encode_with_special_tokens(text).len())
+    use std::sync::LazyLock;
+    static BPE: LazyLock<Option<tiktoken_rs::CoreBPE>> =
+        LazyLock::new(|| tiktoken_rs::cl100k_base().ok());
+
+    BPE.as_ref().map_or(0, |bpe| bpe.encode_with_special_tokens(text).len())
 }
 
 /// Build a directory tree string from discovered file paths.

crates/ast-doc-core/src/lib.rs

@@ -66,6 +66,7 @@ pub struct PipelineResult {
 /// # Errors
 ///
 /// Returns an error if any pipeline stage fails.
+#[cfg_attr(feature = "hotpath", hotpath::measure)]
 pub fn run_pipeline(config: &AstDocConfig) -> eyre::Result<PipelineResult> {
     // Phase 1: Ingestion — file discovery, git metadata, directory tree
     let ingestion = ingestion::run_ingestion(config)?;
@@ -116,6 +117,22 @@ fn compute_base_overhead(ingestion: &IngestionResult) -> usize {
 }
 
 /// Count tokens in a string using `tiktoken-rs`.
+///
+/// Uses a cached BPE instance to avoid repeated initialization.
 fn count_tokens(text: &str) -> usize {
-    tiktoken_rs::cl100k_base().map_or(0, |bpe| bpe.encode_with_special_tokens(text).len())
+    use std::sync::LazyLock;
+    static BPE: LazyLock<Option<tiktoken_rs::CoreBPE>> =
+        LazyLock::new(|| tiktoken_rs::cl100k_base().ok());
+
+    BPE.as_ref().map_or(0, |bpe| bpe.encode_with_special_tokens(text).len())
+}
+
+/// Initialize hotpath guard for tests.
+///
+/// This is called automatically when running tests with the hotpath feature enabled.
+#[cfg(all(test, feature = "hotpath"))]
+#[ctor::ctor]
+fn init_hotpath_for_tests() {
+    let _guard = hotpath::GuardBuilder::new("test").build();
+    std::mem::forget(_guard);
 }

crates/ast-doc-core/src/renderer/mod.rs

@@ -12,6 +12,7 @@ use crate::{config::AstDocConfig, ingestion::IngestionResult, scheduler::Schedul
 /// # Errors
 ///
 /// Returns an error if rendering fails.
+#[cfg_attr(feature = "hotpath", hotpath::measure)]
 pub fn render_llms_txt(
     scheduled: &ScheduleResult,
     ingestion: &IngestionResult,

crates/ast-doc-core/src/scheduler/mod.rs

@@ -48,6 +48,7 @@ pub struct ScheduleResult {
 ///
 /// Returns `AstDocError::BudgetExceeded` if even minimum strategies
 /// exceed the token budget.
+#[cfg_attr(feature = "hotpath", hotpath::measure)]
 pub fn run_scheduler(
     parsed: &[ParsedFile],
     config: &AstDocConfig,

crates/ast-doc-core/tests/bdd.rs

@@ -2,6 +2,13 @@
 
 #![allow(clippy::print_stdout, clippy::print_stderr)]
 
+#[cfg(feature = "hotpath")]
+#[ctor::ctor]
+fn init_hotpath_for_bdd() {
+    let _guard = hotpath::GuardBuilder::new("bdd_test").build();
+    std::mem::forget(_guard);
+}
+
 use std::{fs, path::PathBuf};
 
 use ast_doc_core::{