add tracing to the compressor

Signed-off-by: Connor Tsui <connor.tsui20@gmail.com>

Author: connortsui20

Cargo.lock

@@ -15,6 +15,7 @@ use regex::Regex;
 use vortex::utils::aliases::hash_map::HashMap;
 use vortex_bench::Engine;
 use vortex_bench::Format;
+use vortex_bench::LogFormat;
 use vortex_bench::Target;
 use vortex_bench::compress::CompressMeasurements;
 use vortex_bench::compress::CompressOp;
@@ -39,7 +40,7 @@ use vortex_bench::public_bi::PBIDataset::CMSprovider;
 use vortex_bench::public_bi::PBIDataset::Euro2016;
 use vortex_bench::public_bi::PBIDataset::Food;
 use vortex_bench::public_bi::PBIDataset::HashTags;
-use vortex_bench::setup_logging_and_tracing;
+use vortex_bench::setup_logging_and_tracing_with_format;
 
 #[derive(Parser, Debug)]
 #[command(version, about, long_about = None)]
@@ -69,13 +70,17 @@ struct Args {
     output_path: Option<PathBuf>,
     #[arg(long)]
     tracing: bool,
+    /// Format for the primary stderr log sink. `text` is the default human-readable format;
+    /// `json` emits one JSON object per event, suitable for piping into `jq`.
+    #[arg(long, value_enum, default_value_t = LogFormat::Text)]
+    log_format: LogFormat,
 }
 
 #[tokio::main]
 async fn main() -> anyhow::Result<()> {
     let args = Args::parse();
 
-    setup_logging_and_tracing(args.verbose, args.tracing)?;
+    setup_logging_and_tracing_with_format(args.verbose, args.tracing, args.log_format)?;
 
     run_compress(
         args.iterations,

benchmarks/compress-bench/src/main.rs

@@ -61,6 +61,7 @@ tracing = { workspace = true }
 tracing-perfetto = { workspace = true }
 tracing-subscriber = { workspace = true, features = [
     "env-filter",
+    "json",
     "tracing-log",
 ] }
 url = { workspace = true }

vortex-bench/Cargo.toml

@@ -17,12 +17,13 @@ use vortex_bench::Benchmark;
 use vortex_bench::BenchmarkArg;
 use vortex_bench::CompactionStrategy;
 use vortex_bench::Format;
+use vortex_bench::LogFormat;
 use vortex_bench::Opt;
 use vortex_bench::Opts;
 use vortex_bench::conversions::convert_parquet_directory_to_vortex;
 use vortex_bench::create_benchmark;
 use vortex_bench::generate_duckdb_registration_sql;
-use vortex_bench::setup_logging_and_tracing;
+use vortex_bench::setup_logging_and_tracing_with_format;
 
 #[derive(Parser)]
 #[command(name = "bench-data-gen")]
@@ -37,6 +38,11 @@ struct Args {
     #[arg(long)]
     tracing: bool,
 
+    /// Format for the primary stderr log sink. `text` is the default human-readable format;
+    /// `json` emits one JSON object per event, suitable for piping into `jq`.
+    #[arg(long, value_enum, default_value_t = LogFormat::Text)]
+    log_format: LogFormat,
+
     #[arg(long, value_delimiter = ',', value_parser = value_parser!(Format))]
     formats: Vec<Format>,
 
@@ -49,7 +55,7 @@ async fn main() -> anyhow::Result<()> {
     let args = Args::parse();
     let opts = Opts::from(args.options);
 
-    setup_logging_and_tracing(args.verbose, args.tracing)?;
+    setup_logging_and_tracing_with_format(args.verbose, args.tracing, args.log_format)?;
 
     let benchmark = create_benchmark(args.benchmark, &opts)?;
 

vortex-bench/src/bin/data-gen.rs

@@ -4,34 +4,77 @@
 use std::fs::File;
 use std::io::IsTerminal;
 
+use clap::ValueEnum;
 use tracing::level_filters::LevelFilter;
 use tracing_perfetto::PerfettoLayer;
 use tracing_subscriber::EnvFilter;
+use tracing_subscriber::Layer;
 use tracing_subscriber::prelude::*;
 
-/// Initialize logging/tracing for a benchmark
-pub fn setup_logging_and_tracing(verbose: bool, tracing: bool) -> anyhow::Result<()> {
+/// Format for the primary stderr log sink.
+///
+/// `Text` is the default human-readable formatter matching the historical behavior of this crate.
+/// `Json` emits one newline-delimited JSON object per event, suitable for piping into `jq` or a log
+/// aggregator.
+#[derive(Copy, Clone, Debug, Default, PartialEq, Eq, ValueEnum)]
+pub enum LogFormat {
+    #[default]
+    Text,
+    Json,
+}
+
+/// Initialize logging/tracing for a benchmark, hardcoding [`LogFormat::Text`].
+///
+/// See [`setup_logging_and_tracing_with_format`] if you want to select JSON
+/// output from a CLI flag.
+pub fn setup_logging_and_tracing(verbose: bool, perfetto: bool) -> anyhow::Result<()> {
+    setup_logging_and_tracing_with_format(verbose, perfetto, LogFormat::Text)
+}
+
+/// Initialize logging/tracing for a benchmark with an explicit stderr format.
+///
+/// - `verbose`: when `RUST_LOG` is unset, raises the default filter from `INFO` to `TRACE`. Has no
+///   effect when `RUST_LOG` is set (the env var wins).
+/// - `perfetto`: when `true`, additionally attaches a [`tracing_perfetto::PerfettoLayer`] that
+///   writes span begin/end events to `trace.json` in the current directory. Intended to be loaded
+///   into the Perfetto UI for flamegraph visualization.
+/// - `format`: controls the primary stderr sink's formatting. See [`LogFormat`].
+pub fn setup_logging_and_tracing_with_format(
+    verbose: bool,
+    perfetto: bool,
+    format: LogFormat,
+) -> anyhow::Result<()> {
     let filter = default_env_filter(verbose);
 
-    let fmt_layer = tracing_subscriber::fmt::layer()
-        .with_writer(std::io::stderr)
-        .with_level(true)
-        .with_file(true)
-        .with_line_number(true)
-        .with_ansi(std::io::stderr().is_terminal());
+    let perfetto_layer = perfetto
+        .then(|| {
+            Ok::<_, anyhow::Error>(
+                PerfettoLayer::new(File::create("trace.json")?).with_debug_annotations(true),
+            )
+        })
+        .transpose()?;
+
+    // `fmt::layer()` and `fmt::layer().json()` produce different concrete types,
+    // so erase each to a `dyn Layer` via `.boxed()` and keep the registry uniform.
+    let fmt_layer: Box<dyn Layer<_> + Send + Sync> = match format {
+        LogFormat::Text => tracing_subscriber::fmt::layer()
+            .with_writer(std::io::stderr)
+            .with_level(true)
+            .with_file(true)
+            .with_line_number(true)
+            .with_ansi(std::io::stderr().is_terminal())
+            .boxed(),
+        LogFormat::Json => tracing_subscriber::fmt::layer()
+            .json()
+            .with_writer(std::io::stderr)
+            .with_current_span(true)
+            .with_span_list(true)
+            .boxed(),
+    };
 
     tracing_subscriber::registry()
         .with(filter)
-        .with(
-            tracing
-                .then(|| {
-                    Ok::<_, anyhow::Error>(
-                        PerfettoLayer::new(File::create("trace.json")?)
-                            .with_debug_annotations(true),
-                    )
-                })
-                .transpose()?,
-        )
+        .with(perfetto_layer)
         .with(fmt_layer)
         .init();
 

vortex-bench/src/utils/logging.rs

@@ -307,8 +307,6 @@ impl Scheme for ZigZagScheme {
 
         let compressed = compressor.compress_child(&encoded.into_array(), &ctx, self.id(), 0)?;
 
-        tracing::debug!("zigzag output: {}", compressed.encoding_id());
-
         Ok(ZigZag::try_new(compressed)?.into_array())
     }
 }

vortex-btrblocks/src/schemes/integer.rs

@@ -19,7 +19,7 @@ num-traits = { workspace = true }
 parking_lot = { workspace = true }
 rand = { workspace = true }
 rustc-hash = { workspace = true }
-tracing = { workspace = true }
+tracing = { workspace = true, features = ["std", "attributes"] }
 vortex-array = { workspace = true }
 vortex-buffer = { workspace = true }
 vortex-error = { workspace = true }
@@ -29,6 +29,7 @@ vortex-utils = { workspace = true }
 [dev-dependencies]
 divan = { workspace = true }
 rstest = { workspace = true }
+tracing-subscriber = { workspace = true, features = ["env-filter"] }
 vortex-array = { workspace = true, features = ["_test-harness"] }
 
 [lints]