Add smart export filtering with gyro-based ground test detection

- Add --force-export flag to bypass filtering
- Implement 3-tier filtering: <5s skip, 5-15s check data density, >15s check gyro activity
- Skip logs with <1500fps data density in 5-15s range
- Detect ground tests using gyro variance analysis (threshold 0.3)
- Conservative approach prevents false-skips of legitimate flights
- Update README with filtering documentation and examples

Author: nerdCopter

README.md

@@ -19,6 +19,7 @@ A high-performance Rust library and command-line tool for parsing BBL (Blackbox
 - **CSV Export**: Flight data and header export with blackbox_decode compatibility
 - **GPS Export**: GPX file generation for GPS-enabled flight logs
 - **Event Export**: Flight event data extraction in JSONL format
+- **Smart Export Filtering**: Automatically skips short test flights while preserving high-quality short logs
 - **Command Line Interface**: Glob patterns, debug mode, configurable output directories
 - **Comprehensive Examples**: Practical demonstrations of crate usage with PID display and multi-firmware support
 
@@ -290,8 +291,27 @@ cargo build --release
 
 # Custom output directory
 ./target/release/bbl_parser --csv --output-dir ./output logs/*.BBL
+
+# Force export all logs (bypasses smart filtering)
+./target/release/bbl_parser --csv --force-export logs/*.BBL
 ```
 
+### Smart Export Filtering
+
+By default, the tool uses smart filtering to skip short test flights and focus on meaningful flight data:
+
+- **< 5 seconds**: Always skipped (brief arming, connectivity tests)
+- **5-15 seconds**: Kept only if data density > 1500 fps (sufficient for analysis)
+- **> 15 seconds**: Always kept (normal flight logs)
+
+**Examples:**
+- 0.7s flight with 1500 frames → Skipped (too short)
+- 2.2s flight with 4407 frames (2003 fps) → Kept (good data density)
+- 10s flight with 8000 frames (800 fps) → Skipped (insufficient data density)
+- 20s flight → Always kept (normal duration)
+
+Use `--force-export` to bypass filtering and export all logs regardless of duration or data quality.
+
 ## Output
 
 ### Console Statistics (Always Displayed)

src/main.rs

@@ -190,6 +190,7 @@ struct ExportOptions {
     gpx: bool,
     event: bool,
     output_dir: Option<String>,
+    force_export: bool,
 }
 
 // Pre-computed CSV field mapping for performance
@@ -304,12 +305,19 @@ fn main() -> Result<()> {
                 .help("Export event data (E frames) to JSON files")
                 .action(clap::ArgAction::SetTrue),
         )
+        .arg(
+            Arg::new("force-export")
+                .long("force-export")
+                .help("Force export of all logs, including short flights (bypasses smart filtering: <5s skip, 5-15s needs >1500fps, >15s keep)")
+                .action(clap::ArgAction::SetTrue),
+        )
         .get_matches();
 
     let debug = matches.get_flag("debug");
     let export_csv = matches.get_flag("csv");
     let export_gpx = matches.get_flag("gpx") || matches.get_flag("gps");
     let export_event = matches.get_flag("event");
+    let force_export = matches.get_flag("force-export");
     let output_dir = matches.get_one::<String>("output-dir").cloned();
     let file_patterns: Vec<&String> = matches.get_many::<String>("files").unwrap().collect();
 
@@ -318,6 +326,7 @@ fn main() -> Result<()> {
         gpx: export_gpx,
         event: export_event,
         output_dir: output_dir.clone(),
+        force_export,
     };
 
     // Keep legacy csv_options for compatibility
@@ -1028,6 +1037,160 @@ fn display_log_info(log: &BBLLog) {
     }
 }
 
+/// Determines if a log should be skipped for export based on duration and frame count
+/// Uses smart filtering: <5s always skip, 5-15s keep if good data density (>1500fps), >15s always keep
+fn should_skip_export(log: &BBLLog, force_export: bool) -> (bool, String) {
+    if force_export {
+        return (false, String::new()); // Never skip when forced
+    }
+
+    const VERY_SHORT_DURATION_MS: u64 = 5_000; // 5 seconds - always skip
+    const SHORT_DURATION_MS: u64 = 15_000; // 15 seconds - threshold for normal logs
+    const MIN_DATA_DENSITY_FPS: f64 = 1500.0; // Minimum fps for short logs
+    const FALLBACK_MIN_FRAMES: u32 = 7_500; // ~5 seconds at 1500 fps (fallback when no duration)
+
+    // Check if we have duration information
+    if log.stats.start_time_us > 0 && log.stats.end_time_us > log.stats.start_time_us {
+        let duration_us = log
+            .stats
+            .end_time_us
+            .saturating_sub(log.stats.start_time_us);
+        let duration_ms = duration_us / 1000;
+        let duration_s = duration_ms as f64 / 1000.0;
+        let fps = log.stats.total_frames as f64 / duration_s;
+
+        // Very short logs: < 5 seconds → Always skip
+        if duration_ms < VERY_SHORT_DURATION_MS {
+            return (true, format!("too short ({:.1}s < 5.0s)", duration_s));
+        }
+
+        // Short logs: 5-15 seconds → Keep if sufficient data density (>1500 fps)
+        if duration_ms < SHORT_DURATION_MS {
+            if fps < MIN_DATA_DENSITY_FPS {
+                return (
+                    true,
+                    format!(
+                        "insufficient data density ({:.0}fps < {:.0}fps for {:.1}s log)",
+                        fps, MIN_DATA_DENSITY_FPS, duration_s
+                    ),
+                );
+            }
+            // Good data density, keep it
+            return (false, String::new());
+        }
+
+        // Normal logs: > 15 seconds → Check for minimal gyro activity (ground tests)
+        if duration_ms >= SHORT_DURATION_MS {
+            let (is_minimal_movement, max_variance) = has_minimal_gyro_activity(log);
+            if is_minimal_movement {
+                return (
+                    true,
+                    format!(
+                        "minimal gyro activity ({:.1} variance) - likely ground test",
+                        max_variance
+                    ),
+                );
+            }
+        }
+
+        return (false, String::new());
+    }
+
+    // No duration information available, fall back to frame count
+    // Skip if very low frame count (equivalent to <5s at minimum viable fps)
+    if log.stats.total_frames < FALLBACK_MIN_FRAMES {
+        return (
+            true,
+            format!(
+                "too few frames ({} < {}) and no duration info",
+                log.stats.total_frames, FALLBACK_MIN_FRAMES
+            ),
+        );
+    }
+
+    // Sufficient frames without duration info, keep it
+    (false, String::new())
+}
+
+/// Analyzes gyro variance to detect ground tests vs actual flight
+/// Returns true if the log appears to be a static ground test (minimal movement)
+fn has_minimal_gyro_activity(log: &BBLLog) -> (bool, f64) {
+    // Conservative thresholds to avoid false-skips
+    const MIN_SAMPLES_FOR_ANALYSIS: usize = 15; // Reduced for limited sample data 
+    const VERY_LOW_GYRO_VARIANCE_THRESHOLD: f64 = 0.3; // More aggressive threshold for ground test detection
+    
+    let mut gyro_x_values = Vec::new();
+    let mut gyro_y_values = Vec::new();
+    let mut gyro_z_values = Vec::new();
+    
+    // First try to use debug_frames if available (contains more comprehensive data)
+    if let Some(debug_frames) = &log.debug_frames {
+        // Collect gyro data from I and P frames in debug_frames
+        for (frame_type, frames) in debug_frames {
+            if *frame_type == 'I' || *frame_type == 'P' {
+                for frame in frames {
+                    if let Some(gyro_x) = frame.data.get("gyroADC[0]") {
+                        if let Some(gyro_y) = frame.data.get("gyroADC[1]") {
+                            if let Some(gyro_z) = frame.data.get("gyroADC[2]") {
+                                gyro_x_values.push(*gyro_x as f64);
+                                gyro_y_values.push(*gyro_y as f64);
+                                gyro_z_values.push(*gyro_z as f64);
+                            }
+                        }
+                    }
+                }
+            }
+        }
+    }
+    
+    // Fallback to sample_frames if debug_frames not available or insufficient data
+    if gyro_x_values.len() < MIN_SAMPLES_FOR_ANALYSIS {
+        for frame in &log.sample_frames {
+            if let Some(gyro_x) = frame.data.get("gyroADC[0]") {
+                if let Some(gyro_y) = frame.data.get("gyroADC[1]") {
+                    if let Some(gyro_z) = frame.data.get("gyroADC[2]") {
+                        gyro_x_values.push(*gyro_x as f64);
+                        gyro_y_values.push(*gyro_y as f64);
+                        gyro_z_values.push(*gyro_z as f64);
+                    }
+                }
+            }
+        }
+    }
+    
+    // Need sufficient data points for reliable analysis
+    if gyro_x_values.len() < MIN_SAMPLES_FOR_ANALYSIS {
+        return (false, 0.0); // Not enough data, don't skip (conservative approach)
+    }
+    
+    // Calculate variance for each axis
+    let variance_x = calculate_variance(&gyro_x_values);
+    let variance_y = calculate_variance(&gyro_y_values);  
+    let variance_z = calculate_variance(&gyro_z_values);
+    
+    // Use the maximum variance across all axes
+    let max_variance = variance_x.max(variance_y).max(variance_z);
+    
+    // Very conservative: only skip if ALL axes show extremely low variance
+    let is_minimal = max_variance < VERY_LOW_GYRO_VARIANCE_THRESHOLD;
+    
+    (is_minimal, max_variance)
+}
+
+/// Calculate variance of a dataset
+fn calculate_variance(values: &[f64]) -> f64 {
+    if values.len() < 2 {
+        return 0.0;
+    }
+    
+    let mean = values.iter().sum::<f64>() / values.len() as f64;
+    let variance = values.iter()
+        .map(|x| (x - mean).powi(2))
+        .sum::<f64>() / values.len() as f64;
+    
+    variance
+}
+
 #[allow(dead_code)]
 fn export_logs_to_csv(
     logs: &[BBLLog],
@@ -2454,6 +2617,19 @@ fn parse_bbl_file_streaming(
         // Display log info immediately
         display_log_info(&log);
 
+        // Check if we should skip exports for this log
+        let (should_skip, reason) = should_skip_export(&log, export_options.force_export);
+        if should_skip {
+            println!("Skipping exports for this log: {}", reason);
+            processed_logs += 1;
+
+            // Add separator between logs for clarity
+            if log_index + 1 < log_positions.len() {
+                println!();
+            }
+            continue;
+        }
+
         // Export CSV immediately while data is hot in cache
         if export_options.csv {
             if let Err(e) = export_single_log_to_csv(&log, file_path, csv_options, debug) {