diff --git a/src/export.rs b/src/export.rs index 1a0f9a2..60f1455 100644 --- a/src/export.rs +++ b/src/export.rs @@ -170,31 +170,19 @@ fn export_flight_data_to_csv(log: &BBLLog, output_path: &Path) -> Result<()> { .map(|(csv_name, _)| csv_name.clone()) .collect(); - // Collect all frames in chronological order - let mut all_frames = Vec::new(); - - if let Some(ref debug_frames) = log.debug_frames { - // Collect only I, P frames for CSV export (S frames are merged into I/P frames during parsing) - for frame_type in ['I', 'P'] { - if let Some(frames) = debug_frames.get(&frame_type) { - for frame in frames { - all_frames.push((frame.timestamp_us, frame_type, frame)); - } - } + // Collect all I and P frames in chronological order + let mut all_frames: Vec<(u64, char, &DecodedFrame)> = Vec::new(); + + // Use log.frames which contains all parsed frames + for frame in &log.frames { + if frame.frame_type == 'I' || frame.frame_type == 'P' { + all_frames.push((frame.timestamp_us, frame.frame_type, frame)); } } // Sort by timestamp all_frames.sort_by_key(|(timestamp, _, _)| *timestamp); - if all_frames.is_empty() { - // Write at least the sample frames if no debug frames - for frame in &log.sample_frames { - all_frames.push((frame.timestamp_us, frame.frame_type, frame)); - } - all_frames.sort_by_key(|(timestamp, _, _)| *timestamp); - } - if all_frames.is_empty() { return Ok(()); // No data to export } diff --git a/src/lib.rs b/src/lib.rs index 1ba6df7..d5c4b0f 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -12,7 +12,7 @@ //! //! let export_options = ExportOptions::default(); //! let log = parse_bbl_file(Path::new("flight.BBL"), export_options, false).unwrap(); -//! println!("Found {} frames", log.sample_frames.len()); +//! println!("Found {} frames", log.frames.len()); //! ``` // Module declarations diff --git a/src/main.rs b/src/main.rs index 4105a99..1fa74d9 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,7 +1,6 @@ use anyhow::{Context, Result}; use clap::{Arg, Command}; use glob::glob; -use semver::Version; use std::collections::{HashMap, HashSet}; use std::fs; use std::io::Write; @@ -9,21 +8,36 @@ use std::path::{Path, PathBuf}; // Import conversion functions from crate library to avoid code duplication use bbl_parser::conversion::{ - convert_gps_altitude, convert_gps_coordinate, convert_gps_course, convert_gps_speed, - format_failsafe_phase, format_flight_mode_flags, format_state_flags, generate_gpx_timestamp, + convert_amperage_to_amps, convert_vbat_to_volts, format_failsafe_phase, + format_flight_mode_flags, format_state_flags, generate_gpx_timestamp, }; -// Import parser types from crate library -use bbl_parser::parser::{ - parse_e_frame, parse_frame_data, parse_h_frame, parse_s_frame, BBLDataStream, -}; +// Import parser types from crate library - using crate's unified implementations +use bbl_parser::parser::{parse_frames, parse_headers_from_text}; // Import types from crate library -use bbl_parser::types::{EventFrame, FrameDefinition}; +use bbl_parser::types::{ + BBLHeader, BBLLog, DecodedFrame, EventFrame, GpsCoordinate, GpsHomeCoordinate, +}; + +// Test-only imports +#[cfg(test)] +use bbl_parser::types::{FrameDefinition, FrameStats}; + +// Import ExportOptions from crate library +use bbl_parser::ExportOptions; /// Maximum recursion depth to prevent stack overflow const MAX_RECURSION_DEPTH: usize = 100; +/// Get output directory from export options, falling back to file's parent directory or ".". +fn get_output_dir<'a>(export_options: &'a ExportOptions, file_path: &'a Path) -> &'a str { + export_options + .output_dir + .as_deref() + .unwrap_or_else(|| file_path.parent().and_then(|p| p.to_str()).unwrap_or(".")) +} + /// Expand input paths to a list of BBL files. /// If a path is a file, add it directly (will be filtered later for BBL/BFL/TXT extension). /// If a path is a directory, recursively find all BBL files within it. @@ -230,93 +244,6 @@ fn find_bbl_files_in_dir_with_depth( Ok(bbl_files) } -// FieldDefinition and FrameDefinition now imported from bbl_parser::types - -#[derive(Debug)] -struct BBLHeader { - firmware_revision: String, - board_info: String, - craft_name: String, - data_version: u8, - looptime: u32, - /// Log start datetime from header (ISO 8601 format) - log_start_datetime: Option, - i_frame_def: FrameDefinition, - p_frame_def: FrameDefinition, - s_frame_def: FrameDefinition, - g_frame_def: FrameDefinition, - h_frame_def: FrameDefinition, - sysconfig: HashMap, - all_headers: Vec, -} - -#[derive(Debug, Default)] -struct FrameStats { - i_frames: u32, - p_frames: u32, - h_frames: u32, - g_frames: u32, - e_frames: u32, - s_frames: u32, - total_frames: u32, - total_bytes: u64, - start_time_us: u64, - end_time_us: u64, - failed_frames: u32, - missing_iterations: u64, -} - -#[derive(Debug, Clone)] -struct DecodedFrame { - frame_type: char, - timestamp_us: u64, - #[allow(dead_code)] - loop_iteration: u32, - data: HashMap, -} - -// GPS related structures for GPX export -#[derive(Debug, Clone)] -struct GpsCoordinate { - latitude: f64, - longitude: f64, - altitude: f64, - timestamp_us: u64, - num_sats: Option, - #[allow(dead_code)] - speed: Option, - #[allow(dead_code)] - ground_course: Option, -} - -#[derive(Debug, Clone)] -struct GpsHomeCoordinate { - #[allow(dead_code)] - home_latitude: f64, - #[allow(dead_code)] - home_longitude: f64, - #[allow(dead_code)] - timestamp_us: u64, -} - -#[derive(Debug)] -struct BBLLog { - log_number: usize, - total_logs: usize, - header: BBLHeader, - stats: FrameStats, - sample_frames: Vec, // Only store a few sample frames, not all - debug_frames: Option>>, // Frame data by type for debug output -} - -// Frame history for prediction during parsing -struct FrameHistory { - current_frame: Vec, - previous_frame: Vec, - previous2_frame: Vec, - valid: bool, -} - #[allow(dead_code)] fn should_have_frame(frame_index: u32, sysconfig: &HashMap) -> bool { let frame_interval_i = sysconfig.get("frameIntervalI").copied().unwrap_or(32); @@ -333,15 +260,6 @@ struct CsvExportOptions { output_dir: Option, } -#[derive(Debug, Clone)] -struct ExportOptions { - csv: bool, - gpx: bool, - event: bool, - output_dir: Option, - force_export: bool, -} - // Pre-computed CSV field mapping for performance #[derive(Debug)] struct CsvFieldMap { @@ -669,7 +587,7 @@ fn parse_bbl_file( let log_data = &file_data[start_pos..end_pos]; // Parse this individual log - let (log, _gps_coords, _home_coords, _events) = parse_single_log( + let log = parse_single_log( log_data, log_index + 1, log_positions.len(), @@ -682,21 +600,16 @@ fn parse_bbl_file( Ok(logs) } -// Type alias to reduce complexity -type ParseSingleLogResult = ( - BBLLog, - Vec, - Vec, - Vec, -); - +/// Parse a single log from binary data. +/// +/// Parses all frames and stores them in BBLLog.frames for CSV export. fn parse_single_log( log_data: &[u8], log_number: usize, total_logs: usize, debug: bool, export_options: &ExportOptions, -) -> Result { +) -> Result { // Find where headers end and binary data begins let mut header_end = 0; for i in 1..log_data.len() { @@ -746,7 +659,7 @@ fn parse_single_log( sample_duration / 1000 ); println!( - "DEBUG: Total frames: {}, Sample frames: {}", + "DEBUG: Total frames: {}, Stored frames: {}", stats.total_frames, frames.len() ); @@ -757,224 +670,14 @@ fn parse_single_log( total_logs, header, stats, - sample_frames: frames, + frames, debug_frames, + gps_coordinates: gps_coords, + home_coordinates: home_coords, + event_frames: events, }; - Ok((log, gps_coords, home_coords, events)) -} - -fn parse_headers_from_text(header_text: &str, debug: bool) -> Result { - let mut all_headers = Vec::new(); - let mut firmware_revision = String::new(); - let mut board_info = String::new(); - let mut craft_name = String::new(); - let mut data_version = 2u8; - let mut looptime = 0u32; - let mut log_start_datetime: Option = None; - let mut sysconfig = HashMap::new(); - - // Initialize frame definitions - let mut i_frame_def = FrameDefinition::new(); - let mut p_frame_def = FrameDefinition::new(); - let mut s_frame_def = FrameDefinition::new(); - let mut g_frame_def = FrameDefinition::new(); - let mut h_frame_def = FrameDefinition::new(); - - for line in header_text.lines() { - let line = line.trim(); - if line.is_empty() || !line.starts_with("H ") { - continue; - } - - all_headers.push(line.to_string()); - - // Parse specific headers following JavaScript reference - if line.starts_with("H Firmware revision:") { - firmware_revision = line - .strip_prefix("H Firmware revision:") - .unwrap_or("") - .trim() - .to_string(); - } else if line.starts_with("H Board information:") { - board_info = line - .strip_prefix("H Board information:") - .unwrap_or("") - .trim() - .to_string(); - } else if line.starts_with("H Craft name:") { - craft_name = line - .strip_prefix("H Craft name:") - .unwrap_or("") - .trim() - .to_string(); - } else if line.starts_with("H Data version:") { - if let Ok(version) = line - .strip_prefix("H Data version:") - .unwrap_or("2") - .trim() - .parse() - { - data_version = version; - } - } else if line.starts_with("H Log start datetime:") { - // Parse log start datetime for GPX timestamp generation - // Format: "2024-10-10T18:37:25.559+00:00" or "0000-01-01T00:00:00.000+00:00" if not set - if let Some(datetime_str) = line.strip_prefix("H Log start datetime:") { - log_start_datetime = Some(datetime_str.trim().to_string()); - } - } else if line.starts_with("H looptime:") { - if let Ok(lt) = line - .strip_prefix("H looptime:") - .unwrap_or("0") - .trim() - .parse() - { - looptime = lt; - } - } else if line.starts_with("H Field I name:") { - // Parse I frame field names - if let Some(field_str) = line.strip_prefix("H Field I name:") { - let names: Vec = - field_str.split(',').map(|s| s.trim().to_string()).collect(); - i_frame_def = FrameDefinition::from_field_names(names); - } - } else if line.starts_with("H Field P name:") { - // Parse P frame field names - if let Some(field_str) = line.strip_prefix("H Field P name:") { - let names: Vec = - field_str.split(',').map(|s| s.trim().to_string()).collect(); - p_frame_def = FrameDefinition::from_field_names(names); - } - } else if line.starts_with("H Field S name:") { - // Parse S frame field names - if let Some(field_str) = line.strip_prefix("H Field S name:") { - let names: Vec = - field_str.split(',').map(|s| s.trim().to_string()).collect(); - s_frame_def = FrameDefinition::from_field_names(names); - } - } else if line.starts_with("H Field G name:") { - // Parse G frame field names - if let Some(field_str) = line.strip_prefix("H Field G name:") { - let names: Vec = - field_str.split(',').map(|s| s.trim().to_string()).collect(); - g_frame_def = FrameDefinition::from_field_names(names); - } - } else if line.starts_with("H Field H name:") { - // Parse H frame field names - if let Some(field_str) = line.strip_prefix("H Field H name:") { - let names: Vec = - field_str.split(',').map(|s| s.trim().to_string()).collect(); - h_frame_def = FrameDefinition::from_field_names(names); - } - } else if line.starts_with("H Field I signed:") { - // Parse I frame signed data - if let Some(signed_str) = line.strip_prefix("H Field I signed:") { - let signed_data = parse_signed_data(signed_str); - i_frame_def.update_signed(&signed_data); - } - } else if line.starts_with("H Field I predictor:") { - // Parse I frame predictors - if let Some(pred_str) = line.strip_prefix("H Field I predictor:") { - let predictors = parse_numeric_data(pred_str); - i_frame_def.update_predictors(&predictors); - } - } else if line.starts_with("H Field I encoding:") { - // Parse I frame encodings - if let Some(enc_str) = line.strip_prefix("H Field I encoding:") { - let encodings = parse_numeric_data(enc_str); - i_frame_def.update_encoding(&encodings); - } - } else if line.starts_with("H Field P predictor:") { - // Parse P frame predictors - if let Some(pred_str) = line.strip_prefix("H Field P predictor:") { - let predictors = parse_numeric_data(pred_str); - - // P frames inherit field names from I frames but have their own predictors - if p_frame_def.field_names.is_empty() && !i_frame_def.field_names.is_empty() { - p_frame_def = - FrameDefinition::from_field_names(i_frame_def.field_names.clone()); - } - p_frame_def.update_predictors(&predictors); - } - } else if line.starts_with("H Field P encoding:") { - // Parse P frame encodings - if let Some(enc_str) = line.strip_prefix("H Field P encoding:") { - let encodings = parse_numeric_data(enc_str); - // P frames inherit field names from I frames but have their own encodings - if p_frame_def.field_names.is_empty() && !i_frame_def.field_names.is_empty() { - p_frame_def = - FrameDefinition::from_field_names(i_frame_def.field_names.clone()); - } - p_frame_def.update_encoding(&encodings); - } - } else if line.starts_with("H Field S signed:") { - // Parse S frame signed data - if let Some(signed_str) = line.strip_prefix("H Field S signed:") { - let signed_data = parse_signed_data(signed_str); - s_frame_def.update_signed(&signed_data); - } - } else if line.starts_with("H Field S predictor:") { - // Parse S frame predictors - if let Some(pred_str) = line.strip_prefix("H Field S predictor:") { - let predictors = parse_numeric_data(pred_str); - s_frame_def.update_predictors(&predictors); - } - } else if line.starts_with("H Field S encoding:") { - // Parse S frame encodings - if let Some(enc_str) = line.strip_prefix("H Field S encoding:") { - let encodings = parse_numeric_data(enc_str); - s_frame_def.update_encoding(&encodings); - } - } - - // Parse additional sysconfig values - if let Some(colon_pos) = line.find(':') { - if let Some(field_name) = line.get(2..colon_pos) { - if let Some(field_value) = line.get(colon_pos + 1..) { - let field_name = field_name.trim(); - let field_value = field_value.trim(); - - // Store numeric values that might be useful later - if let Ok(num_value) = field_value.parse::() { - sysconfig.insert(field_name.to_string(), num_value); - if debug && field_name == "vbatref" { - eprintln!("DEBUG: Found vbatref={} in headers", num_value); - } - } - } - } - } - } - - if debug { - println!( - "Parsed headers: Firmware={firmware_revision}, Board={board_info}, Craft={craft_name}" - ); - println!("Data version: {data_version}, Looptime: {looptime}"); - } - - // Extract vbatref from sysconfig for debug output - let vbatref = sysconfig.get("vbatref").copied().unwrap_or(0); - if debug && vbatref > 0 { - eprintln!("DEBUG: Found vbatref={} in headers", vbatref); - } - - Ok(BBLHeader { - firmware_revision, - board_info, - craft_name, - data_version, - looptime, - log_start_datetime, - i_frame_def, - p_frame_def, - s_frame_def, - g_frame_def, - h_frame_def, - sysconfig, - all_headers, - }) + Ok(log) } #[allow(dead_code)] @@ -1304,9 +1007,9 @@ fn has_minimal_gyro_activity(log: &BBLLog) -> (bool, f64) { } } - // Fallback to sample_frames if debug_frames not available or insufficient data + // Fallback to frames if debug_frames not available or insufficient data if gyro_x_values.len() < MIN_SAMPLES_FOR_ANALYSIS { - for frame in &log.sample_frames { + for frame in &log.frames { if let Some(gyro_x) = frame.data.get("gyroADC[0]") { if let Some(gyro_y) = frame.data.get("gyroADC[1]") { if let Some(gyro_z) = frame.data.get("gyroADC[2]") { @@ -1503,32 +1206,19 @@ fn export_flight_data_to_csv(log: &BBLLog, output_path: &Path, debug: bool) -> R .map(|(csv_name, _)| csv_name.clone()) .collect(); - // Collect all frames in chronological order - let mut all_frames = Vec::new(); + // Collect all I and P frames in chronological order + // S frames are merged into I/P frames during parsing, matching blackbox_decode behavior + let mut all_frames: Vec<(u64, char, &DecodedFrame)> = Vec::new(); - if let Some(ref debug_frames) = log.debug_frames { - // Collect only I, P frames for CSV export (S frames are merged into I/P frames during parsing) - // This matches blackbox_decode behavior where S-frame data doesn't create separate CSV rows - for frame_type in ['I', 'P'] { - if let Some(frames) = debug_frames.get(&frame_type) { - for frame in frames { - all_frames.push((frame.timestamp_us, frame_type, frame)); - } - } + for frame in &log.frames { + if frame.frame_type == 'I' || frame.frame_type == 'P' { + all_frames.push((frame.timestamp_us, frame.frame_type, frame)); } } // Sort by timestamp all_frames.sort_by_key(|(timestamp, _, _)| *timestamp); - if all_frames.is_empty() { - // Write at least the sample frames if no debug frames - for frame in &log.sample_frames { - all_frames.push((frame.timestamp_us, frame.frame_type, frame)); - } - all_frames.sort_by_key(|(timestamp, _, _)| *timestamp); - } - if all_frames.is_empty() { return Ok(()); // No data to export } @@ -1641,767 +1331,6 @@ fn export_flight_data_to_csv(log: &BBLLog, output_path: &Path, debug: bool) -> R Ok(()) } -type ParseFramesResult = Result<( - FrameStats, - Vec, - Option>>, - Vec, - Vec, - Vec, -)>; - -fn parse_frames( - binary_data: &[u8], - header: &BBLHeader, - debug: bool, - export_options: &ExportOptions, -) -> ParseFramesResult { - let mut stats = FrameStats::default(); - let mut sample_frames = Vec::new(); - let mut debug_frames: HashMap> = HashMap::new(); - let mut last_main_frame_timestamp = 0u64; // Track timestamp for S frames - - // Track the most recent S-frame data for merging (following JavaScript approach) - let mut last_slow_data: HashMap = HashMap::new(); - - // Decide whether to store all frames based on CSV export requirement - let store_all_frames = export_options.csv; // Store all frames when CSV export is requested - - if debug { - println!("Binary data size: {} bytes", binary_data.len()); - if !binary_data.is_empty() { - println!( - "First 16 bytes: {:02X?}", - &binary_data[..16.min(binary_data.len())] - ); - } - } - - if binary_data.is_empty() { - return Ok(( - stats, - sample_frames, - Some(debug_frames), - Vec::new(), - Vec::new(), - Vec::new(), - )); - } - - // Initialize frame history for proper P-frame parsing - let mut frame_history = FrameHistory { - current_frame: vec![0; header.i_frame_def.count], - previous_frame: vec![0; header.i_frame_def.count], - previous2_frame: vec![0; header.i_frame_def.count], - valid: false, - }; - - // Collections for GPS and Event export - let mut gps_coordinates: Vec = Vec::new(); - let mut home_coordinates: Vec = Vec::new(); - let mut event_frames: Vec = Vec::new(); - - // GPS frame history for differential encoding - let mut gps_frame_history: Vec = Vec::new(); - - let mut stream = BBLDataStream::new(binary_data); - - // Main frame parsing loop - process frames as a stream, don't store all - while !stream.eof { - let frame_start_pos = stream.pos; - - match stream.read_byte() { - Ok(frame_type_byte) => { - let frame_type = match frame_type_byte as char { - 'I' => 'I', - 'P' => 'P', - 'H' => 'H', - 'G' => 'G', - 'E' => 'E', - 'S' => 'S', - _ => { - if debug && stats.failed_frames < 3 { - println!( - "Unknown frame type byte 0x{:02X} ('{:?}') at offset {}", - frame_type_byte, frame_type_byte as char, frame_start_pos - ); - } - stats.failed_frames += 1; - continue; - } - }; - - if debug && stats.total_frames < 3 { - println!("Found frame type '{frame_type}' at offset {frame_start_pos}"); - } - - // Parse frame using proper streaming logic - let mut frame_data = HashMap::new(); - let mut parsing_success = false; - - match frame_type { - 'I' => { - if header.i_frame_def.count > 0 { - // I-frames reset the prediction history - frame_history.current_frame.fill(0); - - if parse_frame_data( - &mut stream, - &header.i_frame_def, - &mut frame_history.current_frame, - None, // I-frames don't use prediction - None, - 0, - false, // Not raw - header.data_version, - &header.sysconfig, - debug, - ) - .is_ok() - { - // Update time and loop iteration from parsed frame - for (i, field_name) in - header.i_frame_def.field_names.iter().enumerate() - { - if i < frame_history.current_frame.len() { - let value = frame_history.current_frame[i]; - frame_data.insert(field_name.clone(), value); - } - } - - // Merge lastSlow data into I-frame (following JavaScript approach) - for (key, value) in &last_slow_data { - frame_data.insert(key.clone(), *value); - } - - if debug && stats.i_frames < 3 { - println!("DEBUG: I-frame merged lastSlow. rxSignalReceived: {:?}, rxFlightChannelsValid: {:?}", - frame_data.get("rxSignalReceived"), frame_data.get("rxFlightChannelsValid")); - } - - // Update history for future P-frames - // Both the previous and previous-previous states become the I-frame, - // because we can't look further into the past than the I-frame - frame_history - .previous_frame - .copy_from_slice(&frame_history.current_frame); - frame_history - .previous2_frame - .copy_from_slice(&frame_history.current_frame); - frame_history.valid = true; - - // **BLACKBOX_DECODE COMPATIBILITY**: Validate frame before accepting - let current_time = - frame_data.get("time").copied().unwrap_or(0) as u64; - let current_loop = - frame_data.get("loopIteration").copied().unwrap_or(0) as u32; - - // Apply minimal validation - blackbox_decode includes frames from loop 0 - // Only reject frames with clearly invalid data (zero time/loop when data should be present) - let is_valid_frame = - current_time > 0 && (current_loop > 0 || current_time > 1000); - - if is_valid_frame { - parsing_success = true; - stats.i_frames += 1; - - if debug && stats.i_frames <= 3 { - println!( - "DEBUG: Accepted I-frame - time:{}, loop:{}", - current_time, current_loop - ); - } - } else { - if debug && stats.i_frames < 5 { - println!( - "DEBUG: Rejected I-frame - time:{}, loop:{} (invalid)", - current_time, current_loop - ); - } - parsing_success = false; - } - } - } - } - 'P' => { - if header.p_frame_def.count > 0 && frame_history.valid { - let mut p_frame_values = vec![0i32; header.p_frame_def.count]; - - if parse_frame_data( - &mut stream, - &header.p_frame_def, - &mut p_frame_values, - Some(&frame_history.previous_frame), - Some(&frame_history.previous2_frame), - 0, // TODO: Calculate skipped frames properly - false, // Not raw - header.data_version, - &header.sysconfig, - debug, - ) - .is_ok() - { - // P-frames: parse_frame_data already computed correct absolute values - // Copy previous frame as base, then update only P-frame fields - frame_history - .current_frame - .copy_from_slice(&frame_history.previous_frame); - - // Update only the fields that are present in P-frame with computed values - for (i, field_name) in - header.p_frame_def.field_names.iter().enumerate() - { - if i < p_frame_values.len() { - // Find corresponding index in I-frame structure - if let Some(i_frame_idx) = header - .i_frame_def - .field_names - .iter() - .position(|name| name == field_name) - { - if i_frame_idx < frame_history.current_frame.len() { - // p_frame_values[i] contains correctly calculated absolute value - frame_history.current_frame[i_frame_idx] = - p_frame_values[i]; - } - } - } - } - - // Copy current frame to output using I-frame field names and structure - for (i, field_name) in - header.i_frame_def.field_names.iter().enumerate() - { - if i < frame_history.current_frame.len() { - let value = frame_history.current_frame[i]; - frame_data.insert(field_name.clone(), value); - } - } - - // Merge lastSlow data into P-frame (following JavaScript approach) - for (key, value) in &last_slow_data { - frame_data.insert(key.clone(), *value); - } - - if debug && stats.p_frames < 3 { - println!("DEBUG: P-frame merged lastSlow. rxSignalReceived: {:?}, rxFlightChannelsValid: {:?}", - frame_data.get("rxSignalReceived"), frame_data.get("rxFlightChannelsValid")); - } - - // Update history - frame_history - .previous2_frame - .copy_from_slice(&frame_history.previous_frame); - frame_history - .previous_frame - .copy_from_slice(&frame_history.current_frame); - - // **BLACKBOX_DECODE COMPATIBILITY**: Validate P-frame before accepting - let current_time = - frame_data.get("time").copied().unwrap_or(0) as u64; - let current_loop = - frame_data.get("loopIteration").copied().unwrap_or(0) as u32; - - // Apply minimal validation - blackbox_decode includes frames from loop 0 - // Only reject frames with clearly invalid data (zero time/loop when data should be present) - let is_valid_frame = - current_time > 0 && (current_loop > 0 || current_time > 1000); - - if is_valid_frame { - parsing_success = true; - stats.p_frames += 1; - - if debug && stats.p_frames <= 3 { - println!( - "DEBUG: Accepted P-frame - time:{}, loop:{}", - current_time, current_loop - ); - } - } else { - if debug && stats.p_frames < 5 { - println!( - "DEBUG: Rejected P-frame - time:{}, loop:{} (invalid)", - current_time, current_loop - ); - } - parsing_success = false; - } - } - } else { - // Skip P-frame if we don't have valid I-frame history - skip_frame(&mut stream, frame_type, debug)?; - stats.failed_frames += 1; - } - } - 'S' => { - if debug && stats.s_frames < 5 { - println!( - "DEBUG: Found S-frame, header.s_frame_def.count={}", - header.s_frame_def.count - ); - } - if header.s_frame_def.count > 0 { - if let Ok(data) = parse_s_frame(&mut stream, &header.s_frame_def, debug) - { - // Following JavaScript approach: update lastSlow data - if debug && stats.s_frames < 3 { - println!("DEBUG: Processing S-frame with data: {data:?}"); - } - - for (key, value) in &data { - last_slow_data.insert(key.clone(), *value); - } - - if debug && stats.s_frames < 3 { - println!( - "DEBUG: S-frame data updated lastSlow: {last_slow_data:?}" - ); - } - - // S-frames don't create separate CSV rows - they only update lastSlow data - // that gets merged into subsequent I/P frames (blackbox_decode compatibility) - stats.s_frames += 1; - - if debug && stats.s_frames <= 3 { - println!("DEBUG: S-frame count incremented to {} (data merged into lastSlow)", stats.s_frames); - } - } else if debug && stats.s_frames < 5 { - println!("DEBUG: S-frame parsing failed"); - } - } else if debug && stats.s_frames < 5 { - println!("DEBUG: Skipping S-frame - header.s_frame_def.count is 0"); - } - } - 'H' => { - if header.h_frame_def.count > 0 { - if let Ok(data) = parse_h_frame(&mut stream, &header.h_frame_def, debug) - { - frame_data = data.clone(); - parsing_success = true; - stats.h_frames += 1; - - // Extract GPS home coordinates for GPX export if enabled - if export_options.gpx { - let timestamp = last_main_frame_timestamp; - - if let (Some(&home_lat_raw), Some(&home_lon_raw)) = ( - frame_data.get("GPS_home[0]"), - frame_data.get("GPS_home[1]"), - ) { - if debug && home_coordinates.is_empty() { - println!("DEBUG: HOME raw values - home_lat_raw: {}, home_lon_raw: {}", home_lat_raw, home_lon_raw); - println!( - "DEBUG: HOME converted - lat: {:.7}, lon: {:.7}", - convert_gps_coordinate(home_lat_raw), - convert_gps_coordinate(home_lon_raw) - ); - } - - let home_coordinate = GpsHomeCoordinate { - home_latitude: convert_gps_coordinate(home_lat_raw), - home_longitude: convert_gps_coordinate(home_lon_raw), - timestamp_us: timestamp, - }; - home_coordinates.push(home_coordinate); - } - } - } - } else { - skip_frame(&mut stream, frame_type, debug)?; - stats.h_frames += 1; - parsing_success = true; - } - } - 'G' => { - if header.g_frame_def.count > 0 { - // Initialize GPS frame history if needed - if gps_frame_history.is_empty() { - gps_frame_history = vec![0i32; header.g_frame_def.count]; - } - - let mut g_frame_values = vec![0i32; header.g_frame_def.count]; - - if parse_frame_data( - &mut stream, - &header.g_frame_def, - &mut g_frame_values, - Some(&gps_frame_history), // Use GPS frame history for differential encoding - None, // GPS frames typically don't use previous2 - 0, // TODO: Calculate skipped frames properly - false, // Not raw - header.data_version, - &header.sysconfig, - debug, - ) - .is_ok() - { - // Update GPS frame history with new values - gps_frame_history.copy_from_slice(&g_frame_values); - - // Copy GPS frame data to output - for (i, field_name) in - header.g_frame_def.field_names.iter().enumerate() - { - if i < g_frame_values.len() { - let value = g_frame_values[i]; - frame_data.insert(field_name.clone(), value); - } - } - - parsing_success = true; - stats.g_frames += 1; - - // Extract GPS coordinates for GPX export if enabled - if export_options.gpx { - let gps_time = - frame_data.get("time").copied().unwrap_or(0) as u64; - let timestamp = if gps_time > 0 { - gps_time - } else { - last_main_frame_timestamp - }; - - if let (Some(&lat_raw), Some(&lon_raw), Some(&alt_raw)) = ( - frame_data.get("GPS_coord[0]"), - frame_data.get("GPS_coord[1]"), - frame_data.get("GPS_altitude"), - ) { - // GPS coordinates are deltas from home position - // Need to add home coordinates to get actual GPS position - let actual_lat = - if let Some(home_coord) = home_coordinates.first() { - home_coord.home_latitude - + convert_gps_coordinate(lat_raw) - } else { - convert_gps_coordinate(lat_raw) - }; - - let actual_lon = - if let Some(home_coord) = home_coordinates.first() { - home_coord.home_longitude - + convert_gps_coordinate(lon_raw) - } else { - convert_gps_coordinate(lon_raw) - }; - - if debug && gps_coordinates.len() < 3 { - println!("DEBUG: GPS raw values - lat_raw: {}, lon_raw: {}, alt_raw: {}", lat_raw, lon_raw, alt_raw); - println!("DEBUG: GPS converted - lat: {:.7}, lon: {:.7}, alt: {:.2}", - actual_lat, actual_lon, - convert_gps_altitude(alt_raw, &header.firmware_revision)); - } - - let coordinate = GpsCoordinate { - latitude: actual_lat, - longitude: actual_lon, - altitude: convert_gps_altitude( - alt_raw, - &header.firmware_revision, - ), - timestamp_us: timestamp, - num_sats: frame_data.get("GPS_numSat").copied(), - speed: frame_data - .get("GPS_speed") - .map(|&s| convert_gps_speed(s)), - ground_course: frame_data - .get("GPS_ground_course") - .map(|&c| convert_gps_course(c)), - }; - gps_coordinates.push(coordinate); - } - } - } - } else { - skip_frame(&mut stream, frame_type, debug)?; - stats.g_frames += 1; - parsing_success = true; - } - } - 'E' => { - if let Ok(mut event_frame) = parse_e_frame(&mut stream, debug) { - // Store event data for potential export - // For now, create a dummy data entry for consistency - frame_data - .insert("event_type".to_string(), event_frame.event_type as i32); - frame_data.insert("event_description".to_string(), 0); // Can't store string in i32 map - parsing_success = true; - stats.e_frames += 1; - - // Collect event frames for JSON export if enabled - if export_options.event { - event_frame.timestamp_us = last_main_frame_timestamp; - event_frames.push(event_frame); - } - - if debug && stats.e_frames <= 3 { - println!( - "DEBUG: Parsed E-frame - Type: {}", - frame_data.get("event_type").unwrap_or(&0) - ); - } - } else { - skip_frame(&mut stream, frame_type, debug)?; - stats.e_frames += 1; - parsing_success = true; - } - } - _ => {} - }; - - if !parsing_success { - stats.failed_frames += 1; - } - - stats.total_frames += 1; - - // Show progress for large files - if (debug && stats.total_frames % 50000 == 0) || stats.total_frames % 100000 == 0 { - println!("Parsed {} frames so far...", stats.total_frames); - std::io::stdout().flush().unwrap_or_default(); - } - - // Store only a few sample frames for display purposes - if parsing_success && sample_frames.len() < 10 { - // Extract timing before moving frame_data - let timestamp_us = frame_data.get("time").copied().unwrap_or(0) as u64; - let loop_iteration = - frame_data.get("loopIteration").copied().unwrap_or(0) as u32; - - // Update last timestamp for main frames (I, P) - if (frame_type == 'I' || frame_type == 'P') && timestamp_us > 0 { - last_main_frame_timestamp = timestamp_us; - } - - // S frames inherit timestamp from last main frame - let final_timestamp = if frame_type == 'S' && timestamp_us == 0 { - last_main_frame_timestamp - } else { - timestamp_us - }; - - if debug && (frame_type == 'I' || frame_type == 'P') && sample_frames.len() < 3 - { - println!( - "DEBUG: Frame {:?} has timestamp {}. Available fields: {:?}", - frame_type, - timestamp_us, - frame_data.keys().collect::>() - ); - if let Some(time_val) = frame_data.get("time") { - println!("DEBUG: 'time' field value: {time_val}"); - } - if let Some(loop_val) = frame_data.get("loopIteration") { - println!("DEBUG: 'loopIteration' field value: {loop_val}"); - } - } - - let decoded_frame = DecodedFrame { - frame_type, - timestamp_us: final_timestamp, - loop_iteration, - data: frame_data.clone(), - }; - sample_frames.push(decoded_frame.clone()); - - // Store debug frames (always store for sample frames) - let debug_frame_list = debug_frames.entry(frame_type).or_default(); - debug_frame_list.push(decoded_frame); - } else if parsing_success && store_all_frames { - // Store ALL frames for CSV export when requested - let debug_frame_list = debug_frames.entry(frame_type).or_default(); - // Store all frames for complete CSV export - memory usage managed by processing in chunks - let timestamp_us = frame_data.get("time").copied().unwrap_or(0) as u64; - let loop_iteration = - frame_data.get("loopIteration").copied().unwrap_or(0) as u32; - - // Update last timestamp for main frames (I, P) - if (frame_type == 'I' || frame_type == 'P') && timestamp_us > 0 { - last_main_frame_timestamp = timestamp_us; - } - - // S frames inherit timestamp from last main frame - let final_timestamp = if frame_type == 'S' && timestamp_us == 0 { - last_main_frame_timestamp - } else { - timestamp_us - }; - - if debug && timestamp_us == 0 && debug_frame_list.len() < 5 { - println!( - "DEBUG: Non-sample frame {:?} has timestamp 0->{}. Fields: {:?}", - frame_type, - final_timestamp, - frame_data.keys().collect::>() - ); - } - - let decoded_frame = DecodedFrame { - frame_type, - timestamp_us: final_timestamp, - loop_iteration, - data: frame_data.clone(), - }; - debug_frame_list.push(decoded_frame); - } - - // Update timing from first and last valid frames with time data - if parsing_success { - if let Some(time_us) = frame_data.get("time") { - let time_val = *time_us as u64; - if stats.start_time_us == 0 { - stats.start_time_us = time_val; - } - stats.end_time_us = time_val; - } - } - } - Err(_) => break, - } - - // More aggressive safety limits to prevent hanging - if stats.total_frames > 1000000 || stats.failed_frames > 10000 { - if debug { - println!("Hit safety limit - stopping frame parsing"); - } - break; - } - } - - stats.total_bytes = binary_data.len() as u64; - - if debug { - println!( - "Parsed {} frames: {} I, {} P, {} H, {} G, {} E, {} S", - stats.total_frames, - stats.i_frames, - stats.p_frames, - stats.h_frames, - stats.g_frames, - stats.e_frames, - stats.s_frames - ); - println!("Failed to parse: {} frames", stats.failed_frames); - } - - Ok(( - stats, - sample_frames, - Some(debug_frames), - gps_coordinates, - home_coordinates, - event_frames, - )) -} - -// Note: parse_g_frame is no longer used - G frames now use differential encoding -// like P frames in the main parsing loop for correct GPS coordinate calculation - -fn skip_frame(stream: &mut BBLDataStream, frame_type: char, debug: bool) -> Result<()> { - if debug { - println!("Skipping {frame_type} frame"); - } - - // Skip frame by reading a few bytes - this is a simple heuristic - // In a full implementation, we'd parse these properly too - match frame_type { - 'E' => { - // Event frames - read event type and some data - let _event_type = stream.read_byte()?; - // Read up to 16 bytes of event data - for _ in 0..16 { - if stream.eof { - break; - } - let _ = stream.read_byte(); - } - } - 'G' | 'H' => { - // GPS frames - read several fields - for _ in 0..7 { - if stream.eof { - break; - } - let _ = stream.read_unsigned_vb(); - } - } - _ => { - // Unknown frame type - read a few bytes - for _ in 0..8 { - if stream.eof { - break; - } - let _ = stream.read_byte(); - } - } - } - - Ok(()) -} - -fn parse_signed_data(signed_data: &str) -> Vec { - signed_data.split(',').map(|s| s.trim() == "1").collect() -} - -fn parse_numeric_data(numeric_data: &str) -> Vec { - numeric_data - .split(',') - .filter_map(|s| s.trim().parse().ok()) - .collect() -} - -/// Converts raw vbatLatest value to volts using firmware-aware scaling. -/// -/// Betaflight < 4.3.0: tenths (0.1V units) -/// Betaflight >= 4.3.0: hundredths (0.01V units) -/// EmuFlight: always tenths (0.1V units) -/// iNav: always hundredths (0.01V units) -fn convert_vbat_to_volts(raw_value: i32, firmware_revision: &str) -> f32 { - // Determine scaling factor based on firmware - let scale_factor = if firmware_revision.contains("EmuFlight") { - // EmuFlight always uses tenths - 0.1 - } else if firmware_revision.contains("iNav") { - // iNav always uses hundredths - 0.01 - } else if firmware_revision.contains("Betaflight") { - // Betaflight version-dependent scaling - if let Some(version) = extract_firmware_version(firmware_revision) { - if version >= Version::new(4, 3, 0) { - 0.01 // hundredths for >= 4.3.0 - } else { - 0.1 // tenths for < 4.3.0 - } - } else { - // Default to modern Betaflight scaling if version can't be parsed - 0.01 - } - } else { - // Unknown firmware, default to hundredths - 0.01 - }; - - raw_value as f32 * scale_factor -} - -/// Extract version from firmware revision string -fn extract_firmware_version(firmware_revision: &str) -> Option { - // Parse version from strings like "Betaflight 4.5.1 (77d01ba3b) AT32F435M" - let words: Vec<&str> = firmware_revision.split_whitespace().collect(); - for (i, word) in words.iter().enumerate() { - if word.to_lowercase().contains("betaflight") && i + 1 < words.len() { - if let Ok(version) = Version::parse(words[i + 1]) { - return Some(version); - } - } - } - None -} - -/// Converts raw amperageLatest value to amps (0.01A units) -fn convert_amperage_to_amps(raw_value: i32) -> f32 { - raw_value as f32 / 100.0 -} - fn parse_bbl_file_streaming( file_path: &Path, debug: bool, @@ -2460,7 +1389,7 @@ fn parse_bbl_file_streaming( let log_data = &file_data[start_pos..end_pos]; // Parse this individual log - let (log, gps_coords, home_coords, events) = parse_single_log( + let log = parse_single_log( log_data, log_index + 1, log_positions.len(), @@ -2499,13 +1428,13 @@ fn parse_bbl_file_streaming( } // Export GPS data to GPX if requested - if export_options.gpx && !gps_coords.is_empty() { + if export_options.gpx && !log.gps_coordinates.is_empty() { if let Err(e) = export_gpx_file( file_path, log_index, log_positions.len(), - &gps_coords, - &home_coords, + &log.gps_coordinates, + &log.home_coordinates, export_options, log.header.log_start_datetime.as_deref(), ) { @@ -2521,12 +1450,12 @@ fn parse_bbl_file_streaming( } // Export event data to JSON if requested - if export_options.event && !events.is_empty() { + if export_options.event && !log.event_frames.is_empty() { if let Err(e) = export_event_file( file_path, log_index, log_positions.len(), - &events, + &log.event_frames, export_options, ) { let filename = file_path @@ -2578,10 +1507,7 @@ fn export_gpx_file( .and_then(|n| n.to_str()) .unwrap_or("unknown"); - let output_dir = export_options - .output_dir - .as_deref() - .unwrap_or_else(|| file_path.parent().unwrap().to_str().unwrap()); + let output_dir = get_output_dir(export_options, file_path); // Use consistent naming: only add suffix for multiple logs let log_suffix = if total_logs > 1 { @@ -2645,10 +1571,7 @@ fn export_event_file( .and_then(|n| n.to_str()) .unwrap_or("unknown"); - let output_dir = export_options - .output_dir - .as_deref() - .unwrap_or_else(|| file_path.parent().unwrap().to_str().unwrap()); + let output_dir = get_output_dir(export_options, file_path); // Use consistent naming: only add suffix for multiple logs let log_suffix = if total_logs > 1 { diff --git a/src/parser/frame.rs b/src/parser/frame.rs index 6ece294..61ffb79 100644 --- a/src/parser/frame.rs +++ b/src/parser/frame.rs @@ -1,3 +1,6 @@ +use crate::conversion::{ + convert_gps_altitude, convert_gps_coordinate, convert_gps_course, convert_gps_speed, +}; use crate::parser::{ decoder::apply_predictor_with_debug, decoder::*, event::parse_e_frame, gps::*, stream::BBLDataStream, @@ -6,15 +9,27 @@ use crate::types::{ DecodedFrame, EventFrame, FrameDefinition, FrameHistory, FrameStats, GpsCoordinate, GpsHomeCoordinate, }; +use crate::ExportOptions; use anyhow::Result; use std::collections::HashMap; +use std::io::Write; /// Parse frames from binary data +/// +/// Parses ALL frames from binary data and stores them for CSV export. +/// This is the unified implementation used by both CLI and crate. +/// +/// # Arguments +/// * `binary_data` - Raw binary frame data +/// * `header` - Parsed BBL header with frame definitions +/// * `debug` - Enable debug output +/// * `export_options` - Export options controlling GPS/event collection #[allow(clippy::type_complexity)] pub fn parse_frames( binary_data: &[u8], header: &crate::types::BBLHeader, debug: bool, + export_options: &ExportOptions, ) -> Result<( FrameStats, Vec, @@ -24,20 +39,12 @@ pub fn parse_frames( Vec, )> { let mut stats = FrameStats::default(); - let mut sample_frames = Vec::new(); - let mut debug_frames: Option>> = - if debug { Some(HashMap::new()) } else { None }; - - // Collections for GPS and Event export - let mut gps_coordinates: Vec = Vec::new(); - let mut home_coordinates: Vec = Vec::new(); - let mut event_frames: Vec = Vec::new(); + let mut frames = Vec::new(); + let mut debug_frames: HashMap> = HashMap::new(); + let mut last_main_frame_timestamp = 0u64; // Track timestamp for S frames - // GPS frame history for differential encoding (like P-frames) - let mut gps_frame_history: Vec = Vec::new(); - - // Track the last main frame timestamp for G/H/E frame timestamping - let mut last_main_frame_timestamp: u64 = 0; + // Track the most recent S-frame data for merging (following JavaScript approach) + let mut last_slow_data: HashMap = HashMap::new(); if debug { println!("Binary data size: {} bytes", binary_data.len()); @@ -52,20 +59,31 @@ pub fn parse_frames( if binary_data.is_empty() { return Ok(( stats, - sample_frames, - debug_frames, - gps_coordinates, - home_coordinates, - event_frames, + frames, + Some(debug_frames), + Vec::new(), + Vec::new(), + Vec::new(), )); } // Initialize frame history for proper P-frame parsing - let mut frame_history = FrameHistory::new(header.i_frame_def.count); - let mut stream = BBLDataStream::new(binary_data); + let mut frame_history = FrameHistory { + current_frame: vec![0; header.i_frame_def.count], + previous_frame: vec![0; header.i_frame_def.count], + previous2_frame: vec![0; header.i_frame_def.count], + valid: false, + }; - // Track the most recent S-frame data for merging (following JavaScript approach) - let mut last_slow_data: HashMap = HashMap::new(); + // Collections for GPS and Event export + let mut gps_coordinates: Vec = Vec::new(); + let mut home_coordinates: Vec = Vec::new(); + let mut event_frames: Vec = Vec::new(); + + // GPS frame history for differential encoding + let mut gps_frame_history: Vec = Vec::new(); + + let mut stream = BBLDataStream::new(binary_data); // Main frame parsing loop - process frames as a stream while !stream.eof { @@ -93,10 +111,7 @@ pub fn parse_frames( }; if debug && stats.total_frames < 3 { - println!( - "Found frame type '{}' at offset {}", - frame_type, frame_start_pos - ); + println!("Found frame type '{frame_type}' at offset {frame_start_pos}"); } // Parse frame using proper streaming logic @@ -123,15 +138,13 @@ pub fn parse_frames( ) .is_ok() { - // Copy parsed data to frame_data HashMap + // Update time and loop iteration from parsed frame for (i, field_name) in header.i_frame_def.field_names.iter().enumerate() { if i < frame_history.current_frame.len() { - frame_data.insert( - field_name.clone(), - frame_history.current_frame[i], - ); + let value = frame_history.current_frame[i]; + frame_data.insert(field_name.clone(), value); } } @@ -140,238 +153,417 @@ pub fn parse_frames( frame_data.insert(key.clone(), *value); } - if debug && stats.i_frames <= 2 { + if debug && stats.i_frames < 3 { println!("DEBUG: I-frame merged lastSlow. rxSignalReceived: {:?}, rxFlightChannelsValid: {:?}", frame_data.get("rxSignalReceived"), frame_data.get("rxFlightChannelsValid")); } // Update history for future P-frames - frame_history.update(frame_history.current_frame.clone()); - parsing_success = true; - stats.i_frames += 1; + frame_history + .previous_frame + .copy_from_slice(&frame_history.current_frame); + frame_history + .previous2_frame + .copy_from_slice(&frame_history.current_frame); + frame_history.valid = true; + + // Validate frame before accepting + let current_time = + frame_data.get("time").copied().unwrap_or(0) as u64; + let current_loop = + frame_data.get("loopIteration").copied().unwrap_or(0) as u32; + + let is_valid_frame = + current_time > 0 && (current_loop > 0 || current_time > 1000); + + if is_valid_frame { + parsing_success = true; + stats.i_frames += 1; - // Update last_main_frame_timestamp for G/H/E frame timestamping - if let Some(&time_val) = frame_data.get("time") { - if time_val > 0 { - last_main_frame_timestamp = time_val as u64; + if debug && stats.i_frames <= 3 { + println!( + "DEBUG: Accepted I-frame - time:{}, loop:{}", + current_time, current_loop + ); } + } else if debug && stats.i_frames < 5 { + println!( + "DEBUG: Rejected I-frame - time:{}, loop:{} (invalid)", + current_time, current_loop + ); } } } } 'P' => { if header.p_frame_def.count > 0 && frame_history.valid { - frame_history.current_frame.fill(0); + let mut p_frame_values = vec![0i32; header.p_frame_def.count]; if parse_frame_data( &mut stream, &header.p_frame_def, - &mut frame_history.current_frame, + &mut p_frame_values, Some(&frame_history.previous_frame), Some(&frame_history.previous2_frame), - 0, // TODO: Calculate skipped frames properly - false, // Not raw + 0, + false, header.data_version, &header.sysconfig, debug, ) .is_ok() { - // Copy parsed data using I-frame field names (P-frames use I-frame structure) + // Copy previous frame as base, then update P-frame fields + frame_history + .current_frame + .copy_from_slice(&frame_history.previous_frame); + + // Update only the fields present in P-frame + for (i, field_name) in + header.p_frame_def.field_names.iter().enumerate() + { + if i < p_frame_values.len() { + if let Some(i_frame_idx) = header + .i_frame_def + .field_names + .iter() + .position(|name| name == field_name) + { + if i_frame_idx < frame_history.current_frame.len() { + frame_history.current_frame[i_frame_idx] = + p_frame_values[i]; + } + } + } + } + + // Copy current frame to output for (i, field_name) in header.i_frame_def.field_names.iter().enumerate() { if i < frame_history.current_frame.len() { - frame_data.insert( - field_name.clone(), - frame_history.current_frame[i], - ); + let value = frame_history.current_frame[i]; + frame_data.insert(field_name.clone(), value); } } - // Merge lastSlow data into P-frame (following JavaScript approach) + // Merge lastSlow data for (key, value) in &last_slow_data { frame_data.insert(key.clone(), *value); } - if debug && stats.p_frames <= 2 { + if debug && stats.p_frames < 3 { println!("DEBUG: P-frame merged lastSlow. rxSignalReceived: {:?}, rxFlightChannelsValid: {:?}", frame_data.get("rxSignalReceived"), frame_data.get("rxFlightChannelsValid")); } // Update history - frame_history.update(frame_history.current_frame.clone()); - parsing_success = true; - stats.p_frames += 1; + frame_history + .previous2_frame + .copy_from_slice(&frame_history.previous_frame); + frame_history + .previous_frame + .copy_from_slice(&frame_history.current_frame); + + // Validate P-frame + let current_time = + frame_data.get("time").copied().unwrap_or(0) as u64; + let current_loop = + frame_data.get("loopIteration").copied().unwrap_or(0) as u32; + + let is_valid_frame = + current_time > 0 && (current_loop > 0 || current_time > 1000); + + if is_valid_frame { + parsing_success = true; + stats.p_frames += 1; - // Update last_main_frame_timestamp for G/H/E frame timestamping - if let Some(&time_val) = frame_data.get("time") { - if time_val > 0 { - last_main_frame_timestamp = time_val as u64; + if debug && stats.p_frames <= 3 { + println!( + "DEBUG: Accepted P-frame - time:{}, loop:{}", + current_time, current_loop + ); } + } else if debug && stats.p_frames < 5 { + println!( + "DEBUG: Rejected P-frame - time:{}, loop:{} (invalid)", + current_time, current_loop + ); } } } else { - // Skip P-frame if we don't have valid I-frame history skip_frame(&mut stream, frame_type, debug)?; stats.failed_frames += 1; } } 'S' => { + if debug && stats.s_frames < 5 { + println!( + "DEBUG: Found S-frame, header.s_frame_def.count={}", + header.s_frame_def.count + ); + } if header.s_frame_def.count > 0 { if let Ok(data) = parse_s_frame(&mut stream, &header.s_frame_def, debug) { - // Following JavaScript approach: update lastSlow data - if debug { - println!("DEBUG: Processing S-frame with data: {:?}", data); + if debug && stats.s_frames < 3 { + println!("DEBUG: Processing S-frame with data: {data:?}"); } for (key, value) in &data { last_slow_data.insert(key.clone(), *value); } - if debug { + if debug && stats.s_frames < 3 { println!( - "DEBUG: S-frame data updated lastSlow: {:?}", - last_slow_data + "DEBUG: S-frame data updated lastSlow: {last_slow_data:?}" ); } - frame_data = data; - parsing_success = true; stats.s_frames += 1; + + if debug && stats.s_frames <= 3 { + println!("DEBUG: S-frame count incremented to {} (data merged into lastSlow)", stats.s_frames); + } + } else if debug && stats.s_frames < 5 { + println!("DEBUG: S-frame parsing failed"); } + } else if debug && stats.s_frames < 5 { + println!("DEBUG: Skipping S-frame - header.s_frame_def.count is 0"); } } - 'G' | 'H' | 'E' => { - match frame_type { - 'H' => { - // Parse H-frame (GPS home position) - if header.h_frame_def.count > 0 { - if let Ok(data) = - parse_h_frame(&mut stream, &header.h_frame_def, debug) - { - frame_data = data.clone(); - parsing_success = true; - stats.h_frames += 1; - - // Extract GPS home coordinates - let timestamp = last_main_frame_timestamp; - if let Some(home_coord) = - extract_home_coordinate(&frame_data, timestamp, debug) - { - home_coordinates.push(home_coord); + 'H' => { + if header.h_frame_def.count > 0 { + if let Ok(data) = parse_h_frame(&mut stream, &header.h_frame_def, debug) + { + frame_data = data.clone(); + parsing_success = true; + stats.h_frames += 1; + + // Extract GPS home coordinates for GPX export if enabled + if export_options.gpx { + let timestamp = last_main_frame_timestamp; + + if let (Some(&home_lat_raw), Some(&home_lon_raw)) = ( + frame_data.get("GPS_home[0]"), + frame_data.get("GPS_home[1]"), + ) { + if debug && home_coordinates.is_empty() { + println!("DEBUG: HOME raw values - home_lat_raw: {}, home_lon_raw: {}", home_lat_raw, home_lon_raw); + println!( + "DEBUG: HOME converted - lat: {:.7}, lon: {:.7}", + convert_gps_coordinate(home_lat_raw), + convert_gps_coordinate(home_lon_raw) + ); } + + let home_coordinate = GpsHomeCoordinate { + home_latitude: convert_gps_coordinate(home_lat_raw), + home_longitude: convert_gps_coordinate(home_lon_raw), + timestamp_us: timestamp, + }; + home_coordinates.push(home_coordinate); } - } else { - skip_frame(&mut stream, frame_type, debug)?; - stats.h_frames += 1; - parsing_success = true; } } - 'G' => { - // Parse G-frame (GPS position data) - if header.g_frame_def.count > 0 { - if let Ok(data) = parse_g_frame( - &mut stream, - &header.g_frame_def, - &mut gps_frame_history, - header.data_version, - &header.sysconfig, - debug, - ) { - frame_data = data.clone(); - parsing_success = true; - stats.g_frames += 1; - - // Extract GPS coordinates - let gps_time = - frame_data.get("time").copied().unwrap_or(0) as u64; - let timestamp = if gps_time > 0 { - gps_time - } else { - last_main_frame_timestamp - }; + } else { + skip_frame(&mut stream, frame_type, debug)?; + stats.h_frames += 1; + parsing_success = true; + } + } + 'G' => { + if header.g_frame_def.count > 0 { + // Initialize GPS frame history if needed + if gps_frame_history.is_empty() { + gps_frame_history = vec![0i32; header.g_frame_def.count]; + } - if let Some(coord) = extract_gps_coordinate( - &frame_data, - &home_coordinates, - timestamp, - &header.firmware_revision, - debug, - ) { - gps_coordinates.push(coord); - } + let mut g_frame_values = vec![0i32; header.g_frame_def.count]; + + if parse_frame_data( + &mut stream, + &header.g_frame_def, + &mut g_frame_values, + Some(&gps_frame_history), + None, + 0, + false, + header.data_version, + &header.sysconfig, + debug, + ) + .is_ok() + { + // Update GPS frame history + gps_frame_history.copy_from_slice(&g_frame_values); + + // Copy GPS frame data to output + for (i, field_name) in + header.g_frame_def.field_names.iter().enumerate() + { + if i < g_frame_values.len() { + let value = g_frame_values[i]; + frame_data.insert(field_name.clone(), value); } - } else { - skip_frame(&mut stream, frame_type, debug)?; - stats.g_frames += 1; - parsing_success = true; } - } - 'E' => { - // Parse E-frame (Event data) - if let Ok(mut event_frame) = parse_e_frame(&mut stream, debug) { - frame_data.insert( - "event_type".to_string(), - event_frame.event_type as i32, - ); - parsing_success = true; - stats.e_frames += 1; - // Set timestamp and collect event frame - event_frame.timestamp_us = last_main_frame_timestamp; - event_frames.push(event_frame); + parsing_success = true; + stats.g_frames += 1; + + // Extract GPS coordinates for GPX export if enabled + if export_options.gpx { + let gps_time = + frame_data.get("time").copied().unwrap_or(0) as u64; + let timestamp = if gps_time > 0 { + gps_time + } else { + last_main_frame_timestamp + }; + + if let (Some(&lat_raw), Some(&lon_raw), Some(&alt_raw)) = ( + frame_data.get("GPS_coord[0]"), + frame_data.get("GPS_coord[1]"), + frame_data.get("GPS_altitude"), + ) { + let actual_lat = + if let Some(home_coord) = home_coordinates.first() { + home_coord.home_latitude + + convert_gps_coordinate(lat_raw) + } else { + convert_gps_coordinate(lat_raw) + }; + + let actual_lon = + if let Some(home_coord) = home_coordinates.first() { + home_coord.home_longitude + + convert_gps_coordinate(lon_raw) + } else { + convert_gps_coordinate(lon_raw) + }; + + if debug && gps_coordinates.len() < 3 { + println!("DEBUG: GPS raw values - lat_raw: {}, lon_raw: {}, alt_raw: {}", lat_raw, lon_raw, alt_raw); + println!("DEBUG: GPS converted - lat: {:.7}, lon: {:.7}, alt: {:.2}", + actual_lat, actual_lon, + convert_gps_altitude(alt_raw, &header.firmware_revision)); + } - if debug && stats.e_frames <= 3 { - println!( - "DEBUG: Parsed E-frame - Type: {}", - frame_data.get("event_type").unwrap_or(&0) - ); + let coordinate = GpsCoordinate { + latitude: actual_lat, + longitude: actual_lon, + altitude: convert_gps_altitude( + alt_raw, + &header.firmware_revision, + ), + timestamp_us: timestamp, + num_sats: frame_data.get("GPS_numSat").copied(), + speed: frame_data + .get("GPS_speed") + .map(|&s| convert_gps_speed(s)), + ground_course: frame_data + .get("GPS_ground_course") + .map(|&c| convert_gps_course(c)), + }; + gps_coordinates.push(coordinate); } - } else { - skip_frame(&mut stream, frame_type, debug)?; - stats.e_frames += 1; - parsing_success = true; } } - _ => {} + } else { + skip_frame(&mut stream, frame_type, debug)?; + stats.g_frames += 1; + parsing_success = true; + } + } + 'E' => { + if let Ok(mut event_frame) = parse_e_frame(&mut stream, debug) { + frame_data + .insert("event_type".to_string(), event_frame.event_type as i32); + frame_data.insert("event_description".to_string(), 0); + parsing_success = true; + stats.e_frames += 1; + + // Collect event frames for JSON export if enabled + if export_options.event { + event_frame.timestamp_us = last_main_frame_timestamp; + event_frames.push(event_frame); + } + + if debug && stats.e_frames <= 3 { + println!( + "DEBUG: Parsed E-frame - Type: {}", + frame_data.get("event_type").unwrap_or(&0) + ); + } + } else { + skip_frame(&mut stream, frame_type, debug)?; + stats.e_frames += 1; + parsing_success = true; } } _ => {} }; - if !parsing_success { + // S-frames don't set parsing_success but are processed successfully + // (they update lastSlow data merged into I/P frames) + if !parsing_success && frame_type != 'S' { stats.failed_frames += 1; } stats.total_frames += 1; // Show progress for large files - if debug && stats.total_frames % 50000 == 0 || stats.total_frames % 100000 == 0 { + if (debug && stats.total_frames % 50000 == 0) || stats.total_frames % 100000 == 0 { println!("Parsed {} frames so far...", stats.total_frames); + std::io::stdout().flush().unwrap_or_default(); } - // Store only a few sample frames for display purposes - if parsing_success && sample_frames.len() < 10 { - let decoded_frame = create_decoded_frame(frame_type, &frame_data); - sample_frames.push(decoded_frame.clone()); + // Store ALL successfully parsed frames + if parsing_success { + let timestamp_us = frame_data.get("time").copied().unwrap_or(0) as u64; + let loop_iteration = + frame_data.get("loopIteration").copied().unwrap_or(0) as u32; - // Store debug frames if debug mode is enabled - if let Some(ref mut debug_map) = debug_frames { - let debug_frame_list = debug_map.entry(frame_type).or_insert_with(Vec::new); - debug_frame_list.push(decoded_frame); + // Update last timestamp for main frames (I, P) + if (frame_type == 'I' || frame_type == 'P') && timestamp_us > 0 { + last_main_frame_timestamp = timestamp_us; } - } else if parsing_success { - // Even if we don't store in sample_frames, still store for debug if enabled - if let Some(ref mut debug_map) = debug_frames { - let debug_frame_list = debug_map.entry(frame_type).or_insert_with(Vec::new); - // Store frames strategically for the display pattern (first/middle/last) - if debug_frame_list.len() < 50 { - let decoded_frame = create_decoded_frame(frame_type, &frame_data); - debug_frame_list.push(decoded_frame); + + // S frames inherit timestamp from last main frame + let final_timestamp = if frame_type == 'S' && timestamp_us == 0 { + last_main_frame_timestamp + } else { + timestamp_us + }; + + if debug && (frame_type == 'I' || frame_type == 'P') && frames.len() < 3 { + println!( + "DEBUG: Frame {:?} has timestamp {}. Available fields: {:?}", + frame_type, + timestamp_us, + frame_data.keys().collect::>() + ); + if let Some(time_val) = frame_data.get("time") { + println!("DEBUG: 'time' field value: {time_val}"); + } + if let Some(loop_val) = frame_data.get("loopIteration") { + println!("DEBUG: 'loopIteration' field value: {loop_val}"); } } + + let decoded_frame = DecodedFrame { + frame_type, + timestamp_us: final_timestamp, + loop_iteration, + data: frame_data.clone(), + }; + frames.push(decoded_frame.clone()); + + // Also store in debug_frames for debug purposes + if debug { + let debug_frame_list = debug_frames.entry(frame_type).or_default(); + debug_frame_list.push(decoded_frame); + } } // Update timing from first and last valid frames with time data @@ -388,7 +580,7 @@ pub fn parse_frames( Err(_) => break, } - // More aggressive safety limits to prevent hanging + // Safety limits to prevent hanging if stats.total_frames > 1000000 || stats.failed_frames > 10000 { if debug { println!("Hit safety limit - stopping frame parsing"); @@ -415,26 +607,14 @@ pub fn parse_frames( Ok(( stats, - sample_frames, - debug_frames, + frames, + Some(debug_frames), gps_coordinates, home_coordinates, event_frames, )) } -fn create_decoded_frame(frame_type: char, frame_data: &HashMap) -> DecodedFrame { - let timestamp_us = frame_data.get("time").copied().unwrap_or(0) as u64; - let loop_iteration = frame_data.get("loopIteration").copied().unwrap_or(0) as u32; - - DecodedFrame { - frame_type, - timestamp_us, - loop_iteration, - data: frame_data.clone(), - } -} - /// Parse frame data using the specified frame definition #[allow(clippy::too_many_arguments)] pub fn parse_frame_data( diff --git a/src/parser/header.rs b/src/parser/header.rs index 51c5c28..e07cfcf 100644 --- a/src/parser/header.rs +++ b/src/parser/header.rs @@ -111,6 +111,13 @@ pub fn parse_headers_from_text(header_text: &str, debug: bool) -> Result Result Result> { if debug { @@ -86,7 +86,13 @@ pub fn parse_bbl_bytes_all_logs( .unwrap_or(data.len()); let log_data = &data[start_pos..end_pos]; - let log = parse_single_log(log_data, log_index + 1, log_positions.len(), debug)?; + let log = parse_single_log( + log_data, + log_index + 1, + log_positions.len(), + debug, + &export_options, + )?; logs.push(log); } @@ -109,11 +115,19 @@ pub fn parse_bbl_bytes( // This is a placeholder for the systematic migration process /// Internal function to parse a single BBL log from binary data +/// +/// # Arguments +/// * `log_data` - Raw log data (headers + binary frames) +/// * `log_number` - 1-based log number +/// * `total_logs` - Total number of logs in the file +/// * `debug` - Enable debug output +/// * `export_options` - Export options controlling GPS/event collection fn parse_single_log( log_data: &[u8], log_number: usize, total_logs: usize, debug: bool, + export_options: &crate::ExportOptions, ) -> Result { // Find where headers end and binary data begins let mut header_end = 0; @@ -134,13 +148,13 @@ fn parse_single_log( // Parse binary frame data let binary_data = &log_data[header_end..]; - let (mut stats, sample_frames, debug_frames, gps_coordinates, home_coordinates, event_frames) = - crate::parser::frame::parse_frames(binary_data, &header, debug)?; + let (mut stats, frames, debug_frames, gps_coordinates, home_coordinates, event_frames) = + crate::parser::frame::parse_frames(binary_data, &header, debug, export_options)?; // Update frame stats timing from actual frame data - if !sample_frames.is_empty() { - stats.start_time_us = sample_frames.first().unwrap().timestamp_us; - stats.end_time_us = sample_frames.last().unwrap().timestamp_us; + if !frames.is_empty() { + stats.start_time_us = frames.first().unwrap().timestamp_us; + stats.end_time_us = frames.last().unwrap().timestamp_us; } let log = BBLLog { @@ -148,7 +162,7 @@ fn parse_single_log( total_logs, header, stats, - sample_frames, + frames, debug_frames, gps_coordinates, home_coordinates, diff --git a/src/types/log.rs b/src/types/log.rs index d673e5b..f3bfa93 100644 --- a/src/types/log.rs +++ b/src/types/log.rs @@ -14,7 +14,8 @@ pub struct BBLLog { pub total_logs: usize, pub header: BBLHeader, pub stats: FrameStats, - pub sample_frames: Vec, + /// All parsed frames (I and P frames with decoded values) + pub frames: Vec, pub debug_frames: Option>>, pub gps_coordinates: Vec, pub home_coordinates: Vec, @@ -28,7 +29,7 @@ impl BBLLog { total_logs, header: BBLHeader::default(), stats: FrameStats::default(), - sample_frames: Vec::new(), + frames: Vec::new(), debug_frames: None, gps_coordinates: Vec::new(), home_coordinates: Vec::new(),