From c86da177e37fd69a3df5a727e985887beb984f3d Mon Sep 17 00:00:00 2001 From: nerdCopter <56646290+nerdCopter@users.noreply.github.com> Date: Mon, 1 Dec 2025 10:18:28 -0600 Subject: [PATCH 01/15] refactor: move sign_extend helpers to crate library (Phase 1) - Create src/parser/helpers.rs with 7 sign_extend functions - Export helpers from parser/mod.rs - Remove bbl_format from lib.rs (will be CLI-only module) - Update stream.rs to use shared helpers - Update bbl_format.rs to use shared helpers - All tests pass, CSV output identical to master --- src/bbl_format.rs | 59 +++---------------- src/lib.rs | 3 - src/parser/helpers.rs | 130 ++++++++++++++++++++++++++++++++++++++++++ src/parser/mod.rs | 2 + src/parser/stream.rs | 72 +++++------------------ 5 files changed, 152 insertions(+), 114 deletions(-) create mode 100644 src/parser/helpers.rs diff --git a/src/bbl_format.rs b/src/bbl_format.rs index 1c75778..44da403 100644 --- a/src/bbl_format.rs +++ b/src/bbl_format.rs @@ -1,6 +1,13 @@ use anyhow::Result; +use bbl_parser::parser::helpers::{ + sign_extend_16bit, sign_extend_24bit, sign_extend_2bit, sign_extend_4bit, sign_extend_6bit, + sign_extend_8bit, +}; use std::collections::HashMap; +// Re-export sign_extend_14bit for backward compatibility with main.rs +pub use bbl_parser::parser::helpers::sign_extend_14bit; + // BBL Encoding constants - directly from JavaScript reference pub const ENCODING_SIGNED_VB: u8 = 0; pub const ENCODING_UNSIGNED_VB: u8 = 1; @@ -266,58 +273,6 @@ impl<'a> BBLDataStream<'a> { } } -// Sign extension functions - exact replicas of JavaScript implementations -pub fn sign_extend_2bit(value: u8) -> i32 { - let val = value as i32; - if (val & 0x02) != 0 { - val | !0x03 - } else { - val & 0x03 - } -} - -pub fn sign_extend_4bit(value: u8) -> i32 { - let val = value as i32; - if (val & 0x08) != 0 { - val | !0x0f - } else { - val & 0x0f - } -} - -pub fn sign_extend_6bit(value: u8) -> i32 { - let val = value as i32; - if (val & 0x20) != 0 { - val | !0x3f - } else { - val & 0x3f - } -} - -pub fn sign_extend_8bit(value: u8) -> i32 { - value as i8 as i32 -} - -pub fn sign_extend_16bit(value: u16) -> i32 { - value as i16 as i32 -} - -pub fn sign_extend_24bit(value: u32) -> i32 { - if (value & 0x800000) != 0 { - (value | 0xff000000) as i32 - } else { - (value & 0x7fffff) as i32 - } -} - -pub fn sign_extend_14bit(value: u16) -> i32 { - if (value & 0x2000) != 0 { - -((value & 0x1fff) as i32) - } else { - (value & 0x1fff) as i32 - } -} - #[allow(clippy::too_many_arguments)] #[allow(dead_code)] pub fn apply_predictor( diff --git a/src/lib.rs b/src/lib.rs index 501ccd5..1ba6df7 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -16,7 +16,6 @@ //! ``` // Module declarations -mod bbl_format; pub mod conversion; pub mod error; pub mod export; @@ -25,8 +24,6 @@ pub mod types; // Re-export everything from modules for convenience #[allow(ambiguous_glob_reexports)] -pub use bbl_format::*; -#[allow(ambiguous_glob_reexports)] pub use conversion::*; #[allow(ambiguous_glob_reexports)] pub use error::*; diff --git a/src/parser/helpers.rs b/src/parser/helpers.rs new file mode 100644 index 0000000..950e350 --- /dev/null +++ b/src/parser/helpers.rs @@ -0,0 +1,130 @@ +//! Helper functions for BBL parsing +//! +//! This module provides sign extension functions used for decoding various +//! fixed-width signed values from the blackbox binary format. + +/// Sign-extend a 2-bit value to i32 +pub fn sign_extend_2bit(value: u8) -> i32 { + let val = value as i32; + if (val & 0x02) != 0 { + val | !0x03 + } else { + val & 0x03 + } +} + +/// Sign-extend a 4-bit value to i32 +pub fn sign_extend_4bit(value: u8) -> i32 { + let val = value as i32; + if (val & 0x08) != 0 { + val | !0x0f + } else { + val & 0x0f + } +} + +/// Sign-extend a 6-bit value to i32 +pub fn sign_extend_6bit(value: u8) -> i32 { + let val = value as i32; + if (val & 0x20) != 0 { + val | !0x3f + } else { + val & 0x3f + } +} + +/// Sign-extend an 8-bit value to i32 +pub fn sign_extend_8bit(value: u8) -> i32 { + value as i8 as i32 +} + +/// Sign-extend a 16-bit value to i32 +pub fn sign_extend_16bit(value: u16) -> i32 { + value as i16 as i32 +} + +/// Sign-extend a 24-bit value to i32 +pub fn sign_extend_24bit(value: u32) -> i32 { + if (value & 0x800000) != 0 { + (value | 0xff000000) as i32 + } else { + (value & 0x7fffff) as i32 + } +} + +/// Sign-extend a 14-bit value to i32 (sign-magnitude format) +/// Bit 13 indicates sign, bits 0-12 are the magnitude. +/// Returns negative value if sign bit is set. +pub fn sign_extend_14bit(value: u16) -> i32 { + if (value & 0x2000) != 0 { + -((value & 0x1fff) as i32) + } else { + (value & 0x1fff) as i32 + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_sign_extend_2bit() { + assert_eq!(sign_extend_2bit(0), 0); + assert_eq!(sign_extend_2bit(1), 1); + assert_eq!(sign_extend_2bit(2), -2); + assert_eq!(sign_extend_2bit(3), -1); + } + + #[test] + fn test_sign_extend_4bit() { + assert_eq!(sign_extend_4bit(0), 0); + assert_eq!(sign_extend_4bit(7), 7); + assert_eq!(sign_extend_4bit(8), -8); + assert_eq!(sign_extend_4bit(15), -1); + } + + #[test] + fn test_sign_extend_6bit() { + assert_eq!(sign_extend_6bit(0), 0); + assert_eq!(sign_extend_6bit(31), 31); + assert_eq!(sign_extend_6bit(32), -32); + assert_eq!(sign_extend_6bit(63), -1); + } + + #[test] + fn test_sign_extend_8bit() { + assert_eq!(sign_extend_8bit(0), 0); + assert_eq!(sign_extend_8bit(127), 127); + assert_eq!(sign_extend_8bit(128), -128); + assert_eq!(sign_extend_8bit(255), -1); + } + + #[test] + fn test_sign_extend_16bit() { + assert_eq!(sign_extend_16bit(0), 0); + assert_eq!(sign_extend_16bit(32767), 32767); + assert_eq!(sign_extend_16bit(32768), -32768); + assert_eq!(sign_extend_16bit(65535), -1); + } + + #[test] + fn test_sign_extend_24bit() { + assert_eq!(sign_extend_24bit(0), 0); + assert_eq!(sign_extend_24bit(0x7FFFFF), 0x7FFFFF); + assert_eq!(sign_extend_24bit(0x800000), -8388608); + assert_eq!(sign_extend_24bit(0xFFFFFF), -1); + } + + #[test] + fn test_sign_extend_14bit() { + // Positive values (bit 13 clear) + assert_eq!(sign_extend_14bit(0), 0); + assert_eq!(sign_extend_14bit(1), 1); + assert_eq!(sign_extend_14bit(0x1FFF), 0x1FFF); // 8191 + + // Negative values (bit 13 set) + assert_eq!(sign_extend_14bit(0x2000), 0); // -0 + assert_eq!(sign_extend_14bit(0x2001), -1); + assert_eq!(sign_extend_14bit(0x3FFF), -8191); + } +} diff --git a/src/parser/mod.rs b/src/parser/mod.rs index fb5025d..1a94493 100644 --- a/src/parser/mod.rs +++ b/src/parser/mod.rs @@ -3,6 +3,7 @@ pub mod event; pub mod frame; pub mod gps; pub mod header; +pub mod helpers; pub mod main; pub mod stream; @@ -11,5 +12,6 @@ pub use event::*; pub use frame::*; pub use gps::*; pub use header::*; +pub use helpers::*; pub use main::*; pub use stream::*; diff --git a/src/parser/stream.rs b/src/parser/stream.rs index b6a8672..422ebbd 100644 --- a/src/parser/stream.rs +++ b/src/parser/stream.rs @@ -1,3 +1,7 @@ +use crate::parser::helpers::{ + sign_extend_14bit, sign_extend_16bit, sign_extend_24bit, sign_extend_2bit, sign_extend_4bit, + sign_extend_6bit, sign_extend_8bit, +}; use anyhow::Result; /// BBL data stream for reading binary data @@ -226,82 +230,32 @@ impl<'a> BBLDataStream<'a> { /// Returns the negated value to match blackbox_decode behavior. pub fn read_neg_14bit(&mut self) -> Result { let unsigned = self.read_unsigned_vb()? as u16; - Ok(-sign_extend_14bit_sign_magnitude(unsigned)) - } -} - -/// Sign-magnitude 14-bit encoding (matches bbl_format::sign_extend_14bit and blackbox_decode) -/// Bit 13 indicates sign, bits 0-12 are the magnitude -fn sign_extend_14bit_sign_magnitude(value: u16) -> i32 { - if (value & 0x2000) != 0 { - -((value & 0x1fff) as i32) - } else { - (value & 0x1fff) as i32 - } -} - -// Sign extension helper functions - exact replicas of JavaScript implementation -fn sign_extend_2bit(value: u8) -> i32 { - if (value & 0x02) != 0 { - (value as i32) | !0x03 - } else { - value as i32 - } -} - -fn sign_extend_4bit(value: u8) -> i32 { - if (value & 0x08) != 0 { - (value as i32) | !0x0f - } else { - value as i32 - } -} - -fn sign_extend_6bit(value: u8) -> i32 { - if (value & 0x20) != 0 { - (value as i32) | !0x3f - } else { - value as i32 - } -} - -fn sign_extend_8bit(value: u8) -> i32 { - value as i8 as i32 -} - -fn sign_extend_16bit(value: u16) -> i32 { - value as i16 as i32 -} - -fn sign_extend_24bit(value: u32) -> i32 { - if (value & 0x800000) != 0 { - (value as i32) | !0xffffff - } else { - value as i32 + Ok(-sign_extend_14bit(unsigned)) } } #[cfg(test)] mod tests { use super::*; + use crate::parser::helpers::sign_extend_14bit; #[test] fn test_sign_extend_14bit_sign_magnitude_positive() { // Positive values have bit 13 = 0 (sign bit clear) - assert_eq!(sign_extend_14bit_sign_magnitude(0x0000), 0); // 0 - assert_eq!(sign_extend_14bit_sign_magnitude(0x0001), 1); // 1 - assert_eq!(sign_extend_14bit_sign_magnitude(0x1FFF), 0x1FFF); // 8191 (max positive magnitude) + assert_eq!(sign_extend_14bit(0x0000), 0); // 0 + assert_eq!(sign_extend_14bit(0x0001), 1); // 1 + assert_eq!(sign_extend_14bit(0x1FFF), 0x1FFF); // 8191 (max positive magnitude) } #[test] fn test_sign_extend_14bit_sign_magnitude_negative() { // Negative values have bit 13 = 1 (sign bit set), magnitude in bits 0-12 // 0x2000 = bit 13 set, magnitude 0 -> returns -0 = 0 (actually negative zero) - assert_eq!(sign_extend_14bit_sign_magnitude(0x2000), 0); // -0 - // 0x2001 = bit 13 set, magnitude 1 -> returns -1 - assert_eq!(sign_extend_14bit_sign_magnitude(0x2001), -1); + assert_eq!(sign_extend_14bit(0x2000), 0); // -0 + // 0x2001 = bit 13 set, magnitude 1 -> returns -1 + assert_eq!(sign_extend_14bit(0x2001), -1); // 0x3FFF = bit 13 set, magnitude 0x1FFF (8191) -> returns -8191 - assert_eq!(sign_extend_14bit_sign_magnitude(0x3FFF), -8191); + assert_eq!(sign_extend_14bit(0x3FFF), -8191); } #[test] From 1dca257249fb5d7c1f07c3b02bd27a10bf2a8af5 Mon Sep 17 00:00:00 2001 From: nerdCopter <56646290+nerdCopter@users.noreply.github.com> Date: Mon, 1 Dec 2025 14:56:32 -0600 Subject: [PATCH 02/15] refactor: unify parse_frame_data and BBLDataStream (Phase 2+3) - Add debug parameter to crate's parse_frame_data - Add apply_predictor_with_debug with field_names, skipped_frames support - Add vbatLatest corruption prevention logic to crate - Add read_tag8_8svb_counted method with proper group counting - Fix ENCODING_TAG8_8SVB handling to count consecutive fields - Update main.rs to import from bbl_parser crate: * BBLDataStream, parse_frame_data, FrameDefinition * sign_extend_14bit, encoding constants - Remove duplicate FieldDefinition/FrameDefinition from main.rs - Delete bbl_format.rs entirely (509 lines removed) - All CSV outputs verified identical to master branch --- src/bbl_format.rs | 610 ------------------------------------------ src/main.rs | 129 +++------ src/parser/decoder.rs | 180 ++++++++++--- src/parser/frame.rs | 89 +++--- src/parser/gps.rs | 1 + src/parser/stream.rs | 24 ++ 6 files changed, 262 insertions(+), 771 deletions(-) delete mode 100644 src/bbl_format.rs diff --git a/src/bbl_format.rs b/src/bbl_format.rs deleted file mode 100644 index 44da403..0000000 --- a/src/bbl_format.rs +++ /dev/null @@ -1,610 +0,0 @@ -use anyhow::Result; -use bbl_parser::parser::helpers::{ - sign_extend_16bit, sign_extend_24bit, sign_extend_2bit, sign_extend_4bit, sign_extend_6bit, - sign_extend_8bit, -}; -use std::collections::HashMap; - -// Re-export sign_extend_14bit for backward compatibility with main.rs -pub use bbl_parser::parser::helpers::sign_extend_14bit; - -// BBL Encoding constants - directly from JavaScript reference -pub const ENCODING_SIGNED_VB: u8 = 0; -pub const ENCODING_UNSIGNED_VB: u8 = 1; -pub const ENCODING_NEG_14BIT: u8 = 3; -#[allow(dead_code)] -pub const ENCODING_TAG8_8SVB: u8 = 6; -#[allow(dead_code)] -pub const ENCODING_TAG2_3S32: u8 = 7; -#[allow(dead_code)] -pub const ENCODING_TAG8_4S16: u8 = 8; -pub const ENCODING_NULL: u8 = 9; -#[allow(dead_code)] -pub const ENCODING_TAG2_3SVARIABLE: u8 = 10; - -// Predictor constants - directly from JavaScript reference -#[allow(dead_code)] -pub const PREDICT_0: u8 = 0; -#[allow(dead_code)] -pub const PREDICT_PREVIOUS: u8 = 1; -#[allow(dead_code)] -pub const PREDICT_STRAIGHT_LINE: u8 = 2; -#[allow(dead_code)] -pub const PREDICT_AVERAGE_2: u8 = 3; -#[allow(dead_code)] -pub const PREDICT_MINTHROTTLE: u8 = 4; -#[allow(dead_code)] -pub const PREDICT_MOTOR_0: u8 = 5; -#[allow(dead_code)] -pub const PREDICT_INC: u8 = 6; -#[allow(dead_code)] -pub const PREDICT_HOME_COORD: u8 = 7; -#[allow(dead_code)] -pub const PREDICT_1500: u8 = 8; -#[allow(dead_code)] -pub const PREDICT_VBATREF: u8 = 9; -#[allow(dead_code)] -pub const PREDICT_LAST_MAIN_FRAME_TIME: u8 = 10; -#[allow(dead_code)] -pub const PREDICT_MINMOTOR: u8 = 11; - -pub struct BBLDataStream<'a> { - data: &'a [u8], - pub pos: usize, - end: usize, - pub eof: bool, -} - -impl<'a> BBLDataStream<'a> { - pub fn new(data: &'a [u8]) -> Self { - Self { - data, - pos: 0, - end: data.len(), - eof: false, - } - } - - #[allow(dead_code)] - pub fn set_position(&mut self, pos: usize) { - self.pos = pos; - self.eof = pos >= self.end; - } - - pub fn read_byte(&mut self) -> Result { - if self.pos < self.end { - let byte = self.data[self.pos]; - self.pos += 1; - Ok(byte) - } else { - self.eof = true; - Err(anyhow::anyhow!("EOF")) - } - } - - #[allow(dead_code)] - pub fn read_char(&mut self) -> Result { - Ok(self.read_byte()? as char) - } - - // Read unsigned variable byte - exact replica of JavaScript implementation - pub fn read_unsigned_vb(&mut self) -> Result { - let mut result = 0u32; - let mut shift = 0; - - // 5 bytes is enough to encode 32-bit unsigned quantities - for _ in 0..5 { - let b = match self.read_byte() { - Ok(byte) => byte, - Err(_) => return Ok(0), - }; - - result |= ((b & !0x80) as u32) << shift; - - // Final byte? - if b < 128 { - return Ok(result); - } - - shift += 7; - } - - // This VB-encoded int is too long! - Ok(0) - } - - // Read signed variable byte - exact replica of JavaScript implementation - pub fn read_signed_vb(&mut self) -> Result { - let unsigned = self.read_unsigned_vb()?; - - // Apply ZigZag decoding to recover the signed value - Ok(((unsigned >> 1) as i32) ^ -((unsigned & 1) as i32)) - } - - // Read Tag8_4S16 encoding - exact replica of JavaScript implementation - pub fn read_tag8_4s16_v2(&mut self, values: &mut [i32]) -> Result<()> { - let selector = self.read_byte()?; - let mut nibble_index = 0; - let mut buffer = 0u8; - - #[allow(clippy::needless_range_loop)] - for i in 0..4 { - let field_type = (selector >> (i * 2)) & 0x03; - - match field_type { - 0 => values[i] = 0, // FIELD_ZERO - 1 => { - // FIELD_4BIT - if nibble_index == 0 { - buffer = self.read_byte()?; - values[i] = sign_extend_4bit(buffer >> 4); - nibble_index = 1; - } else { - values[i] = sign_extend_4bit(buffer & 0x0f); - nibble_index = 0; - } - } - 2 => { - // FIELD_8BIT - if nibble_index == 0 { - values[i] = sign_extend_8bit(self.read_byte()?); - } else { - let mut char1 = (buffer & 0x0f) << 4; - buffer = self.read_byte()?; - char1 |= buffer >> 4; - values[i] = sign_extend_8bit(char1); - } - } - 3 => { - // FIELD_16BIT - if nibble_index == 0 { - let char1 = self.read_byte()?; - let char2 = self.read_byte()?; - values[i] = sign_extend_16bit(((char1 as u16) << 8) | (char2 as u16)); - } else { - let char1 = self.read_byte()?; - let char2 = self.read_byte()?; - values[i] = sign_extend_16bit( - (((buffer & 0x0f) as u16) << 12) - | ((char1 as u16) << 4) - | ((char2 as u16) >> 4), - ); - buffer = char2; - } - } - _ => unreachable!(), - } - } - - Ok(()) - } - - // Read Tag2_3S32 encoding - exact replica of JavaScript implementation - pub fn read_tag2_3s32(&mut self, values: &mut [i32]) -> Result<()> { - let lead_byte = self.read_byte()?; - - match lead_byte >> 6 { - 0 => { - // 2-bit fields - values[0] = sign_extend_2bit((lead_byte >> 4) & 0x03); - values[1] = sign_extend_2bit((lead_byte >> 2) & 0x03); - values[2] = sign_extend_2bit(lead_byte & 0x03); - } - 1 => { - // 4-bit fields - values[0] = sign_extend_4bit(lead_byte & 0x0f); - let second_byte = self.read_byte()?; - values[1] = sign_extend_4bit(second_byte >> 4); - values[2] = sign_extend_4bit(second_byte & 0x0f); - } - 2 => { - // 6-bit fields - values[0] = sign_extend_6bit(lead_byte & 0x3f); - let byte2 = self.read_byte()?; - values[1] = sign_extend_6bit(byte2 & 0x3f); - let byte3 = self.read_byte()?; - values[2] = sign_extend_6bit(byte3 & 0x3f); - } - 3 => { - // 8, 16 or 24 bit fields - let mut selector = lead_byte; - #[allow(clippy::needless_range_loop)] - for i in 0..3 { - match selector & 0x03 { - 0 => { - // 8-bit - let byte1 = self.read_byte()?; - values[i] = sign_extend_8bit(byte1); - } - 1 => { - // 16-bit - let byte1 = self.read_byte()?; - let byte2 = self.read_byte()?; - values[i] = sign_extend_16bit((byte1 as u16) | ((byte2 as u16) << 8)); - } - 2 => { - // 24-bit - let byte1 = self.read_byte()?; - let byte2 = self.read_byte()?; - let byte3 = self.read_byte()?; - values[i] = sign_extend_24bit( - (byte1 as u32) | ((byte2 as u32) << 8) | ((byte3 as u32) << 16), - ); - } - 3 => { - // 32-bit - let byte1 = self.read_byte()?; - let byte2 = self.read_byte()?; - let byte3 = self.read_byte()?; - let byte4 = self.read_byte()?; - values[i] = (byte1 as i32) - | ((byte2 as i32) << 8) - | ((byte3 as i32) << 16) - | ((byte4 as i32) << 24); - } - _ => unreachable!(), - } - selector >>= 2; - } - } - _ => unreachable!(), - } - - Ok(()) - } - - // Read Tag8_8SVB encoding - exact replica of JavaScript implementation - pub fn read_tag8_8svb(&mut self, values: &mut [i32], value_count: usize) -> Result<()> { - if value_count == 1 { - values[0] = self.read_signed_vb()?; - } else { - let mut header = self.read_byte()?; - #[allow(clippy::needless_range_loop)] - for i in 0..8.min(value_count) { - values[i] = if header & 0x01 != 0 { - self.read_signed_vb()? - } else { - 0 - }; - header >>= 1; - } - } - Ok(()) - } -} - -#[allow(clippy::too_many_arguments)] -#[allow(dead_code)] -pub fn apply_predictor( - field_index: usize, - predictor: u8, - raw_value: i32, - current_frame: &[i32], - previous_frame: Option<&[i32]>, - previous2_frame: Option<&[i32]>, - skipped_frames: u32, - sysconfig: &HashMap, - field_names: &[String], - debug: bool, -) -> i32 { - match predictor { - PREDICT_0 => raw_value, - - PREDICT_PREVIOUS => { - if let Some(prev) = previous_frame { - if field_index < prev.len() { - let result = prev[field_index] + raw_value; - - // CRITICAL FIX: Prevent corruption propagation for vbatLatest - if field_names - .get(field_index) - .map(|name| name == "vbatLatest") - .unwrap_or(false) - { - // Check if previous value is corrupted (way too high for voltage) - if prev[field_index] > 1000 { - if debug { - eprintln!("DEBUG: Fixed corrupted vbatLatest previous value {} replaced with reasonable estimate", prev[field_index]); - } - // Use a reasonable voltage estimate based on vbatref - let vbatref = sysconfig.get("vbatref").copied().unwrap_or(4095); - return vbatref + raw_value; // Use vbatref as baseline + current delta - } - } - - result - } else { - raw_value - } - } else { - raw_value - } - } - - PREDICT_STRAIGHT_LINE => { - if let (Some(prev), Some(prev2)) = (previous_frame, previous2_frame) { - if field_index < prev.len() && field_index < prev2.len() { - raw_value + 2 * prev[field_index] - prev2[field_index] - } else { - raw_value - } - } else { - raw_value - } - } - - PREDICT_AVERAGE_2 => { - if let (Some(prev), Some(prev2)) = (previous_frame, previous2_frame) { - if field_index < prev.len() && field_index < prev2.len() { - raw_value + ((prev[field_index] + prev2[field_index]) / 2) - } else { - raw_value - } - } else { - raw_value - } - } - - PREDICT_MINTHROTTLE => { - let minthrottle = sysconfig.get("minthrottle").copied().unwrap_or(1150); - raw_value + minthrottle - } - - PREDICT_MOTOR_0 => { - // Find motor[0] field index - if let Some(motor0_idx) = field_names.iter().position(|name| name == "motor[0]") { - if motor0_idx < current_frame.len() { - current_frame[motor0_idx] + raw_value - } else { - raw_value - } - } else { - raw_value - } - } - - PREDICT_INC => { - let mut result = skipped_frames as i32 + 1; - if let Some(prev) = previous_frame { - if field_index < prev.len() { - result += prev[field_index]; - } - } - result - } - - PREDICT_1500 => raw_value + 1500, - - PREDICT_VBATREF => { - let vbatref = sysconfig.get("vbatref").copied().unwrap_or(4095); - - // CRITICAL FIX: Check for corrupted raw values in vbatLatest - // Normal vbatLatest raw values should be small deltas (-50 to +50) or small absolute values (<1000) - // Large values (>4000) indicate stream parsing corruption or wrong predictor application - if field_names - .get(field_index) - .map(|name| name == "vbatLatest") - .unwrap_or(false) - && !(-1000..=4000).contains(&raw_value) - { - // This is clearly a corrupted value - likely caused by stream parsing error - // Instead of propagating corruption, use a safe default value - if debug { - eprintln!( - "DEBUG: Fixed corrupted vbatLatest raw_value {} replaced with 0", - raw_value - ); - } - return vbatref; // Return just vbatref (safe default) - } - - raw_value + vbatref - } - - PREDICT_MINMOTOR => { - // Get the min motor value from motorOutput "min,max" format - let minmotor = if let Some(motor_output) = sysconfig.get("motorOutput") { - // Parse "48,2047" format to get first value (48) - let motor_output_str = motor_output.to_string(); - if let Some(comma_pos) = motor_output_str.find(',') { - motor_output_str[..comma_pos].parse().unwrap_or(48) - } else { - motor_output_str.parse().unwrap_or(48) - } - } else { - 48 // Default min motor output value - }; - raw_value + minmotor - } - - _ => raw_value, - } -} - -pub fn decode_frame_field( - stream: &mut BBLDataStream, - encoding: u8, - _data_version: u8, -) -> Result { - match encoding { - ENCODING_SIGNED_VB => stream.read_signed_vb(), - - ENCODING_UNSIGNED_VB => Ok(stream.read_unsigned_vb()? as i32), - - ENCODING_NEG_14BIT => { - let value = stream.read_unsigned_vb()? as u16; - Ok(-sign_extend_14bit(value)) - } - - ENCODING_NULL => Ok(0), - - _ => Err(anyhow::anyhow!("Unsupported encoding: {}", encoding)), - } -} - -#[allow(clippy::too_many_arguments)] -#[allow(dead_code)] -pub fn parse_frame_data( - stream: &mut BBLDataStream, - frame_def: &crate::FrameDefinition, - current_frame: &mut [i32], - previous_frame: Option<&[i32]>, - previous2_frame: Option<&[i32]>, - skipped_frames: u32, - raw: bool, - data_version: u8, - sysconfig: &HashMap, - debug: bool, -) -> Result<()> { - let mut i = 0; - let mut values = [0i32; 8]; - - while i < frame_def.fields.len() { - let field = &frame_def.fields[i]; - - if field.predictor == PREDICT_INC { - current_frame[i] = apply_predictor( - i, - field.predictor, - 0, - current_frame, - previous_frame, - previous2_frame, - skipped_frames, - sysconfig, - &frame_def.field_names, - debug, - ); - i += 1; - continue; - } - - match field.encoding { - ENCODING_TAG8_4S16 => { - if data_version < 2 { - // v1 implementation would be different but we'll use v2 - } - stream.read_tag8_4s16_v2(&mut values)?; - - // Apply predictors for the 4 fields - for j in 0..4 { - if i + j >= frame_def.fields.len() { - break; - } - let predictor = if raw { - PREDICT_0 - } else { - frame_def.fields[i + j].predictor - }; - - current_frame[i + j] = apply_predictor( - i + j, - predictor, - values[j], - current_frame, - previous_frame, - previous2_frame, - skipped_frames, - sysconfig, - &frame_def.field_names, - debug, - ); - } - i += 4; - continue; - } - - ENCODING_TAG2_3S32 => { - stream.read_tag2_3s32(&mut values)?; - - // Apply predictors for the 3 fields - for j in 0..3 { - if i + j >= frame_def.fields.len() { - break; - } - let predictor = if raw { - PREDICT_0 - } else { - frame_def.fields[i + j].predictor - }; - current_frame[i + j] = apply_predictor( - i + j, - predictor, - values[j], - current_frame, - previous_frame, - previous2_frame, - skipped_frames, - sysconfig, - &frame_def.field_names, - debug, - ); - } - i += 3; - continue; - } - - ENCODING_TAG8_8SVB => { - // Count how many fields use this encoding - let mut group_count = 1; - for j in i + 1..i + 8.min(frame_def.fields.len() - i) { - if frame_def.fields[j].encoding != ENCODING_TAG8_8SVB { - break; - } - group_count += 1; - } - - stream.read_tag8_8svb(&mut values, group_count)?; - - // Apply predictors for the group - for j in 0..group_count { - if i + j >= frame_def.fields.len() { - break; - } - let predictor = if raw { - PREDICT_0 - } else { - frame_def.fields[i + j].predictor - }; - - current_frame[i + j] = apply_predictor( - i + j, - predictor, - values[j], - current_frame, - previous_frame, - previous2_frame, - skipped_frames, - sysconfig, - &frame_def.field_names, - debug, - ); - } - i += group_count; - continue; - } - - _ => { - let raw_value = decode_frame_field(stream, field.encoding, data_version)?; - let predictor = if raw { PREDICT_0 } else { field.predictor }; - - current_frame[i] = apply_predictor( - i, - predictor, - raw_value, - current_frame, - previous_frame, - previous2_frame, - skipped_frames, - sysconfig, - &frame_def.field_names, - debug, - ); - } - } - - i += 1; - } - - Ok(()) -} diff --git a/src/main.rs b/src/main.rs index dc43911..698f2bf 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,5 +1,3 @@ -mod bbl_format; - use anyhow::{Context, Result}; use clap::{Arg, Command}; use glob::glob; @@ -15,6 +13,16 @@ use bbl_parser::conversion::{ format_failsafe_phase, format_flight_mode_flags, format_state_flags, }; +// Import parser types from crate library +use bbl_parser::parser::helpers::sign_extend_14bit; +use bbl_parser::parser::{ + parse_frame_data, BBLDataStream, ENCODING_NEG_14BIT, ENCODING_NULL, ENCODING_SIGNED_VB, + ENCODING_TAG2_3S32, ENCODING_UNSIGNED_VB, +}; + +// Import types from crate library +use bbl_parser::types::FrameDefinition; + /// Maximum recursion depth to prevent stack overflow const MAX_RECURSION_DEPTH: usize = 100; @@ -224,72 +232,7 @@ fn find_bbl_files_in_dir_with_depth( Ok(bbl_files) } -#[derive(Debug, Clone)] -struct FieldDefinition { - name: String, - signed: bool, - predictor: u8, - encoding: u8, -} - -#[derive(Debug, Clone)] -struct FrameDefinition { - fields: Vec, - field_names: Vec, - count: usize, -} - -impl FrameDefinition { - fn new() -> Self { - Self { - fields: Vec::new(), - field_names: Vec::new(), - count: 0, - } - } - - fn from_field_names(names: Vec) -> Self { - let fields = names - .iter() - .map(|name| FieldDefinition { - name: name.clone(), - signed: false, - predictor: 0, - encoding: 0, - }) - .collect(); - let count = names.len(); - Self { - fields, - field_names: names, - count, - } - } - - fn update_signed(&mut self, signed_data: &[bool]) { - for (i, field) in self.fields.iter_mut().enumerate() { - if i < signed_data.len() { - field.signed = signed_data[i]; - } - } - } - - fn update_predictors(&mut self, predictors: &[u8]) { - for (i, field) in self.fields.iter_mut().enumerate() { - if i < predictors.len() { - field.predictor = predictors[i]; - } - } - } - - fn update_encoding(&mut self, encodings: &[u8]) { - for (i, field) in self.fields.iter_mut().enumerate() { - if i < encodings.len() { - field.encoding = encodings[i]; - } - } - } -} +// FieldDefinition and FrameDefinition now imported from bbl_parser::types #[derive(Debug)] struct BBLHeader { @@ -1763,7 +1706,7 @@ fn parse_frames( // GPS frame history for differential encoding let mut gps_frame_history: Vec = Vec::new(); - let mut stream = bbl_format::BBLDataStream::new(binary_data); + let mut stream = BBLDataStream::new(binary_data); // Main frame parsing loop - process frames as a stream, don't store all while !stream.eof { @@ -1804,7 +1747,7 @@ fn parse_frames( // I-frames reset the prediction history frame_history.current_frame.fill(0); - if bbl_format::parse_frame_data( + if parse_frame_data( &mut stream, &header.i_frame_def, &mut frame_history.current_frame, @@ -1886,7 +1829,7 @@ fn parse_frames( if header.p_frame_def.count > 0 && frame_history.valid { let mut p_frame_values = vec![0i32; header.p_frame_def.count]; - if bbl_format::parse_frame_data( + if parse_frame_data( &mut stream, &header.p_frame_def, &mut p_frame_values, @@ -2080,7 +2023,7 @@ fn parse_frames( let mut g_frame_values = vec![0i32; header.g_frame_def.count]; - if bbl_format::parse_frame_data( + if parse_frame_data( &mut stream, &header.g_frame_def, &mut g_frame_values, @@ -2354,7 +2297,7 @@ fn parse_frames( #[allow(dead_code)] fn parse_i_frame( - stream: &mut bbl_format::BBLDataStream, + stream: &mut BBLDataStream, frame_def: &FrameDefinition, debug: bool, ) -> Result> { @@ -2363,12 +2306,10 @@ fn parse_i_frame( // Parse each field according to the frame definition for field in &frame_def.fields { let value = match field.encoding { - bbl_format::ENCODING_SIGNED_VB => stream.read_signed_vb()?, - bbl_format::ENCODING_UNSIGNED_VB => stream.read_unsigned_vb()? as i32, - bbl_format::ENCODING_NEG_14BIT => { - -(bbl_format::sign_extend_14bit(stream.read_unsigned_vb()? as u16)) - } - bbl_format::ENCODING_NULL => 0, + ENCODING_SIGNED_VB => stream.read_signed_vb()?, + ENCODING_UNSIGNED_VB => stream.read_unsigned_vb()? as i32, + ENCODING_NEG_14BIT => -(sign_extend_14bit(stream.read_unsigned_vb()? as u16)), + ENCODING_NULL => 0, _ => { if debug { println!( @@ -2387,7 +2328,7 @@ fn parse_i_frame( } fn parse_s_frame( - stream: &mut bbl_format::BBLDataStream, + stream: &mut BBLDataStream, frame_def: &FrameDefinition, debug: bool, ) -> Result> { @@ -2398,22 +2339,22 @@ fn parse_s_frame( let field = &frame_def.fields[field_index]; match field.encoding { - bbl_format::ENCODING_SIGNED_VB => { + ENCODING_SIGNED_VB => { let value = stream.read_signed_vb()?; data.insert(field.name.clone(), value); field_index += 1; } - bbl_format::ENCODING_UNSIGNED_VB => { + ENCODING_UNSIGNED_VB => { let value = stream.read_unsigned_vb()? as i32; data.insert(field.name.clone(), value); field_index += 1; } - bbl_format::ENCODING_NEG_14BIT => { - let value = -(bbl_format::sign_extend_14bit(stream.read_unsigned_vb()? as u16)); + ENCODING_NEG_14BIT => { + let value = -(sign_extend_14bit(stream.read_unsigned_vb()? as u16)); data.insert(field.name.clone(), value); field_index += 1; } - bbl_format::ENCODING_TAG2_3S32 => { + ENCODING_TAG2_3S32 => { // This encoding handles 3 fields at once let mut values = [0i32; 8]; stream.read_tag2_3s32(&mut values)?; @@ -2427,7 +2368,7 @@ fn parse_s_frame( } field_index += 3; } - bbl_format::ENCODING_NULL => { + ENCODING_NULL => { data.insert(field.name.clone(), 0); field_index += 1; } @@ -2450,7 +2391,7 @@ fn parse_s_frame( } fn parse_h_frame( - stream: &mut bbl_format::BBLDataStream, + stream: &mut BBLDataStream, frame_def: &FrameDefinition, debug: bool, ) -> Result> { @@ -2467,12 +2408,10 @@ fn parse_h_frame( } let value = match field.encoding { - bbl_format::ENCODING_SIGNED_VB => stream.read_signed_vb()?, - bbl_format::ENCODING_UNSIGNED_VB => stream.read_unsigned_vb()? as i32, - bbl_format::ENCODING_NEG_14BIT => { - -(bbl_format::sign_extend_14bit(stream.read_unsigned_vb()? as u16)) - } - bbl_format::ENCODING_NULL => 0, + ENCODING_SIGNED_VB => stream.read_signed_vb()?, + ENCODING_UNSIGNED_VB => stream.read_unsigned_vb()? as i32, + ENCODING_NEG_14BIT => -(sign_extend_14bit(stream.read_unsigned_vb()? as u16)), + ENCODING_NULL => 0, _ => { if debug { println!( @@ -2494,7 +2433,7 @@ fn parse_h_frame( // like P frames in the main parsing loop for correct GPS coordinate calculation // Parse E frames (Event frames) - based on C reference implementation -fn parse_e_frame(stream: &mut bbl_format::BBLDataStream, debug: bool) -> Result { +fn parse_e_frame(stream: &mut BBLDataStream, debug: bool) -> Result { if debug { println!("Parsing E frame (Event frame)"); } @@ -2660,7 +2599,7 @@ fn parse_e_frame(stream: &mut bbl_format::BBLDataStream, debug: bool) -> Result< }) } -fn skip_frame(stream: &mut bbl_format::BBLDataStream, frame_type: char, debug: bool) -> Result<()> { +fn skip_frame(stream: &mut BBLDataStream, frame_type: char, debug: bool) -> Result<()> { if debug { println!("Skipping {frame_type} frame"); } diff --git a/src/parser/decoder.rs b/src/parser/decoder.rs index 7e06a78..2ac3dff 100644 --- a/src/parser/decoder.rs +++ b/src/parser/decoder.rs @@ -52,6 +52,9 @@ pub fn decode_field_value( Ok(()) } +/// Apply predictor to decode frame field value +/// Enhanced version with debug support, field names lookup, and corruption prevention +#[allow(clippy::too_many_arguments)] pub fn apply_predictor( predictor: u8, value: i32, @@ -61,72 +64,177 @@ pub fn apply_predictor( previous2_frame: &[i32], sysconfig: &std::collections::HashMap, ) -> Result { + // Call the enhanced version with default parameters + Ok(apply_predictor_with_debug( + field_index, + predictor, + value, + current_frame, + Some(previous_frame), + Some(previous2_frame), + 0, + sysconfig, + &[], + false, + )) +} + +/// Enhanced apply_predictor with debug support, field names lookup, and corruption prevention +/// This matches the CLI implementation's full feature set +#[allow(clippy::too_many_arguments)] +pub fn apply_predictor_with_debug( + field_index: usize, + predictor: u8, + raw_value: i32, + current_frame: &[i32], + previous_frame: Option<&[i32]>, + previous2_frame: Option<&[i32]>, + skipped_frames: u32, + sysconfig: &std::collections::HashMap, + field_names: &[String], + debug: bool, +) -> i32 { match predictor { - PREDICT_0 => Ok(value), + PREDICT_0 => raw_value, + PREDICT_PREVIOUS => { - if field_index < previous_frame.len() { - Ok(value + previous_frame[field_index]) + if let Some(prev) = previous_frame { + if field_index < prev.len() { + let result = prev[field_index] + raw_value; + + // CRITICAL FIX: Prevent corruption propagation for vbatLatest + if field_names + .get(field_index) + .map(|name| name == "vbatLatest") + .unwrap_or(false) + { + // Check if previous value is corrupted (way too high for voltage) + if prev[field_index] > 1000 { + if debug { + eprintln!("DEBUG: Fixed corrupted vbatLatest previous value {} replaced with reasonable estimate", prev[field_index]); + } + // Use a reasonable voltage estimate based on vbatref + let vbatref = sysconfig.get("vbatref").copied().unwrap_or(4095); + return vbatref + raw_value; + } + } + + result + } else { + raw_value + } } else { - Ok(value) + raw_value } } + PREDICT_STRAIGHT_LINE => { - if field_index < previous_frame.len() && field_index < previous2_frame.len() { - let prediction = 2 * previous_frame[field_index] - previous2_frame[field_index]; - Ok(value + prediction) - } else if field_index < previous_frame.len() { - Ok(value + previous_frame[field_index]) + if let (Some(prev), Some(prev2)) = (previous_frame, previous2_frame) { + if field_index < prev.len() && field_index < prev2.len() { + raw_value + 2 * prev[field_index] - prev2[field_index] + } else { + raw_value + } } else { - Ok(value) + raw_value } } + PREDICT_AVERAGE_2 => { - if field_index < previous_frame.len() && field_index < previous2_frame.len() { - let average = (previous_frame[field_index] + previous2_frame[field_index]) / 2; - Ok(value + average) - } else if field_index < previous_frame.len() { - Ok(value + previous_frame[field_index]) + if let (Some(prev), Some(prev2)) = (previous_frame, previous2_frame) { + if field_index < prev.len() && field_index < prev2.len() { + raw_value + ((prev[field_index] + prev2[field_index]) / 2) + } else { + raw_value + } } else { - Ok(value) + raw_value } } + PREDICT_MINTHROTTLE => { - let minthrottle = sysconfig.get("minthrottle").copied().unwrap_or(1000); - Ok(value + minthrottle) + let minthrottle = sysconfig.get("minthrottle").copied().unwrap_or(1150); + raw_value + minthrottle } + PREDICT_MOTOR_0 => { - // motor[1], motor[2], motor[3] are predicted based on motor[0] - // Find motor[0] field index (typically field 39 in I-frame) - // For now, use current_frame[39] as motor[0] position based on header analysis - let motor0_index = 39; // Based on field analysis: motor[0] is at position 39 + // Find motor[0] field index dynamically if field_names available + if !field_names.is_empty() { + if let Some(motor0_idx) = field_names.iter().position(|name| name == "motor[0]") { + if motor0_idx < current_frame.len() { + return current_frame[motor0_idx] + raw_value; + } + } + } + // Fallback: use hardcoded position (typically field 39 in I-frame) + let motor0_index = 39; if motor0_index < current_frame.len() { - Ok(value + current_frame[motor0_index]) + current_frame[motor0_index] + raw_value } else { - Ok(value) + raw_value } } + PREDICT_INC => { - if field_index < previous_frame.len() { - Ok(previous_frame[field_index] + value) - } else { - Ok(value) + let mut result = skipped_frames as i32 + 1; + if let Some(prev) = previous_frame { + if field_index < prev.len() { + result += prev[field_index]; + } } + result } + PREDICT_HOME_COORD => { // GPS home coordinate prediction - for now just return value - Ok(value) + raw_value } - PREDICT_1500 => Ok(value + 1500), + + PREDICT_1500 => raw_value + 1500, + PREDICT_VBATREF => { let vbatref = sysconfig.get("vbatref").copied().unwrap_or(4095); - Ok(value + vbatref) + + // CRITICAL FIX: Check for corrupted raw values in vbatLatest + if !field_names.is_empty() + && field_names + .get(field_index) + .map(|name| name == "vbatLatest") + .unwrap_or(false) + && !(-1000..=4000).contains(&raw_value) + { + if debug { + eprintln!( + "DEBUG: Fixed corrupted vbatLatest raw_value {} replaced with 0", + raw_value + ); + } + return vbatref; + } + + raw_value + vbatref } + PREDICT_MINMOTOR => { - // predictor 11 - // motor[0] prediction: value + motorOutput[0] (minimum motor output) - let motor_output_min = sysconfig.get("motorOutput[0]").copied().unwrap_or(48); - Ok(value + motor_output_min) // Force signed 32-bit like Betaflight + // Get the min motor value from motorOutput or motorOutput[0] + let minmotor = sysconfig + .get("motorOutput[0]") + .copied() + .or_else(|| { + sysconfig.get("motorOutput").and_then(|&val| { + // Parse "48,2047" format to get first value + let motor_output_str = val.to_string(); + if let Some(comma_pos) = motor_output_str.find(',') { + motor_output_str[..comma_pos].parse().ok() + } else { + motor_output_str.parse().ok() + } + }) + }) + .unwrap_or(48); + raw_value + minmotor } - _ => Err(anyhow::anyhow!("Invalid predictor type: {}", predictor)), + + _ => raw_value, } } diff --git a/src/parser/frame.rs b/src/parser/frame.rs index b29e6ab..8ffff7a 100644 --- a/src/parser/frame.rs +++ b/src/parser/frame.rs @@ -116,6 +116,7 @@ pub fn parse_frames( false, // Not raw header.data_version, &header.sysconfig, + debug, ) .is_ok() { @@ -169,6 +170,7 @@ pub fn parse_frames( false, // Not raw header.data_version, &header.sysconfig, + debug, ) .is_ok() { @@ -438,11 +440,14 @@ pub fn parse_frame_data( current_frame: &mut [i32], previous_frame: Option<&[i32]>, previous2_frame: Option<&[i32]>, - _skipped_frames: u32, + skipped_frames: u32, raw: bool, _data_version: u8, sysconfig: &HashMap, + debug: bool, ) -> Result<()> { + use crate::parser::decoder::apply_predictor_with_debug; + let mut i = 0; let mut values = [0i32; 8]; @@ -450,15 +455,18 @@ pub fn parse_frame_data( let field = &frame_def.fields[i]; if field.predictor == PREDICT_INC { - current_frame[i] = apply_predictor( + current_frame[i] = apply_predictor_with_debug( + i, field.predictor, 0, - i, current_frame, - previous_frame.unwrap_or(&[]), - previous2_frame.unwrap_or(&[]), + previous_frame, + previous2_frame, + skipped_frames, sysconfig, - )?; + &frame_def.field_names, + debug, + ); i += 1; continue; } @@ -477,15 +485,18 @@ pub fn parse_frame_data( } else { frame_def.fields[i + j].predictor }; - current_frame[i + j] = apply_predictor( + current_frame[i + j] = apply_predictor_with_debug( + i + j, predictor, values[j], - i + j, current_frame, - previous_frame.unwrap_or(&[]), - previous2_frame.unwrap_or(&[]), + previous_frame, + previous2_frame, + skipped_frames, sysconfig, - )?; + &frame_def.field_names, + debug, + ); } i += 4; continue; @@ -504,25 +515,37 @@ pub fn parse_frame_data( } else { frame_def.fields[i + j].predictor }; - current_frame[i + j] = apply_predictor( + current_frame[i + j] = apply_predictor_with_debug( + i + j, predictor, values[j], - i + j, current_frame, - previous_frame.unwrap_or(&[]), - previous2_frame.unwrap_or(&[]), + previous_frame, + previous2_frame, + skipped_frames, sysconfig, - )?; + &frame_def.field_names, + debug, + ); } i += 3; continue; } ENCODING_TAG8_8SVB => { - stream.read_tag8_8svb(&mut values)?; + // Count how many consecutive fields use this encoding + let mut group_count = 1; + for j in i + 1..i + 8.min(frame_def.fields.len() - i) { + if frame_def.fields[j].encoding != ENCODING_TAG8_8SVB { + break; + } + group_count += 1; + } + + stream.read_tag8_8svb_counted(&mut values, group_count)?; - // Apply predictors for the 8 fields - for j in 0..8 { + // Apply predictors for the group + for j in 0..group_count { if i + j >= frame_def.fields.len() { break; } @@ -531,17 +554,20 @@ pub fn parse_frame_data( } else { frame_def.fields[i + j].predictor }; - current_frame[i + j] = apply_predictor( + current_frame[i + j] = apply_predictor_with_debug( + i + j, predictor, values[j], - i + j, current_frame, - previous_frame.unwrap_or(&[]), - previous2_frame.unwrap_or(&[]), + previous_frame, + previous2_frame, + skipped_frames, sysconfig, - )?; + &frame_def.field_names, + debug, + ); } - i += 8; + i += group_count; continue; } @@ -549,15 +575,18 @@ pub fn parse_frame_data( decode_field_value(stream, field.encoding, &mut values, 0)?; let raw_value = values[0]; let predictor = if raw { PREDICT_0 } else { field.predictor }; - current_frame[i] = apply_predictor( + current_frame[i] = apply_predictor_with_debug( + i, predictor, raw_value, - i, current_frame, - previous_frame.unwrap_or(&[]), - previous2_frame.unwrap_or(&[]), + previous_frame, + previous2_frame, + skipped_frames, sysconfig, - )?; + &frame_def.field_names, + debug, + ); } } diff --git a/src/parser/gps.rs b/src/parser/gps.rs index 977b920..3e4af95 100644 --- a/src/parser/gps.rs +++ b/src/parser/gps.rs @@ -129,6 +129,7 @@ pub fn parse_g_frame( false, // Not raw data_version, sysconfig, + false, // debug - GPS parsing doesn't need verbose output )?; // Update GPS frame history with new values diff --git a/src/parser/stream.rs b/src/parser/stream.rs index 422ebbd..bf8687f 100644 --- a/src/parser/stream.rs +++ b/src/parser/stream.rs @@ -209,8 +209,11 @@ impl<'a> BBLDataStream<'a> { } /// Read Tag8_8SVB encoding - exact replica of JavaScript implementation + /// When value_count is 1, reads single signed VB without header byte. + /// Otherwise reads header byte followed by up to 8 values based on header bits. #[allow(clippy::needless_range_loop)] pub fn read_tag8_8svb(&mut self, values: &mut [i32]) -> Result<()> { + // Fixed 8-value version for internal use let selector = self.read_byte()?; for i in 0..8 { @@ -224,6 +227,27 @@ impl<'a> BBLDataStream<'a> { Ok(()) } + /// Read Tag8_8SVB encoding with variable count + /// When value_count is 1, reads single signed VB without header byte. + /// Otherwise reads header byte followed by up to value_count values based on header bits. + #[allow(clippy::needless_range_loop)] + pub fn read_tag8_8svb_counted(&mut self, values: &mut [i32], value_count: usize) -> Result<()> { + if value_count == 1 { + values[0] = self.read_signed_vb()?; + } else { + let mut header = self.read_byte()?; + for i in 0..8.min(value_count) { + values[i] = if header & 0x01 != 0 { + self.read_signed_vb()? + } else { + 0 + }; + header >>= 1; + } + } + Ok(()) + } + /// Read negative 14-bit encoding (sign-magnitude format) /// Reads an unsigned variable byte and interprets it as a 14-bit sign-magnitude value. /// Bit 13 is the sign bit, bits 0-12 are the magnitude. From 34f0bb95082213e81dbce43666c19626eb24a0a1 Mon Sep 17 00:00:00 2001 From: nerdCopter <56646290+nerdCopter@users.noreply.github.com> Date: Mon, 1 Dec 2025 15:47:48 -0600 Subject: [PATCH 03/15] fix: simplify PREDICT_MINMOTOR logic (remove dead code) Remove nonsensical string parsing of i32 values in PREDICT_MINMOTOR. Since sysconfig is HashMap, the values are already integers and don't need comma-separated string parsing. Addresses CodeRabbit review feedback. --- src/parser/decoder.rs | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/src/parser/decoder.rs b/src/parser/decoder.rs index 2ac3dff..4c85201 100644 --- a/src/parser/decoder.rs +++ b/src/parser/decoder.rs @@ -216,21 +216,11 @@ pub fn apply_predictor_with_debug( } PREDICT_MINMOTOR => { - // Get the min motor value from motorOutput or motorOutput[0] + // Get the min motor value from motorOutput[0] or motorOutput let minmotor = sysconfig .get("motorOutput[0]") + .or_else(|| sysconfig.get("motorOutput")) .copied() - .or_else(|| { - sysconfig.get("motorOutput").and_then(|&val| { - // Parse "48,2047" format to get first value - let motor_output_str = val.to_string(); - if let Some(comma_pos) = motor_output_str.find(',') { - motor_output_str[..comma_pos].parse().ok() - } else { - motor_output_str.parse().ok() - } - }) - }) .unwrap_or(48); raw_value + minmotor } From ed5bc26c2fbf0d3fe5cdf4b04a871d3bf2310c56 Mon Sep 17 00:00:00 2001 From: nerdCopter <56646290+nerdCopter@users.noreply.github.com> Date: Tue, 2 Dec 2025 07:42:46 -0600 Subject: [PATCH 04/15] refactor: address CodeRabbit nitpick feedback - Extract magic number 1000 to MAX_REASONABLE_VBAT_RAW constant - Add debug logging for PREDICT_MOTOR_0 hardcoded fallback - Remove redundant function-scoped import (decoder::* glob already includes it) All tests pass, no clippy warnings. --- src/parser/decoder.rs | 11 ++++++++++- src/parser/frame.rs | 7 ++++--- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/src/parser/decoder.rs b/src/parser/decoder.rs index 4c85201..d6da5f0 100644 --- a/src/parser/decoder.rs +++ b/src/parser/decoder.rs @@ -13,6 +13,9 @@ pub const ENCODING_TAG2_3SVARIABLE: u8 = 10; // Predictor constants - directly from JavaScript reference pub const PREDICT_0: u8 = 0; + +// Maximum reasonable raw vbatLatest value before considering it corrupted +const MAX_REASONABLE_VBAT_RAW: i32 = 1000; pub const PREDICT_PREVIOUS: u8 = 1; pub const PREDICT_STRAIGHT_LINE: u8 = 2; pub const PREDICT_AVERAGE_2: u8 = 3; @@ -109,7 +112,7 @@ pub fn apply_predictor_with_debug( .unwrap_or(false) { // Check if previous value is corrupted (way too high for voltage) - if prev[field_index] > 1000 { + if prev[field_index] > MAX_REASONABLE_VBAT_RAW { if debug { eprintln!("DEBUG: Fixed corrupted vbatLatest previous value {} replaced with reasonable estimate", prev[field_index]); } @@ -169,6 +172,12 @@ pub fn apply_predictor_with_debug( // Fallback: use hardcoded position (typically field 39 in I-frame) let motor0_index = 39; if motor0_index < current_frame.len() { + if debug { + eprintln!( + "DEBUG: PREDICT_MOTOR_0 using hardcoded fallback index {}", + motor0_index + ); + } current_frame[motor0_index] + raw_value } else { raw_value diff --git a/src/parser/frame.rs b/src/parser/frame.rs index 8ffff7a..2b75b72 100644 --- a/src/parser/frame.rs +++ b/src/parser/frame.rs @@ -1,4 +1,7 @@ -use crate::parser::{decoder::*, event::parse_e_frame, gps::*, stream::BBLDataStream}; +use crate::parser::{ + decoder::apply_predictor_with_debug, decoder::*, event::parse_e_frame, gps::*, + stream::BBLDataStream, +}; use crate::types::{ DecodedFrame, EventFrame, FrameDefinition, FrameHistory, FrameStats, GpsCoordinate, GpsHomeCoordinate, @@ -446,8 +449,6 @@ pub fn parse_frame_data( sysconfig: &HashMap, debug: bool, ) -> Result<()> { - use crate::parser::decoder::apply_predictor_with_debug; - let mut i = 0; let mut values = [0i32; 8]; From 1150bb7e7675ac9376ed2202eaa974339a3b01eb Mon Sep 17 00:00:00 2001 From: nerdCopter <56646290+nerdCopter@users.noreply.github.com> Date: Tue, 2 Dec 2025 08:47:27 -0600 Subject: [PATCH 05/15] refactor: reorganize MAX_REASONABLE_VBAT_RAW constant placement Move MAX_REASONABLE_VBAT_RAW constant to after predictor constants for better organization. Groups domain-specific constants together and improves code clarity by placing the constant near related corruption detection logic. --- src/parser/decoder.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/parser/decoder.rs b/src/parser/decoder.rs index d6da5f0..bd563c2 100644 --- a/src/parser/decoder.rs +++ b/src/parser/decoder.rs @@ -13,9 +13,6 @@ pub const ENCODING_TAG2_3SVARIABLE: u8 = 10; // Predictor constants - directly from JavaScript reference pub const PREDICT_0: u8 = 0; - -// Maximum reasonable raw vbatLatest value before considering it corrupted -const MAX_REASONABLE_VBAT_RAW: i32 = 1000; pub const PREDICT_PREVIOUS: u8 = 1; pub const PREDICT_STRAIGHT_LINE: u8 = 2; pub const PREDICT_AVERAGE_2: u8 = 3; @@ -28,6 +25,10 @@ pub const PREDICT_VBATREF: u8 = 9; pub const PREDICT_LAST_MAIN_FRAME_TIME: u8 = 10; pub const PREDICT_MINMOTOR: u8 = 11; +// Domain-specific constants for corruption detection +// Maximum reasonable raw vbatLatest value before considering it corrupted +const MAX_REASONABLE_VBAT_RAW: i32 = 1000; + /// Decode a field value using the specified encoding pub fn decode_field_value( stream: &mut BBLDataStream, From bad06e347140abdeaf2ef04239792893392307da Mon Sep 17 00:00:00 2001 From: nerdCopter <56646290+nerdCopter@users.noreply.github.com> Date: Tue, 2 Dec 2025 09:17:14 -0600 Subject: [PATCH 06/15] docs: add detailed documentation for MAX_REASONABLE_VBAT_RAW constant MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Explain voltage mapping from raw counts to volts using Betaflight's default vbat_scale (110). Document that 1000 raw counts ≈ 9.0V and 1420 ≈ 12.6V (fully charged 3S LiPo). Clarify threshold reasoning: 1000 was chosen as a conservative corruption detection threshold. Note that this is intentionally strict and may require configurability for operational safety in future versions. --- src/parser/decoder.rs | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/src/parser/decoder.rs b/src/parser/decoder.rs index bd563c2..13a2812 100644 --- a/src/parser/decoder.rs +++ b/src/parser/decoder.rs @@ -26,7 +26,19 @@ pub const PREDICT_LAST_MAIN_FRAME_TIME: u8 = 10; pub const PREDICT_MINMOTOR: u8 = 11; // Domain-specific constants for corruption detection -// Maximum reasonable raw vbatLatest value before considering it corrupted +// +// MAX_REASONABLE_VBAT_RAW: Maximum reasonable raw vbatLatest value before considering it corrupted. +// +// Voltage Mapping (using Betaflight's default vbat_scale of 110): +// Voltage = (raw_value * vbat_scale) / 4095 * 3.3V * (10+1)/1 [typical voltage divider] +// Simplified: raw_value of 1000 ≈ 9.0V, 1420 ≈ 12.6V (fully charged 3S LiPo) +// +// Threshold Reasoning: +// 1000 was chosen as a conservative corruption detection threshold at the low edge. +// Values outside the symmetric range (-MAX..=+MAX) are treated as corrupted data. +// This is intentionally strict to catch obvious corruption; operators needing to +// adjust the margin for operational safety may require this to be configurable +// in a future version. const MAX_REASONABLE_VBAT_RAW: i32 = 1000; /// Decode a field value using the specified encoding From 1bd2f86d8ee4c1a46c3548cd64328c38e04089ed Mon Sep 17 00:00:00 2001 From: nerdCopter <56646290+nerdCopter@users.noreply.github.com> Date: Tue, 2 Dec 2025 09:18:11 -0600 Subject: [PATCH 07/15] refactor: replace magic numbers with MAX_REASONABLE_VBAT_RAW constant Replace hard-coded range -1000..=4000 with symmetric range using MAX_REASONABLE_VBAT_RAW constant (-MAX..=+MAX). Update debug message to include the constant value for clarity. Add inline comment explaining the symmetric range approach. --- src/parser/decoder.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/parser/decoder.rs b/src/parser/decoder.rs index 13a2812..34cdb28 100644 --- a/src/parser/decoder.rs +++ b/src/parser/decoder.rs @@ -218,17 +218,18 @@ pub fn apply_predictor_with_debug( let vbatref = sysconfig.get("vbatref").copied().unwrap_or(4095); // CRITICAL FIX: Check for corrupted raw values in vbatLatest + // Uses symmetric range based on MAX_REASONABLE_VBAT_RAW constant if !field_names.is_empty() && field_names .get(field_index) .map(|name| name == "vbatLatest") .unwrap_or(false) - && !(-1000..=4000).contains(&raw_value) + && !(-MAX_REASONABLE_VBAT_RAW..=MAX_REASONABLE_VBAT_RAW).contains(&raw_value) { if debug { eprintln!( - "DEBUG: Fixed corrupted vbatLatest raw_value {} replaced with 0", - raw_value + "DEBUG: Fixed corrupted vbatLatest raw_value {} (outside +/-{}) replaced with vbatref", + raw_value, MAX_REASONABLE_VBAT_RAW ); } return vbatref; From caa2bc2d11f3c597f4f1fe5d5a4088cc3156199d Mon Sep 17 00:00:00 2001 From: nerdCopter <56646290+nerdCopter@users.noreply.github.com> Date: Tue, 2 Dec 2025 09:30:23 -0600 Subject: [PATCH 08/15] refactor: propagate debug parameter in GPS frame parsing Replace hardcoded false with debug parameter to enable consistent debug output when debug mode is enabled, matching the function's debug parameter signature. --- src/parser/gps.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/parser/gps.rs b/src/parser/gps.rs index 3e4af95..df4a0de 100644 --- a/src/parser/gps.rs +++ b/src/parser/gps.rs @@ -129,7 +129,7 @@ pub fn parse_g_frame( false, // Not raw data_version, sysconfig, - false, // debug - GPS parsing doesn't need verbose output + debug, )?; // Update GPS frame history with new values From 298074ace98a0ed0c5ea0feb8482597274b51de3 Mon Sep 17 00:00:00 2001 From: nerdCopter <56646290+nerdCopter@users.noreply.github.com> Date: Tue, 2 Dec 2025 09:33:51 -0600 Subject: [PATCH 09/15] refactor: elevate logging level for motor[0] fallback handling Replace DEBUG with WARNING prefix and enhance message clarity to highlight when motor[0] field is not found and fallback to hardcoded index 39 is used. Add comment noting this is frame-definition-dependent and may not work for all firmware versions. This increases visibility of a fragile fallback that should be investigated when encountered in production. --- src/parser/decoder.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/parser/decoder.rs b/src/parser/decoder.rs index 34cdb28..118ed28 100644 --- a/src/parser/decoder.rs +++ b/src/parser/decoder.rs @@ -183,11 +183,12 @@ pub fn apply_predictor_with_debug( } } // Fallback: use hardcoded position (typically field 39 in I-frame) + // This is frame-definition-dependent and may not be correct for all firmware versions let motor0_index = 39; if motor0_index < current_frame.len() { if debug { eprintln!( - "DEBUG: PREDICT_MOTOR_0 using hardcoded fallback index {}", + "WARNING: PREDICT_MOTOR_0 falling back to hardcoded index {} (motor[0] not found in field_names)", motor0_index ); } From 6d4e9a85fb70c33da21599afa732dd91cb23afc7 Mon Sep 17 00:00:00 2001 From: nerdCopter <56646290+nerdCopter@users.noreply.github.com> Date: Tue, 2 Dec 2025 09:36:59 -0600 Subject: [PATCH 10/15] refactor: use stream.read_neg_14bit() for ENCODING_NEG_14BIT handling Replace manual reimplementation of ENCODING_NEG_14BIT handling (-(sign_extend_14bit(stream.read_unsigned_vb()? as u16))) with the existing stream.read_neg_14bit() method in parse_i_frame, parse_s_frame, and parse_h_frame functions. This provides consistency with decoder.rs, gps.rs, and frame.rs which already use the stream method. Remove unused sign_extend_14bit import. --- src/main.rs | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/main.rs b/src/main.rs index 698f2bf..4ffd6f4 100644 --- a/src/main.rs +++ b/src/main.rs @@ -14,7 +14,6 @@ use bbl_parser::conversion::{ }; // Import parser types from crate library -use bbl_parser::parser::helpers::sign_extend_14bit; use bbl_parser::parser::{ parse_frame_data, BBLDataStream, ENCODING_NEG_14BIT, ENCODING_NULL, ENCODING_SIGNED_VB, ENCODING_TAG2_3S32, ENCODING_UNSIGNED_VB, @@ -2308,7 +2307,7 @@ fn parse_i_frame( let value = match field.encoding { ENCODING_SIGNED_VB => stream.read_signed_vb()?, ENCODING_UNSIGNED_VB => stream.read_unsigned_vb()? as i32, - ENCODING_NEG_14BIT => -(sign_extend_14bit(stream.read_unsigned_vb()? as u16)), + ENCODING_NEG_14BIT => stream.read_neg_14bit()?, ENCODING_NULL => 0, _ => { if debug { @@ -2350,7 +2349,7 @@ fn parse_s_frame( field_index += 1; } ENCODING_NEG_14BIT => { - let value = -(sign_extend_14bit(stream.read_unsigned_vb()? as u16)); + let value = stream.read_neg_14bit()?; data.insert(field.name.clone(), value); field_index += 1; } @@ -2410,7 +2409,7 @@ fn parse_h_frame( let value = match field.encoding { ENCODING_SIGNED_VB => stream.read_signed_vb()?, ENCODING_UNSIGNED_VB => stream.read_unsigned_vb()? as i32, - ENCODING_NEG_14BIT => -(sign_extend_14bit(stream.read_unsigned_vb()? as u16)), + ENCODING_NEG_14BIT => stream.read_neg_14bit()?, ENCODING_NULL => 0, _ => { if debug { From f254f88424320f33c4e859abec76f1654d7b2831 Mon Sep 17 00:00:00 2001 From: nerdCopter <56646290+nerdCopter@users.noreply.github.com> Date: Tue, 2 Dec 2025 09:48:40 -0600 Subject: [PATCH 11/15] feat: use 'Log start datetime' header for GPX timestamps Parse the 'H Log start datetime:' header from BBL files and use it to generate absolute timestamps in GPX exports, following betaflight's blackbox_decode approach of combining the log start datetime with frame timestamps. When the FC clock wasn't set (0000-01-01...), falls back to relative timestamps from Unix epoch. Changes: - Add log_start_datetime field to BBLHeader (types/header.rs) - Parse 'H Log start datetime:' header (parser/header.rs, main.rs) - Add datetime parsing and epoch conversion helpers - Update export_gpx_file to use parsed datetime for timestamps - Remove hardcoded 2025-03-26 date workaround --- src/main.rs | 211 ++++++++++++++++++++++++++++++++++++++++--- src/parser/header.rs | 8 ++ src/types/header.rs | 5 + 3 files changed, 213 insertions(+), 11 deletions(-) diff --git a/src/main.rs b/src/main.rs index 4ffd6f4..2e3813d 100644 --- a/src/main.rs +++ b/src/main.rs @@ -240,6 +240,8 @@ struct BBLHeader { craft_name: String, data_version: u8, looptime: u32, + /// Log start datetime from header (ISO 8601 format) + log_start_datetime: Option, i_frame_def: FrameDefinition, p_frame_def: FrameDefinition, s_frame_def: FrameDefinition, @@ -780,6 +782,7 @@ fn parse_headers_from_text(header_text: &str, debug: bool) -> Result let mut craft_name = String::new(); let mut data_version = 2u8; let mut looptime = 0u32; + let mut log_start_datetime: Option = None; let mut sysconfig = HashMap::new(); // Initialize frame definitions @@ -825,6 +828,12 @@ fn parse_headers_from_text(header_text: &str, debug: bool) -> Result { data_version = version; } + } else if line.starts_with("H Log start datetime:") { + // Parse log start datetime for GPX timestamp generation + // Format: "2024-10-10T18:37:25.559+00:00" or "0000-01-01T00:00:00.000+00:00" if not set + if let Some(datetime_str) = line.strip_prefix("H Log start datetime:") { + log_start_datetime = Some(datetime_str.trim().to_string()); + } } else if line.starts_with("H looptime:") { if let Ok(lt) = line .strip_prefix("H looptime:") @@ -968,6 +977,7 @@ fn parse_headers_from_text(header_text: &str, debug: bool) -> Result craft_name, data_version, looptime, + log_start_datetime, i_frame_def, p_frame_def, s_frame_def, @@ -2809,6 +2819,7 @@ fn parse_bbl_file_streaming( &gps_coords, &home_coords, export_options, + log.header.log_start_datetime.as_deref(), ) { let filename = file_path .file_name() @@ -2860,6 +2871,188 @@ fn parse_bbl_file_streaming( // GPS/GPX export functions // Note: GPS conversion functions now imported from bbl_parser::conversion module +/// Generate GPX timestamp from log_start_datetime header + frame timestamp. +/// Following blackbox_decode approach: dateTime + (gpsFrameTime / 1000000) +/// If log_start_datetime is not available or invalid, falls back to relative time from epoch. +fn generate_gpx_timestamp(log_start_datetime: Option<&str>, frame_timestamp_us: u64) -> String { + let total_seconds = frame_timestamp_us / 1_000_000; + let microseconds = frame_timestamp_us % 1_000_000; + + // Try to parse the log start datetime if available + if let Some(datetime_str) = log_start_datetime { + // Check for placeholder datetime (clock not set on FC) + if datetime_str.starts_with("0000-01-01") { + // FC clock wasn't set, fall back to relative time + return format_relative_timestamp(total_seconds, microseconds); + } + + // Parse ISO 8601 datetime: "2024-10-10T18:37:25.559+00:00" + // We only need the date and base time parts for combining with frame offset + if let Some(base_time) = parse_datetime_to_epoch(datetime_str) { + let absolute_secs = base_time + total_seconds; + + // Convert back to date/time components + let secs_per_minute = 60u64; + let secs_per_hour = 3600u64; + let secs_per_day = 86400u64; + + // Calculate time components + let time_of_day = absolute_secs % secs_per_day; + let hours = (time_of_day / secs_per_hour) % 24; + let minutes = (time_of_day % secs_per_hour) / secs_per_minute; + let seconds = time_of_day % secs_per_minute; + + // Calculate date components (days since epoch 1970-01-01) + let days_since_epoch = absolute_secs / secs_per_day; + let (year, month, day) = days_to_ymd(days_since_epoch); + + return format!( + "{:04}-{:02}-{:02}T{:02}:{:02}:{:02}.{:06}Z", + year, month, day, hours, minutes, seconds, microseconds + ); + } + } + + // Fallback: use relative time from epoch + format_relative_timestamp(total_seconds, microseconds) +} + +/// Format a relative timestamp (when no absolute datetime is available) +fn format_relative_timestamp(total_seconds: u64, microseconds: u64) -> String { + // Use 1970-01-01 as base, add the relative seconds + let secs_per_minute = 60u64; + let secs_per_hour = 3600u64; + let secs_per_day = 86400u64; + + let days = total_seconds / secs_per_day; + let time_of_day = total_seconds % secs_per_day; + let hours = time_of_day / secs_per_hour; + let minutes = (time_of_day % secs_per_hour) / secs_per_minute; + let seconds = time_of_day % secs_per_minute; + + let (year, month, day) = days_to_ymd(days); + format!( + "{:04}-{:02}-{:02}T{:02}:{:02}:{:02}.{:06}Z", + year, month, day, hours, minutes, seconds, microseconds + ) +} + +/// Parse ISO 8601 datetime string to seconds since Unix epoch (1970-01-01T00:00:00Z) +fn parse_datetime_to_epoch(datetime_str: &str) -> Option { + // Format: "2024-10-10T18:37:25.559+00:00" or "2024-10-10T18:37:25.559Z" + // We ignore timezone and treat as UTC for simplicity + + // Split into date and time parts + let datetime_part = datetime_str.split('+').next()?.split('Z').next()?; + let parts: Vec<&str> = datetime_part.split('T').collect(); + if parts.len() != 2 { + return None; + } + + let date_parts: Vec = parts[0].split('-').filter_map(|s| s.parse().ok()).collect(); + if date_parts.len() != 3 { + return None; + } + + let time_part = parts[1].split('.').next()?; // Ignore fractional seconds + let time_parts: Vec = time_part + .split(':') + .filter_map(|s| s.parse().ok()) + .collect(); + if time_parts.len() != 3 { + return None; + } + + let year = date_parts[0]; + let month = date_parts[1]; + let day = date_parts[2]; + let hour = time_parts[0]; + let minute = time_parts[1]; + let second = time_parts[2]; + + // Convert to days since epoch (simplified, doesn't handle all edge cases) + let days = ymd_to_days(year, month, day)?; + let secs = + (days as u64) * 86400 + (hour as u64) * 3600 + (minute as u64) * 60 + (second as u64); + + Some(secs) +} + +/// Convert year/month/day to days since Unix epoch (1970-01-01) +fn ymd_to_days(year: u32, month: u32, day: u32) -> Option { + if !(1..=12).contains(&month) || !(1..=31).contains(&day) { + return None; + } + + // Days in each month (non-leap year) + let days_in_month = [0, 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31]; + + let mut total_days: i64 = 0; + + // Add days for complete years since 1970 + for y in 1970..year { + total_days += if is_leap_year(y) { 366 } else { 365 }; + } + + // Add days for complete months in current year + for m in 1..month { + total_days += days_in_month[m as usize] as i64; + if m == 2 && is_leap_year(year) { + total_days += 1; + } + } + + // Add days in current month + total_days += (day - 1) as i64; + + if total_days >= 0 { + Some(total_days as u64) + } else { + None + } +} + +/// Convert days since Unix epoch to year/month/day +fn days_to_ymd(days: u64) -> (u32, u32, u32) { + let mut remaining_days = days as i64; + let mut year = 1970u32; + + // Find the year + loop { + let days_in_year = if is_leap_year(year) { 366 } else { 365 }; + if remaining_days < days_in_year { + break; + } + remaining_days -= days_in_year; + year += 1; + } + + // Days in each month (non-leap year) + let mut days_in_month = [0, 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31]; + if is_leap_year(year) { + days_in_month[2] = 29; + } + + // Find the month + let mut month = 1u32; + for (m, &days) in days_in_month.iter().enumerate().skip(1) { + if remaining_days < days as i64 { + month = m as u32; + break; + } + remaining_days -= days as i64; + } + + let day = (remaining_days + 1) as u32; + + (year, month, day) +} + +/// Check if a year is a leap year +fn is_leap_year(year: u32) -> bool { + (year % 4 == 0 && year % 100 != 0) || (year % 400 == 0) +} + fn export_gpx_file( file_path: &Path, log_number: usize, @@ -2867,6 +3060,7 @@ fn export_gpx_file( gps_coords: &[GpsCoordinate], _home_coords: &[GpsHomeCoordinate], // TODO: Use home coordinates for reference point export_options: &ExportOptions, + log_start_datetime: Option<&str>, ) -> Result<()> { if gps_coords.is_empty() { return Ok(()); @@ -2910,20 +3104,14 @@ fn export_gpx_file( } } - // Convert timestamp to ISO format - // Simplified timestamp calculation to approximate BBD format - let total_seconds = coord.timestamp_us / 1_000_000; - let microseconds = coord.timestamp_us % 1_000_000; - - // Use March 26, 2025 as base date to match BBD format more closely - let hours = 5 + (total_seconds / 3600) % 24; // Start at 05:xx like BBD - let minutes = (total_seconds % 3600) / 60; - let seconds = total_seconds % 60; + // Generate GPX timestamp from log_start_datetime + frame timestamp + // Following blackbox_decode approach: dateTime + (gpsFrameTime / 1000000) + let timestamp_str = generate_gpx_timestamp(log_start_datetime, coord.timestamp_us); writeln!( gpx_file, - r#" {:.2}"#, - coord.latitude, coord.longitude, coord.altitude, hours, minutes, seconds, microseconds + r#" {:.2}"#, + coord.latitude, coord.longitude, coord.altitude, timestamp_str )?; } @@ -3092,6 +3280,7 @@ mod tests { craft_name: "TestCraft".to_string(), data_version: 2, looptime: 500, + log_start_datetime: None, i_frame_def: FrameDefinition::new(), p_frame_def: FrameDefinition::new(), s_frame_def: FrameDefinition::new(), diff --git a/src/parser/header.rs b/src/parser/header.rs index 137a5a9..51c5c28 100644 --- a/src/parser/header.rs +++ b/src/parser/header.rs @@ -46,6 +46,14 @@ pub fn parse_headers_from_text(header_text: &str, debug: bool) -> Result, pub i_frame_def: FrameDefinition, pub p_frame_def: FrameDefinition, pub s_frame_def: FrameDefinition, @@ -30,6 +34,7 @@ impl Default for BBLHeader { craft_name: String::new(), data_version: 2, looptime: 0, + log_start_datetime: None, i_frame_def: FrameDefinition::new(), p_frame_def: FrameDefinition::new(), s_frame_def: FrameDefinition::new(), From 4c85196c666cd4c89610ae6a226751f42aa8c2dd Mon Sep 17 00:00:00 2001 From: nerdCopter <56646290+nerdCopter@users.noreply.github.com> Date: Tue, 2 Dec 2025 09:54:12 -0600 Subject: [PATCH 12/15] refactor: move GPX timestamp functions to conversion.rs library Move datetime/timestamp conversion functions from main.rs to conversion.rs library module so they are available for both CLI and crate users. Moved functions: - generate_gpx_timestamp() - main entry point for GPX timestamp generation - format_relative_timestamp() - fallback for relative time - parse_datetime_to_epoch() - ISO 8601 datetime parsing - ymd_to_days() / days_to_ymd() - date conversion helpers - is_leap_year() - leap year calculation main.rs now imports generate_gpx_timestamp from bbl_parser::conversion --- src/conversion.rs | 186 ++++++++++++++++++++++++++++++++++++++++++++++ src/main.rs | 185 +-------------------------------------------- 2 files changed, 188 insertions(+), 183 deletions(-) diff --git a/src/conversion.rs b/src/conversion.rs index 19e048a..f499c3e 100644 --- a/src/conversion.rs +++ b/src/conversion.rs @@ -202,3 +202,189 @@ pub fn format_failsafe_phase(phase: i32) -> String { _ => phase.to_string(), } } + +// ============================================================================ +// GPX Timestamp Generation (for GPS export) +// ============================================================================ + +/// Generate GPX timestamp from log_start_datetime header + frame timestamp. +/// Following blackbox_decode approach: dateTime + (gpsFrameTime / 1000000) +/// If log_start_datetime is not available or invalid, falls back to relative time from epoch. +pub fn generate_gpx_timestamp(log_start_datetime: Option<&str>, frame_timestamp_us: u64) -> String { + let total_seconds = frame_timestamp_us / 1_000_000; + let microseconds = frame_timestamp_us % 1_000_000; + + // Try to parse the log start datetime if available + if let Some(datetime_str) = log_start_datetime { + // Check for placeholder datetime (clock not set on FC) + if datetime_str.starts_with("0000-01-01") { + // FC clock wasn't set, fall back to relative time + return format_relative_timestamp(total_seconds, microseconds); + } + + // Parse ISO 8601 datetime: "2024-10-10T18:37:25.559+00:00" + // We only need the date and base time parts for combining with frame offset + if let Some(base_time) = parse_datetime_to_epoch(datetime_str) { + let absolute_secs = base_time + total_seconds; + + // Convert back to date/time components + let secs_per_minute = 60u64; + let secs_per_hour = 3600u64; + let secs_per_day = 86400u64; + + // Calculate time components + let time_of_day = absolute_secs % secs_per_day; + let hours = (time_of_day / secs_per_hour) % 24; + let minutes = (time_of_day % secs_per_hour) / secs_per_minute; + let seconds = time_of_day % secs_per_minute; + + // Calculate date components (days since epoch 1970-01-01) + let days_since_epoch = absolute_secs / secs_per_day; + let (year, month, day) = days_to_ymd(days_since_epoch); + + return format!( + "{:04}-{:02}-{:02}T{:02}:{:02}:{:02}.{:06}Z", + year, month, day, hours, minutes, seconds, microseconds + ); + } + } + + // Fallback: use relative time from epoch + format_relative_timestamp(total_seconds, microseconds) +} + +/// Format a relative timestamp (when no absolute datetime is available) +fn format_relative_timestamp(total_seconds: u64, microseconds: u64) -> String { + // Use 1970-01-01 as base, add the relative seconds + let secs_per_minute = 60u64; + let secs_per_hour = 3600u64; + let secs_per_day = 86400u64; + + let days = total_seconds / secs_per_day; + let time_of_day = total_seconds % secs_per_day; + let hours = time_of_day / secs_per_hour; + let minutes = (time_of_day % secs_per_hour) / secs_per_minute; + let seconds = time_of_day % secs_per_minute; + + let (year, month, day) = days_to_ymd(days); + format!( + "{:04}-{:02}-{:02}T{:02}:{:02}:{:02}.{:06}Z", + year, month, day, hours, minutes, seconds, microseconds + ) +} + +/// Parse ISO 8601 datetime string to seconds since Unix epoch (1970-01-01T00:00:00Z) +fn parse_datetime_to_epoch(datetime_str: &str) -> Option { + // Format: "2024-10-10T18:37:25.559+00:00" or "2024-10-10T18:37:25.559Z" + // We ignore timezone and treat as UTC for simplicity + + // Split into date and time parts + let datetime_part = datetime_str.split('+').next()?.split('Z').next()?; + let parts: Vec<&str> = datetime_part.split('T').collect(); + if parts.len() != 2 { + return None; + } + + let date_parts: Vec = parts[0].split('-').filter_map(|s| s.parse().ok()).collect(); + if date_parts.len() != 3 { + return None; + } + + let time_part = parts[1].split('.').next()?; // Ignore fractional seconds + let time_parts: Vec = time_part + .split(':') + .filter_map(|s| s.parse().ok()) + .collect(); + if time_parts.len() != 3 { + return None; + } + + let year = date_parts[0]; + let month = date_parts[1]; + let day = date_parts[2]; + let hour = time_parts[0]; + let minute = time_parts[1]; + let second = time_parts[2]; + + // Convert to days since epoch (simplified, doesn't handle all edge cases) + let days = ymd_to_days(year, month, day)?; + let secs = + (days as u64) * 86400 + (hour as u64) * 3600 + (minute as u64) * 60 + (second as u64); + + Some(secs) +} + +/// Convert year/month/day to days since Unix epoch (1970-01-01) +fn ymd_to_days(year: u32, month: u32, day: u32) -> Option { + if !(1..=12).contains(&month) || !(1..=31).contains(&day) { + return None; + } + + // Days in each month (non-leap year) + let days_in_month = [0, 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31]; + + let mut total_days: i64 = 0; + + // Add days for complete years since 1970 + for y in 1970..year { + total_days += if is_leap_year(y) { 366 } else { 365 }; + } + + // Add days for complete months in current year + for m in 1..month { + total_days += days_in_month[m as usize] as i64; + if m == 2 && is_leap_year(year) { + total_days += 1; + } + } + + // Add days in current month + total_days += (day - 1) as i64; + + if total_days >= 0 { + Some(total_days as u64) + } else { + None + } +} + +/// Convert days since Unix epoch to year/month/day +fn days_to_ymd(days: u64) -> (u32, u32, u32) { + let mut remaining_days = days as i64; + let mut year = 1970u32; + + // Find the year + loop { + let days_in_year = if is_leap_year(year) { 366 } else { 365 }; + if remaining_days < days_in_year { + break; + } + remaining_days -= days_in_year; + year += 1; + } + + // Days in each month (non-leap year) + let mut days_in_month = [0, 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31]; + if is_leap_year(year) { + days_in_month[2] = 29; + } + + // Find the month + let mut month = 1u32; + for (m, &days) in days_in_month.iter().enumerate().skip(1) { + if remaining_days < days as i64 { + month = m as u32; + break; + } + remaining_days -= days as i64; + } + + let day = (remaining_days + 1) as u32; + + (year, month, day) +} + +/// Check if a year is a leap year +fn is_leap_year(year: u32) -> bool { + (year % 4 == 0 && year % 100 != 0) || (year % 400 == 0) +} diff --git a/src/main.rs b/src/main.rs index 2e3813d..106d45b 100644 --- a/src/main.rs +++ b/src/main.rs @@ -10,7 +10,7 @@ use std::path::{Path, PathBuf}; // Import conversion functions from crate library to avoid code duplication use bbl_parser::conversion::{ convert_gps_altitude, convert_gps_coordinate, convert_gps_course, convert_gps_speed, - format_failsafe_phase, format_flight_mode_flags, format_state_flags, + format_failsafe_phase, format_flight_mode_flags, format_state_flags, generate_gpx_timestamp, }; // Import parser types from crate library @@ -2870,188 +2870,7 @@ fn parse_bbl_file_streaming( // GPS/GPX export functions // Note: GPS conversion functions now imported from bbl_parser::conversion module - -/// Generate GPX timestamp from log_start_datetime header + frame timestamp. -/// Following blackbox_decode approach: dateTime + (gpsFrameTime / 1000000) -/// If log_start_datetime is not available or invalid, falls back to relative time from epoch. -fn generate_gpx_timestamp(log_start_datetime: Option<&str>, frame_timestamp_us: u64) -> String { - let total_seconds = frame_timestamp_us / 1_000_000; - let microseconds = frame_timestamp_us % 1_000_000; - - // Try to parse the log start datetime if available - if let Some(datetime_str) = log_start_datetime { - // Check for placeholder datetime (clock not set on FC) - if datetime_str.starts_with("0000-01-01") { - // FC clock wasn't set, fall back to relative time - return format_relative_timestamp(total_seconds, microseconds); - } - - // Parse ISO 8601 datetime: "2024-10-10T18:37:25.559+00:00" - // We only need the date and base time parts for combining with frame offset - if let Some(base_time) = parse_datetime_to_epoch(datetime_str) { - let absolute_secs = base_time + total_seconds; - - // Convert back to date/time components - let secs_per_minute = 60u64; - let secs_per_hour = 3600u64; - let secs_per_day = 86400u64; - - // Calculate time components - let time_of_day = absolute_secs % secs_per_day; - let hours = (time_of_day / secs_per_hour) % 24; - let minutes = (time_of_day % secs_per_hour) / secs_per_minute; - let seconds = time_of_day % secs_per_minute; - - // Calculate date components (days since epoch 1970-01-01) - let days_since_epoch = absolute_secs / secs_per_day; - let (year, month, day) = days_to_ymd(days_since_epoch); - - return format!( - "{:04}-{:02}-{:02}T{:02}:{:02}:{:02}.{:06}Z", - year, month, day, hours, minutes, seconds, microseconds - ); - } - } - - // Fallback: use relative time from epoch - format_relative_timestamp(total_seconds, microseconds) -} - -/// Format a relative timestamp (when no absolute datetime is available) -fn format_relative_timestamp(total_seconds: u64, microseconds: u64) -> String { - // Use 1970-01-01 as base, add the relative seconds - let secs_per_minute = 60u64; - let secs_per_hour = 3600u64; - let secs_per_day = 86400u64; - - let days = total_seconds / secs_per_day; - let time_of_day = total_seconds % secs_per_day; - let hours = time_of_day / secs_per_hour; - let minutes = (time_of_day % secs_per_hour) / secs_per_minute; - let seconds = time_of_day % secs_per_minute; - - let (year, month, day) = days_to_ymd(days); - format!( - "{:04}-{:02}-{:02}T{:02}:{:02}:{:02}.{:06}Z", - year, month, day, hours, minutes, seconds, microseconds - ) -} - -/// Parse ISO 8601 datetime string to seconds since Unix epoch (1970-01-01T00:00:00Z) -fn parse_datetime_to_epoch(datetime_str: &str) -> Option { - // Format: "2024-10-10T18:37:25.559+00:00" or "2024-10-10T18:37:25.559Z" - // We ignore timezone and treat as UTC for simplicity - - // Split into date and time parts - let datetime_part = datetime_str.split('+').next()?.split('Z').next()?; - let parts: Vec<&str> = datetime_part.split('T').collect(); - if parts.len() != 2 { - return None; - } - - let date_parts: Vec = parts[0].split('-').filter_map(|s| s.parse().ok()).collect(); - if date_parts.len() != 3 { - return None; - } - - let time_part = parts[1].split('.').next()?; // Ignore fractional seconds - let time_parts: Vec = time_part - .split(':') - .filter_map(|s| s.parse().ok()) - .collect(); - if time_parts.len() != 3 { - return None; - } - - let year = date_parts[0]; - let month = date_parts[1]; - let day = date_parts[2]; - let hour = time_parts[0]; - let minute = time_parts[1]; - let second = time_parts[2]; - - // Convert to days since epoch (simplified, doesn't handle all edge cases) - let days = ymd_to_days(year, month, day)?; - let secs = - (days as u64) * 86400 + (hour as u64) * 3600 + (minute as u64) * 60 + (second as u64); - - Some(secs) -} - -/// Convert year/month/day to days since Unix epoch (1970-01-01) -fn ymd_to_days(year: u32, month: u32, day: u32) -> Option { - if !(1..=12).contains(&month) || !(1..=31).contains(&day) { - return None; - } - - // Days in each month (non-leap year) - let days_in_month = [0, 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31]; - - let mut total_days: i64 = 0; - - // Add days for complete years since 1970 - for y in 1970..year { - total_days += if is_leap_year(y) { 366 } else { 365 }; - } - - // Add days for complete months in current year - for m in 1..month { - total_days += days_in_month[m as usize] as i64; - if m == 2 && is_leap_year(year) { - total_days += 1; - } - } - - // Add days in current month - total_days += (day - 1) as i64; - - if total_days >= 0 { - Some(total_days as u64) - } else { - None - } -} - -/// Convert days since Unix epoch to year/month/day -fn days_to_ymd(days: u64) -> (u32, u32, u32) { - let mut remaining_days = days as i64; - let mut year = 1970u32; - - // Find the year - loop { - let days_in_year = if is_leap_year(year) { 366 } else { 365 }; - if remaining_days < days_in_year { - break; - } - remaining_days -= days_in_year; - year += 1; - } - - // Days in each month (non-leap year) - let mut days_in_month = [0, 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31]; - if is_leap_year(year) { - days_in_month[2] = 29; - } - - // Find the month - let mut month = 1u32; - for (m, &days) in days_in_month.iter().enumerate().skip(1) { - if remaining_days < days as i64 { - month = m as u32; - break; - } - remaining_days -= days as i64; - } - - let day = (remaining_days + 1) as u32; - - (year, month, day) -} - -/// Check if a year is a leap year -fn is_leap_year(year: u32) -> bool { - (year % 4 == 0 && year % 100 != 0) || (year % 400 == 0) -} +// (generate_gpx_timestamp for GPX timestamp generation) fn export_gpx_file( file_path: &Path, From 1c557226085acc47b414e4662d65f8a5283115c4 Mon Sep 17 00:00:00 2001 From: nerdCopter <56646290+nerdCopter@users.noreply.github.com> Date: Tue, 2 Dec 2025 10:10:23 -0600 Subject: [PATCH 13/15] fix: handle timezone offsets in ISO 8601 datetime parsing Previously the datetime parser ignored timezone offsets and treated all times as UTC, which would cause incorrect timestamps when logs include real-time data from GPS, RTC, or radio sources with non-UTC timezone configurations. Now properly parses and handles: - UTC indicator 'Z' - Positive offsets like '+02:00' (local time ahead of UTC) - Negative offsets like '-05:00' (local time behind UTC) Converts local time to UTC for consistent GPX timestamp generation. --- src/conversion.rs | 67 ++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 60 insertions(+), 7 deletions(-) diff --git a/src/conversion.rs b/src/conversion.rs index f499c3e..382cdd7 100644 --- a/src/conversion.rs +++ b/src/conversion.rs @@ -274,13 +274,47 @@ fn format_relative_timestamp(total_seconds: u64, microseconds: u64) -> String { } /// Parse ISO 8601 datetime string to seconds since Unix epoch (1970-01-01T00:00:00Z) +/// Handles timezone offsets like "+02:00" or "-05:00" by adjusting the result to UTC. fn parse_datetime_to_epoch(datetime_str: &str) -> Option { - // Format: "2024-10-10T18:37:25.559+00:00" or "2024-10-10T18:37:25.559Z" - // We ignore timezone and treat as UTC for simplicity + // Format: "2024-10-10T18:37:25.559+02:00" or "2024-10-10T18:37:25.559Z" + // Parse timezone offset if present, then convert local time to UTC + + // Extract timezone offset in seconds (positive = ahead of UTC, negative = behind) + let tz_offset_secs: i64 = if datetime_str.contains('Z') { + 0 // UTC, no offset + } else if let Some(plus_pos) = datetime_str.rfind('+') { + // Positive offset like "+02:00" means local time is ahead of UTC + parse_tz_offset(&datetime_str[plus_pos + 1..]).unwrap_or(0) + } else if let Some(minus_pos) = datetime_str.rfind('-') { + // Check if this is a date separator or timezone offset + // Timezone offset format: "-HH:MM" at end of string + let potential_tz = &datetime_str[minus_pos + 1..]; + if potential_tz.contains(':') && potential_tz.len() <= 6 { + // Negative offset like "-05:00" means local time is behind UTC + -parse_tz_offset(potential_tz).unwrap_or(0) + } else { + 0 // Date separator, assume UTC + } + } else { + 0 // No timezone info, assume UTC + }; + + // Strip timezone suffix to get clean datetime for parsing + let datetime_clean = if datetime_str.contains('Z') { + datetime_str.split('Z').next()? + } else if datetime_str.contains('+') { + datetime_str.split('+').next()? + } else { + // Handle negative offset: find last '-' that's part of timezone + let parts: Vec<&str> = datetime_str.rsplitn(2, '-').collect(); + if parts.len() == 2 && parts[0].contains(':') && parts[0].len() <= 5 { + parts[1] + } else { + datetime_str + } + }; - // Split into date and time parts - let datetime_part = datetime_str.split('+').next()?.split('Z').next()?; - let parts: Vec<&str> = datetime_part.split('T').collect(); + let parts: Vec<&str> = datetime_clean.split('T').collect(); if parts.len() != 2 { return None; } @@ -308,10 +342,29 @@ fn parse_datetime_to_epoch(datetime_str: &str) -> Option { // Convert to days since epoch (simplified, doesn't handle all edge cases) let days = ymd_to_days(year, month, day)?; - let secs = + let local_secs = (days as u64) * 86400 + (hour as u64) * 3600 + (minute as u64) * 60 + (second as u64); - Some(secs) + // Convert local time to UTC by subtracting the offset + // If offset is +02:00, local time is 2 hours ahead of UTC, so subtract 2 hours + let utc_secs = if tz_offset_secs >= 0 { + local_secs.saturating_sub(tz_offset_secs as u64) + } else { + local_secs.saturating_add((-tz_offset_secs) as u64) + }; + + Some(utc_secs) +} + +/// Parse timezone offset string like "02:00" or "05:30" to seconds +fn parse_tz_offset(tz_str: &str) -> Option { + let parts: Vec<&str> = tz_str.split(':').collect(); + if parts.len() != 2 { + return None; + } + let hours: i64 = parts[0].parse().ok()?; + let minutes: i64 = parts[1].parse().ok()?; + Some(hours * 3600 + minutes * 60) } /// Convert year/month/day to days since Unix epoch (1970-01-01) From 7d36b2550dd8f139314fe7378ca5fcc3c85a5974 Mon Sep 17 00:00:00 2001 From: nerdCopter <56646290+nerdCopter@users.noreply.github.com> Date: Tue, 2 Dec 2025 10:30:17 -0600 Subject: [PATCH 14/15] docs: document datetime parsing format and add unit tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add comprehensive documentation to parse_datetime_to_epoch explaining: - Exact Betaflight datetime format: YYYY-MM-DDTHH:MM:SS.mmm±HH:MM - Supported timezone formats (only colon-separated HH:MM) - Known limitations (no compact offsets like -0500, no region names) - Fractional seconds truncation behavior - Handling of 0000-01-01 placeholder datetime Add 13 unit tests covering: - UTC with Z suffix - Positive/negative timezone offsets - Zero offset equivalence - No timezone (treated as UTC) - Fractional seconds truncation - Betaflight default placeholder - Half-hour timezone offsets (e.g., +05:30) - Invalid format handling - Compact offset limitation documentation - GPX timestamp generation scenarios --- src/conversion.rs | 174 +++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 172 insertions(+), 2 deletions(-) diff --git a/src/conversion.rs b/src/conversion.rs index 382cdd7..8e988a3 100644 --- a/src/conversion.rs +++ b/src/conversion.rs @@ -273,8 +273,27 @@ fn format_relative_timestamp(total_seconds: u64, microseconds: u64) -> String { ) } -/// Parse ISO 8601 datetime string to seconds since Unix epoch (1970-01-01T00:00:00Z) -/// Handles timezone offsets like "+02:00" or "-05:00" by adjusting the result to UTC. +/// Parse ISO 8601 datetime string to seconds since Unix epoch (1970-01-01T00:00:00Z). +/// +/// This function handles the datetime format used by Betaflight's blackbox logs: +/// `YYYY-MM-DDTHH:MM:SS.mmm±HH:MM` (e.g., `2024-10-10T18:37:25.559+00:00`) +/// +/// # Accepted Input Formats +/// - `YYYY-MM-DDTHH:MM:SS.mmmZ` - UTC with 'Z' suffix +/// - `YYYY-MM-DDTHH:MM:SS.mmm+HH:MM` - With positive timezone offset (e.g., `+02:00`) +/// - `YYYY-MM-DDTHH:MM:SS.mmm-HH:MM` - With negative timezone offset (e.g., `-05:00`) +/// - `YYYY-MM-DDTHH:MM:SS` - No timezone (treated as UTC) +/// +/// # Format Limitations +/// - Compact timezone formats like `-0500` are NOT supported (only colon-separated `HH:MM`) +/// - Region-based timezones like `America/New_York` are NOT supported +/// - Fractional seconds are truncated to whole seconds for epoch calculation +/// - Strings without timezone info are treated as UTC +/// +/// # Betaflight Header Context +/// When the flight controller's RTC is not set, Betaflight outputs `0000-01-01T00:00:00.000` +/// as the default datetime. This value should be detected by the caller (via +/// `starts_with("0000-01-01")`) and handled as "no valid datetime available". fn parse_datetime_to_epoch(datetime_str: &str) -> Option { // Format: "2024-10-10T18:37:25.559+02:00" or "2024-10-10T18:37:25.559Z" // Parse timezone offset if present, then convert local time to UTC @@ -441,3 +460,154 @@ fn days_to_ymd(days: u64) -> (u32, u32, u32) { fn is_leap_year(year: u32) -> bool { (year % 4 == 0 && year % 100 != 0) || (year % 400 == 0) } + +#[cfg(test)] +mod tests { + use super::*; + + // Tests for parse_datetime_to_epoch - locking in Betaflight datetime parsing behavior + + #[test] + fn test_parse_datetime_utc_z_suffix() { + // Standard Betaflight format with Z suffix (UTC) + let result = parse_datetime_to_epoch("2024-10-10T18:37:25.559Z"); + assert!(result.is_some()); + // 2024-10-10 18:37:25 UTC + // Expected: days since epoch * 86400 + time of day in seconds + let epoch = result.unwrap(); + assert!(epoch > 0); + // Verify it's in the right ballpark (2024 is ~54 years after 1970) + assert!(epoch > 1700000000); // After 2023 + assert!(epoch < 1800000000); // Before 2027 + } + + #[test] + fn test_parse_datetime_positive_offset() { + // Betaflight format with positive offset (+02:00) + // 2024-10-10T18:37:25+02:00 means UTC is 16:37:25 + let with_offset = parse_datetime_to_epoch("2024-10-10T18:37:25.559+02:00"); + let utc_time = parse_datetime_to_epoch("2024-10-10T16:37:25.559Z"); + + assert!(with_offset.is_some()); + assert!(utc_time.is_some()); + // Both should result in the same UTC epoch + assert_eq!(with_offset.unwrap(), utc_time.unwrap()); + } + + #[test] + fn test_parse_datetime_negative_offset() { + // Betaflight format with negative offset (-05:00) + // 2024-10-10T18:37:25-05:00 means UTC is 23:37:25 + let with_offset = parse_datetime_to_epoch("2024-10-10T18:37:25.559-05:00"); + let utc_time = parse_datetime_to_epoch("2024-10-10T23:37:25.559Z"); + + assert!(with_offset.is_some()); + assert!(utc_time.is_some()); + // Both should result in the same UTC epoch + assert_eq!(with_offset.unwrap(), utc_time.unwrap()); + } + + #[test] + fn test_parse_datetime_zero_offset() { + // +00:00 should be same as Z + let with_offset = parse_datetime_to_epoch("2024-10-10T18:37:25.559+00:00"); + let utc_z = parse_datetime_to_epoch("2024-10-10T18:37:25.559Z"); + + assert!(with_offset.is_some()); + assert!(utc_z.is_some()); + assert_eq!(with_offset.unwrap(), utc_z.unwrap()); + } + + #[test] + fn test_parse_datetime_no_timezone_treated_as_utc() { + // No timezone info - treated as UTC + let no_tz = parse_datetime_to_epoch("2024-10-10T18:37:25"); + let utc_z = parse_datetime_to_epoch("2024-10-10T18:37:25.000Z"); + + assert!(no_tz.is_some()); + assert!(utc_z.is_some()); + assert_eq!(no_tz.unwrap(), utc_z.unwrap()); + } + + #[test] + fn test_parse_datetime_fractional_seconds_truncated() { + // Fractional seconds are truncated (not rounded) + let with_millis = parse_datetime_to_epoch("2024-10-10T18:37:25.999Z"); + let without_millis = parse_datetime_to_epoch("2024-10-10T18:37:25.000Z"); + + assert!(with_millis.is_some()); + assert!(without_millis.is_some()); + // Both should be the same (fractional seconds truncated) + assert_eq!(with_millis.unwrap(), without_millis.unwrap()); + } + + #[test] + fn test_parse_datetime_betaflight_default_placeholder() { + // Betaflight default when RTC not set: 0000-01-01T00:00:00.000 + // This should parse (returns Some) but caller should detect via starts_with("0000-01-01") + let result = parse_datetime_to_epoch("0000-01-01T00:00:00.000+00:00"); + // The function may return None for year 0000 since it's before 1970 + // or it may return Some with an invalid value - either is acceptable + // The important thing is the caller checks for "0000-01-01" prefix first + // This test documents the current behavior + assert!(result.is_none() || result == Some(0)); + } + + #[test] + fn test_parse_datetime_half_hour_offset() { + // Some timezones have 30-minute offsets (e.g., India +05:30) + let with_offset = parse_datetime_to_epoch("2024-10-10T18:37:25.559+05:30"); + let utc_time = parse_datetime_to_epoch("2024-10-10T13:07:25.559Z"); + + assert!(with_offset.is_some()); + assert!(utc_time.is_some()); + assert_eq!(with_offset.unwrap(), utc_time.unwrap()); + } + + #[test] + fn test_parse_datetime_invalid_format_returns_none() { + // Invalid formats should return None + assert!(parse_datetime_to_epoch("not-a-datetime").is_none()); + assert!(parse_datetime_to_epoch("2024-10-10").is_none()); // Missing time + assert!(parse_datetime_to_epoch("18:37:25").is_none()); // Missing date + assert!(parse_datetime_to_epoch("").is_none()); + } + + #[test] + fn test_parse_datetime_compact_offset_not_supported() { + // Compact offset format like -0500 is NOT supported (only HH:MM with colon) + // This test documents the limitation - it will be treated as no timezone + let compact = parse_datetime_to_epoch("2024-10-10T18:37:25.559-0500"); + // The -0500 won't be parsed as a timezone, so it's treated as local time without offset + // This means it differs from the colon-separated version + let _colon_sep = parse_datetime_to_epoch("2024-10-10T18:37:25.559-05:00"); + + // Both should parse, but may have different values + // The compact form will be treated as UTC (offset not recognized) + assert!(compact.is_some() || compact.is_none()); // May or may not parse + } + + // Tests for generate_gpx_timestamp + + #[test] + fn test_generate_gpx_timestamp_with_valid_datetime() { + // When log_start_datetime is valid, should produce absolute timestamp + let timestamp = generate_gpx_timestamp(Some("2024-10-10T18:37:25.559+00:00"), 1_000_000); + assert!(timestamp.contains("2024-10-10T18:37:26")); // 1 second after start + } + + #[test] + fn test_generate_gpx_timestamp_placeholder_datetime() { + // When FC clock wasn't set, should fall back to relative time + let timestamp = generate_gpx_timestamp(Some("0000-01-01T00:00:00.000+00:00"), 1_000_000); + // Should use 1970-01-01 as base (relative time) + assert!(timestamp.contains("1970-01-01")); + } + + #[test] + fn test_generate_gpx_timestamp_no_datetime() { + // When no datetime provided, should use relative time from epoch + let timestamp = generate_gpx_timestamp(None, 0); + assert!(timestamp.contains("1970-01-01T00:00:00")); + } +} From d311302f5e051a8816c3b47ae4c623f7fbb38a0b Mon Sep 17 00:00:00 2001 From: nerdCopter <56646290+nerdCopter@users.noreply.github.com> Date: Tue, 2 Dec 2025 10:51:12 -0600 Subject: [PATCH 15/15] refactor: extract shared timestamp formatting logic - Create epoch_seconds_to_iso8601() helper to eliminate duplication between generate_gpx_timestamp() and format_relative_timestamp() - Remove redundant format_relative_timestamp() function - Add defensive month assignment in days_to_ymd() for extra safety Addresses code quality nitpicks from review. --- src/conversion.rs | 71 +++++++++++++++++++---------------------------- 1 file changed, 29 insertions(+), 42 deletions(-) diff --git a/src/conversion.rs b/src/conversion.rs index 8e988a3..bd63c1c 100644 --- a/src/conversion.rs +++ b/src/conversion.rs @@ -207,6 +207,27 @@ pub fn format_failsafe_phase(phase: i32) -> String { // GPX Timestamp Generation (for GPS export) // ============================================================================ +/// Convert epoch seconds and microseconds to ISO 8601 string. +/// Helper function to eliminate duplication between timestamp formatting functions. +fn epoch_seconds_to_iso8601(total_seconds: u64, microseconds: u64) -> String { + let secs_per_minute = 60u64; + let secs_per_hour = 3600u64; + let secs_per_day = 86400u64; + + let time_of_day = total_seconds % secs_per_day; + let hours = (time_of_day / secs_per_hour) % 24; + let minutes = (time_of_day % secs_per_hour) / secs_per_minute; + let seconds = time_of_day % secs_per_minute; + + let days_since_epoch = total_seconds / secs_per_day; + let (year, month, day) = days_to_ymd(days_since_epoch); + + format!( + "{:04}-{:02}-{:02}T{:02}:{:02}:{:02}.{:06}Z", + year, month, day, hours, minutes, seconds, microseconds + ) +} + /// Generate GPX timestamp from log_start_datetime header + frame timestamp. /// Following blackbox_decode approach: dateTime + (gpsFrameTime / 1000000) /// If log_start_datetime is not available or invalid, falls back to relative time from epoch. @@ -219,58 +240,19 @@ pub fn generate_gpx_timestamp(log_start_datetime: Option<&str>, frame_timestamp_ // Check for placeholder datetime (clock not set on FC) if datetime_str.starts_with("0000-01-01") { // FC clock wasn't set, fall back to relative time - return format_relative_timestamp(total_seconds, microseconds); + return epoch_seconds_to_iso8601(total_seconds, microseconds); } // Parse ISO 8601 datetime: "2024-10-10T18:37:25.559+00:00" // We only need the date and base time parts for combining with frame offset if let Some(base_time) = parse_datetime_to_epoch(datetime_str) { let absolute_secs = base_time + total_seconds; - - // Convert back to date/time components - let secs_per_minute = 60u64; - let secs_per_hour = 3600u64; - let secs_per_day = 86400u64; - - // Calculate time components - let time_of_day = absolute_secs % secs_per_day; - let hours = (time_of_day / secs_per_hour) % 24; - let minutes = (time_of_day % secs_per_hour) / secs_per_minute; - let seconds = time_of_day % secs_per_minute; - - // Calculate date components (days since epoch 1970-01-01) - let days_since_epoch = absolute_secs / secs_per_day; - let (year, month, day) = days_to_ymd(days_since_epoch); - - return format!( - "{:04}-{:02}-{:02}T{:02}:{:02}:{:02}.{:06}Z", - year, month, day, hours, minutes, seconds, microseconds - ); + return epoch_seconds_to_iso8601(absolute_secs, microseconds); } } // Fallback: use relative time from epoch - format_relative_timestamp(total_seconds, microseconds) -} - -/// Format a relative timestamp (when no absolute datetime is available) -fn format_relative_timestamp(total_seconds: u64, microseconds: u64) -> String { - // Use 1970-01-01 as base, add the relative seconds - let secs_per_minute = 60u64; - let secs_per_hour = 3600u64; - let secs_per_day = 86400u64; - - let days = total_seconds / secs_per_day; - let time_of_day = total_seconds % secs_per_day; - let hours = time_of_day / secs_per_hour; - let minutes = (time_of_day % secs_per_hour) / secs_per_minute; - let seconds = time_of_day % secs_per_minute; - - let (year, month, day) = days_to_ymd(days); - format!( - "{:04}-{:02}-{:02}T{:02}:{:02}:{:02}.{:06}Z", - year, month, day, hours, minutes, seconds, microseconds - ) + epoch_seconds_to_iso8601(total_seconds, microseconds) } /// Parse ISO 8601 datetime string to seconds since Unix epoch (1970-01-01T00:00:00Z). @@ -450,6 +432,11 @@ fn days_to_ymd(days: u64) -> (u32, u32, u32) { } remaining_days -= days as i64; } + // Defensive: if somehow we exhausted all months (shouldn't happen), default to December + // This satisfies the year-loop invariant that remaining_days < 365/366 + if remaining_days >= days_in_month[month as usize] as i64 { + month = 12; + } let day = (remaining_days + 1) as u32;