feat(quote)!: remove the derived QuoteTick.midpoint column (#1201)

* feat(quote)!: remove the derived QuoteTick.midpoint column

The engine-vs-terminal audit flagged QuoteTick.midpoint as a client-side derivation: the parser computed (bid + ask) / 2 and surfaced it as a column the wire never sends. The terminal's gRPC bridge is pure pass-through and emits no midpoint for quotes — this is the same class as the already-removed derive_ohlcvc and local greeks calc. The SDK's contract is to deliver exactly what the terminal delivers, so the derived column is removed.

Removed the midpoint synthesis from every tick codegen surface (parser, tdbe struct, rust/python arrow frames, python/typescript classes, cpp layout asserts) and the columns presence helper's derive_midpoint path, then regenerated. The QuoteTick struct, its Arrow/Polars column, the per-binding class fields, the C ABI struct (tail padding grows to keep the align(64) size), the MCP/server JSON serialisers, and the benches all drop it. IvTick.midpoint and MarketValueTick's integer midpoint are real server-sent values and are unchanged.

Breaking: the quote history / snapshot Arrow schema and the QuoteTick binding types no longer carry a midpoint column; compute (bid + ask) / 2 caller-side if needed.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

* test: drop residual QuoteTick.midpoint references in projection tests

test_column_projection built two QuoteTick values with a midpoint field and
asserted midpoint in the projected column set. QuoteTick no longer carries that
column, so `cargo check --all-targets --features arrow,polars` failed with
E0560. Remove the field from both constructions and drop midpoint from the
expected column lists; the projection assertions now match the wire schema.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

* style: rustfmt projection-test comment reflow

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

---------

Co-authored-by: preview <noreply@anthropic.com>

Author: userFRM

thetadatadx-cpp/include/thetadatadx.h

@@ -603,7 +603,7 @@ THETADATADX_ALIGN64_BEGIN typedef struct {
 } ThetaDataDxIndexPriceAtTimeTick THETADATADX_ALIGN64_END;
 
 /* NBBO quote tick (*_history_quote): the bid/ask quote with sizes,
- * exchanges, conditions, and a derived midpoint. */
+ * exchanges, and conditions. */
 THETADATADX_ALIGN64_BEGIN typedef struct {
     int32_t ms_of_day;
     int32_t bid_size;
@@ -624,9 +624,7 @@ THETADATADX_ALIGN64_BEGIN typedef struct {
      * 'P' (80) for a put, 0 when contract identity is absent
      * (single-contract queries). Cast to char for display. */
     uint32_t right;
-    /* 4 bytes padding before the double field */
-    double midpoint;
-    uint8_t _tail_padding[40];
+    uint8_t _tail_padding[52];
 } ThetaDataDxQuoteTick THETADATADX_ALIGN64_END;
 
 /* Trade-with-quote tick (*_history_trade_quote): each trade print fused

thetadatadx-cpp/include/tick_layout_asserts.hpp.inc

@@ -462,8 +462,6 @@ static_assert(offsetof(QuoteTick, strike) == 64,
               "ThetaDataDxQuoteTick.strike offset drifted from Rust");
 static_assert(offsetof(QuoteTick, right) == 72,
               "ThetaDataDxQuoteTick.right offset drifted from Rust");
-static_assert(offsetof(QuoteTick, midpoint) == 80,
-              "ThetaDataDxQuoteTick.midpoint offset drifted from Rust");
 static_assert(sizeof(TradeGreeksAllTick) == 320 && alignof(TradeGreeksAllTick) == 64,
               "ThetaDataDxTradeGreeksAllTick layout drifted from Rust");
 static_assert(offsetof(TradeGreeksAllTick, ms_of_day) == 0,

thetadatadx-py/src/_generated/tick_arrow.rs

@@ -264,7 +264,6 @@ pub(crate) fn arrow_schema_for_qualname(qualname: &str) -> Option<Arc<Schema>> {
             Field::new("ask", DataType::Float64, false),
             Field::new("ask_condition", DataType::Int32, false),
             Field::new("date", DataType::Int32, false),
-            Field::new("midpoint", DataType::Float64, false),
             Field::new("expiration", DataType::Int32, true),
             Field::new("strike", DataType::Float64, true),
             Field::new("right", DataType::Utf8, true),
@@ -2310,7 +2309,6 @@ pub(crate) mod slice_arrow {
         let has_expiration = present.contains("expiration");
         let has_strike = present.contains("strike");
         let has_right = present.contains("right");
-        let has_midpoint = present.contains("midpoint");
         let mut col_ms_of_day: Vec<i32> = Vec::with_capacity(if has_ms_of_day { n } else { 0 });
         let mut col_bid_size: Vec<i32> = Vec::with_capacity(if has_bid_size { n } else { 0 });
         let mut col_bid_exchange: Vec<i32> = Vec::with_capacity(if has_bid_exchange { n } else { 0 });
@@ -2324,7 +2322,6 @@ pub(crate) mod slice_arrow {
         let mut col_expiration: Vec<Option<i32>> = Vec::with_capacity(if has_expiration { n } else { 0 });
         let mut col_strike: Vec<Option<f64>> = Vec::with_capacity(if has_strike { n } else { 0 });
         let mut col_right: Vec<Option<String>> = Vec::with_capacity(if has_right { n } else { 0 });
-        let mut col_midpoint: Vec<f64> = Vec::with_capacity(if has_midpoint { n } else { 0 });
         for t in ticks {
             if has_ms_of_day { col_ms_of_day.push(t.ms_of_day); }
             if has_bid_size { col_bid_size.push(t.bid_size); }
@@ -2339,7 +2336,6 @@ pub(crate) mod slice_arrow {
             if has_expiration { col_expiration.push(t.has_contract_id().then_some(t.expiration)); }
             if has_strike { col_strike.push(t.has_contract_id().then_some(t.strike)); }
             if has_right { col_right.push(if t.right == '\0' { None } else { Some(t.right.to_string()) }); }
-            if has_midpoint { col_midpoint.push(t.midpoint); }
         }
         let mut fields: Vec<Field> = Vec::new();
         let mut columns: Vec<ArrayRef> = Vec::new();
@@ -2402,10 +2398,6 @@ pub(crate) mod slice_arrow {
             fields.push(Field::new("right", DataType::Utf8, true));
             columns.push(Arc::new(StringArray::from(col_right)) as ArrayRef);
         }
-        if has_midpoint {
-            fields.push(Field::new("midpoint", DataType::Float64, false));
-            columns.push(Arc::new(Float64Array::from(col_midpoint)) as ArrayRef);
-        }
         RecordBatch::try_new_with_options(Arc::new(Schema::new(fields)), columns, &RecordBatchOptions::new().with_row_count(Some(n))).map_err(|e| pyo3::exceptions::PyRuntimeError::new_err(e.to_string()))
     }
 

thetadatadx-py/src/_generated/tick_classes.rs

@@ -991,16 +991,15 @@ pub(crate) struct QuoteTick {
     #[pyo3(get)] pub ask: f64,
     #[pyo3(get)] pub ask_condition: i32,
     #[pyo3(get)] pub date: i32,
-    #[pyo3(get)] pub midpoint: f64,
     #[pyo3(get)] pub expiration: Option<i32>,
     #[pyo3(get)] pub strike: Option<f64>,
     #[pyo3(get)] pub right: Option<String>,
 }
 #[pymethods]
 impl QuoteTick {
     #[new]
-    #[pyo3(signature = (*, ms_of_day = 0i32, bid_size = 0i32, bid_exchange = 0i32, bid = 0.0f64, bid_condition = 0i32, ask_size = 0i32, ask_exchange = 0i32, ask = 0.0f64, ask_condition = 0i32, date = 0i32, midpoint = 0.0f64, expiration = None, strike = None, right = None))]
-    fn new(ms_of_day: i32, bid_size: i32, bid_exchange: i32, bid: f64, bid_condition: i32, ask_size: i32, ask_exchange: i32, ask: f64, ask_condition: i32, date: i32, midpoint: f64, expiration: Option<i32>, strike: Option<f64>, right: Option<String>) -> Self {
+    #[pyo3(signature = (*, ms_of_day = 0i32, bid_size = 0i32, bid_exchange = 0i32, bid = 0.0f64, bid_condition = 0i32, ask_size = 0i32, ask_exchange = 0i32, ask = 0.0f64, ask_condition = 0i32, date = 0i32, expiration = None, strike = None, right = None))]
+    fn new(ms_of_day: i32, bid_size: i32, bid_exchange: i32, bid: f64, bid_condition: i32, ask_size: i32, ask_exchange: i32, ask: f64, ask_condition: i32, date: i32, expiration: Option<i32>, strike: Option<f64>, right: Option<String>) -> Self {
         Self {
             ms_of_day,
             bid_size,
@@ -1012,7 +1011,6 @@ impl QuoteTick {
             ask,
             ask_condition,
             date,
-            midpoint,
             expiration,
             strike,
             right,
@@ -4940,7 +4938,6 @@ impl QuoteTickList {
                 ask: t.ask,
                 ask_condition: t.ask_condition,
                 date: t.date,
-                midpoint: t.midpoint,
                 expiration: t.expiration.unwrap_or(0),
                 strike: t.strike.unwrap_or(0.0),
                 right: match t.right.as_deref() { Some("C") => 'C', Some("P") => 'P', None | Some("") => '\0', Some(other) => return Err(pyo3::exceptions::PyValueError::new_err(format!("right must be \"C\" or \"P\", got {other:?}"))) },
@@ -5005,7 +5002,6 @@ impl QuoteTickList {
                 ask: t.ask,
                 ask_condition: t.ask_condition,
                 date: t.date,
-                midpoint: t.midpoint,
                 expiration: t.has_contract_id().then_some(t.expiration),
                 strike: t.has_contract_id().then_some(t.strike),
                 right: if t.right == '\0' { None } else { Some(t.right.to_string()) },
@@ -5035,7 +5031,6 @@ impl QuoteTickList {
                     ask: t.ask,
                     ask_condition: t.ask_condition,
                     date: t.date,
-                    midpoint: t.midpoint,
                     expiration: t.has_contract_id().then_some(t.expiration),
                     strike: t.has_contract_id().then_some(t.strike),
                     right: if t.right == '\0' { None } else { Some(t.right.to_string()) },
@@ -5111,7 +5106,6 @@ impl QuoteTickListIter {
                 ask: t.ask,
                 ask_condition: t.ask_condition,
                 date: t.date,
-                midpoint: t.midpoint,
                 expiration: t.has_contract_id().then_some(t.expiration),
                 strike: t.has_contract_id().then_some(t.strike),
                 right: if t.right == '\0' { None } else { Some(t.right.to_string()) },
@@ -7485,7 +7479,6 @@ pub(crate) fn quote_ticks_vec_to_pylist(py: Python<'_>, ticks: thetadatadx::Tick
                 ask: t.ask,
                 ask_condition: t.ask_condition,
                 date: t.date,
-                midpoint: t.midpoint,
                 expiration: t.has_contract_id().then_some(t.expiration),
                 strike: t.has_contract_id().then_some(t.strike),
                 right: if t.right == '\0' { None } else { Some(t.right.to_string()) },

thetadatadx-py/tests/test_dataframe_arrow.py

@@ -261,13 +261,6 @@ def test_ohlc_tick_to_arrow_schema():
     _assert_arrow_types_match(table, expected)
 
 
-def test_quote_tick_to_arrow_has_midpoint():
-    tick = thetadatadx.QuoteTick(ms_of_day=34_200_000, bid=99.95, ask=100.05, midpoint=100.0)
-    lst = thetadatadx.QuoteTickList([tick])
-    table = lst.to_arrow()
-    schema = {f.name: str(f.type) for f in table.schema}
-    assert "midpoint" in schema
-    assert schema["midpoint"] == "double"
 
 
 def test_trade_tick_to_arrow_schema():

thetadatadx-rs/benches/bench_tick.rs

@@ -1,7 +1,7 @@
 use criterion::{criterion_group, criterion_main, Criterion};
 use std::hint::black_box;
 
-use thetadatadx::{OhlcTick, QuoteTick, TradeTick};
+use thetadatadx::{OhlcTick, TradeTick};
 
 // ═══════════════════════════════════════════════════════════════════════════
 //  Tick operation benchmarks
@@ -39,30 +39,6 @@ fn bench_trade_tick_price_access(c: &mut Criterion) {
     });
 }
 
-fn bench_quote_tick_midpoint(c: &mut Criterion) {
-    let tick = QuoteTick {
-        ms_of_day: 34_200_000,
-        bid_size: 50,
-        bid_exchange: 4,
-        bid: 150.20,
-        bid_condition: 1,
-        ask_size: 30,
-        ask_exchange: 4,
-        ask: 150.30,
-        ask_condition: 1,
-        midpoint: 150.25,
-        date: 20240315,
-        expiration: 0,
-        strike: 0.0,
-        right: '\0',
-    };
-    c.bench_function("quote_tick_midpoint", |b| {
-        b.iter(|| {
-            black_box(black_box(&tick).midpoint);
-        });
-    });
-}
-
 fn bench_ohlc_tick_all_prices(c: &mut Criterion) {
     let tick = OhlcTick {
         ms_of_day: 34_200_000,
@@ -93,7 +69,6 @@ fn bench_ohlc_tick_all_prices(c: &mut Criterion) {
 criterion_group!(
     tick_benches,
     bench_trade_tick_price_access,
-    bench_quote_tick_midpoint,
     bench_ohlc_tick_all_prices,
 );
 

thetadatadx-rs/build_support/ticks/parser.rs

@@ -166,9 +166,6 @@ fn generate_parser(out: &mut String, type_name: &str, def: &TickTypeDef) {
         out.push_str("    let _cid_right_idx = h.iter().position(|c| *c == \"right\");\n");
     }
 
-    // Check if this is QuoteTick (needs midpoint computation).
-    let is_quote_tick = type_name == "QuoteTick";
-
     out.push('\n');
 
     // Phase 2: seed the output, then bulk-extract column by column in
@@ -187,9 +184,6 @@ fn generate_parser(out: &mut String, type_name: &str, def: &TickTypeDef) {
         out.push_str("            strike: 0.0,\n");
         out.push_str("            right: '\\0',\n");
     }
-    if is_quote_tick {
-        out.push_str("            midpoint: 0.0,\n");
-    }
     out.push_str("        };\n");
     out.push_str("        rows.len()\n");
     out.push_str("    ];\n");
@@ -236,13 +230,6 @@ fn generate_parser(out: &mut String, type_name: &str, def: &TickTypeDef) {
         out.push_str("        }\n");
     }
 
-    if is_quote_tick {
-        // QuoteTick gets midpoint computed from bid + ask.
-        out.push_str("        for tick in ticks.iter_mut() {\n");
-        out.push_str("            tick.midpoint = (tick.bid + tick.ask) / 2.0;\n");
-        out.push_str("        }\n");
-    }
-
     out.push_str("        row_base += rows.len();\n");
     out.push_str("    }\n");
     out.push_str("    Ok(ticks)\n");
@@ -259,10 +246,9 @@ fn generate_parser(out: &mut String, type_name: &str, def: &TickTypeDef) {
 /// resolves several schema names to the same wire spelling), and the parser
 /// fills every such field from that one column. `present_columns_from` counts
 /// each physical header once (first-claim, exact matches before aliases) but
-/// exempts the two derived fields that are not their own physical column:
-/// `date` (the `YYYYMMDD` split of a `Timestamp` header, present whenever it
-/// resolves) and `QuoteTick.midpoint` (computed from `bid` + `ask`, present
-/// whenever both inputs are). The EOD wire sends `created`: the exact match
+/// exempts the derived `date` field that is not its own physical column:
+/// the `YYYYMMDD` split of a `Timestamp` header, present whenever it
+/// resolves. The EOD wire sends `created`: the exact match
 /// (`created` -> `created_ms_of_day`) claims the header, and `date` rides the
 /// same column via the exemption instead of being dropped.
 ///
@@ -273,7 +259,6 @@ fn generate_parser(out: &mut String, type_name: &str, def: &TickTypeDef) {
 /// `is_open` and `status`, so its generated impl delegates to the matching
 /// calendar-specific presence table instead of the generic first-claim helper.
 fn generate_present_columns(out: &mut String, type_name: &str, def: &TickTypeDef) {
-    let derive_midpoint = type_name == "QuoteTick";
     writeln!(out, "impl crate::columns::WireColumns for {type_name} {{").unwrap();
     out.push_str("    fn present_columns(headers: &[&str]) -> crate::columns::ColumnPresence {\n");
     if type_name == "CalendarDay" {
@@ -292,15 +277,15 @@ fn generate_present_columns(out: &mut String, type_name: &str, def: &TickTypeDef
         out.push_str("        ];\n");
         writeln!(
             out,
-            "        crate::columns::present_columns_from(headers, COLS, {}, {derive_midpoint})",
+            "        crate::columns::present_columns_from(headers, COLS, {})",
             def.contract_id,
         )
         .unwrap();
     }
     out.push_str("    }\n\n");
 
     // all_columns(): the full-schema column set (schema columns + the
-    // contract-id trio + the QuoteTick `midpoint` derived tail), matching
+    // contract-id trio), matching
     // the column order the full `to_arrow` / `to_polars` builders emit.
     out.push_str("    fn all_columns() -> crate::columns::ColumnPresence {\n");
     out.push_str("        crate::columns::ColumnPresence::from_names([\n");
@@ -312,9 +297,6 @@ fn generate_present_columns(out: &mut String, type_name: &str, def: &TickTypeDef
         out.push_str("            \"strike\",\n");
         out.push_str("            \"right\",\n");
     }
-    if type_name == "QuoteTick" {
-        out.push_str("            \"midpoint\",\n");
-    }
     out.push_str("        ])\n");
     out.push_str("    }\n");
     out.push_str("}\n\n");

thetadatadx-rs/build_support_bin/ticks/cpp.rs

@@ -26,7 +26,7 @@ pub(super) fn render_cpp_tick_layout_asserts(schema: &Schema) -> String {
             continue;
         }
         let def = &schema.types[type_name];
-        let (size, align) = tick_ffi_size_and_align(type_name, def);
+        let (size, align) = tick_ffi_size_and_align(def);
         // C++ wrapper exposes the schema type name verbatim under
         // `thetadatadx::cxx`; the C mirror is `ThetaDataDx<TypeName>`. The wrapper
         // alias is `using <TypeName> = ThetaDataDx<TypeName>;` so `sizeof(alias)`
@@ -40,10 +40,9 @@ pub(super) fn render_cpp_tick_layout_asserts(schema: &Schema) -> String {
         .unwrap();
         writeln!(out, "              \"{c_name} layout drifted from Rust\");").unwrap();
         // Per-field offset asserts. Same field set/order as the Rust
-        // offset table (column tail + contract_id triple +
-        // QuoteTick.midpoint). Field-offset drift can sneak past total-
+        // offset table (column tail + contract_id triple). Field-offset drift can sneak past total-
         // size asserts when two same-size fields swap order.
-        for (field, offset) in tick_ffi_offsets(type_name, def) {
+        for (field, offset) in tick_ffi_offsets(def) {
             let field = c_field_ident(&field);
             writeln!(out, "static_assert(offsetof({alias}, {field}) == {offset},").unwrap();
             writeln!(

thetadatadx-rs/build_support_bin/ticks/layout.rs

@@ -9,12 +9,11 @@
 use super::schema::TickTypeDef;
 
 /// `(field, byte_offset)` pairs for every column the parser fills, in
-/// declaration order, including the `contract_id` triple and
-/// `QuoteTick.midpoint` tail.
-pub(super) fn tick_ffi_offsets(type_name: &str, def: &TickTypeDef) -> Vec<(String, usize)> {
+/// declaration order, including the `contract_id` triple.
+pub(super) fn tick_ffi_offsets(def: &TickTypeDef) -> Vec<(String, usize)> {
     let mut offsets = Vec::new();
     let mut size = 0usize;
-    for (field_name, field_type) in tick_ffi_fields(type_name, def) {
+    for (field_name, field_type) in tick_ffi_fields(def) {
         let (field_size, field_align) = tick_ffi_field_layout(field_type);
         size = size.next_multiple_of(field_align);
         offsets.push((field_name.to_string(), size));
@@ -28,18 +27,18 @@ pub(super) fn tick_ffi_offsets(type_name: &str, def: &TickTypeDef) -> Vec<(Strin
 /// the max of the schema's `align` directive and every field's natural
 /// alignment, and size is rounded up to a multiple of that alignment to
 /// reproduce Rust's struct tail padding.
-pub(super) fn tick_ffi_size_and_align(type_name: &str, def: &TickTypeDef) -> (usize, usize) {
+pub(super) fn tick_ffi_size_and_align(def: &TickTypeDef) -> (usize, usize) {
     let mut size = 0usize;
     let mut struct_align = def.align.unwrap_or(1) as usize;
-    for (_, field_type) in tick_ffi_fields(type_name, def) {
+    for (_, field_type) in tick_ffi_fields(def) {
         let (field_size, field_align) = tick_ffi_field_layout(field_type);
         struct_align = struct_align.max(field_align);
         size = size.next_multiple_of(field_align) + field_size;
     }
     (size.next_multiple_of(struct_align), struct_align)
 }
 
-fn tick_ffi_fields<'a>(type_name: &'a str, def: &'a TickTypeDef) -> Vec<(&'a str, &'a str)> {
+fn tick_ffi_fields(def: &TickTypeDef) -> Vec<(&str, &str)> {
     let mut fields = def
         .columns
         .iter()
@@ -50,9 +49,6 @@ fn tick_ffi_fields<'a>(type_name: &'a str, def: &'a TickTypeDef) -> Vec<(&'a str
         fields.push(("strike", "price"));
         fields.push(("right", "right"));
     }
-    if type_name == "QuoteTick" {
-        fields.push(("midpoint", "price"));
-    }
     fields
 }