From 796ce66a9ff55a942644989e78d1a85b23e07ab1 Mon Sep 17 00:00:00 2001 From: userFRM Date: Thu, 7 May 2026 00:35:27 +0200 Subject: [PATCH] refactor(wave1): vocab + version refs + ParsedRight wire-up + json_canon fold + ADRs (refs #500) Bundle five low-risk, mechanical refactors: 1. Drop banned vocabulary "firehose" from source comments, README.md, and ROADMAP.md. Replace with "full-stream" / "full-type". CHANGELOG history is intentionally untouched. 2. Drop internal version references from source comments (`v8.0.10`, `v7.2.0`, `v8.0.3`, `v8.0.2`, `v6.0.1+`). Semantic content preserved; release metadata removed where it adds no maintenance value. 3. Wire `tdbe::right::ParsedRight::from_wire_byte` at the four `is_call` / `is_put` sites in `crates/tdbe/src/types/tick.rs`. The `right == 67` / `right == 80` magic-number comparisons go through the typed parser; behaviour is identical, the magic numbers are gone. 4. Fold `crates/json_canon` into `tdbe::json_canon`. Workspace member removed; `sonic-rs` dep moved to `tdbe`; CLI / MCP / server callers updated to `tdbe::json_canon::*`. 5. Add seven Architecture Decision Records under `docs/architecture/`: ADR-001 (Java terminal parity sourcing), ADR-002 (FPSS ring power-of- two capacity), ADR-003 (MDDS 2^tier concurrent-request mapping), ADR-004 (Eastern-time DST cutover), ADR-005 (OCC-21 century scope), ADR-006 (FPSS reconnect policy), ADR-007 (flatfiles MDDS SPKI pin). Version bumped 8.0.32 -> 8.0.33 via scripts/bump_version.py. Refs #500. Co-Authored-By: Claude Opus 4.7 (1M context) --- CHANGELOG.md | 33 ++++ Cargo.lock | 15 +- Cargo.toml | 2 +- README.md | 2 +- ROADMAP.md | 8 +- crates/json_canon/Cargo.toml | 16 -- crates/tdbe/Cargo.toml | 1 + crates/tdbe/benches/bench_tick.rs | 2 +- .../src/lib.rs => tdbe/src/json_canon.rs} | 0 crates/tdbe/src/lib.rs | 1 + crates/tdbe/src/types/tick.rs | 18 +- crates/thetadatadx/Cargo.toml | 2 +- .../build_support/endpoints/render/python.rs | 6 +- .../build_support/ticks/python_arrow.rs | 6 +- crates/thetadatadx/src/decode.rs | 2 +- crates/thetadatadx/src/fpss/mod.rs | 4 +- crates/thetadatadx/src/unified.rs | 6 +- crates/thetadatadx/src/wire_semantics.rs | 4 +- docs-site/docs/changelog.md | 33 ++++ .../ADR-001-java-terminal-parity.md | 39 ++++ .../ADR-002-fpss-ring-power-of-two.md | 39 ++++ .../ADR-003-mdds-semaphore-pow2-tier.md | 45 +++++ .../ADR-004-eastern-time-dst-cutover.md | 51 +++++ .../ADR-005-occ21-century-scope.md | 42 ++++ .../ADR-006-fpss-reconnect-policy.md | 49 +++++ .../ADR-007-flatfiles-mdds-spki-pin.md | 45 +++++ ffi/Cargo.toml | 2 +- sdks/python/Cargo.lock | 160 ++++++++++++++- sdks/python/Cargo.toml | 2 +- sdks/typescript/Cargo.lock | 185 +++++++++++++++++- sdks/typescript/Cargo.toml | 2 +- sdks/typescript/npm/darwin-arm64/package.json | 2 +- .../typescript/npm/linux-x64-gnu/package.json | 2 +- .../npm/win32-x64-msvc/package.json | 2 +- sdks/typescript/package.json | 8 +- tools/cli/Cargo.toml | 3 +- tools/cli/src/main.rs | 2 +- tools/mcp/Cargo.lock | 13 +- tools/mcp/Cargo.toml | 3 +- tools/mcp/src/main.rs | 4 +- tools/server/Cargo.lock | 13 +- tools/server/Cargo.toml | 3 +- tools/server/src/format.rs | 2 +- tools/server/src/handler.rs | 6 +- 44 files changed, 781 insertions(+), 104 deletions(-) delete mode 100644 crates/json_canon/Cargo.toml rename crates/{json_canon/src/lib.rs => tdbe/src/json_canon.rs} (100%) create mode 100644 docs/architecture/ADR-001-java-terminal-parity.md create mode 100644 docs/architecture/ADR-002-fpss-ring-power-of-two.md create mode 100644 docs/architecture/ADR-003-mdds-semaphore-pow2-tier.md create mode 100644 docs/architecture/ADR-004-eastern-time-dst-cutover.md create mode 100644 docs/architecture/ADR-005-occ21-century-scope.md create mode 100644 docs/architecture/ADR-006-fpss-reconnect-policy.md create mode 100644 docs/architecture/ADR-007-flatfiles-mdds-spki-pin.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 8fbca68ad..29837eda6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,39 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [8.0.33] - 2026-05-07 + +### Added + +- **Seven Architecture Decision Records** under `docs/architecture/`: + ADR-001 (Java terminal parity sourcing), ADR-002 (FPSS ring power-of-two + capacity), ADR-003 (MDDS `2^tier` concurrent-request mapping), ADR-004 + (Eastern-time DST cutover), ADR-005 (OCC-21 century scope, expires + 2099-12-31), ADR-006 (FPSS reconnect policy with rate-limit-aware backoff), + ADR-007 (flatfiles MDDS SPKI pin rotation policy). +- **`tdbe::json_canon`** — JSON canonicalisation (non-finite f64 to `null`) + is now a `tdbe` submodule. Existing `json_canon::*` callers in the CLI, + MCP, and server crates migrate to `tdbe::json_canon::*`. + +### Changed + +- **`ParsedRight::from_wire_byte` is now wired at the four `is_call` / + `is_put` magic-number sites in `crates/tdbe/src/types/tick.rs`.** The + `right == 67` / `right == 80` raw integer comparisons go through the + typed parser; behaviour is identical, the magic numbers are gone. + +### Removed + +- **`crates/json_canon` workspace member** folded into `tdbe::json_canon`. + External consumers should switch to `tdbe = "0.12.9"` and import + `tdbe::json_canon::{canonicalize, canonicalize_and_serialize, finite_or_null}`. +- **Internal version references in source comments** (`v8.0.10`, `v7.2.0`, + `v8.0.3`, `v8.0.2`, `v6.0.1+`). Semantic content preserved; release + metadata pruned out of code paths where it adds no maintenance value. +- **Banned vocabulary `firehose`** removed from source doc comments, + README.md, and ROADMAP.md. Replaced with `full-stream` / `full-type`. + CHANGELOG history is intentionally untouched. + ## [8.0.32] - 2026-05-06 ### Fixed diff --git a/Cargo.lock b/Cargo.lock index a383cb062..b2c3fe1c4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1488,13 +1488,6 @@ dependencies = [ "wasm-bindgen", ] -[[package]] -name = "json_canon" -version = "0.1.0" -dependencies = [ - "sonic-rs", -] - [[package]] name = "lazy_static" version = "1.5.0" @@ -3085,6 +3078,7 @@ name = "tdbe" version = "0.12.9" dependencies = [ "criterion", + "sonic-rs", "thiserror 2.0.18", ] @@ -3103,7 +3097,7 @@ dependencies = [ [[package]] name = "thetadatadx" -version = "8.0.32" +version = "8.0.33" dependencies = [ "arrow-array", "arrow-schema", @@ -3143,11 +3137,10 @@ dependencies = [ [[package]] name = "thetadatadx-cli" -version = "8.0.32" +version = "8.0.33" dependencies = [ "clap", "comfy-table", - "json_canon", "sonic-rs", "tdbe", "thetadatadx", @@ -3156,7 +3149,7 @@ dependencies = [ [[package]] name = "thetadatadx-ffi" -version = "8.0.32" +version = "8.0.33" dependencies = [ "tdbe", "thetadatadx", diff --git a/Cargo.toml b/Cargo.toml index 54960707e..01f5f446f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,5 @@ [workspace] -members = ["crates/json_canon", "crates/tdbe", "crates/thetadatadx", "tools/cli", "ffi"] +members = ["crates/tdbe", "crates/thetadatadx", "tools/cli", "ffi"] exclude = ["sdks/python", "sdks/typescript", "tools/mcp", "tools/server"] resolver = "2" diff --git a/README.md b/README.md index 2bb82434c..289a34136 100644 --- a/README.md +++ b/README.md @@ -158,7 +158,7 @@ All prices (`bid`, `ask`, `price`, `open`, `high`, `low`, `close`) are `f64`, de All endpoints return fully typed data in every language. See the [API Reference](docs/api-reference.md) for the complete method list. -**Additional surfaces** (not REST/gRPC endpoints): FPSS real-time streaming (7 subscribe/unsubscribe methods per contract and per firehose type) and a local Greeks calculator (22 Black-Scholes Greeks plus an IV solver, callable individually or batched). +**Additional surfaces** (not REST/gRPC endpoints): FPSS real-time streaming (7 subscribe/unsubscribe methods per contract and per full-stream type) and a local Greeks calculator (22 Black-Scholes Greeks plus an IV solver, callable individually or batched). ## Architecture diff --git a/ROADMAP.md b/ROADMAP.md index aa7949852..3fd7bcbf7 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -11,7 +11,7 @@ The SDK exposes three independent ways to consume data: | Surface | Purpose | Status | |---------|---------|--------| | **MDDS** | Request / response history + reference data | Verified | -| **FPSS** | Live streaming firehose (trades, quotes, OI) | Verified | +| **FPSS** | Live streaming full-stream (trades, quotes, OI) | Verified | | **FLATFILES** | Server-pre-built whole-universe daily blobs delivered over the legacy MDDS auth path | Verified | Each surface has a separate authentication flow and a separate transport. All three are reachable from the same `ThetaDataDx` client; consumers pick which surface they need on a per-call basis. @@ -156,9 +156,9 @@ Server retention window: 7 calendar days. Older history: contact ThetaData sales | Option quote subscription | Standard | Verified | | Option trade subscription | Standard | Verified | | Open interest subscription | Pro | Verified | -| Full trade firehose (Option) | Pro | Verified | -| Full trade firehose (Stock) | Pro | Verified | -| Full OI firehose | Pro | Verified | +| Full trade stream (Option) | Pro | Verified | +| Full trade stream (Stock) | Pro | Verified | +| Full OI stream | Pro | Verified | | Index price subscription | Indices subscription | Subscription-tier-blocked | | Dev server replay | — | Verified | | Reconnection (per-read deadline, Java-parity) | — | Verified | diff --git a/crates/json_canon/Cargo.toml b/crates/json_canon/Cargo.toml deleted file mode 100644 index b3435960f..000000000 --- a/crates/json_canon/Cargo.toml +++ /dev/null @@ -1,16 +0,0 @@ -[package] -name = "json_canon" -version = "0.1.0" -edition.workspace = true -rust-version.workspace = true -authors.workspace = true -license.workspace = true -homepage.workspace = true -repository.workspace = true -description = "JSON canonicalisation for non-finite f64 (NaN / +Inf / -Inf -> null) shared by cli, server, and mcp." - -[lints] -workspace = true - -[dependencies] -sonic-rs = "0.5.8" diff --git a/crates/tdbe/Cargo.toml b/crates/tdbe/Cargo.toml index d91428f35..d56e73b3f 100644 --- a/crates/tdbe/Cargo.toml +++ b/crates/tdbe/Cargo.toml @@ -16,6 +16,7 @@ categories = ["encoding", "finance"] workspace = true [dependencies] +sonic-rs = "0.5.8" thiserror = "2.0.18" [dev-dependencies] diff --git a/crates/tdbe/benches/bench_tick.rs b/crates/tdbe/benches/bench_tick.rs index ef03bc08d..8340a7ec7 100644 --- a/crates/tdbe/benches/bench_tick.rs +++ b/crates/tdbe/benches/bench_tick.rs @@ -6,7 +6,7 @@ use tdbe::types::tick::{OhlcTick, QuoteTick, TradeTick}; // ═══════════════════════════════════════════════════════════════════════════ // Tick operation benchmarks // -// Post-f64 migration (v6.0.1+): price fields are `f64`, decoded at parse time. +// Post-f64 migration: price fields are `f64`, decoded at parse time. // These benches measure direct field access and simple arithmetic rather than // the old helper-method decoding path. // ═══════════════════════════════════════════════════════════════════════════ diff --git a/crates/json_canon/src/lib.rs b/crates/tdbe/src/json_canon.rs similarity index 100% rename from crates/json_canon/src/lib.rs rename to crates/tdbe/src/json_canon.rs diff --git a/crates/tdbe/src/lib.rs b/crates/tdbe/src/lib.rs index 9171351aa..8abcf4eb1 100644 --- a/crates/tdbe/src/lib.rs +++ b/crates/tdbe/src/lib.rs @@ -19,6 +19,7 @@ pub mod error; pub mod exchange; pub mod flags; pub mod greeks; +pub mod json_canon; pub mod latency; pub mod right; pub mod sequences; diff --git a/crates/tdbe/src/types/tick.rs b/crates/tdbe/src/types/tick.rs index 980490624..f143b713e 100644 --- a/crates/tdbe/src/types/tick.rs +++ b/crates/tdbe/src/types/tick.rs @@ -16,6 +16,8 @@ include!("tick_generated.rs"); +pub use crate::right::ParsedRight; + // ───────────────────────────────────────────────────────────────────────────── // Contract identification helpers // ───────────────────────────────────────────────────────────────────────────── @@ -23,15 +25,15 @@ include!("tick_generated.rs"); macro_rules! impl_contract_id { ($ty:ident) => { impl $ty { - /// `true` when `right` == 'C' (ASCII 67). + /// `true` when `right` decodes to `Call` (ASCII `'C'`). #[inline] pub fn is_call(&self) -> bool { - self.right == 67 + ParsedRight::from_wire_byte(self.right) == Some(ParsedRight::Call) } - /// `true` when `right` == 'P' (ASCII 80). + /// `true` when `right` decodes to `Put` (ASCII `'P'`). #[inline] pub fn is_put(&self) -> bool { - self.right == 80 + ParsedRight::from_wire_byte(self.right) == Some(ParsedRight::Put) } /// `true` when the server populated contract identification fields. #[inline] @@ -95,15 +97,15 @@ impl TradeTick { } impl OptionContract { - /// `true` when `right` == 'C' (ASCII 67). + /// `true` when `right` decodes to `Call` (ASCII `'C'`). #[inline] pub fn is_call(&self) -> bool { - self.right == 67 + ParsedRight::from_wire_byte(self.right) == Some(ParsedRight::Call) } - /// `true` when `right` == 'P' (ASCII 80). + /// `true` when `right` decodes to `Put` (ASCII `'P'`). #[inline] pub fn is_put(&self) -> bool { - self.right == 80 + ParsedRight::from_wire_byte(self.right) == Some(ParsedRight::Put) } } diff --git a/crates/thetadatadx/Cargo.toml b/crates/thetadatadx/Cargo.toml index 05fe06d78..101fb0c7a 100644 --- a/crates/thetadatadx/Cargo.toml +++ b/crates/thetadatadx/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "thetadatadx" -version = "8.0.32" +version = "8.0.33" edition.workspace = true rust-version.workspace = true authors.workspace = true diff --git a/crates/thetadatadx/build_support/endpoints/render/python.rs b/crates/thetadatadx/build_support/endpoints/render/python.rs index dee9e6115..09c3c4a89 100644 --- a/crates/thetadatadx/build_support/endpoints/render/python.rs +++ b/crates/thetadatadx/build_support/endpoints/render/python.rs @@ -10,7 +10,7 @@ //! //! # Three variants per endpoint (sync / async / fluent builder) //! -//! The v8.0.2 Python-UX surface exposes every historical endpoint in three +//! The Python-UX surface exposes every historical endpoint in three //! forms without duplicating the spec: //! //! 1. **Sync `fn name(...)`** — original blocking signature; every kwarg is @@ -204,7 +204,7 @@ pub(super) fn render_python_historical_methods(endpoints: &[GeneratedEndpoint]) .iter() .filter(|endpoint| !is_streaming_endpoint(endpoint)) { - // Sync — preserves the pre-v8.0.2 signature byte-for-byte. + // Sync — preserves the original blocking signature byte-for-byte. out.push_str(&render_python_endpoint_sync(endpoint)); out.push('\n'); // Async — awaitable via pyo3-async-runtimes. @@ -218,7 +218,7 @@ pub(super) fn render_python_historical_methods(endpoints: &[GeneratedEndpoint]) out } -// ───────────────────────── Sync (pre-v8.0.2 signature) ───────────────────── +// ───────────────────────── Sync (original blocking signature) ───────────── fn render_python_endpoint_sync(endpoint: &GeneratedEndpoint) -> String { let method_params = method_params(endpoint); diff --git a/crates/thetadatadx/build_support/ticks/python_arrow.rs b/crates/thetadatadx/build_support/ticks/python_arrow.rs index a2d34da9c..7c2ba1b19 100644 --- a/crates/thetadatadx/build_support/ticks/python_arrow.rs +++ b/crates/thetadatadx/build_support/ticks/python_arrow.rs @@ -53,9 +53,9 @@ pub(super) fn render_python_tick_arrow(schema: &Schema) -> String { // Schema map + slice-to-Arrow converters. The pyclass-list dispatcher // and the `ArrowFromPyclassList` trait that used to live here have - // been removed along with every free-function DataFrame surface in - // v8.0.3 — the chained `List` terminals are the sole path - // from decoded ticks to DataFrame output. + // been removed along with every free-function DataFrame surface — the + // chained `List` terminals are the sole path from decoded + // ticks to DataFrame output. out.push_str(&render_python_arrow_schema_map(schema)); out.push('\n'); out.push_str(&render_record_batch_to_pyarrow_helper()); diff --git a/crates/thetadatadx/src/decode.rs b/crates/thetadatadx/src/decode.rs index c77316680..3da82e352 100644 --- a/crates/thetadatadx/src/decode.rs +++ b/crates/thetadatadx/src/decode.rs @@ -1731,7 +1731,7 @@ mod tests { #[test] fn parse_greeks_all_ticks_decodes_price_encoded_greeks() { - // Regression: v7.2.0 strict decode rejected Price cells for Greek + // Regression: an earlier strict decode rejected Price cells for Greek // columns, but the v3 MDDS server sends Greeks as Price-encoded // values (mirroring Java's `dataValue2Object` -> BigDecimal path). // Live run #24520486541 on main surfaced this as diff --git a/crates/thetadatadx/src/fpss/mod.rs b/crates/thetadatadx/src/fpss/mod.rs index f20fac68c..e6f94d21f 100644 --- a/crates/thetadatadx/src/fpss/mod.rs +++ b/crates/thetadatadx/src/fpss/mod.rs @@ -161,7 +161,7 @@ pub struct FpssClient { next_req_id: AtomicI32, /// Active per-contract subscriptions for reconnection. pub(in crate::fpss) active_subs: Arc>>, - /// Active full-type (firehose) subscriptions for reconnection. + /// Active full-type (full-stream) subscriptions for reconnection. pub(in crate::fpss) active_full_subs: Arc>>, /// Server-assigned contract ID mapping. @@ -774,7 +774,7 @@ impl FpssClient { .clone() } - /// Get a snapshot of currently active full-type (firehose) subscriptions. + /// Get a snapshot of currently active full-type (full-stream) subscriptions. pub fn active_full_subscriptions( &self, ) -> Vec<(SubscriptionKind, tdbe::types::enums::SecType)> { diff --git a/crates/thetadatadx/src/unified.rs b/crates/thetadatadx/src/unified.rs index ae0645ffa..6135885d0 100644 --- a/crates/thetadatadx/src/unified.rs +++ b/crates/thetadatadx/src/unified.rs @@ -337,7 +337,7 @@ impl ThetaDataDx { self.with_streaming(|s| s.subscribe_all(contract)) } - /// Subscribe to all trades for a security type (firehose). + /// Subscribe to all trades for a security type (full-stream). /// # Errors /// /// Returns an error on network, authentication, or parsing failure. @@ -345,7 +345,7 @@ impl ThetaDataDx { self.with_streaming(|s| s.subscribe_full_trades(sec_type)) } - /// Subscribe to all open interest for a security type (firehose). + /// Subscribe to all open interest for a security type (full-stream). /// # Errors /// /// Returns an error on network, authentication, or parsing failure. @@ -424,7 +424,7 @@ impl ThetaDataDx { self.with_streaming(|s| Ok(s.active_subscriptions())) } - /// Get all active full-type (firehose) subscriptions. + /// Get all active full-type (full-stream) subscriptions. /// # Errors /// /// Returns an error on network, authentication, or parsing failure. diff --git a/crates/thetadatadx/src/wire_semantics.rs b/crates/thetadatadx/src/wire_semantics.rs index 12c1172e7..1781ac582 100644 --- a/crates/thetadatadx/src/wire_semantics.rs +++ b/crates/thetadatadx/src/wire_semantics.rs @@ -6,9 +6,9 @@ //! `#[path]` reuse of this file) // The `venue=nqb` default used to live here as a runtime constant applied -// at query-assembly time. In v8.0.10 the default moved into the SSOT +// at query-assembly time. The default now lives in the SSOT // (`endpoint_surface.toml` -> `stock_venue_filter.default = "nqb"`) so it -// flows through every emitted SDK builder uniformly. `modes.rs` now reads +// flows through every emitted SDK builder uniformly. `modes.rs` reads // `param.default` directly; no runtime bridge needed. /// Canonicalize the `expiration` parameter for the MDDS server. diff --git a/docs-site/docs/changelog.md b/docs-site/docs/changelog.md index 8fbca68ad..29837eda6 100644 --- a/docs-site/docs/changelog.md +++ b/docs-site/docs/changelog.md @@ -5,6 +5,39 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [8.0.33] - 2026-05-07 + +### Added + +- **Seven Architecture Decision Records** under `docs/architecture/`: + ADR-001 (Java terminal parity sourcing), ADR-002 (FPSS ring power-of-two + capacity), ADR-003 (MDDS `2^tier` concurrent-request mapping), ADR-004 + (Eastern-time DST cutover), ADR-005 (OCC-21 century scope, expires + 2099-12-31), ADR-006 (FPSS reconnect policy with rate-limit-aware backoff), + ADR-007 (flatfiles MDDS SPKI pin rotation policy). +- **`tdbe::json_canon`** — JSON canonicalisation (non-finite f64 to `null`) + is now a `tdbe` submodule. Existing `json_canon::*` callers in the CLI, + MCP, and server crates migrate to `tdbe::json_canon::*`. + +### Changed + +- **`ParsedRight::from_wire_byte` is now wired at the four `is_call` / + `is_put` magic-number sites in `crates/tdbe/src/types/tick.rs`.** The + `right == 67` / `right == 80` raw integer comparisons go through the + typed parser; behaviour is identical, the magic numbers are gone. + +### Removed + +- **`crates/json_canon` workspace member** folded into `tdbe::json_canon`. + External consumers should switch to `tdbe = "0.12.9"` and import + `tdbe::json_canon::{canonicalize, canonicalize_and_serialize, finite_or_null}`. +- **Internal version references in source comments** (`v8.0.10`, `v7.2.0`, + `v8.0.3`, `v8.0.2`, `v6.0.1+`). Semantic content preserved; release + metadata pruned out of code paths where it adds no maintenance value. +- **Banned vocabulary `firehose`** removed from source doc comments, + README.md, and ROADMAP.md. Replaced with `full-stream` / `full-type`. + CHANGELOG history is intentionally untouched. + ## [8.0.32] - 2026-05-06 ### Fixed diff --git a/docs/architecture/ADR-001-java-terminal-parity.md b/docs/architecture/ADR-001-java-terminal-parity.md new file mode 100644 index 000000000..1429d2d14 --- /dev/null +++ b/docs/architecture/ADR-001-java-terminal-parity.md @@ -0,0 +1,39 @@ +# ADR-001: Java terminal parity sourcing + +## Status + +Accepted. + +## Context + +ThetaDataDx is reverse-engineered from the official Java terminal. Prior to this +ADR, every Rust module that mirrored a Java class carried inline +`Source: .method()` breadcrumbs. That repeated metadata leaked +vendor-internal class names into the public source surface and tied per-line +maintenance to upstream binary changes the public crate cannot describe. + +## Decision + +- Every Rust module whose behaviour mirrors an upstream Java class carries a + single doc-header link back to this ADR — never per-line breadcrumbs. +- The authoritative reverse-engineering ledger lives at + `private/JAVA_TERMINAL_FINDINGS.md` and `private/JAVA_TERMINAL_MAPPING.md`. + Both are gitignored. They map every public Rust entity to the Java class / + method that informed it. +- Per-line `Source:` breadcrumbs in source files are deleted. The ADR + private + ledger replace them. + +## Alternatives + +- **Keep inline breadcrumbs.** Rejected: leaks vendor class names into the + public source tree; rotates with every upstream binary version. +- **Drop the ledger entirely.** Rejected: reproducibility for new contributors + evaporates the moment the original author rotates off. + +## Consequences + +- Public source surface stays clean of vendor class names. +- Reproducibility for the reverse-engineering work is preserved through the + private ledger. +- New mirroring work updates the private ledger first; the source files stay + vocabulary-neutral with one ADR-001 link in the module header. diff --git a/docs/architecture/ADR-002-fpss-ring-power-of-two.md b/docs/architecture/ADR-002-fpss-ring-power-of-two.md new file mode 100644 index 000000000..47bcd1cf6 --- /dev/null +++ b/docs/architecture/ADR-002-fpss-ring-power-of-two.md @@ -0,0 +1,39 @@ +# ADR-002: FPSS ring buffer power-of-two capacity + +## Status + +Accepted. + +## Context + +`crates/thetadatadx/src/fpss/ring.rs` provides the single-producer / single- +consumer ring buffer that bridges the FPSS read loop and the streaming +dispatcher. Hot-path indexing happens on every tick. + +A ring whose capacity is a power of two can wrap an index with `i & (cap - 1)` +— a single AND, branchless. A capacity that is not a power of two requires a +modulo against a non-constant divisor (~20 cycles on x86_64) or a comparison- +plus-conditional-subtract pair, both of which break the steady-state ILP the +read path relies on. + +## Decision + +Ring buffer construction enforces `capacity.is_power_of_two()`. Anything else +returns an error at construction; there is no silent rounding. The invariant +is documented at the type and re-asserted in the constructor body so removing +the check leaves a compile-time tombstone. + +## Alternatives + +- **Round up to the next power of two on construction.** Rejected: silently + changes the caller's stated buffer budget. Bloomberg-grade APIs do not + silently rewrite caller intent. +- **Use modulo on every wrap.** Rejected: ~20 cycles per tick on the read + path is unacceptable on the steady-state quote feed. + +## Consequences + +- Index wrap is one AND, branchless, on every read and write. +- Non-power-of-two sizes are a hard error at construction; callers see the + failure immediately rather than discovering a perf cliff under load. +- The ring's documented capacity equals what the caller passed in. diff --git a/docs/architecture/ADR-003-mdds-semaphore-pow2-tier.md b/docs/architecture/ADR-003-mdds-semaphore-pow2-tier.md new file mode 100644 index 000000000..2ec32859b --- /dev/null +++ b/docs/architecture/ADR-003-mdds-semaphore-pow2-tier.md @@ -0,0 +1,45 @@ +# ADR-003: MDDS concurrent-request limit as `2^tier` + +## Status + +Accepted. + +## Context + +MDDS subscription tiers (`Free`, `Value`, `Standard`, `Pro`) gate the maximum +number of in-flight requests a single client may hold against the gRPC server. +The historical mapping was Free=1, Value=2, Standard=4, Pro=8 — a `2^tier` +progression. Until this ADR the mapping lived as a magic-number table inside +`mdds/client.rs` with a comment line referencing the four discriminant values +by hand. + +## Decision + +The mapping is codified at the type level: + +```rust +pub fn max_concurrent_requests(self) -> usize { 1usize << self as u32 } +``` + +`SubscriptionTier` is a typed enum (variants `Free=0`, `Value=1`, `Standard=2`, +`Pro=3`) replacing the previous `Option` field on the client. Any wire +integer outside `0..=3` is rejected at construction via +`SubscriptionTier::from_wire(i32) -> Option`. + +The Wave 3 PR moves the existing magic-number table into this single shift. + +## Alternatives + +- **Keep the magic-number table.** Rejected: invites silent drift if upstream + ever tweaks one tier's limit; the comment is the only spec. +- **Hard-code via `match self { Free => 1, ... }`.** Rejected: hides the + `2^tier` semantic behind a switch statement. The shift makes the + doubling-per-tier rule visible at the call site. + +## Consequences + +- Tier semantics are explicit at the type level instead of in comments. +- Adding a future tier becomes "add a discriminant," with the limit derived + for free as long as the doubling rule holds. +- A regression that breaks the `2^tier` invariant becomes a one-line fix in + one place. diff --git a/docs/architecture/ADR-004-eastern-time-dst-cutover.md b/docs/architecture/ADR-004-eastern-time-dst-cutover.md new file mode 100644 index 000000000..bfbbd9709 --- /dev/null +++ b/docs/architecture/ADR-004-eastern-time-dst-cutover.md @@ -0,0 +1,51 @@ +# ADR-004: Eastern-time DST cutover handling + +## Status + +Accepted. + +## Context + +ThetaDataDx timestamps are emitted in U.S. Eastern time. Eastern DST rules +changed on 2007-08-08 (effective 2007 calendar year): + +- **Pre-2007:** DST started on the first Sunday of April and ended on the + last Sunday of October. +- **2007-onward:** DST starts on the second Sunday of March and ends on the + first Sunday of November. + +Three surfaces need to apply the same cutover: flatfiles (historical archive +decoding back to 2003), MDDS (live response decoding), and FPSS (real-time +tick wall-clock derivation). Until this ADR, the DST math lived inside +`crates/thetadatadx/src/decode.rs` and was not reachable from flatfiles or +fpss without crate-level cycles. + +## Decision + +A single canonical implementation lives in `tdbe::time`: + +- `eastern_offset_ms(ms_since_unix_epoch: i64) -> i64` +- `march_second_sunday_utc(year)`, `november_first_sunday_utc(year)` + (post-2007 anchors) +- `april_first_sunday_utc(year)`, `october_last_sunday_utc(year)` + (pre-2007 anchors) +- `civil_to_epoch_days(year, month, day)` and `timestamp_to_*` helpers + +Wave 2 lifts these from `decode.rs` into `tdbe::time` so flatfiles, mdds, and +fpss share the same module without a reverse dependency. + +## Alternatives + +- **Vendor `chrono-tz` / IANA tzdata.** Rejected: pulls a multi-MB tzdata + blob into the binary core for a single timezone the SDK hard-codes + anyway; rebuild cadence drifts independently of upstream Java parity. +- **Duplicate the math per surface.** Rejected: how the bug got introduced + in the first place. Three implementations means three opportunities for + silent drift. + +## Consequences + +- Flatfiles, MDDS, and FPSS share one DST module with one source of truth. +- The 2007 cutover and the pre-2007 / post-2007 fork are tested in `tdbe` + against a hand-built fixture of NYSE half-day calendars. +- Any future U.S. Congress DST tweak edits one file. diff --git a/docs/architecture/ADR-005-occ21-century-scope.md b/docs/architecture/ADR-005-occ21-century-scope.md new file mode 100644 index 000000000..89efb4b06 --- /dev/null +++ b/docs/architecture/ADR-005-occ21-century-scope.md @@ -0,0 +1,42 @@ +# ADR-005: OCC-21 contract ID century scope + +## Status + +Accepted. + +## Context + +OCC-21 contract identifiers (the 21-character symbology used by the OCC and +mirrored on the FPSS wire) encode the option's expiration year as a two-digit +field `YY`. There is no century qualifier on the wire. + +The SDK has to pick a century interpretation. U.S. listed options expire on +the third Friday of the contract month, with LEAPS extending up to ~3 years +out. Today (2026) the longest live LEAPS expire in 2028. OCC has never issued +options with expirations later than 2099-12-31, and there is no proposed +cutover to a three-digit year field. + +## Decision + +OCC-21 parsing inside `Contract::parse_occ21` interprets `YY` as `2000 + YY`. +The accepted century is `2000-2099`; expirations outside this range are +rejected at the input boundary with a structured error. + +## Alternatives + +- **Sliding 50-year window.** Rejected: the wire format is OCC's, OCC has + not published a sliding-window rule, and a sliding window in the SDK + would silently disagree with OCC's own systems for far-out LEAPS. +- **Accept any `YY` and round.** Rejected: silent rounding violates the + Bloomberg-grade rule that data parsers do not invent input. + +## Consequences + +- Inputs outside 2000-2099 are rejected at the parse boundary; no silent + reinterpretation reaches FPSS. +- OCC-21 parsing is a pure function of the wire bytes, no clock dependency. + +## When this exception expires + +2099-12-31. If OCC-21 outlives this ADR, this file is the spec for the +follow-up decision. diff --git a/docs/architecture/ADR-006-fpss-reconnect-policy.md b/docs/architecture/ADR-006-fpss-reconnect-policy.md new file mode 100644 index 000000000..5b931412a --- /dev/null +++ b/docs/architecture/ADR-006-fpss-reconnect-policy.md @@ -0,0 +1,49 @@ +# ADR-006: FPSS reconnect policy + +## Status + +Accepted. + +## Context + +The FPSS streaming surface holds a long-lived TLS socket against the upstream +data centre. Disconnects come in two flavours: + +1. **Generic transient drops** (network blip, TLS renegotiation, peer + restart). The right behaviour is a short backoff and retry. +2. **Rate-limited disconnects** (HTTP-429-equivalent on the FPSS handshake, + or the upstream's per-account concurrent-session limit). The right + behaviour is a long enough wait that a reconnect storm cannot escalate + the rate-limit window. + +Hammering the upstream after a 429 burns the account's reconnect budget and +ends with a multi-minute lockout. Hammering a transient drop, on the other +hand, recovers the stream in single-digit seconds. + +## Decision + +`ReconnectPolicy` exposes two timings and an attempt cap: + +- `wait_ms` — default backoff for generic disconnects. +- `wait_rate_limited_ms` — 130 seconds, used only after a rate-limit + signal from the FPSS handshake. +- Maximum reconnect attempts capped at 5 before the dispatcher surfaces a + hard failure to the caller. + +The rate-limit wait is intentionally larger than the upstream's published +rate-limit window so a single reconnect storm cannot bracket a second +window. + +## Alternatives + +- **Single backoff for all disconnect causes.** Rejected: a generic + exponential backoff long enough to absorb a 429 is unacceptable for + transient drops; one short enough for transient drops re-triggers the 429. +- **Unbounded retries.** Rejected: a permanent upstream issue would burn + the reconnect budget silently and never surface to the operator. + +## Consequences + +- Rate-limited bursts back off without runaway reconnect storms. +- Transient drops recover in `wait_ms` rather than `wait_rate_limited_ms`. +- Hard failures surface to the caller after 5 attempts; no silent stalls. diff --git a/docs/architecture/ADR-007-flatfiles-mdds-spki-pin.md b/docs/architecture/ADR-007-flatfiles-mdds-spki-pin.md new file mode 100644 index 000000000..e1cd913e9 --- /dev/null +++ b/docs/architecture/ADR-007-flatfiles-mdds-spki-pin.md @@ -0,0 +1,45 @@ +# ADR-007: Flatfiles MDDS SPKI pin + +## Status + +Accepted. + +## Context + +The MDDS flatfiles surface authenticates against a long-lived data-centre +endpoint that, prior to this ADR, was reachable only via a static TLS +handshake. The standard public-CA PKI chain is insufficient: a CA compromise +or a country-level CA injection could MITM a multi-gigabyte historical +download without surfacing any error to the SDK. + +To close that window, the flatfiles client pins a specific Subject Public +Key Info (SPKI) value at +`crates/thetadatadx/src/flatfiles/mdds_spki.rs`. Any handshake whose leaf +certificate's SPKI does not match the pinned value is rejected before any +request bytes are sent. + +## Decision + +- The SPKI pin is hard-coded at the source level, not configurable at + runtime, so a compromised process cannot relax it. +- Rotating the pin requires a coordinated PR carrying the new SPKI value, + staged through dev → stage → prod, with the old pin retained as a + short-window fallback during the cutover. +- The PR itself is the rotation gate; there is no out-of-band override. + +## Alternatives + +- **Rely on public CA chain only.** Rejected: a CA compromise silently + breaks data integrity for the most expensive surface (multi-gigabyte + historical downloads). +- **Runtime-configurable pin.** Rejected: a process compromised at + runtime can also rewrite the pin, defeating the whole point. A source- + level pin requires a code release to weaken. + +## Consequences + +- MITM resistance against CA-level compromise on the flatfiles path. +- Rotation is intentional friction — by design, not by accident. +- A cert rotation that catches the SDK off-guard manifests as a hard + flatfiles failure, surfaced to the operator immediately rather than + silently downloading from the wrong endpoint. diff --git a/ffi/Cargo.toml b/ffi/Cargo.toml index 2c8d3d797..1f00d91fa 100644 --- a/ffi/Cargo.toml +++ b/ffi/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "thetadatadx-ffi" -version = "8.0.32" +version = "8.0.33" edition.workspace = true rust-version.workspace = true authors.workspace = true diff --git a/sdks/python/Cargo.lock b/sdks/python/Cargo.lock index 2092dea35..12876611a 100644 --- a/sdks/python/Cargo.lock +++ b/sdks/python/Cargo.lock @@ -709,6 +709,18 @@ version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6" +[[package]] +name = "faststr" +version = "0.2.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ca7d44d22004409a61c393afb3369c8f7bb74abcae49fe249ee01dcc3002113" +dependencies = [ + "bytes", + "rkyv", + "serde", + "simdutf8", +] + [[package]] name = "find-msvc-tools" version = "0.1.9" @@ -1447,6 +1459,26 @@ version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d87ecb2933e8aeadb3e3a02b828fed80a7528047e68b4f424523a0981a3a084" +[[package]] +name = "munge" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e17401f259eba956ca16491461b6e8f72913a0a114e39736ce404410f915a0c" +dependencies = [ + "munge_macro", +] + +[[package]] +name = "munge_macro" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4568f25ccbd45ab5d5603dc34318c1ec56b117531781260002151b8530a9f931" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "nom" version = "7.1.3" @@ -1689,6 +1721,26 @@ dependencies = [ "prost", ] +[[package]] +name = "ptr_meta" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b9a0cf95a1196af61d4f1cbdab967179516d9a4a4312af1f31948f8f6224a79" +dependencies = [ + "ptr_meta_derive", +] + +[[package]] +name = "ptr_meta_derive" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7347867d0a7e1208d93b46767be83e2b8f978c3dad35f775ac8d8847551d6fe1" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "pulldown-cmark" version = "0.13.3" @@ -1871,6 +1923,15 @@ version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" +[[package]] +name = "rancor" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a063ea72381527c2a0561da9c80000ef822bdd7c3241b1cc1b12100e3df081ee" +dependencies = [ + "ptr_meta", +] + [[package]] name = "rand" version = "0.9.4" @@ -1909,6 +1970,26 @@ dependencies = [ "rustversion", ] +[[package]] +name = "ref-cast" +version = "1.0.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f354300ae66f76f1c85c5f84693f0ce81d747e2c3f21a45fef496d89c960bf7d" +dependencies = [ + "ref-cast-impl", +] + +[[package]] +name = "ref-cast-impl" +version = "1.0.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7186006dcb21920990093f30e3dea63b7d6e977bf1256be20c3563a5db070da" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "regex" version = "1.12.3" @@ -1938,6 +2019,12 @@ version = "0.8.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a" +[[package]] +name = "rend" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cadadef317c2f20755a64d7fdc48f9e7178ee6b0e1f7fce33fa60f1d68a276e6" + [[package]] name = "reqwest" version = "0.13.3" @@ -1989,6 +2076,35 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "rkyv" +version = "0.8.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73389e0c99e664f919275ab5b5b0471391fe9a8de61e1dff9b1eaf56a90f16e3" +dependencies = [ + "bytes", + "hashbrown 0.17.0", + "indexmap", + "munge", + "ptr_meta", + "rancor", + "rend", + "rkyv_derive", + "tinyvec", + "uuid", +] + +[[package]] +name = "rkyv_derive" +version = "0.8.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d2ed0b54125315fb36bd021e82d314d1c126548f871634b483f46b31d13cac6" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "rustc-hash" version = "2.1.2" @@ -2268,6 +2384,45 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "sonic-number" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3775c3390edf958191f1ab1e8c5c188907feebd0f3ce1604cb621f72961dbf32" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "sonic-rs" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d971cc77a245ccf1756dbd1a87c3e7f709c0191464096510d43eec056d0f2c4f" +dependencies = [ + "ahash", + "bumpalo", + "bytes", + "cfg-if", + "faststr", + "itoa", + "ref-cast", + "serde", + "simdutf8", + "sonic-number", + "sonic-simd", + "thiserror 2.0.18", + "zmij", +] + +[[package]] +name = "sonic-simd" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f99e664ecd2d85a68c87e3c7a3cfe691f647ea9e835de984aba4d54a41f817d4" +dependencies = [ + "cfg-if", +] + [[package]] name = "stable_deref_trait" version = "1.2.1" @@ -2321,6 +2476,7 @@ checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca" name = "tdbe" version = "0.12.9" dependencies = [ + "sonic-rs", "thiserror 2.0.18", ] @@ -2339,7 +2495,7 @@ dependencies = [ [[package]] name = "thetadatadx" -version = "8.0.32" +version = "8.0.33" dependencies = [ "crossbeam-channel", "disruptor", @@ -2374,7 +2530,7 @@ dependencies = [ [[package]] name = "thetadatadx-py" -version = "8.0.32" +version = "8.0.33" dependencies = [ "arrow", "arrow-array", diff --git a/sdks/python/Cargo.toml b/sdks/python/Cargo.toml index 9303385f5..f554d8edc 100644 --- a/sdks/python/Cargo.toml +++ b/sdks/python/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "thetadatadx-py" -version = "8.0.32" +version = "8.0.33" edition = "2021" description = "Python bindings for thetadatadx — native ThetaData SDK powered by Rust" license = "Apache-2.0" diff --git a/sdks/typescript/Cargo.lock b/sdks/typescript/Cargo.lock index 57dd9d734..729975d7d 100644 --- a/sdks/typescript/Cargo.lock +++ b/sdks/typescript/Cargo.lock @@ -2,6 +2,19 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "ahash" +version = "0.8.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" +dependencies = [ + "cfg-if", + "getrandom 0.3.4", + "once_cell", + "version_check", + "zerocopy", +] + [[package]] name = "aho-corasick" version = "1.1.4" @@ -450,6 +463,18 @@ version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6" +[[package]] +name = "faststr" +version = "0.2.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ca7d44d22004409a61c393afb3369c8f7bb74abcae49fe249ee01dcc3002113" +dependencies = [ + "bytes", + "rkyv", + "serde", + "simdutf8", +] + [[package]] name = "find-msvc-tools" version = "0.1.9" @@ -1135,6 +1160,26 @@ version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d87ecb2933e8aeadb3e3a02b828fed80a7528047e68b4f424523a0981a3a084" +[[package]] +name = "munge" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e17401f259eba956ca16491461b6e8f72913a0a114e39736ce404410f915a0c" +dependencies = [ + "munge_macro", +] + +[[package]] +name = "munge_macro" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4568f25ccbd45ab5d5603dc34318c1ec56b117531781260002151b8530a9f931" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "napi" version = "3.8.5" @@ -1434,6 +1479,26 @@ dependencies = [ "prost", ] +[[package]] +name = "ptr_meta" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b9a0cf95a1196af61d4f1cbdab967179516d9a4a4312af1f31948f8f6224a79" +dependencies = [ + "ptr_meta_derive", +] + +[[package]] +name = "ptr_meta_derive" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7347867d0a7e1208d93b46767be83e2b8f978c3dad35f775ac8d8847551d6fe1" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "pulldown-cmark" version = "0.13.3" @@ -1531,6 +1596,15 @@ version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" +[[package]] +name = "rancor" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a063ea72381527c2a0561da9c80000ef822bdd7c3241b1cc1b12100e3df081ee" +dependencies = [ + "ptr_meta", +] + [[package]] name = "rand" version = "0.9.4" @@ -1569,6 +1643,26 @@ dependencies = [ "rustversion", ] +[[package]] +name = "ref-cast" +version = "1.0.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f354300ae66f76f1c85c5f84693f0ce81d747e2c3f21a45fef496d89c960bf7d" +dependencies = [ + "ref-cast-impl", +] + +[[package]] +name = "ref-cast-impl" +version = "1.0.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7186006dcb21920990093f30e3dea63b7d6e977bf1256be20c3563a5db070da" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "regex" version = "1.12.3" @@ -1598,6 +1692,12 @@ version = "0.8.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a" +[[package]] +name = "rend" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cadadef317c2f20755a64d7fdc48f9e7178ee6b0e1f7fce33fa60f1d68a276e6" + [[package]] name = "reqwest" version = "0.13.3" @@ -1649,6 +1749,35 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "rkyv" +version = "0.8.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73389e0c99e664f919275ab5b5b0471391fe9a8de61e1dff9b1eaf56a90f16e3" +dependencies = [ + "bytes", + "hashbrown 0.17.0", + "indexmap", + "munge", + "ptr_meta", + "rancor", + "rend", + "rkyv_derive", + "tinyvec", + "uuid", +] + +[[package]] +name = "rkyv_derive" +version = "0.8.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d2ed0b54125315fb36bd021e82d314d1c126548f871634b483f46b31d13cac6" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "rustc-hash" version = "2.1.2" @@ -1876,6 +2005,12 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" +[[package]] +name = "simdutf8" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" + [[package]] name = "slab" version = "0.4.12" @@ -1898,6 +2033,45 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "sonic-number" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3775c3390edf958191f1ab1e8c5c188907feebd0f3ce1604cb621f72961dbf32" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "sonic-rs" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d971cc77a245ccf1756dbd1a87c3e7f709c0191464096510d43eec056d0f2c4f" +dependencies = [ + "ahash", + "bumpalo", + "bytes", + "cfg-if", + "faststr", + "itoa", + "ref-cast", + "serde", + "simdutf8", + "sonic-number", + "sonic-simd", + "thiserror 2.0.18", + "zmij", +] + +[[package]] +name = "sonic-simd" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f99e664ecd2d85a68c87e3c7a3cfe691f647ea9e835de984aba4d54a41f817d4" +dependencies = [ + "cfg-if", +] + [[package]] name = "stable_deref_trait" version = "1.2.1" @@ -1945,6 +2119,7 @@ dependencies = [ name = "tdbe" version = "0.12.9" dependencies = [ + "sonic-rs", "thiserror 2.0.18", ] @@ -1963,7 +2138,7 @@ dependencies = [ [[package]] name = "thetadatadx" -version = "8.0.32" +version = "8.0.33" dependencies = [ "crossbeam-channel", "disruptor", @@ -1998,7 +2173,7 @@ dependencies = [ [[package]] name = "thetadatadx-napi" -version = "8.0.32" +version = "8.0.33" dependencies = [ "chrono", "napi", @@ -2427,6 +2602,12 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + [[package]] name = "walkdir" version = "2.5.0" diff --git a/sdks/typescript/Cargo.toml b/sdks/typescript/Cargo.toml index 8f98748a0..1eff188ca 100644 --- a/sdks/typescript/Cargo.toml +++ b/sdks/typescript/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "thetadatadx-napi" -version = "8.0.32" +version = "8.0.33" edition = "2021" description = "TypeScript/Node.js bindings for thetadatadx — native ThetaData SDK powered by Rust" license = "Apache-2.0" diff --git a/sdks/typescript/npm/darwin-arm64/package.json b/sdks/typescript/npm/darwin-arm64/package.json index 341aa1fbd..a17d0674e 100644 --- a/sdks/typescript/npm/darwin-arm64/package.json +++ b/sdks/typescript/npm/darwin-arm64/package.json @@ -1,6 +1,6 @@ { "name": "thetadatadx-darwin-arm64", - "version": "8.0.32", + "version": "8.0.33", "os": [ "darwin" ], diff --git a/sdks/typescript/npm/linux-x64-gnu/package.json b/sdks/typescript/npm/linux-x64-gnu/package.json index 2c29c58f7..ff440a049 100644 --- a/sdks/typescript/npm/linux-x64-gnu/package.json +++ b/sdks/typescript/npm/linux-x64-gnu/package.json @@ -1,6 +1,6 @@ { "name": "thetadatadx-linux-x64-gnu", - "version": "8.0.32", + "version": "8.0.33", "os": [ "linux" ], diff --git a/sdks/typescript/npm/win32-x64-msvc/package.json b/sdks/typescript/npm/win32-x64-msvc/package.json index 2d413acac..05af0ec97 100644 --- a/sdks/typescript/npm/win32-x64-msvc/package.json +++ b/sdks/typescript/npm/win32-x64-msvc/package.json @@ -1,6 +1,6 @@ { "name": "thetadatadx-win32-x64-msvc", - "version": "8.0.32", + "version": "8.0.33", "os": [ "win32" ], diff --git a/sdks/typescript/package.json b/sdks/typescript/package.json index 1042a3535..069868ead 100644 --- a/sdks/typescript/package.json +++ b/sdks/typescript/package.json @@ -1,6 +1,6 @@ { "name": "thetadatadx", - "version": "8.0.32", + "version": "8.0.33", "description": "Native ThetaData SDK for Node.js — powered by Rust via napi-rs", "license": "Apache-2.0", "repository": { @@ -30,9 +30,9 @@ "@napi-rs/cli": "^3.6.2" }, "optionalDependencies": { - "thetadatadx-linux-x64-gnu": "8.0.32", - "thetadatadx-darwin-arm64": "8.0.32", - "thetadatadx-win32-x64-msvc": "8.0.32" + "thetadatadx-linux-x64-gnu": "8.0.33", + "thetadatadx-darwin-arm64": "8.0.33", + "thetadatadx-win32-x64-msvc": "8.0.33" }, "engines": { "node": ">= 20" diff --git a/tools/cli/Cargo.toml b/tools/cli/Cargo.toml index 3de8d27f3..53db62a14 100644 --- a/tools/cli/Cargo.toml +++ b/tools/cli/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "thetadatadx-cli" -version = "8.0.32" +version = "8.0.33" edition.workspace = true rust-version.workspace = true authors.workspace = true @@ -22,7 +22,6 @@ path = "src/main.rs" [dependencies] thetadatadx = { path = "../../crates/thetadatadx" } tdbe = { version = "0.12.9", path = "../../crates/tdbe" } -json_canon = { version = "0.1.0", path = "../../crates/json_canon" } clap = { version = "4.6.1", features = ["derive"] } tokio = { version = "1.52.1", features = ["rt-multi-thread", "macros"] } sonic-rs = "0.5.8" diff --git a/tools/cli/src/main.rs b/tools/cli/src/main.rs index c211d43e3..4f5b10343 100644 --- a/tools/cli/src/main.rs +++ b/tools/cli/src/main.rs @@ -480,7 +480,7 @@ fn raw_ms(ms: i32) -> sonic_rs::Value { /// the REST and MCP frontends through the `json_canon` crate so all three /// produce byte-identical output for the same tick payload. fn raw_f64(value: f64) -> sonic_rs::Value { - json_canon::finite_or_null(value) + tdbe::json_canon::finite_or_null(value) } /// Raw integer value as JSON number. diff --git a/tools/mcp/Cargo.lock b/tools/mcp/Cargo.lock index 753b08798..0be3e784e 100644 --- a/tools/mcp/Cargo.lock +++ b/tools/mcp/Cargo.lock @@ -928,13 +928,6 @@ dependencies = [ "wasm-bindgen", ] -[[package]] -name = "json_canon" -version = "0.1.0" -dependencies = [ - "sonic-rs", -] - [[package]] name = "lazy_static" version = "1.5.0" @@ -1947,6 +1940,7 @@ dependencies = [ name = "tdbe" version = "0.12.9" dependencies = [ + "sonic-rs", "thiserror 2.0.18", ] @@ -1965,7 +1959,7 @@ dependencies = [ [[package]] name = "thetadatadx" -version = "8.0.32" +version = "8.0.33" dependencies = [ "crossbeam-channel", "disruptor", @@ -2000,9 +1994,8 @@ dependencies = [ [[package]] name = "thetadatadx-mcp" -version = "8.0.32" +version = "8.0.33" dependencies = [ - "json_canon", "serde", "sonic-rs", "tdbe", diff --git a/tools/mcp/Cargo.toml b/tools/mcp/Cargo.toml index e32ac3ce3..cceb01532 100644 --- a/tools/mcp/Cargo.toml +++ b/tools/mcp/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "thetadatadx-mcp" -version = "8.0.32" +version = "8.0.33" edition = "2021" description = "MCP server for ThetaDataDx — gives LLMs instant access to ThetaData market data" license = "Apache-2.0" @@ -13,7 +13,6 @@ path = "src/main.rs" [dependencies] thetadatadx = { path = "../../crates/thetadatadx" } tdbe = { version = "0.12.9", path = "../../crates/tdbe" } -json_canon = { version = "0.1.0", path = "../../crates/json_canon" } tokio = { version = "1.52.1", features = ["rt-multi-thread", "macros", "io-util", "io-std"] } serde = { version = "1.0.228", features = ["derive"] } sonic-rs = "0.5.8" diff --git a/tools/mcp/src/main.rs b/tools/mcp/src/main.rs index 828899480..3288dba20 100644 --- a/tools/mcp/src/main.rs +++ b/tools/mcp/src/main.rs @@ -951,7 +951,7 @@ async fn handle_request( /// `tools/call` arm so the regression test for issue #459 can exercise the /// canonicalisation path without spinning up a live `ThetaDataDx` client. fn build_tool_call_response(id: Value, result: &mut Value) -> JsonRpcResponse { - match json_canon::canonicalize_and_serialize(result) { + match tdbe::json_canon::canonicalize_and_serialize(result) { Ok(text) => JsonRpcResponse::success( id, json!({ @@ -1584,7 +1584,7 @@ mod tests { }); if let Some(ticks) = tool_result.get_mut("ticks").and_then(|v| v.as_array_mut()) { if let Some(row) = ticks.first_mut().and_then(|v| v.as_object_mut()) { - row.insert("vega", json_canon::finite_or_null(f64::NAN)); + row.insert("vega", tdbe::json_canon::finite_or_null(f64::NAN)); } } diff --git a/tools/server/Cargo.lock b/tools/server/Cargo.lock index 4fbe61232..796d47b83 100644 --- a/tools/server/Cargo.lock +++ b/tools/server/Cargo.lock @@ -1167,13 +1167,6 @@ dependencies = [ "wasm-bindgen", ] -[[package]] -name = "json_canon" -version = "0.1.0" -dependencies = [ - "sonic-rs", -] - [[package]] name = "lazy_static" version = "1.5.0" @@ -2340,6 +2333,7 @@ dependencies = [ name = "tdbe" version = "0.12.9" dependencies = [ + "sonic-rs", "thiserror 2.0.18", ] @@ -2358,7 +2352,7 @@ dependencies = [ [[package]] name = "thetadatadx" -version = "8.0.32" +version = "8.0.33" dependencies = [ "crossbeam-channel", "disruptor", @@ -2393,11 +2387,10 @@ dependencies = [ [[package]] name = "thetadatadx-server" -version = "8.0.32" +version = "8.0.33" dependencies = [ "axum", "clap", - "json_canon", "rustls", "serde", "serde_urlencoded", diff --git a/tools/server/Cargo.toml b/tools/server/Cargo.toml index 5cba6b454..930a39aac 100644 --- a/tools/server/Cargo.toml +++ b/tools/server/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "thetadatadx-server" -version = "8.0.32" +version = "8.0.33" edition = "2021" rust-version = "1.85" authors = ["userFRM"] @@ -22,7 +22,6 @@ path = "src/main.rs" thetadatadx = { path = "../../crates/thetadatadx", features = ["config-file"] } rustls = { version = "0.23.38", features = ["ring"] } tdbe = { version = "0.12.9", path = "../../crates/tdbe" } -json_canon = { version = "0.1.0", path = "../../crates/json_canon" } axum = { version = "0.8.9", features = ["ws"] } tokio = { version = "1.52.1", features = ["full"] } sonic-rs = "0.5.8" diff --git a/tools/server/src/format.rs b/tools/server/src/format.rs index d0ab80b6e..8f2b63ebd 100644 --- a/tools/server/src/format.rs +++ b/tools/server/src/format.rs @@ -571,7 +571,7 @@ fn render_csv_value(value: &sonic_rs::Value) -> String { // an explicit sentinel string so the CSV column is observable rather // than silently empty. let mut owned = value.clone(); - json_canon::canonicalize(&mut owned); + tdbe::json_canon::canonicalize(&mut owned); match sonic_rs::to_string(&owned) { Ok(rendered) => escape_csv_field(&rendered), Err(err) => { diff --git a/tools/server/src/handler.rs b/tools/server/src/handler.rs index 1a74d08a6..bd7a1933a 100644 --- a/tools/server/src/handler.rs +++ b/tools/server/src/handler.rs @@ -32,7 +32,7 @@ use crate::validation; /// see structured JSON they can parse. fn error_response(status: StatusCode, error_type: &str, msg: &str) -> Response { let mut body = format::error_envelope(error_type, msg); - let json_bytes = json_canon::canonicalize_and_serialize(&mut body).unwrap_or_else(|err| { + let json_bytes = tdbe::json_canon::canonicalize_and_serialize(&mut body).unwrap_or_else(|err| { tracing::error!( error = %err, "error envelope failed to serialise; emitting minimal fallback" @@ -62,7 +62,7 @@ fn error_response(status: StatusCode, error_type: &str, msg: &str) -> Response { /// structured `500` carrying the underlying error message rather than an /// empty `200 OK` body. fn json_response(val: &mut sonic_rs::Value) -> Response { - match json_canon::canonicalize_and_serialize(val) { + match tdbe::json_canon::canonicalize_and_serialize(val) { Ok(json_bytes) => ( StatusCode::OK, [( @@ -541,7 +541,7 @@ mod tests { "vega": Value::new_null(), }); if let Some(o) = row.as_object_mut() { - o.insert(&"vega", json_canon::finite_or_null(f64::NAN)); + o.insert(&"vega", tdbe::json_canon::finite_or_null(f64::NAN)); } let mut envelope = format::ok_envelope(vec![row]);