-
Notifications
You must be signed in to change notification settings - Fork 6
Expand file tree
/
Copy pathcheck_binding_parity.py
More file actions
6815 lines (6162 loc) · 289 KB
/
Copy pathcheck_binding_parity.py
File metadata and controls
6815 lines (6162 loc) · 289 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
#!/usr/bin/env python3
"""Cross-binding parity check (Gate 2 / issue #545 + #595).
Reads `sdks/parity.toml` — the declared cross-binding presence matrix
— and compares each row's `python` / `typescript` / `cpp` claims to
the actual binding state extracted from:
Class-level rows (no dot in `name`):
- Python: every `m.add_class::<T>()` registered in `lib.rs` + helper
`register_*` calls, expanded statically by parsing the Rust source.
Mirrors the regex powering `test_no_pyclass_name_collisions.py`.
- TypeScript: `export declare class X` / `export class X` declarations
in `sdks/typescript/index.d.ts`.
- C++: `^class X` / `^struct X` declarations in
`sdks/cpp/include/thetadatadx.hpp`. The `.h` header is C-only and not
considered for parity.
Field-level rows (dotted `name`, e.g. `ReconnectConfig.wait_ms`):
- Python: `#[setter] fn set_<canonical>` and `#[getter] fn <canonical>`
parsed from `sdks/python/src/*.rs`. The canonical name composes the
struct prefix (e.g. `reconnect_`) with the row suffix (`wait_ms`).
- TypeScript napi: `#[napi(js_name = "set<CamelCase>")]` and the
matching getter declaration in `sdks/typescript/src/*.rs`. The
CamelCase form lifts the snake_case canonical name.
- C++: `set_<canonical>` / `get_<canonical>` member functions on the
`class Config { ... }` body in `thetadatadx.hpp` PLUS the matching
`thetadatadx_config_set_<canonical>` C-ABI declaration in `thetadatadx.h`.
- FFI: `thetadatadx_config_set_<canonical>` AND
`thetadatadx_config_get_<canonical>` (or the `_explicit` widened-ABI shape)
parsed from `ffi/src/*.rs`. Any binding flagged `true` on a field
row implies the FFI symbol exists, because every higher-level
binding forwards into the same C ABI.
Rust-only rows: a dotted row with `rust_only = true` MUST cite an
`issue = "#N"` tracking number. The script enforces both — a
`rust_only` flag with no issue or an `issue` flag with no `rust_only`
fails the gate.
Historical endpoint families (`[[historical_base]]` /
`[[historical_async]]` / `[[historical_streaming]]`): the buffered,
async-query, and server-stream surfaces per endpoint. Each carries a
`rust` column whose source of truth is the registry of record,
`crates/thetadatadx/endpoint_surface.toml` — the file the build pipeline
generates every binding's historical method from. The Rust buffered
surface is every `[[endpoints]]` entry except the four `*_stream` FPSS
subscription endpoints (61 endpoints); the Rust streaming subset mirrors
the build's `endpoint_streams` SSOT (list / snapshot / calendar endpoints
get no server-stream terminal). `[[historical_base]]` additionally pins
the C-ABI `thetadatadx_<endpoint>_with_options` base symbol read from the
SHIPPED header (`sdks/cpp/include/endpoint_with_options.h.inc`) and
cross-checks the shipped header, the `ffi/src` source, and the registry
for agreement so a stale regenerated header is caught. This is the core
"every endpoint exists on all five surfaces" guarantee.
Exits non-zero on any mismatch. Run from the repo root.
A `--selftest` switch runs an in-process synthetic-source matrix
covering positive (all-bound) and negative (missing-on-TS,
missing-on-C++, missing-on-FFI, undocumented-orphan, rust_only-
without-issue) cases. The selftest is registered with the
audit-protocol convention for CI gates.
"""
from __future__ import annotations
import pathlib
import re
import sys
import tempfile
import tomllib
from typing import Any
REPO_ROOT = pathlib.Path(__file__).resolve().parents[1]
PARITY_TOML = REPO_ROOT / "sdks" / "parity.toml"
PY_SRC = REPO_ROOT / "sdks" / "python" / "src"
TS_DTS = REPO_ROOT / "sdks" / "typescript" / "index.d.ts"
TS_SRC = REPO_ROOT / "sdks" / "typescript" / "src"
CPP_HPP = REPO_ROOT / "sdks" / "cpp" / "include" / "thetadatadx.hpp"
CPP_H = REPO_ROOT / "sdks" / "cpp" / "include" / "thetadatadx.h"
FFI_SRC = REPO_ROOT / "ffi" / "src"
CONFIG_DIR = REPO_ROOT / "crates" / "thetadatadx" / "src" / "config"
# Core Rust streaming surfaces whose public observability accessors must
# each carry a parity row. The unified surface lives on the
# `StreamSurface` view returned by `Client::stream()`; the standalone
# surface is the `StreamingClient` FPSS client.
CORE_CLIENT_RS = REPO_ROOT / "crates" / "thetadatadx" / "src" / "client.rs"
CORE_FPSS_MOD_RS = REPO_ROOT / "crates" / "thetadatadx" / "src" / "fpss" / "mod.rs"
# The canonical endpoint registry of record. The build pipeline generates
# the Rust historical surface (`HistoricalClient::<endpoint>` methods + the
# per-endpoint streaming builders) from this file; the parity gate reads it
# directly so a dropped or renamed Rust historical endpoint trips without a
# full build. Every `[[endpoints]]` entry not named `*_stream` (the four FPSS
# real-time subscription endpoints) is one of the buffered historical
# endpoints (the 61-endpoint base surface).
ENDPOINT_SURFACE_TOML = (
REPO_ROOT / "crates" / "thetadatadx" / "endpoint_surface.toml"
)
# The shipped C-ABI base header fragment. `thetadatadx.h` includes it; it
# declares one `thetadatadx_<endpoint>_with_options` extern "C" symbol per
# buffered endpoint. Reading the SHIPPED header (not just the `ffi/src`
# source) catches a stale regenerated header that drifted from the Rust
# source of truth.
ENDPOINT_WITH_OPTIONS_INC = (
REPO_ROOT / "sdks" / "cpp" / "include" / "endpoint_with_options.h.inc"
)
# ─── Public-surface vocabulary guard ────────────────────────────────
#
# OUR Rust implementation-detail names that must never appear inside a
# PUBLIC client identifier (class / method / field / setter / getter /
# exported type). The bindings legitimately USE the async runtime, the
# lock-free ring, and the lock primitives internally — those uses live
# in implementation code and are out of scope here. This guard fires
# only on the identifiers the parity collectors already harvest, i.e.
# the names a user types. It catches the leak class structurally (a
# banned token embedded in a snake_case / camelCase identifier) where a
# word-boundary (`\bword\b`) text rule cannot, without false-positives
# on internal code.
#
# ALLOW-LIST: `mdds` and `fpss` are ThetaData's PROPRIETARY PROTOCOL names (the
# vendor this SDK wraps). They are NOT impl-detail leaks; the public
# surface stays aligned with the vendor's vocabulary. They are NOT
# listed below and MUST NOT be flagged. Only OUR own implementation
# details (the async runtime, the disruptor-style ring, the lock
# primitives, the I/O-bridge calls) are leaks.
#
# Lowercased; matched as a substring against the lowercased identifier
# (so `Tokio`, `tokio`, `TOKIO`, and an embedded `_tokio_` all hit).
BANNED_SURFACE_TOKENS: tuple[str, ...] = (
"tokio",
"disruptor",
"crossbeam",
"parking_lot",
"parkinglot",
"block_on",
"blockon",
"allow_threads",
"allowthreads",
"os_pipe",
"ospipe",
)
def _surface_token_hit(identifier: str) -> str | None:
"""Return the banned implementation-detail token embedded in
`identifier`, or ``None`` if the identifier is clean.
The match is case-insensitive and substring-based so a token buried
inside a snake_case or camelCase name (`set_tokio_worker_threads`,
`TokioWorkerThreadsSetting`) is caught — exactly the blind spot in a
word-boundary text rule. Vendor protocol names
(`mdds`, `fpss`) are intentionally absent from the token list, so a
`mdds_client` / `fpss` engine stem is never flagged.
"""
lowered = identifier.lower()
for token in BANNED_SURFACE_TOKENS:
if token in lowered:
return token
return None
# ─── Class-level discovery (legacy / non-dotted rows) ───────────────
PYCLASS_RE = re.compile(
r"#\[pyclass(?:\(([^)]*)\))?\][^{]*?"
r"(?:pub(?:\(crate\))?\s+)?(?:struct|enum)\s+(\w+)",
re.MULTILINE | re.DOTALL,
)
NAME_ATTR_RE = re.compile(r'name\s*=\s*"([^"]+)"')
def _python_name(attrs: str | None, struct_name: str) -> str:
if attrs:
m = NAME_ATTR_RE.search(attrs)
if m:
return m.group(1)
return struct_name.removeprefix("Py")
def collect_python_classes(py_src: pathlib.Path) -> set[str]:
"""Python-side pyclasses, in the same way `m.add_class::<T>()`
would surface them."""
out: set[str] = set()
for rs in py_src.rglob("*.rs"):
text = rs.read_text(encoding="utf-8")
for m in PYCLASS_RE.finditer(text):
out.add(_python_name(m.group(1), m.group(2)))
errors_rs = py_src / "errors.rs"
if errors_rs.is_file():
for m in re.finditer(r'm\.add\(\s*"(\w+)"\s*,', errors_rs.read_text(encoding="utf-8")):
out.add(m.group(1))
return out
TS_CLASS_RE = re.compile(r"export\s+declare\s+class\s+(\w+)")
TS_INTERFACE_RE = re.compile(r"export\s+(?:declare\s+)?interface\s+(\w+)")
def collect_typescript_classes(ts_dts: pathlib.Path) -> set[str]:
out: set[str] = set()
if not ts_dts.is_file():
return out
text = ts_dts.read_text(encoding="utf-8")
for m in TS_CLASS_RE.finditer(text):
out.add(m.group(1))
for m in TS_INTERFACE_RE.finditer(text):
out.add(m.group(1))
js_path = ts_dts.with_name("index.js")
if js_path.is_file():
for m in re.finditer(r"exports\.(\w+)\s*=\s*\w+", js_path.read_text(encoding="utf-8")):
out.add(m.group(1))
return out
CPP_CLASS_RE = re.compile(r"^(?:class|struct)\s+(\w+)", re.MULTILINE)
CPP_USING_RE = re.compile(r"^using\s+(\w+)\s*=", re.MULTILINE)
def collect_cpp_classes(cpp_hpp: pathlib.Path) -> set[str]:
out: set[str] = set()
if not cpp_hpp.is_file():
return out
text = cpp_hpp.read_text(encoding="utf-8")
for m in CPP_CLASS_RE.finditer(text):
out.add(m.group(1))
for m in CPP_USING_RE.finditer(text):
out.add(m.group(1))
return out
CPP_ALIASES: dict[str, str] = {
"FlatFilesNamespace": "FlatFiles",
"Contract": "FluentContract",
"Subscription": "FluentSubscription",
"SecType": "FluentSecType",
"ParseError": "StreamParseError",
# The unified client's sub-namespace views carry the Python / TypeScript
# canonical names in `parity.toml`; the C++ header names them without the
# `View` suffix (`client.historical()` -> `Historical`, `client.stream()`
# -> `Stream`).
"HistoricalView": "Historical",
"StreamView": "Stream",
}
def _cpp_class_for(class_name: str) -> str:
"""Resolve a parity-toml `class` field to its C++ class symbol.
Honors `CPP_ALIASES` so a row carrying the Python/TS canonical
name (`Contract`) routes to the corresponding C++ class body
(`FluentContract`).
"""
return CPP_ALIASES.get(class_name, class_name)
def cpp_has(symbol: str, cpp: set[str]) -> bool:
if symbol in cpp:
return True
alias = CPP_ALIASES.get(symbol)
if alias and alias in cpp:
return True
return False
# Parity-toml `class` field → the Rust struct name the Python method
# collector keys on. The collector harvests methods under the bare Rust
# struct identifier (`impl PyContract`), while the cross-binding row uses
# the canonical pyclass `name` (`Contract`). The fluent builders are the
# load-bearing case: `Contract.quote()` lives on `impl PyContract`,
# `SecType.fullTrades()` on `impl PySecType`. Routing the row through this
# table lets a single canonical row resolve against the Python source.
PY_CLASS_ALIASES: dict[str, str] = {
"Contract": "PyContract",
"SecType": "PySecType",
"Subscription": "PySubscription",
}
# Parity-toml `class` field → the Rust struct name the TypeScript method
# collector keys on (lifted from `impl <Name>`). The per-contract fluent
# builders live on the napi `ContractRef` impl, which the cross-binding
# matrix tracks under the canonical `Contract` name (the `ContractRef`
# class itself carries its own `[[class]]` row). The full-stream builders
# live on `impl SecType`, already the canonical name.
TS_CLASS_ALIASES: dict[str, str] = {
"Contract": "ContractRef",
}
def _py_class_for(class_name: str) -> str:
"""Resolve a parity-toml `class` field to the Python collector's key.
Honors `PY_CLASS_ALIASES` so a row carrying the canonical pyclass name
(`Contract`) routes to the Rust struct the collector harvests
(`PyContract`).
"""
return PY_CLASS_ALIASES.get(class_name, class_name)
def _ts_class_for(class_name: str) -> str:
"""Resolve a parity-toml `class` field to the TypeScript collector's key.
Honors `TS_CLASS_ALIASES` so a row carrying the canonical name
(`Contract`) routes to the napi impl the collector harvests
(`ContractRef`).
"""
return TS_CLASS_ALIASES.get(class_name, class_name)
def _py_methods_for(class_name: str, py_methods: dict[str, set[str]]) -> set[str]:
"""Python method set for a parity-toml `class`, trying the aliased
struct key first and falling back to the bare canonical name.
The fallback keeps synthetic selftest matrices keyed by the canonical
name (`{"Contract": {"quote"}}`) resolving while the production
collector keys by the Rust struct (`PyContract`).
"""
aliased = _py_class_for(class_name)
if aliased in py_methods:
return py_methods[aliased]
return py_methods.get(class_name, set())
def _ts_methods_for(class_name: str, ts_methods: dict[str, set[str]]) -> set[str]:
"""TypeScript method set for a parity-toml `class`, trying the aliased
impl key first and falling back to the bare canonical name."""
aliased = _ts_class_for(class_name)
if aliased in ts_methods:
return ts_methods[aliased]
return ts_methods.get(class_name, set())
def _is_implicitly_tracked(name: str) -> bool:
if name.endswith("Tick") or name.endswith("TickList") or name.endswith("TickListIter"):
return True
if name.endswith("Builder"):
return True
if name.endswith("List") or name.endswith("ListIter"):
return True
if name in {
"Quote",
"Trade",
"Ohlcvc",
"OpenInterest",
"MarketValue",
"ContractAssigned",
"Connected",
"Disconnected",
"Error",
"LoginSuccess",
"MarketOpen",
"MarketClose",
"Ping",
"Reconnected",
"ReconnectedServer",
"Reconnecting",
"ReconnectsExhausted",
"ReqResponse",
"Restart",
"ServerError",
"UnknownControl",
"UnknownFrame",
"OptionContract",
"AllGreeks",
}:
return True
return False
# ─── Field-level discovery (per-setter granularity / #595) ──────────
# Struct → setter-name prefix. The Rust struct lives on
# `DirectConfig.<accessor>`, but the binding-side setter name combines
# the prefix with the row's field suffix. E.g. `ReconnectConfig.wait_ms`
# resolves to Python `set_reconnect_wait_ms`, TS `setReconnectWaitMs`,
# C++ `set_reconnect_wait_ms`, FFI `thetadatadx_config_set_reconnect_wait_ms`.
STRUCT_TO_PREFIX: dict[str, str] = {
"HistoricalConfig": "",
"StreamingConfig": "",
"FlatFilesConfig": "flatfiles_",
"ReconnectConfig": "reconnect_",
"RuntimeConfig": "",
"RetryPolicy": "retry_",
"AuthConfig": "",
"MetricsConfig": "metrics_",
}
def _snake_to_camel(snake: str) -> str:
"""`reconnect_wait_ms` → `reconnectWaitMs`."""
head, *rest = snake.split("_")
return head + "".join(part.capitalize() for part in rest)
def _canonical_setter(struct_name: str, suffix: str) -> str | None:
"""Compose the binding-side canonical setter name from the struct
prefix and the row suffix. Returns ``None`` for unknown structs
so the caller surfaces a clear diagnostic.
"""
prefix = STRUCT_TO_PREFIX.get(struct_name)
if prefix is None:
return None
return f"{prefix}{suffix}"
# Some FFI / C++ setters use the widened `_explicit(has_value, n)` ABI
# shape for `Option<usize>` fields (`RuntimeConfig.tokio_worker_threads`,
# `MetricsConfig.port`). The
# parity row uses the bare field name, but the binding exposes
# `thetadatadx_config_set_<field>_explicit` as the canonical setter. Accept
# either shape when matching.
FFI_EXPLICIT_SUFFIXES = ("_explicit",)
def _collect_python_setters(py_src: pathlib.Path) -> set[str]:
"""Setter names on the `Config` pyclass. The pyo3 macro pattern is
``#[setter] fn set_<name>`` (or ``fn <name>``). Field-level parity
requires the setter — getter presence is a UX nicety but several
write-only knobs (e.g. ``reconnect_max_attempts``) have no
getter on any binding by design.
"""
setters: set[str] = set()
if not py_src.is_dir():
return setters
for rs in py_src.rglob("*.rs"):
text = rs.read_text(encoding="utf-8")
for m in re.finditer(r"#\[setter\][^}]*?fn\s+(\w+)", text, re.DOTALL):
name = m.group(1)
if name.startswith("set_"):
setters.add(name[4:])
else:
setters.add(name)
return setters
# TypeScript camelCase compound-word aliases. Multi-word terms that
# the cross-binding contract names as a single snake_case token (e.g.
# `flatfiles` in Python / C++ / FFI) get camelCased as a multi-word
# `FlatFiles` in the napi `js_name`. The alias table records the
# canonical snake form so the parity gate accepts both conventions.
TS_CAMEL_COMPOUNDS: dict[str, str] = {
"flat_files": "flatfiles",
}
def _camel_to_snake_with_aliases(camel: str) -> set[str]:
"""`FlatFilesMaxAttempts` → both `flat_files_max_attempts` and
`flatfiles_max_attempts`. Returns every plausible snake-case
rendering so the parity gate accepts the cross-binding
convention regardless of which form is canonical.
"""
base = re.sub(r"(?<!^)([A-Z])", r"_\1", camel).lower()
renderings = {base}
for source, target in TS_CAMEL_COMPOUNDS.items():
if source in base:
renderings.add(base.replace(source, target))
return renderings
def _collect_typescript_setters(ts_src: pathlib.Path) -> set[str]:
"""napi `set<CamelName>` setter declarations on the napi `Config`
impl block. Returns a set of canonical snake_case names. Getter
presence is intentionally not gated — several write-only knobs
(e.g. `setReconnectMaxAttempts`) have no getter on any binding
by design.
"""
setters_camel: set[str] = set()
if not ts_src.is_dir():
return set()
for rs in ts_src.rglob("*.rs"):
text = rs.read_text(encoding="utf-8")
# Scope to `impl Config { ... }` blocks so a `set<X>` napi method
# on a non-Config class (e.g. the live `StreamView` streaming
# knobs like `setSlowCallbackThresholdUs`) is not mistaken for a
# Config knob setter. This mirrors the Python collector, which
# relies on `#[setter]` being a Config-property-only attribute,
# and the C++/FFI collectors, which intersect with the
# `thetadatadx_config_set_*` C ABI surface.
for body in _iter_impl_config_bodies(text):
# `#[napi(js_name = "setX")]` → setter `X` (drop the `set` prefix).
for m in re.finditer(
r'#\[napi\([^)]*\bjs_name\s*=\s*"set([A-Z]\w*)"[^)]*\)\]',
body,
):
setters_camel.add(m.group(1))
# Lift to snake_case for parity-row matching. Every camelCase
# name renders to one or more snake-case candidates (the bare
# snake form plus any compound-word alias rendering); the gate
# accepts a match against any rendering.
setters_snake: set[str] = set()
for name in setters_camel:
setters_snake.update(_camel_to_snake_with_aliases(name))
return setters_snake
def _collect_cpp_setters(cpp_hpp: pathlib.Path, cpp_h: pathlib.Path) -> set[str]:
"""C++ wrapper exposes setters as inline `set_<name>(<type>)` on
the `class Config { ... }` body. The matching
`thetadatadx_config_set_<name>` declaration in `thetadatadx.h` is the C ABI
surface the wrapper forwards to; the parity gate requires both
halves so a forgotten C header declaration trips at link time.
Getter presence is not gated — several write-only knobs have no
C++ getter by design (matching the FFI / Python / TS contract).
"""
cpp_setters: set[str] = set()
if cpp_hpp.is_file():
text = cpp_hpp.read_text(encoding="utf-8")
for m in re.finditer(r"\bvoid\s+set_(\w+)\s*\(", text):
cpp_setters.add(m.group(1))
# Some C++ setters return `int32_t` for status codes (the
# `_explicit` widened-ABI shape on `Option<usize>` fields).
for m in re.finditer(r"\bint32_t\s+set_(\w+)\s*\(", text):
cpp_setters.add(m.group(1))
h_setters: set[str] = set()
if cpp_h.is_file():
text = cpp_h.read_text(encoding="utf-8")
for m in re.finditer(r"\bthetadatadx_config_set_(\w+)\s*\(", text):
h_setters.add(m.group(1))
return cpp_setters & h_setters
def _collect_ffi_setters(ffi_src: pathlib.Path) -> set[str]:
"""FFI extern C setter declarations in `ffi/src/*.rs`. The
convention is ``thetadatadx_config_set_<name>``. Getter presence is not
gated — several write-only knobs (e.g. the per-class reconnect
budgets) have no FFI getter by design.
"""
setters: set[str] = set()
if not ffi_src.is_dir():
return setters
for rs in ffi_src.rglob("*.rs"):
text = rs.read_text(encoding="utf-8")
for m in re.finditer(r"\bfn\s+thetadatadx_config_set_(\w+)\s*\(", text):
setters.add(m.group(1))
return setters
def _setter_present(canonical: str, setters: set[str]) -> bool:
"""True if `canonical` (or the `_explicit` widened-ABI variant) is
in `setters`.
"""
if canonical in setters:
return True
for suffix in FFI_EXPLICIT_SUFFIXES:
if f"{canonical}{suffix}" in setters:
return True
return False
# ─── Config getter collection (read-back accessor roster) ───────────
#
# The setter collectors above harvest the write side of the Config knob
# roster. These collectors harvest the read-back side: every binding that
# exposes a getter for a knob in one language must expose it (idiomatic
# name) in all. The naming conventions mirror the setters:
# * Python: `#[getter] fn get_<name>` (pyo3 strips `get_`, so the
# property is the bare `<name>`).
# * TypeScript: `#[napi(getter, js_name = "<camelCase>")]`.
# * C++: a `get_<name>(...)` member on `class Config`.
# * C ABI: `thetadatadx_config_get_<name>`.
def _iter_impl_config_bodies(text: str) -> list[str]:
"""Return the body text of every `impl Config { ... }` block in
`text`, bounded by a brace counter. Used to scope the getter
collectors to the `Config` knob roster — getters live on many
pyclasses / napi classes (tick structs, the fluent `Subscription`),
but only the `Config` read-back accessors are part of the
cross-binding knob roster the setter check's read-side complements.
"""
bodies: list[str] = []
for header in re.finditer(r"impl\s+Config\s*\{", text):
bodies.append(_balanced_body(text, header.end()))
return bodies
def _collect_python_getters(py_src: pathlib.Path) -> set[str]:
"""Read-back getter names on the `Config` pyclass.
Scoped to `impl Config { ... }` blocks so tick-class / fluent-value
getters (`QuoteTick.quote_timestamp_ms`, `Subscription.kind`) do not
leak into the Config knob roster. The pyo3 pattern is ``#[getter] fn
get_<name>``; the `get_` prefix the Rust fn name carries is stripped to
the bare canonical name (the Python property spelling).
"""
getters: set[str] = set()
if not py_src.is_dir():
return getters
for rs in py_src.rglob("*.rs"):
text = rs.read_text(encoding="utf-8")
for body in _iter_impl_config_bodies(text):
for m in re.finditer(r"#\[getter\][^}]*?fn\s+(\w+)", body, re.DOTALL):
name = m.group(1)
getters.add(name[4:] if name.startswith("get_") else name)
return getters
def _collect_typescript_getters(ts_src: pathlib.Path) -> set[str]:
"""napi `#[napi(getter, js_name = "<camelCase>")]` read-back accessors
on the `Config` napi class.
Scoped to `impl Config { ... }` blocks (the fluent `Subscription`
getters `isFull` / `secType` and tick-object fields are not Config
knobs). Returns canonical snake_case names — every camelCase `js_name`
lifted back through the compound-word alias table, exactly like the
setter collector.
"""
getters_camel: set[str] = set()
if not ts_src.is_dir():
return set()
for rs in ts_src.rglob("*.rs"):
text = rs.read_text(encoding="utf-8")
for body in _iter_impl_config_bodies(text):
for m in re.finditer(
r'#\[napi\([^)]*\bgetter\b[^)]*\bjs_name\s*=\s*"([a-zA-Z_]\w*)"[^)]*\)\]',
body,
):
getters_camel.add(m.group(1))
getters_snake: set[str] = set()
for name in getters_camel:
getters_snake.update(_camel_to_snake_with_aliases(name))
return getters_snake
def _collect_cpp_getters(cpp_hpp: pathlib.Path) -> set[str]:
"""C++ `get_<name>(...)` read-back members on the `class Config` body.
The wrapper exposes each read-back as a `get_<name>()` member. The
bare `<name>` is the canonical form (the `get_` prefix is the C++
convention, stripped here to compare against the cross-binding roster).
Restricted to the `class Config { ... }` body so unrelated `get_*`
members on other classes are not counted.
"""
getters: set[str] = set()
if not cpp_hpp.is_file():
return getters
text = _expand_cpp_includes(cpp_hpp.read_text(encoding="utf-8"), cpp_hpp.parent)
m = re.search(r"^class\s+Config\s*(?::[^{]*)?\{", text, re.MULTILINE)
if not m:
return getters
body = _balanced_body(text, m.end())
for fm in re.finditer(r"\bget_(\w+)\s*\(", body):
getters.add(fm.group(1))
return getters
def _collect_ffi_getters(ffi_src: pathlib.Path) -> set[str]:
"""FFI `thetadatadx_config_get_<name>` extern C read-back declarations.
Mirrors `_collect_ffi_setters` on the read side. Returns the bare
canonical names with the `thetadatadx_config_get_` prefix stripped.
"""
getters: set[str] = set()
if not ffi_src.is_dir():
return getters
for rs in ffi_src.rglob("*.rs"):
text = rs.read_text(encoding="utf-8")
for m in re.finditer(r"\bfn\s+thetadatadx_config_get_(\w+)\s*\(", text):
getters.add(m.group(1))
return getters
# ─── Client-facing setter-set parity ────────────────────────────────
# Per-binding spelling differences that are pure transport / language
# idiom, not a semantic divergence. Folding them away lets the four
# setter sets be compared for exact equality:
#
# * `_explicit` — the widened `(has_value, n)` ABI variant a binding
# emits for an `Option<usize>` field (`worker_threads_explicit` on
# the C ABI / C++ / napi vs the bare `worker_threads` on Python).
# Same knob, transport-only suffix.
# * `flat_files` → `flatfiles` — napi auto-camelCases `setFlatFiles*`
# to a multi-word `FlatFiles`, which lifts back to BOTH
# `flat_files_*` and `flatfiles_*`; the cross-binding canonical
# form is the single-token `flatfiles_*`.
def _normalize_setter(name: str) -> str:
"""Collapse a per-binding setter spelling to its cross-binding
canonical form so the four setter sets compare for equality.
"""
for suffix in FFI_EXPLICIT_SUFFIXES:
if name.endswith(suffix):
name = name[: -len(suffix)]
break
name = name.replace("flat_files", "flatfiles")
return name
# Client-facing setters that legitimately exist on only some bindings.
# Each entry maps the canonical (normalized) setter name to a written
# reason. This is the documented per-language-idiom carve-out the gate
# tolerates — every entry is a reviewed decision, not a silencer.
#
# Empty today: the `historical_host` / `historical_port` advanced endpoint
# overrides are now bound on every binding (Python / TypeScript / C++ /
# the C ABI), so no carve-out is required. Add an entry here only when a
# setter is intentionally exposed on a strict subset of bindings.
SETTER_PARITY_EXEMPT: dict[str, str] = {}
# Read-back getter equivalent of `SETTER_PARITY_EXEMPT`. A knob exposed
# read-only / write-only on a strict subset of bindings on purpose lists
# its canonical (normalized) name here with a written reason. Several
# write-only knobs (the per-class reconnect budgets) have no getter on ANY
# binding — those never enter the getter universe and need no entry.
#
# Empty today: every knob that exposes a read-back getter exposes it on
# all four bindings.
GETTER_PARITY_EXEMPT: dict[str, str] = {}
def _check_accessor_set_parity(
accessors: dict[str, set[str]],
exempt: dict[str, str],
noun: str,
exempt_const_name: str,
) -> list[str]:
"""Assert a client-facing accessor SET matches across Python /
TypeScript / C++ / the C ABI after normalization.
`accessors` maps each binding name to its raw accessor-name set;
`noun` is the accessor kind for diagnostics (`"setter"` / `"getter"`);
`exempt_const_name` names the carve-out constant in the diagnostic.
Genuine per-language idioms are folded by `_normalize_setter` (the
`_explicit` widened-ABI suffix and the `flat_files`↔`flatfiles`
camelCase split apply identically to setters and getters). Anything
still divergent must be listed in `exempt` with a reason or it fails
the gate. A stale exemption (now uniformly bound everywhere) is itself
flagged so the carve-out list never rots.
"""
norm = {lang: {_normalize_setter(a) for a in names} for lang, names in accessors.items()}
universe: set[str] = set().union(*norm.values()) if norm else set()
errors: list[str] = []
for accessor in sorted(universe - set(exempt)):
present_on = [lang for lang, names in norm.items() if accessor in names]
if len(present_on) != len(norm):
missing = [lang for lang in norm if lang not in present_on]
errors.append(
f" {noun} `{accessor}`: present on {sorted(present_on)}, "
f"missing on {sorted(missing)}. Bind it on every binding, "
f"or add it to {exempt_const_name} with a per-language-idiom "
f"reason."
)
for accessor, reason in exempt.items():
present_on = [lang for lang, names in norm.items() if accessor in names]
if present_on and len(present_on) == len(norm):
errors.append(
f" {noun} `{accessor}`: listed in {exempt_const_name} "
f"({reason!r}) but is now uniformly bound on every binding. "
f"Drop the stale exemption."
)
return errors
def _check_setter_set_parity(
py_setters: set[str],
ts_setters: set[str],
cpp_setters: set[str],
ffi_setters: set[str],
exempt: dict[str, str] | None = None,
) -> list[str]:
"""Assert the client-facing setter SET matches across Python /
TypeScript / C++ / the C ABI after normalization.
The per-row dotted check (`_check_dotted_rows`) verifies each
declared knob resolves on the bindings it claims; this set-level
check is the complementary direction — it catches a knob that
landed on some bindings but silently never made it into the parity
matrix on the others (the `derive_ohlcvc`-missing-on-TS defect
class). Genuine per-language idioms are folded by
`_normalize_setter`; anything still divergent must be listed in
`exempt` (defaults to `SETTER_PARITY_EXEMPT`) with a reason or it
fails the gate. The `exempt` parameter is injectable so the
selftest can exercise the logic with synthetic carve-out lists.
"""
if exempt is None:
exempt = SETTER_PARITY_EXEMPT
return _check_accessor_set_parity(
{
"python": py_setters,
"typescript": ts_setters,
"cpp": cpp_setters,
"ffi": ffi_setters,
},
exempt,
"setter",
"SETTER_PARITY_EXEMPT",
)
def _check_getter_set_parity(
py_getters: set[str],
ts_getters: set[str],
cpp_getters: set[str],
ffi_getters: set[str],
exempt: dict[str, str] | None = None,
) -> list[str]:
"""Assert the client-facing read-back getter SET matches across
Python / TypeScript / C++ / the C ABI after normalization.
The complement to `_check_setter_set_parity` on the read side: a knob
that exposes a getter on some bindings but silently never grew one on
the others trips the gate. Together the two checks pin the full Config
knob roster — both write and read accessors — across every binding.
Per-language idioms fold via `_normalize_setter`; intentional
subset-of-bindings getters list in `exempt` (defaults to
`GETTER_PARITY_EXEMPT`) with a reason.
"""
if exempt is None:
exempt = GETTER_PARITY_EXEMPT
return _check_accessor_set_parity(
{
"python": py_getters,
"typescript": ts_getters,
"cpp": cpp_getters,
"ffi": ffi_getters,
},
exempt,
"getter",
"GETTER_PARITY_EXEMPT",
)
# ─── Public-surface identifier collection (vocab guard) ─────────────
def _check_public_surface_vocab(
py_classes: set[str],
ts_classes: set[str],
cpp_classes: set[str],
py_setters: set[str],
ts_setters: set[str],
cpp_setters: set[str],
ffi_setters: set[str],
py_methods: dict[str, set[str]],
ts_methods: dict[str, set[str]],
cpp_methods: dict[str, set[str]],
) -> list[str]:
"""Assert no PUBLIC client identifier embeds a banned architecture
token.
Scans the identifier sets the parity collectors already harvest —
classes, config setters, and per-class methods on every binding —
for an internal-architecture token (`tokio`, `mdds`, `disruptor`,
...) buried inside the name. This is the structural counterpart to
the text scrubber: it sees only declared public API names, so it
never false-positives on the bindings' legitimate internal use of
the runtime / ring / lock primitives, and it catches a banned token
embedded in a snake_case / camelCase identifier that the scrubber's
`\\bword\\b` rule misses.
"""
errors: list[str] = []
def _check(identifier: str, where: str) -> None:
token = _surface_token_hit(identifier)
if token is not None:
errors.append(
f" {where}: public identifier `{identifier}` embeds "
f"banned architecture token `{token}`. Rename to a "
f"neutral client-facing name (the user concept, not the "
f"implementation)."
)
for cls in sorted(py_classes):
_check(cls, "python class")
for cls in sorted(ts_classes):
_check(cls, "typescript class")
for cls in sorted(cpp_classes):
_check(cls, "cpp class")
for setter in sorted(py_setters):
_check(setter, "python setter")
for setter in sorted(ts_setters):
_check(setter, "typescript setter")
for setter in sorted(cpp_setters):
_check(setter, "cpp setter")
for setter in sorted(ffi_setters):
_check(setter, "ffi setter")
for cls, methods in sorted(py_methods.items()):
for method in sorted(methods):
_check(method, f"python method {cls}.")
for cls, methods in sorted(ts_methods.items()):
for method in sorted(methods):
_check(method, f"typescript method {cls}.")
for cls, methods in sorted(cpp_methods.items()):
for method in sorted(methods):
_check(method, f"cpp method {cls}.")
return errors
# ─── Rust field discovery (reverse-direction orphan check) ──────────
# Structs we consider in scope for the field-level gate. Adding a new
# struct here is one half of the binding-sweep workflow; the other
# half is adding rows to `parity.toml` for every pub field on the new
# struct (or marking each as `rust_only = true, issue = "#N"`).
#
# `ReconnectAttemptLimits` is intentionally NOT scoped here even
# though it carries `pub max_attempts / max_rate_limited_attempts /
# stable_window` fields; those mirror onto the bindings via
# `ReconnectConfig.max_attempts` etc. (the inner `limits` struct is
# wrapped in `ReconnectPolicy::Auto(...)` and the binding setters
# write through to it). The parity-toml rows under `ReconnectConfig.*`
# already cover the cross-binding contract for those fields.
SCOPED_STRUCTS: tuple[str, ...] = (
"HistoricalConfig",
"StreamingConfig",
"FlatFilesConfig",
"ReconnectConfig",
"RuntimeConfig",
"RetryPolicy",
"AuthConfig",
"MetricsConfig",
)
STRUCT_HEADER_RE = re.compile(
r"pub\s+struct\s+(\w+)\s*\{",
)
PUB_FIELD_RE = re.compile(
r"^\s+pub\s+(\w+)\s*:",
re.MULTILINE,
)
def _collect_rust_pub_fields(config_dir: pathlib.Path) -> dict[str, set[str]]:
"""Return `{struct_name: {field, ...}}` for every scoped struct.
Parses `crates/thetadatadx/src/config/*.rs`. Skips fields on
structs not listed in `SCOPED_STRUCTS` — `DirectConfig`'s pub
fields are nested-struct accessors that the class-level gate
already covers.
"""
out: dict[str, set[str]] = {}
if not config_dir.is_dir():
return out
for rs in config_dir.rglob("*.rs"):
text = rs.read_text(encoding="utf-8")
# Find every `pub struct X {` block and walk forward until the
# closing brace. The structs in `crates/thetadatadx/src/config/`
# never nest other struct definitions; a depth=1 brace counter
# suffices.
for header in STRUCT_HEADER_RE.finditer(text):
struct_name = header.group(1)
if struct_name not in SCOPED_STRUCTS:
continue
body_start = header.end()
depth = 1
i = body_start
while i < len(text) and depth > 0:
c = text[i]
if c == "{":
depth += 1
elif c == "}":
depth -= 1
i += 1
body = text[body_start : i - 1]
for fm in PUB_FIELD_RE.finditer(body):
out.setdefault(struct_name, set()).add(fm.group(1))
return out
# Some Rust pub field names differ from the binding-side setter
# suffix:
#
# - `ReconnectConfig.stable_window: Duration` → setter
# `set_reconnect_stable_window_secs` (the binding takes a `u64`
# seconds value; the row name carries the unit suffix).
# - `FlatFilesConfig.initial_backoff: Duration` →
# `set_flatfiles_initial_backoff_secs`.
# - `RetryPolicy.initial_delay: Duration` →
# `set_retry_initial_delay_ms`.
#
# The table below records the rust-field → binding-suffix renames.
# Fields not listed are 1:1.
RUST_FIELD_RENAMES: dict[tuple[str, str], str] = {
# The async worker-thread count is stored internally on
# `RuntimeConfig.tokio_worker_threads`, but the public client setter
# is named `worker_threads` on every binding (the implementation
# runtime name never reaches the user surface). The parity row is
# keyed by the public concept; this mapping bridges the internal
# storage field to that row for the reverse-direction orphan check.
("RuntimeConfig", "tokio_worker_threads"): "worker_threads",
("ReconnectConfig", "stable_window"): "stable_window_secs",
("ReconnectAttemptLimits", "stable_window"): "stable_window_secs",
("ReconnectAttemptLimits", "max_elapsed"): "max_elapsed_secs",
("FlatFilesConfig", "initial_backoff"): "initial_backoff_secs",
("FlatFilesConfig", "max_backoff"): "max_backoff_secs",
("RetryPolicy", "initial_delay"): "initial_delay_ms",
("RetryPolicy", "max_delay"): "max_delay_ms",
("RetryPolicy", "max_elapsed"): "max_elapsed_secs",
# StreamingConfig scalar knobs carry a `streaming_` prefix at the
# binding surface so the generic field names (`timeout_ms`, `ring_size`)
# stay unambiguous against sibling sub-configs.
("StreamingConfig", "timeout_ms"): "streaming_timeout_ms",
("StreamingConfig", "ring_size"): "streaming_ring_size",
("StreamingConfig", "ping_interval_ms"): "streaming_ping_interval_ms",