Skip to content

Commit 9112985

Browse files
steven-vargasteven-varga
committed
Merge branch '30-blog-when-a-week-of-market-data-vanishes-into-1969' into release
2 parents a74e314 + b6d4c37 commit 9112985

16 files changed

Lines changed: 8326 additions & 0 deletions

‎docs/assets/social/iex-timestamp-glitch-wireshark.png‎

267 KB
Loading

‎docs/blog/casts/iex-termshark.cast‎

Lines changed: 700 additions & 0 deletions
Large diffs are not rendered by default.

‎docs/blog/casts/iex-termshark.script‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
<script>
2+
document.addEventListener("DOMContentLoaded", function () {
3+
function waitForAsciinemaPlayer(attempts = 10) {
4+
if (typeof AsciinemaPlayer !== "undefined") {
5+
AsciinemaPlayer.create('/blog/casts/iex-termshark.cast', document.getElementById('asciicast-iex-termshark'), {
6+
cols: 120, rows: 37, autoPlay: true, loop: false, speed: 1.3, idleTimeLimit: .5, controls: false
7+
});
8+
} else if (attempts > 0) {
9+
setTimeout(() => waitForAsciinemaPlayer(attempts - 1), 200);
10+
} else {
11+
console.error("AsciinemaPlayer failed to load.");
12+
}
13+
}
14+
waitForAsciinemaPlayer();
15+
});
16+
</script>
17+

‎docs/blog/casts/iex2h5-issue-93-problem.cast‎

Lines changed: 7185 additions & 0 deletions
Large diffs are not rendered by default.

‎docs/blog/casts/iex2h5-issue-93-problem.script‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
<script>
2+
document.addEventListener("DOMContentLoaded", function () {
3+
function waitForAsciinemaPlayer(attempts = 10) {
4+
if (typeof AsciinemaPlayer !== "undefined") {
5+
AsciinemaPlayer.create('/blog/casts/iex2h5-issue-93-problem.cast', document.getElementById('asciicast-iex2h5-issue-93-problem'), {
6+
cols: 110, rows: 20, autoPlay: true, loop: false, speed: 8.0, idleTimeLimit: 1, controls: false
7+
});
8+
} else if (attempts > 0) {
9+
setTimeout(() => waitForAsciinemaPlayer(attempts - 1), 200);
10+
} else {
11+
console.error("AsciinemaPlayer failed to load.");
12+
}
13+
}
14+
waitForAsciinemaPlayer();
15+
});
16+
</script>

‎docs/blog/examples/iex2h5-issue-93-01.sh‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
```bash
2+
steven@saturn:~$ iex2h5 -c irts -o ~/output.h5 scratch/research/*.pcap
3+
[iex2h5] Converting 9 files using backend: hdf5 — using 1 thread — © Varga Consulting, 2017–2025
4+
[iex2h5] Visit https://vargaconsulting.github.io/iex2h5/ — Star it, Share it, Support Open Tools ⭐️
5+
▫ 2024-11-07 14:30:00 21:00:00 ✓
6+
▫ 2024-11-08 14:30:00 21:00:00 ✓
7+
▫ 1969-12-31 23:59:59 ✓
8+
▫ 1969-12-31 23:59:59 ✓
9+
▪ 1969-12-31 23:59:59 ✓
10+
▪ 1969-12-31 23:59:59 ✓
11+
▪ 1969-12-31 23:59:59 ✓
12+
▪ 1969-12-31 23:59:59 ✓
13+
▫ 2024-11-18 14:30:00 21:00:00 ✓
14+
▫ 2024-11-19 14:30:00 21:00:00 ✓
15+
benchmark: 1357269997 events in 434891ms 3.1 Mticks/s, 0.320000 µs/tick latency, 286.28 GiB input converted into 9.37 GiB output
16+
[iex2h5] Conversion complete — all files processed successfully
17+
[iex2h5] Market data © IEX — Investors Exchange. Attribution required. See https://iextrading.com
18+
19+
```

‎docs/blog/examples/iex2h5-issue-93-02.sh‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
```bash
2+
steven@saturn:~/projects/iex2h5/build$ ./iex2h5 -o ~/tmp.h5 -c none ~/scratch/research/sample_00000_20241111113746.pcap | slowcat --line 100
3+
[iex2h5] Converting 1 file using backend: hdf5 — using 1 thread — © Varga Consulting, 2017–2025
4+
[iex2h5] Visit https://vargaconsulting.github.io/iex2h5/ — Star it, Share it, Support Open Tools ⭐️
5+
Packet timestamp: 1970-01-01 01:38:21 UTC
6+
Packet timestamp: 1970-01-01 01:38:26 UTC
7+
Packet timestamp: 1970-01-01 01:38:31 UTC
8+
Packet timestamp: 1970-01-01 01:38:36 UTC
9+
Packet timestamp: 1970-01-01 01:38:42 UTC
10+
Packet timestamp: 1970-01-01 01:38:47 UTC
11+
Packet timestamp: 1970-01-01 01:39:17 UTC
12+
Packet timestamp: 1970-01-01 01:39:48 UTC
13+
Packet timestamp: 1970-01-01 01:58:53 UTC
14+
Packet timestamp: 1970-01-01 01:58:54 UTC
15+
Packet timestamp: 1970-01-01 01:58:55 UTC
16+
Packet timestamp: 1970-01-01 01:58:56 UTC
17+
Packet timestamp: 1970-01-01 01:58:58 UTC
18+
```

‎docs/blog/examples/iex2h5-issue-93-03.sh‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
steven@saturn:~$ tshark -r scratch/research/TOPS-2024-11-08.pcap -T fields -e frame.time_epoch -e data -c 20 | slowcat --line 200 --truncate=101
2+
1731069116.206792000 01000380010000000000434e0000000000000000000000000100000000000000ab1d1e8930fe0518
3+
1731069117.188933000 01000380010000000000434e9705380000000000000000000100000000000000ace6d2bd30fe0518
4+
1731069117.191213000 01000380010000000000434e8b05370097050000000000003900000000000000fb52d3bd30fe0518
5+
1731069117.193291000 01000380010000000000434e8b053700220b0000000000007000000000000000e0add3bd30fe0518
6+
1731069117.194838000 01000380010000000000434e0a053200ad10000000000000a7000000000000003f02d4bd30fe0518
7+
1731069117.196675000 01000380010000000000434e8b053700b715000000000000d9000000000000003e4fd4bd30fe0518
8+
1731069117.198506000 01000380010000000000434e8b053700421b000000000000100100000000000004b7d4bd30fe0518
9+
1731069117.199851000 01000380010000000000434e8b053700cd2000000000000047010000000000007f0fd5bd30fe0518
10+
1731069117.201351000 01000380010000000000434e8b05370058260000000000007e01000000000000366dd5bd30fe0518
11+
1731069117.202738000 01000380010000000000434e8b053700e32b000000000000b5010000000000004ad1d5bd30fe0518
12+
1731069117.204130000 01000380010000000000434e8b0537006e31000000000000ec010000000000004f38d6bd30fe0518
13+
1731069117.205377000 01000380010000000000434e8b053700f93600000000000023020000000000009785d6bd30fe0518
14+
1731069117.206583000 01000380010000000000434e8b053700843c0000000000005a020000000000009dedd6bd30fe0518
15+
1731069117.207852000 01000380010000000000434e8b0537000f420000000000009102000000000000ad55d7bd30fe0518
16+
1731069117.209046000 01000380010000000000434e8b0537009a47000000000000c802000000000000f7bbd7bd30fe0518
17+
1731069117.210103000 01000380010000000000434e8b053700254d000000000000ff02000000000000050ed8bd30fe0518
18+
1731069117.211103000 01000380010000000000434e8b053700b0520000000000003603000000000000eb6dd8bd30fe0518
19+
1731069117.211960000 01000380010000000000434e8b0537003b580000000000006d0300000000000016cfd8bd30fe0518
20+
1731069117.212890000 01000380010000000000434e8b053700c65d000000000000a4030000000000006024d9bd30fe0518
21+
1731069117.214016000 01000380010000000000434e8b0537005163000000000000db03000000000000bda1d9bd30fe0518
22+
23+
steven@saturn:~$ tshark -r scratch/research/TOPS-2024-11-11.pcap -T fields -e frame.time_epoch -e data -c 20 | slowcat --line 200 --truncate=101
24+
1731325066.340402000 0100038001000000000000000000000000000000000000000100000000000000ffffffffffffffff
25+
1731325071.584101000 0100038001000000000000000000000000000000000000000100000000000000ffffffffffffffff
26+
1731325076.717536000 0100038001000000000000000000000000000000000000000100000000000000ffffffffffffffff
27+
1731325081.950439000 0100038001000000000000000000000000000000000000000100000000000000ffffffffffffffff
28+
1731325087.283595000 0100038001000000000000000000000000000000000000000100000000000000ffffffffffffffff
29+
1731325092.516808000 0100038001000000000000000000000000000000000000000100000000000000ffffffffffffffff
30+
1731325122.849754000 0100038001000000000000000000000000000000000000000100000000000000ffffffffffffffff
31+
1731325153.083934000 0100038001000000000000000000000000000000000000000100000000000000ffffffffffffffff
32+
1731326298.667396000 01000380010000000000464e00000000000000000000000001000000000000006bdb33df0fe80618
33+
1731326299.681534000 01000380010000000000464e00000000000000000000000001000000000000007d0c6f1d10e80618
34+
1731326300.887641000 01000380010000000000464e00000000000000000000000001000000000000002ded526510e80618
35+
1731326301.931621000 01000380010000000000464e00000000000000000000000001000000000000006c9c8ca310e80618
36+
1731326303.137630000 01000380010000000000464e0000000000000000000000000100000000000000f9366feb10e80618
37+
1731326304.181700000 01000380010000000000464e0000000000000000000000000100000000000000cb63aa2911e80618
38+
1731326305.387631000 01000380010000000000464e000000000000000000000000010000000000000096808b7111e80618
39+
1731326306.431776000 01000380010000000000464e000000000000000000000000010000000000000056e9c7af11e80618
40+
1731326307.431820000 01000380010000000000464e0000000000000000000000000100000000000000cc2a63eb11e80618
41+
1731326308.431842000 01000380010000000000464e00000000000000000000000001000000000000003f92fe2612e80618
42+
1731326309.431877000 01000380010000000000464e000000000000000000000000010000000000000044cf996212e80618
43+
1731326310.431900000 01000380010000000000464e00000000000000000000000001000000000000006dff349e12e80618

‎docs/blog/examples/iex2h5-issue-93-04.sh‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
```bash
2+
steven@saturn:~$ steven@saturn:~/projects/iex2h5/build$ ./iex2h5 -c none -o tmp.h5 ~/scratch/research/TOPS-2024-11-11.pcap | slowcat --line 100
3+
[iex2h5] Converting 1 file using backend: hdf5 — using 1 thread — © Varga Consulting, 2017–2025
4+
[iex2h5] Visit https://vargaconsulting.github.io/iex2h5/ — Star it, Share it, Support Open Tools ⭐�
5+
1969-12-31 23:59:59.999999999 ffffffffffffffff
6+
1969-12-31 23:59:59.999999999 ffffffffffffffff
7+
1969-12-31 23:59:59.999999999 ffffffffffffffff
8+
1969-12-31 23:59:59.999999999 ffffffffffffffff
9+
1969-12-31 23:59:59.999999999 ffffffffffffffff
10+
1969-12-31 23:59:59.999999999 ffffffffffffffff
11+
1969-12-31 23:59:59.999999999 ffffffffffffffff
12+
1969-12-31 23:59:59.999999999 ffffffffffffffff
13+
2024-11-11 11:57:41.637405547 1806e80fdf33db6b
14+
2024-11-11 11:57:42.681472125 1806e8101d6f0c7d
15+
2024-11-11 11:57:43.887588653 1806e8106552ed2d
16+
2024-11-11 11:57:44.931556460 1806e810a38c9c6c
17+
2024-11-11 11:57:46.137589497 1806e810eb6f36f9
18+
2024-11-11 11:57:47.181654987 1806e81129aa63cb
19+
2024-11-11 11:57:48.387590294 1806e811718b8096
20+
2024-11-11 11:57:49.431736662 1806e811afc7e956
21+
```

‎docs/blog/examples/iex2h5-issue-93-05.sh‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
```diff
2+
steven@saturn:~/projects/iex2h5/build$ git diff --staged
3+
diff --git a/include/iex.hpp b/include/iex.hpp
4+
index 3c8fba2..a138bde 100644
5+
--- a/include/iex.hpp
6+
+++ b/include/iex.hpp
7+
@@ -392,7 +392,7 @@ namespace iex {
8+
void transport_handler( const iex::transport::header* segment ){
9+
using namespace std;
10+
using namespace date;
11+
-
12+
+ if(segment->time == (~0ULL)) return; // 0xffffffffffffffffULL denotes invalid packet see issue #93
13+
if( !count ) today = date::floor<date::days>( time_point(duration( segment->time) ) );
14+
auto now = time_point(duration( segment->time) );
15+
16+
17+
```

0 commit comments

Comments
 (0)