Skip to content

Commit 75ab815

Browse files
committed
feat(stats): Implement QuanuX-Statistics & Fix C++ Builds
- Implemented QuanuX-Statistics Node: - Added StatsEngine with NATS ingestion and DuckDB storage - Implemented Welford's Algorithm (Online Variance) - Implemented Pairwise Correlation tracking - Added Z-Score signal publishing - Fixed Execution Node Builds: - Resolved StrategyContext ambiguity in ping_pong.cpp - Fixed cpp-httplib linker target (httplib::httplib) - Added missing Simulator headers (SimulatedExchange.h, DuckDBFeeder.h) - Fixed IDE lint errors (include paths) - Documentation: - Added QuanuX-Statistics/SKILL.md - Updated execution-node/SKILL.md with HFT & Simulator guides - Added scripts/test_pub.py for market data simulation
1 parent cb51fac commit 75ab815

12 files changed

Lines changed: 545 additions & 13 deletions

File tree

QuanuX-Statistics/SKILL.md

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
---
2+
name: manage-quanux-statistics
3+
description: Guide for developing and operating the QuanuX Statistics & Research Node.
4+
---
5+
6+
# QuanuX Statistics Node
7+
8+
The **QuanuX Statistics Node** (`quanux_stats`) is a purpose-built service for real-time market analysis and signal generation. It bridges the gap between raw market data and high-level strategy logic by providing pre-calculated metrics.
9+
10+
## Capabilities
11+
12+
1. **Online Statistics**:
13+
- Uses **Welford's Algorithm** to track Variance and Standard Deviation with $O(1)$ complexity/storage per tick.
14+
- Tracks **Z-scores** relative to a rolling window (default 100 ticks).
15+
- Calculates **Pairwise Correlation** matrices in real-time.
16+
17+
2. **Data Persistence**:
18+
- Ingests `MARKET.*` data from NATS (JSON format).
19+
- Writes to **DuckDB** (`market_stats.duckdb`) for offline research and backtesting.
20+
- *Future*: Will support Parquet export for S3 archival.
21+
22+
3. **Signal Publishing**:
23+
- Publishes derived metrics to `STATS.<SYMBOL>` on NATS.
24+
- Format: `{"symbol": "ES", "volatility": 12.5, "z_score": 1.2, ...}`
25+
26+
## Architecture
27+
28+
- **Language**: C++20
29+
- **Core**: `StatsEngine` class.
30+
- **Storage**: `stats_map_` (Red-Black Tree) for per-symbol state.
31+
- **Concurrency**: Single-threaded NATS consumer (for now) protected by `std::mutex` for future expansion.
32+
33+
## Development Guide
34+
35+
### Adding a New Metric
36+
1. **Update `InstrumentStats`** in `include/stats_engine.h`:
37+
Add a new accumulator (e.g., `double sum_log_return`).
38+
2. **Update `update()`** in `src/stats_engine.cpp`:
39+
Implement the online update formula.
40+
3. **Publish**:
41+
Add the new field to the `json derived` object in the NATS callback.
42+
43+
### Running Locally
44+
```bash
45+
# Build
46+
cd QuanuX-Statistics/cpp/build
47+
make quanux_stats
48+
49+
# Run (Ensure NATS is up)
50+
./quanux_stats
51+
```
52+
53+
## Dependencies
54+
- **DuckDB**: Embedded DB for storage.
55+
- **NATS C Client**: High-performance messaging.
56+
- **Eigen**: Linear algebra (Matrix operations).
57+
- **nlohmann/json**: Serialization.
Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
cmake_minimum_required(VERSION 3.20)
2+
project(quanux_stats CXX)
3+
4+
set(CMAKE_CXX_STANDARD 20)
5+
set(CMAKE_CXX_STANDARD_REQUIRED ON)
6+
7+
# Optimization
8+
if(NOT CMAKE_BUILD_TYPE)
9+
set(CMAKE_BUILD_TYPE Release)
10+
endif()
11+
if (UNIX)
12+
set(CMAKE_CXX_FLAGS_RELEASE "-O3 -march=native -mtune=native -fno-omit-frame-pointer")
13+
endif()
14+
15+
# Dependencies
16+
include(FetchContent)
17+
18+
# 1. NATS (Telemetry/Data)
19+
set(NATS_BUILD_STREAMING OFF CACHE BOOL "" FORCE)
20+
FetchContent_Declare(
21+
cnats
22+
GIT_REPOSITORY https://github.com/nats-io/nats.c.git
23+
GIT_TAG v3.9.0
24+
)
25+
FetchContent_MakeAvailable(cnats)
26+
27+
# 2. DuckDB (Storage/Analytics)
28+
FetchContent_Declare(
29+
duckdb
30+
GIT_REPOSITORY https://github.com/duckdb/duckdb.git
31+
GIT_TAG v1.1.3
32+
GIT_SHALLOW TRUE
33+
)
34+
set(BUILD_UNITTESTS OFF CACHE BOOL "" FORCE)
35+
set(BUILD_SHELL OFF CACHE BOOL "" FORCE)
36+
set(BUILD_EXTENSIONS "parquet;json" CACHE STRING "" FORCE)
37+
FetchContent_MakeAvailable(duckdb)
38+
39+
# 3. Eigen (Math/Stats)
40+
FetchContent_Declare(
41+
eigen
42+
GIT_REPOSITORY https://gitlab.com/libeigen/eigen.git
43+
GIT_TAG 3.4.0
44+
)
45+
FetchContent_MakeAvailable(eigen)
46+
47+
# 4. JSON
48+
FetchContent_Declare(
49+
nlohmann_json
50+
GIT_REPOSITORY https://github.com/nlohmann/json.git
51+
GIT_TAG v3.11.3
52+
)
53+
FetchContent_MakeAvailable(nlohmann_json)
54+
55+
# Includes
56+
include_directories(include)
57+
include_directories(../../QuanuX-Common/cpp/include)
58+
59+
# Core Library
60+
add_library(stats_core STATIC
61+
src/stats_engine.cpp
62+
src/time_series_store.cpp
63+
)
64+
target_link_libraries(stats_core PUBLIC
65+
nats_static
66+
duckdb_static
67+
Eigen3::Eigen
68+
nlohmann_json::nlohmann_json
69+
pthread
70+
dl
71+
)
72+
73+
# Main Service
74+
add_executable(quanux_stats src/main.cpp)
75+
target_link_libraries(quanux_stats PRIVATE stats_core)
Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,76 @@
1+
#pragma once
2+
#include <Eigen/Dense>
3+
#include <atomic>
4+
#include <map>
5+
#include <memory>
6+
#include <mutex>
7+
#include <string>
8+
#include <utility>
9+
#include <vector>
10+
11+
// Forward decls
12+
namespace duckdb {
13+
class Connection;
14+
class DuckDB;
15+
} // namespace duckdb
16+
17+
class StatsEngine {
18+
public:
19+
StatsEngine();
20+
~StatsEngine();
21+
22+
void connect_nats(const std::string &url);
23+
void connect_db(const std::string &path);
24+
void run();
25+
void stop();
26+
27+
private:
28+
std::atomic<bool> running_{false};
29+
std::unique_ptr<duckdb::DuckDB> db_;
30+
std::unique_ptr<duckdb::Connection> conn_;
31+
32+
// NATS handle (void* to keep header clean of C includes if desired, but
33+
// sticking to simple for now) We'll impl in cpp
34+
struct NatsContext;
35+
std::unique_ptr<NatsContext> nats_;
36+
37+
// Online Stats State (Example: Keeping a rolling window for correlation)
38+
// Vector of recent prices for correlation calculation
39+
struct InstrumentStats {
40+
uint64_t count = 0;
41+
double mean = 0.0;
42+
double M2 = 0.0; // Welford's algorithm accumulator
43+
44+
// Z-Score window
45+
std::vector<double> window;
46+
size_t window_size = 100;
47+
48+
void update(double price);
49+
double variance() const;
50+
double std_dev() const;
51+
};
52+
53+
// Online Correlation State (Pairwise)
54+
struct CorrelationStats {
55+
uint64_t count = 0;
56+
double mean_x = 0.0;
57+
double mean_y = 0.0;
58+
double C_xy = 0.0; // Co-moment accumulator
59+
60+
void update(double x, double y);
61+
double covariance() const;
62+
double correlation(double std_dev_x, double std_dev_y) const;
63+
};
64+
65+
// Map instrument_id (symbol) -> Stats
66+
std::map<std::string, InstrumentStats> stats_map_;
67+
68+
// Map (symbolA, symbolB) -> CorrelationStats
69+
// Key is sorted: pair(min(A,B), max(A,B))
70+
std::map<std::pair<std::string, std::string>, CorrelationStats> corr_map_;
71+
std::mutex
72+
stats_mutex_; // Protect maps from NATS callbacks if multithreaded (NATS C
73+
// is usually single threaded callback context, but safer)
74+
75+
void update_correlation(const std::string &symbol, double price);
76+
};

QuanuX-Statistics/cpp/src/main.cpp

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
#include "stats_engine.h"
2+
#include <cstdlib>
3+
#include <iostream>
4+
5+
int main(int argc, char **argv) {
6+
std::cout << R"(
7+
___ _ __ __
8+
/ _ \ _ _ __ _ _ __ _ | | \ \ / /
9+
| | | | | | |/ _` | '_ \| | | | \ \/ /
10+
| |_| | |_| | (_| | | | | |_ | | / /\ \
11+
\__\_\\__,_|\__,_|_| |_|\__,|_| /_/ \_\
12+
13+
QuanuX Statistics & Research Node
14+
)" << std::endl;
15+
16+
StatsEngine engine;
17+
18+
const char *nats_url = std::getenv("QUANUX_NATS_URL");
19+
const char *db_path = std::getenv("QUANUX_DB_PATH");
20+
21+
engine.connect_nats(nats_url ? nats_url : "nats://localhost:4222");
22+
engine.connect_db(db_path ? db_path : "market_stats.duckdb");
23+
24+
engine.run();
25+
26+
return 0;
27+
}

0 commit comments

Comments
 (0)