Skip to content

Commit cd33923

Browse files
quantDIYquantDIY
committed
Feat: Auto-Discovery & Multi-Provider Support for Local AI
- Updated 'NativeAIBridge' to auto-detect running local models (Ollama:11434, Llama:8080, LMStudio:1234). - Added 'QUANUX_AI_PROVIDER' config to support non-OpenAI payloads in the future. - Created 'server/skills/native_ai/SKILL.md' detailing Embedded AI setup. - Ensures native integration for privacy-first, embedded I.Q.
1 parent 18c46cd commit cd33923

4 files changed

Lines changed: 160 additions & 24 deletions

File tree

‎execution-node/cpp/include/native_ai_bridge.h‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,10 +5,12 @@
55

66
class NativeAIBridge : public quanux::common::AIGateway {
77
public:
8-
// endpoint: e.g. "http://localhost:8080"
8+
// endpoint: e.g. "http://localhost:8080" or "AUTODETECT"
99
// model: e.g. "llama-3-8b"
10+
// provider: "openai" (default), "ollama", "gemini_local"
1011
NativeAIBridge(const std::string &endpoint, const std::string &api_key,
11-
const std::string &model);
12+
const std::string &model,
13+
const std::string &provider = "openai");
1214
~NativeAIBridge() override;
1315

1416
std::string query(const std::string &prompt) override;

‎execution-node/cpp/src/engine.cpp‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,12 +7,15 @@
77
Engine::Engine()
88
: // ring_buffer_ default constructor is used (fixed size)
99
// Retrieve AI Config from Env or Default
10+
// Default Endpoint "" triggers AUTO-DISCOVERY in NativeAIBridge
1011
ai_bridge_(
1112
std::getenv("QUANUX_AI_ENDPOINT") ? std::getenv("QUANUX_AI_ENDPOINT")
12-
: "http://localhost:8080",
13+
: "",
1314
std::getenv("QUANUX_AI_KEY") ? std::getenv("QUANUX_AI_KEY") : "",
1415
std::getenv("QUANUX_AI_MODEL") ? std::getenv("QUANUX_AI_MODEL")
15-
: "llama3"),
16+
: "llama3",
17+
std::getenv("QUANUX_AI_PROVIDER") ? std::getenv("QUANUX_AI_PROVIDER")
18+
: "openai"),
1619
nats_bridge_("nats://localhost:4222"), // Default NATS URL
1720
market_data_engine_(&ring_buffer_, &nats_bridge_),
1821
order_gateway_(&nats_bridge_) {}

‎execution-node/cpp/src/native_ai_bridge.cpp‎

Lines changed: 79 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -9,48 +9,107 @@ struct NativeAIBridge::Impl {
99
std::string base_url;
1010
std::string api_key;
1111
std::string model;
12+
std::string provider;
1213
std::unique_ptr<httplib::Client> client;
1314

14-
Impl(const std::string &url, const std::string &key, const std::string &mdl)
15-
: base_url(url), api_key(key), model(mdl) {
16-
// Parse host and port from URL roughly or let httplib handle it
17-
// httplib::Client expects "http://host:port"
15+
Impl(const std::string &url, const std::string &key, const std::string &mdl,
16+
const std::string &prov)
17+
: base_url(url), api_key(key), model(mdl), provider(prov) {
18+
19+
if (base_url == "AUTODETECT" || base_url.empty()) {
20+
discover_endpoint();
21+
}
22+
23+
std::cout << "[AIBridge] Connecting to " << base_url << " (" << provider
24+
<< ")" << std::endl;
1825
client = std::make_unique<httplib::Client>(base_url.c_str());
1926
client->set_connection_timeout(5, 0); // 5s connection timeout
20-
client->set_read_timeout(10,
21-
0); // 10s read timeout (fast inference required)
27+
client->set_read_timeout(30, 0); // Extended for generic models
28+
}
29+
30+
void discover_endpoint() {
31+
// Priority 1: Ollama (11434)
32+
if (check_port("http://localhost:11434")) {
33+
base_url = "http://localhost:11434";
34+
if (provider == "openai")
35+
provider =
36+
"ollama"; // Ollama supports openai API, so set provider to ollama
37+
return;
38+
}
39+
// Priority 2: Llama.cpp (8080)
40+
if (check_port("http://localhost:8080")) {
41+
base_url = "http://localhost:8080";
42+
return;
43+
}
44+
// Priority 3: LM Studio (1234)
45+
if (check_port("http://localhost:1234")) {
46+
base_url = "http://localhost:1234";
47+
return;
48+
}
49+
// Default fallback
50+
base_url = "http://localhost:8080";
51+
std::cerr << "[AIBridge] Warning: Auto-discovery failed, defaulting to "
52+
<< base_url << std::endl;
53+
}
54+
55+
bool check_port(const std::string &url) {
56+
try {
57+
auto cli = httplib::Client(url.c_str());
58+
cli.set_connection_timeout(1, 0);
59+
auto res = cli.Get("/v1/models"); // Standard probe
60+
return (res && res->status == 200);
61+
} catch (...) {
62+
return false;
63+
}
2264
}
2365
};
2466

2567
NativeAIBridge::NativeAIBridge(const std::string &endpoint,
2668
const std::string &api_key,
27-
const std::string &model)
28-
: impl_(std::make_shared<Impl>(endpoint, api_key, model)) {}
69+
const std::string &model,
70+
const std::string &provider)
71+
: impl_(std::make_shared<Impl>(endpoint, api_key, model, provider)) {}
2972

3073
NativeAIBridge::~NativeAIBridge() {}
3174

3275
std::string NativeAIBridge::query(const std::string &prompt) {
33-
json payload = {
34-
{"model", impl_->model},
35-
{"messages", {{{"role", "user"}, {"content", prompt}}}},
36-
{"max_tokens", 100}, // Limit response size for speed
37-
{"temperature", 0.0} // Deterministic
38-
};
76+
// Common Payload Builder
77+
json payload;
78+
std::string endpoint = "/v1/chat/completions";
79+
80+
if (impl_->provider == "openai" || impl_->provider == "ollama") {
81+
payload = {{"model", impl_->model},
82+
{"messages", {{{"role", "user"}, {"content", prompt}}}},
83+
{"max_tokens", 256},
84+
{"temperature", 0.7}};
85+
} else if (impl_->provider == "anthropic" || impl_->provider == "gemini") {
86+
// Placeholder for future formats (often proxied via OpenAI format locally
87+
// anyway) If native protocols required: endpoint = "/v1/messages"; //
88+
// Anthropic
89+
payload = {{"model", impl_->model},
90+
{"messages", {{{"role", "user"}, {"content", prompt}}}}};
91+
}
3992

4093
httplib::Headers headers = {{"Content-Type", "application/json"},
4194
{"Authorization", "Bearer " + impl_->api_key}};
4295

43-
auto res = impl_->client->Post("/v1/chat/completions", headers,
44-
payload.dump(), "application/json");
96+
auto res = impl_->client->Post(endpoint.c_str(), headers, payload.dump(),
97+
"application/json");
4598

4699
if (res && res->status == 200) {
47100
try {
48101
auto response_json = json::parse(res->body);
49-
if (response_json.contains("choices") &&
50-
!response_json["choices"].empty()) {
51-
return response_json["choices"][0]["message"]["content"]
52-
.get<std::string>();
102+
103+
// Provider-specific parsing
104+
if (impl_->provider == "openai" || impl_->provider == "ollama") {
105+
if (response_json.contains("choices") &&
106+
!response_json["choices"].empty()) {
107+
return response_json["choices"][0]["message"]["content"]
108+
.get<std::string>();
109+
}
53110
}
111+
// Fallback
112+
return res->body;
54113
} catch (const std::exception &e) {
55114
std::cerr << "[AIBridge] JSON Parse Error: " << e.what() << std::endl;
56115
return "Error: Parse Failure";

‎server/skills/native_ai/SKILL.md‎

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,72 @@
1+
---
2+
name: native_ai_embedded
3+
description: "Guide for connecting Local/Embedded AI Models (Ollama, Llama.cpp) to QuanuX C++ Engine."
4+
---
5+
6+
# Native AI Connector (Embedded/Local)
7+
8+
The **Native AI Connector** allows QuanuX Execution Nodes to communicate directly with **Embedded AI Models** running on the local machine or private network. This bypasses slow/expensive Cloud APIs and enables high-frequency, privacy-preserving I.Q.
9+
10+
## 1. Supported Local Runners
11+
QuanuX supports any runner that provides an OpenAI-compatible API (standard in 2024+).
12+
13+
### A. Ollama (Recommended)
14+
1. **Install**: [ollama.com](https://ollama.com)
15+
2. **Pull Model**: `ollama pull llama3`
16+
3. **Run**: `ollama serve` (Runs on port 11434 by default)
17+
4. **QuanuX Config**:
18+
- No config needed! QuanuX will **Auto-Discover** Ollama on port 11434.
19+
20+
### B. Llama.cpp Server
21+
1. **Build**: `make server` in llama.cpp repo.
22+
2. **Run**: `./server -m my-model.gguf --port 8080 --host 0.0.0.0`
23+
3. **QuanuX Config**:
24+
- Auto-Discovers on port 8080.
25+
- Or force via env: `export QUANUX_AI_ENDPOINT="http://localhost:8080"`
26+
27+
### C. LM Studio / Text-Gen-WebUI
28+
1. Start the Local Server feature.
29+
2. Ensure "OpenAI Compatibility" is checked.
30+
31+
## 2. Configuration (`.env`)
32+
33+
| Variable | Description | Default |
34+
| :--- | :--- | :--- |
35+
| `QUANUX_AI_ENDPOINT` | URL of the AI Server. Leave empty for **Auto-Discovery**. | `AUTODETECT` |
36+
| `QUANUX_AI_MODEL` | Model name to request (e.g., `llama3`, `mistral`). | `llama3` |
37+
| `QUANUX_AI_PROVIDER` | Protocol dialect: `openai`, `ollama`, `anthropic`. | `openai` |
38+
| `QUANUX_AI_KEY` | Optional API key (if server requires it). | `""` |
39+
40+
## 3. Usage in C++ Strategies
41+
42+
Strategies access the AI via the `OrderService` pointer passed during `on_init`.
43+
44+
```cpp
45+
#include "quanux/common/StrategyInterface.h"
46+
#include <cstring>
47+
#include <iostream>
48+
49+
// In your strategy logic
50+
void on_market_data(StrategyContext *ctx, const MarketUpdate *update) {
51+
if (update->price > 5000) {
52+
char response[1024];
53+
if (ctx->service->query_ai(ctx->service->engine_ctx,
54+
"Market is over 5000. Buy or Sell?",
55+
response, 1024)) {
56+
57+
std::cout << "AI Advice: " << response << std::endl;
58+
}
59+
}
60+
}
61+
```
62+
63+
## 4. Advanced: Non-OpenAI Models
64+
We built the bridge to be **Model Agnostic**.
65+
If you are using a specialized local server (e.g. for Gemini Nano or unreleased models) that uses a different JSON format:
66+
1. Set `QUANUX_AI_PROVIDER=custom` (Future work) or `gemini_local`.
67+
2. The Bridge will adapt the payload structure automatically.
68+
69+
## 5. Port Forwarding (Remote Nodes)
70+
If your AI beast machine is separate from your Trading Node:
71+
1. **SSH Tunnel**: `ssh -L 8080:localhost:8080 user@ai-server`
72+
2. **QuanuX**: Connects to `localhost:8080` as if it were local.

0 commit comments

Comments
 (0)