Skip to content

Commit a28c491

Browse files
committed
feat(gateway): replace SQL-string forwarding with plan-based gateway routing
Introduces the Gateway (control/gateway/) as the single cluster-aware execution path for all protocol handlers. Instead of serialising a raw SQL string and re-planning it on the remote node, handlers now ship the pre-planned PhysicalPlan via ExecuteRequest over QUIC. Removed: - control/forward.rs — LocalForwarder (SQL-string execution handler) - control/cluster_forwarder.rs — thin wrapper around ForwardRequest - server/pgwire/handler/routing/forward.rs — forward_sql / remote_leader_for_tasks - server/pgwire/handler/retry.rs — ad-hoc retry wrapper superseded by gateway retry All protocol handlers (pgwire, native, HTTP, RESP, ILP, WebSocket) are updated to call gateway.execute() where available, with a local SPSC fallback for single-node boot before the gateway is installed. CDC consume_remote and topic publish are updated to route through gateway.execute_sql instead of the old ForwardRequest path.
1 parent 71972b2 commit a28c491

44 files changed

Lines changed: 3923 additions & 974 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

nodedb/src/control/cluster_forwarder.rs

Lines changed: 0 additions & 134 deletions
This file was deleted.
Lines changed: 179 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,179 @@
1+
//! Local execution of incoming `ExecuteRequest` RPCs.
2+
//!
3+
//! When a remote node sends an `ExecuteRequest` to this node (because this
4+
//! node is the leader for the target vShard), the [`LocalPlanExecutor`]
5+
//! validates descriptor versions, decodes the `PhysicalPlan`, dispatches
6+
//! it through the local SPSC bridge, and returns an `ExecuteResponse`.
7+
//!
8+
//! Unlike the retired SQL-string forwarding path, this path skips planning
9+
//! entirely — the plan is already encoded by the sender.
10+
11+
use std::sync::Arc;
12+
use std::sync::atomic::{AtomicU64, Ordering};
13+
use std::time::{Duration, Instant};
14+
15+
use nodedb_cluster::forward::PlanExecutor;
16+
use nodedb_cluster::rpc_codec::{ExecuteRequest, ExecuteResponse, TypedClusterError};
17+
18+
use crate::bridge::envelope::{Priority, Request};
19+
use crate::bridge::physical_plan::wire as plan_wire;
20+
use crate::control::state::SharedState;
21+
use crate::types::{ReadConsistency, RequestId};
22+
23+
/// Numeric code for `TypedClusterError::Internal` when plan bytes fail to decode.
24+
const PLAN_DECODE_FAILED: u32 = nodedb_cluster::rpc_codec::PLAN_DECODE_FAILED;
25+
26+
/// Executes pre-planned `PhysicalPlan` on the local Data Plane.
27+
pub struct LocalPlanExecutor {
28+
state: Arc<SharedState>,
29+
next_request_id: AtomicU64,
30+
}
31+
32+
impl LocalPlanExecutor {
33+
pub fn new(state: Arc<SharedState>) -> Self {
34+
Self {
35+
state,
36+
// Offset to avoid collision with direct client and forwarded request IDs.
37+
next_request_id: AtomicU64::new(2_000_000_000),
38+
}
39+
}
40+
41+
fn next_request_id(&self) -> RequestId {
42+
RequestId::new(self.next_request_id.fetch_add(1, Ordering::Relaxed))
43+
}
44+
}
45+
46+
impl PlanExecutor for LocalPlanExecutor {
47+
async fn execute_plan(&self, req: ExecuteRequest) -> ExecuteResponse {
48+
// ── 1. Deadline check ─────────────────────────────────────────────────
49+
if req.deadline_remaining_ms == 0 {
50+
return ExecuteResponse::err(TypedClusterError::DeadlineExceeded { elapsed_ms: 0 });
51+
}
52+
53+
let deadline = Duration::from_millis(req.deadline_remaining_ms).min(Duration::from_secs(
54+
self.state.tuning.network.default_deadline_secs,
55+
));
56+
57+
// ── 2. Descriptor version validation ──────────────────────────────────
58+
//
59+
// For each (collection, version) pair the caller sent, look up the local
60+
// descriptor version from SystemCatalog. If any version differs, the
61+
// caller's plan was built against a stale schema — reject with a typed
62+
// error so they re-plan against fresh leases.
63+
let catalog_ref = self.state.credentials.catalog();
64+
if let Some(catalog) = catalog_ref.as_ref() {
65+
for entry in &req.descriptor_versions {
66+
match catalog.get_collection(req.tenant_id, &entry.collection) {
67+
Ok(Some(stored)) => {
68+
// Version 0 is the pre-B.1 sentinel; treat as 1 (same
69+
// floor the drain gate uses).
70+
let actual = if stored.descriptor_version == 0 {
71+
1
72+
} else {
73+
stored.descriptor_version
74+
};
75+
if actual != entry.version {
76+
return ExecuteResponse::err(TypedClusterError::DescriptorMismatch {
77+
collection: entry.collection.clone(),
78+
expected_version: entry.version,
79+
actual_version: actual,
80+
});
81+
}
82+
}
83+
Ok(None) => {
84+
// Collection not found locally — could be a new collection
85+
// the follower saw but we haven't applied yet, or a race.
86+
// Treat as DescriptorMismatch so the caller re-plans.
87+
if entry.version != 0 {
88+
return ExecuteResponse::err(TypedClusterError::DescriptorMismatch {
89+
collection: entry.collection.clone(),
90+
expected_version: entry.version,
91+
actual_version: 0,
92+
});
93+
}
94+
}
95+
Err(e) => {
96+
return ExecuteResponse::err(TypedClusterError::Internal {
97+
code: PLAN_DECODE_FAILED,
98+
message: format!("catalog lookup failed: {e}"),
99+
});
100+
}
101+
}
102+
}
103+
}
104+
105+
// ── 3. Decode the PhysicalPlan ────────────────────────────────────────
106+
let plan = match plan_wire::decode(&req.plan_bytes) {
107+
Ok(p) => p,
108+
Err(e) => {
109+
return ExecuteResponse::err(TypedClusterError::Internal {
110+
code: PLAN_DECODE_FAILED,
111+
message: format!("plan decode failed: {e}"),
112+
});
113+
}
114+
};
115+
116+
// ── 4. Dispatch through local SPSC bridge ─────────────────────────────
117+
//
118+
// Build a Request, register a oneshot tracker, dispatch, and await the response.
119+
let request_id = self.next_request_id();
120+
let tenant_id = crate::types::TenantId::new(req.tenant_id);
121+
122+
let request = Request {
123+
request_id,
124+
tenant_id,
125+
// Use the first vshard_id from the plan — the sender already routed
126+
// this to the correct node. Use 0 as the default if the plan doesn't
127+
// embed vshard info directly; the Data Plane ignores it for local exec.
128+
vshard_id: crate::types::VShardId::new(0),
129+
plan,
130+
deadline: Instant::now() + deadline,
131+
priority: Priority::Normal,
132+
trace_id: req.trace_id,
133+
consistency: ReadConsistency::Strong,
134+
idempotency_key: None,
135+
event_source: crate::event::EventSource::User,
136+
user_roles: Vec::new(),
137+
};
138+
139+
let rx = self.state.tracker.register_oneshot(request_id);
140+
141+
let dispatch_result = match self.state.dispatcher.lock() {
142+
Ok(mut d) => d.dispatch(request),
143+
Err(poisoned) => poisoned.into_inner().dispatch(request),
144+
};
145+
146+
if let Err(e) = dispatch_result {
147+
return ExecuteResponse::err(TypedClusterError::Internal {
148+
code: PLAN_DECODE_FAILED,
149+
message: format!("dispatch failed: {e}"),
150+
});
151+
}
152+
153+
// ── 5. Collect response payloads ──────────────────────────────────────
154+
match tokio::time::timeout(deadline, rx).await {
155+
Ok(Ok(resp)) => {
156+
if resp.status == crate::bridge::envelope::Status::Error {
157+
let msg = resp
158+
.error_code
159+
.as_ref()
160+
.map(|c| format!("{c:?}"))
161+
.unwrap_or_else(|| "unknown error".into());
162+
ExecuteResponse::err(TypedClusterError::Internal {
163+
code: PLAN_DECODE_FAILED,
164+
message: msg,
165+
})
166+
} else {
167+
ExecuteResponse::ok(vec![resp.payload.to_vec()])
168+
}
169+
}
170+
Ok(Err(_)) => ExecuteResponse::err(TypedClusterError::Internal {
171+
code: PLAN_DECODE_FAILED,
172+
message: "response channel closed".into(),
173+
}),
174+
Err(_) => ExecuteResponse::err(TypedClusterError::DeadlineExceeded {
175+
elapsed_ms: deadline.as_millis() as u64,
176+
}),
177+
}
178+
}
179+
}

0 commit comments

Comments
 (0)