Skip to content

Commit 3ad3b8d

Browse files
Merge pull request #9 from PlotJuggler/feat/size-class-frames
Isolate heavy topics into size-class frames + shed under backpressure (Phase 1)
2 parents a1fd0f1 + 6ce5dd3 commit 3ad3b8d

21 files changed

Lines changed: 1258 additions & 151 deletions

CLAUDE.md

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -238,6 +238,7 @@ min_qos_depth: 1 # Minimum KEEP_LAST subscription depth after aggr
238238
max_qos_depth: 100 # Maximum KEEP_LAST subscription depth after aggregating publisher depths
239239
topic_poll_interval: 1.0 # Seconds between topics_changed notification polls; 0 disables polling
240240
client_backlog_size: 100 # Max frames queued per slow client before dropping the oldest (must be > 0)
241+
heavy_frame_threshold_bytes: 262144 # Isolate messages >= this size into their own size-class frame; 0 disables
241242
tls: false # Enable TLS (wss://); requires certfile and keyfile
242243
certfile: "" # TLS server certificate file
243244
keyfile: "" # TLS private key file
@@ -247,14 +248,14 @@ keyfile: "" # TLS private key file
247248
```bash
248249
pj_bridge_rti --domains 0 1 --port 9090 --publish-rate 50 --session-timeout 10 \
249250
--topic-whitelist ".*" --topic-poll-interval 1.0 --client-backlog-size 100 \
250-
--certfile cert.pem --keyfile key.pem
251+
--heavy-frame-threshold-bytes 262144 --certfile cert.pem --keyfile key.pem
251252
```
252253

253254
### FastDDS (via CLI flags):
254255
```bash
255256
pj_bridge_fastdds --domains 0 1 --port 9090 --publish-rate 50 --session-timeout 10 \
256257
--topic-whitelist ".*" --topic-poll-interval 1.0 --client-backlog-size 100 \
257-
--certfile cert.pem --keyfile key.pem
258+
--heavy-frame-threshold-bytes 262144 --certfile cert.pem --keyfile key.pem
258259
```
259260

260261
See `docs/API.md` for full semantics of each option (topic whitelist matching rules,

CMakeLists.txt

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -256,6 +256,7 @@ if(BUILD_TESTING AND ament_cmake_FOUND)
256256

257257
ament_add_gtest(${PROJECT_NAME}_tests
258258
tests/unit/test_websocket_middleware.cpp
259+
tests/unit/test_backpressure.cpp
259260
tests/unit/test_bounded_frame_queue.cpp
260261
tests/unit/test_message_buffer.cpp
261262
tests/unit/test_session_manager.cpp

README.md

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,7 @@ independently.
5454
| `max_qos_depth` | int | 100 | ROS2 only: maximum KEEP_LAST subscription depth after aggregating publisher depths |
5555
| `topic_poll_interval` | double | 1.0 | Seconds between `topics_changed` notification polls; `0` disables polling |
5656
| `client_backlog_size` | int | 100 | Max binary frames queued per slow client before the oldest is dropped (must be `> 0`) |
57+
| `heavy_frame_threshold_bytes` | int | 262144 | Isolate messages ≥ this size (bytes) into their own size-class frame so they don't starve small topics; `0` disables (must be `>= 0`) |
5758
| `tls` | bool | false | Enable TLS (`wss://`); requires `certfile` and `keyfile` |
5859
| `certfile` | string | `""` | TLS server certificate file |
5960
| `keyfile` | string | `""` | TLS private key file |
@@ -71,6 +72,7 @@ independently.
7172
| `--topic-whitelist` | string list | `.*` | Full-match regex patterns (ECMAScript), repeatable |
7273
| `--topic-poll-interval` | double | 1.0 | Seconds between `topics_changed` notification polls; `0` disables polling |
7374
| `--client-backlog-size` | int | 100 | Max binary frames queued per slow client before the oldest is dropped (range `1`-`1000000`) |
75+
| `--heavy-frame-threshold-bytes` | int | 262144 | Isolate messages ≥ this size (bytes) into their own size-class frame; `0` disables (range `0`-`1000000000`) |
7476
| `--certfile` | string | (none) | TLS server certificate file; enables `wss://`, requires `--keyfile` |
7577
| `--keyfile` | string | (none) | TLS private key file; enables `wss://`, requires `--certfile` |
7678
| `--qos-profile` | string | (none) | RTI only: QoS profile XML file path |

app/include/pj_bridge/bridge_server.hpp

Lines changed: 21 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@
2828

2929
#include "pj_bridge/message_buffer.hpp"
3030
#include "pj_bridge/middleware/middleware_interface.hpp"
31+
#include "pj_bridge/protocol_constants.hpp"
3132
#include "pj_bridge/session_manager.hpp"
3233
#include "pj_bridge/subscription_manager_interface.hpp"
3334
#include "pj_bridge/topic_source_interface.hpp"
@@ -47,6 +48,21 @@ namespace pj_bridge {
4748
* Thread-safe for concurrent client connections.
4849
* Event loop is driven externally (no internal timers).
4950
*/
51+
52+
/// Tunable configuration for BridgeServer (backend-agnostic). Bundled into one
53+
/// struct so entry points construct the server with a single named aggregate
54+
/// rather than a long positional argument list.
55+
struct BridgeServerConfig {
56+
int port = 9090; ///< WebSocket port
57+
double session_timeout = 10.0; ///< client session timeout, seconds
58+
double publish_rate = 50.0; ///< message aggregation/publish rate, Hz
59+
WhitelistFilter whitelist = {}; ///< topic whitelist (default: matches everything)
60+
/// Per-message byte size at or above which a topic's message is isolated into
61+
/// its own size-class ("heavy") frame instead of being aggregated with light
62+
/// topics. 0 disables splitting (single aggregated frame). Default: 256 KiB.
63+
size_t heavy_frame_threshold_bytes = kDefaultHeavyFrameThresholdBytes;
64+
};
65+
5066
class BridgeServer {
5167
public:
5268
struct StatsSnapshot {
@@ -61,16 +77,12 @@ class BridgeServer {
6177
* @param topic_source Backend-specific topic discovery and schema provider
6278
* @param subscription_manager Backend-specific subscription manager
6379
* @param middleware Middleware interface for network communication
64-
* @param port Server port (default: 9090)
65-
* @param session_timeout Session timeout in seconds (default: 10.0)
66-
* @param publish_rate Message aggregation publish rate in Hz (default: 50.0)
67-
* @param whitelist Topic whitelist filter (default: matches everything)
80+
* @param config Tunable server configuration (see BridgeServerConfig)
6881
*/
6982
explicit BridgeServer(
7083
std::shared_ptr<TopicSourceInterface> topic_source,
7184
std::shared_ptr<SubscriptionManagerInterface> subscription_manager,
72-
std::shared_ptr<MiddlewareInterface> middleware, int port = 9090, double session_timeout = 10.0,
73-
double publish_rate = 50.0, WhitelistFilter whitelist = {});
85+
std::shared_ptr<MiddlewareInterface> middleware, BridgeServerConfig config = {});
7486

7587
/// Shuts down middleware before members are destroyed, preventing
7688
/// disconnect callbacks from firing into a partially destroyed object.
@@ -211,6 +223,9 @@ class BridgeServer {
211223
double session_timeout_;
212224
double publish_rate_;
213225
WhitelistFilter whitelist_;
226+
// Per-message byte size at or above which a topic is isolated into its own
227+
// size-class ("heavy") frame; 0 disables splitting. See publish_aggregated_messages().
228+
size_t heavy_frame_threshold_bytes_;
214229

215230
// State
216231
std::atomic<bool> initialized_;

app/include/pj_bridge/message_serializer.hpp

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -89,12 +89,14 @@ class AggregatedMessageSerializer {
8989
* - Offset 0: magic (uint32_t "PJRB" = 0x42524A50, little-endian)
9090
* - Offset 4: message_count (uint32_t, little-endian)
9191
* - Offset 8: uncompressed_size (uint32_t, little-endian)
92-
* - Offset 12: flags (uint32_t, reserved = 0)
92+
* - Offset 12: flags (uint32_t; bit0 = heavy frame, else reserved = 0)
9393
* - Offset 16+: ZSTD-compressed payload
9494
*
95+
* @param flags Header flag bits written at offset 12 (default 0). Use
96+
* kFrameFlagHeavy to mark an isolated large/size-class frame.
9597
* @return Vector containing header + compressed payload
9698
*/
97-
std::vector<uint8_t> finalize();
99+
std::vector<uint8_t> finalize(uint32_t flags = 0);
98100

99101
/**
100102
* @brief Compress data using ZSTD (compression level 1)
Lines changed: 105 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,105 @@
1+
/*
2+
* Copyright (C) 2026 Davide Faconti
3+
*
4+
* This file is part of pj_bridge.
5+
*
6+
* pj_bridge is free software: you can redistribute it and/or modify
7+
* it under the terms of the GNU Affero General Public License as published by
8+
* the Free Software Foundation, either version 3 of the License, or
9+
* (at your option) any later version.
10+
*
11+
* pj_bridge is distributed in the hope that it will be useful,
12+
* but WITHOUT ANY WARRANTY; without even the implied warranty of
13+
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14+
* GNU Affero General Public License for more details.
15+
*
16+
* You should have received a copy of the GNU Affero General Public License
17+
* along with pj_bridge. If not, see <https://www.gnu.org/licenses/>.
18+
*/
19+
20+
#pragma once
21+
22+
#include <cstddef>
23+
#include <optional>
24+
#include <vector>
25+
26+
#include "pj_bridge/middleware/middleware_interface.hpp"
27+
28+
namespace pj_bridge {
29+
30+
/// Outcome of run_backpressure() for one outgoing frame. The disposition is a
31+
/// single discriminant (no contradictory flag combinations are representable);
32+
/// the counters are companion data.
33+
struct SendOutcome {
34+
SendResult result = SendResult::kClientGone; ///< how the current frame was handled
35+
size_t frames_flushed = 0; ///< backlog frames flushed to the socket this call
36+
size_t dropped = 0; ///< backlog frames evicted on overflow (only when result == kQueued)
37+
};
38+
39+
/// Socket-agnostic backpressure policy shared by the send path. The caller
40+
/// injects the socket/queue primitives (as callables — templated to avoid
41+
/// std::function type-erasure on the hot path) so the policy is unit-testable
42+
/// without a real connection:
43+
/// - @p buffered_amount : `size_t()` — current socket buffer bytes
44+
/// - @p pop_pending : `std::optional<vector<uint8_t>>()` — pop oldest queued frame (nullopt if empty)
45+
/// - @p send : `bool(const vector<uint8_t>&)` — transmit a frame; false = client gone
46+
/// - @p queue_pending : `std::optional<size_t>(const vector<uint8_t>&)` — enqueue (drop-oldest),
47+
/// returns #dropped, or nullopt if the client is gone
48+
///
49+
/// Policy: first flush the backlog while the socket has room (re-checking the
50+
/// watermark each iteration so an already-congested socket is never fed
51+
/// further, and never flushing more than @p max_flush frames so a concurrent
52+
/// producer cannot make one call flush forever); then handle the current
53+
/// frame — send it if there is room (kDelivered), else drop a kHeavy frame
54+
/// before transmit (kShed) or enqueue a kNormal frame (kQueued). A vanished
55+
/// client surfaces as kClientGone.
56+
///
57+
/// @param max_flush upper bound on frames flushed this call — pass the backlog
58+
/// size observed at call start.
59+
template <class BufferedAmount, class PopPending, class Send, class QueuePending>
60+
SendOutcome run_backpressure(
61+
FramePriority priority, const std::vector<uint8_t>& frame, size_t watermark, size_t max_flush,
62+
const BufferedAmount& buffered_amount, const PopPending& pop_pending, const Send& send,
63+
const QueuePending& queue_pending) {
64+
SendOutcome out;
65+
66+
// Flush queued frames to the socket, re-checking the watermark each iteration
67+
// so a socket that fills up mid-flush is never fed further (the flush-recheck
68+
// fix: the old loop computed the flush count once and could dump a burst of
69+
// stale frames onto an already-congested socket). Bounded by max_flush so a
70+
// producer enqueueing concurrently cannot keep this single call flushing.
71+
while (out.frames_flushed < max_flush && buffered_amount() < watermark) {
72+
std::optional<std::vector<uint8_t>> queued = pop_pending();
73+
if (!queued) {
74+
break;
75+
}
76+
if (!send(*queued)) {
77+
out.result = SendResult::kClientGone;
78+
return out;
79+
}
80+
out.frames_flushed++;
81+
}
82+
83+
// Handle the current frame against the live socket buffer.
84+
if (buffered_amount() < watermark) {
85+
out.result = send(frame) ? SendResult::kDelivered : SendResult::kClientGone;
86+
return out;
87+
}
88+
89+
// Socket congested: shed heavy frames before transmit; queue normal frames.
90+
if (priority == FramePriority::kHeavy) {
91+
out.result = SendResult::kShed;
92+
return out;
93+
}
94+
95+
std::optional<size_t> dropped = queue_pending(frame);
96+
if (!dropped) {
97+
out.result = SendResult::kClientGone;
98+
return out;
99+
}
100+
out.dropped = *dropped;
101+
out.result = SendResult::kQueued;
102+
return out;
103+
}
104+
105+
} // namespace pj_bridge

app/include/pj_bridge/middleware/middleware_interface.hpp

Lines changed: 25 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,23 @@
2828

2929
namespace pj_bridge {
3030

31+
/// Delivery priority for a per-client binary frame. Under socket congestion a
32+
/// `kHeavy` (large/size-class) frame is dropped before transmit rather than
33+
/// queued, so one big frame cannot starve the small frames behind it; a
34+
/// `kNormal` frame instead falls back to the queue-with-drop-oldest backlog.
35+
/// (When the socket has room, both are sent immediately.) See docs/API.md.
36+
enum class FramePriority { kNormal, kHeavy };
37+
38+
/// Outcome of send_binary(): how the transport handled the frame. Only
39+
/// `kDelivered` and `kQueued` are (or will be) put on the wire; `kShed` and
40+
/// `kClientGone` are never delivered, so callers must NOT count them as sent.
41+
enum class SendResult {
42+
kDelivered, ///< written to the socket now (or flushed from the backlog)
43+
kQueued, ///< enqueued for later delivery (kNormal frame, socket congested)
44+
kShed, ///< dropped before transmit (kHeavy frame, socket congested)
45+
kClientGone, ///< the client disconnected; nothing was sent
46+
};
47+
3148
/// Abstract transport layer between BridgeServer and clients.
3249
///
3350
/// Implementations handle connection management and bidirectional messaging.
@@ -65,8 +82,14 @@ class MiddlewareInterface {
6582
virtual bool publish_data(const std::vector<uint8_t>& data) = 0;
6683

6784
/// Send binary data to a specific client (used for per-client aggregated frames).
68-
/// @return true if the message was sent, false if the client is gone.
69-
virtual bool send_binary(const std::string& client_identity, const std::vector<uint8_t>& data) = 0;
85+
/// @param priority kHeavy frames are shed before transmit under congestion
86+
/// instead of queued (default kNormal preserves the legacy behavior).
87+
/// @return how the frame was handled (see SendResult). Callers counting
88+
/// forwarded bytes/messages must treat only kDelivered/kQueued as
89+
/// forwarded — kShed and kClientGone never reach the client.
90+
virtual SendResult send_binary(
91+
const std::string& client_identity, const std::vector<uint8_t>& data,
92+
FramePriority priority = FramePriority::kNormal) = 0;
7093

7194
/// Discard any queued outbound data for this client (e.g. when its session
7295
/// is destroyed server-side while the socket stays open). Default no-op for

app/include/pj_bridge/middleware/websocket_middleware.hpp

Lines changed: 30 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@
3333
#include <unordered_map>
3434
#include <vector>
3535

36+
#include "pj_bridge/middleware/backpressure.hpp"
3637
#include "pj_bridge/middleware/bounded_frame_queue.hpp"
3738
#include "pj_bridge/middleware/middleware_interface.hpp"
3839

@@ -48,7 +49,13 @@ struct TlsConfig {
4849

4950
class WebSocketMiddleware : public MiddlewareInterface {
5051
public:
51-
explicit WebSocketMiddleware(size_t client_backlog_size = 100, std::optional<TlsConfig> tls = std::nullopt);
52+
/// @param socket_buffer_watermark bytes of outgoing socket buffer at/above
53+
/// which a client is considered congested (frames queue or shed).
54+
/// Defaults to kSocketBufferHighWatermark; primarily overridden in
55+
/// tests to exercise the congested path deterministically.
56+
explicit WebSocketMiddleware(
57+
size_t client_backlog_size = 100, std::optional<TlsConfig> tls = std::nullopt,
58+
size_t socket_buffer_watermark = kSocketBufferHighWatermark);
5259
~WebSocketMiddleware() override;
5360

5461
WebSocketMiddleware(const WebSocketMiddleware&) = delete;
@@ -61,7 +68,11 @@ class WebSocketMiddleware : public MiddlewareInterface {
6168
bool receive_request(std::vector<uint8_t>& data, std::string& client_identity) override;
6269
bool send_reply(const std::string& client_identity, const std::vector<uint8_t>& data) override;
6370
bool publish_data(const std::vector<uint8_t>& data) override;
64-
bool send_binary(const std::string& client_identity, const std::vector<uint8_t>& data) override;
71+
// NOTE: the FramePriority default lives only on the base MiddlewareInterface
72+
// declaration (defaults are bound statically, so repeating it here could
73+
// silently diverge).
74+
SendResult send_binary(
75+
const std::string& client_identity, const std::vector<uint8_t>& data, FramePriority priority) override;
6576
bool is_ready() const override;
6677
void set_on_connect(ConnectionCallback callback) override;
6778
void set_on_disconnect(ConnectionCallback callback) override;
@@ -71,6 +82,17 @@ class WebSocketMiddleware : public MiddlewareInterface {
7182
/// across all clients (currently connected and already disconnected).
7283
uint64_t dropped_frame_count() const;
7384

85+
/// Total number of kHeavy frames shed before transmit under congestion
86+
/// (dropped instead of queued), summed over the middleware's lifetime.
87+
uint64_t heavy_shed_count() const;
88+
89+
/// Lossy-send policy watermark adapted from foxglove_bridge (MIT License,
90+
/// Copyright (c) Foxglove Technologies Inc): once a client's outgoing socket
91+
/// buffer reaches this many bytes, further frames are queued (kNormal) or shed
92+
/// (kHeavy) instead of blocking or disconnecting the client. Public so entry
93+
/// points can sanity-check a configured heavy-frame threshold against it.
94+
static constexpr size_t kSocketBufferHighWatermark = 1u << 20; // 1 MiB
95+
7496
private:
7597
struct IncomingRequest {
7698
std::string client_id;
@@ -98,7 +120,13 @@ class WebSocketMiddleware : public MiddlewareInterface {
98120
// total). Guarded by clients_mutex_.
99121
uint64_t dropped_from_disconnected_{0};
100122

123+
// Lifetime count of kHeavy frames shed before transmit under congestion
124+
// (dropped rather than queued). Distinct from dropped_frame_count(), which
125+
// counts backlog-overflow drops of kNormal frames. Guarded by clients_mutex_.
126+
uint64_t heavy_shed_total_{0};
127+
101128
size_t client_backlog_size_;
129+
size_t socket_buffer_watermark_;
102130
std::optional<TlsConfig> tls_;
103131

104132
ConnectionCallback on_connect_;
@@ -111,12 +139,6 @@ class WebSocketMiddleware : public MiddlewareInterface {
111139

112140
static constexpr int kShutdownTimeoutSeconds = 3;
113141
static constexpr size_t kMaxIncomingQueueSize = 1024;
114-
115-
// Lossy-send policy adapted from foxglove_bridge (MIT License, Copyright
116-
// (c) Foxglove Technologies Inc): once a client's outgoing socket buffer
117-
// exceeds this watermark, new frames are queued (dropping the oldest on
118-
// overflow) instead of blocking or disconnecting the client.
119-
static constexpr size_t kSocketBufferHighWatermark = 1u << 20; // 1 MiB
120142
static constexpr int kDropWarnIntervalSeconds = 30;
121143
};
122144

app/include/pj_bridge/protocol_constants.hpp

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,21 @@ static constexpr uint32_t kBinaryFrameMagic = 0x42524A50;
3333
/// Size of the binary frame header in bytes
3434
static constexpr size_t kBinaryHeaderSize = 16;
3535

36+
/// Binary frame header flag bit (offset 12 of the 16-byte header) reserved for a
37+
/// future "heavy" (isolated large/size-class message) marker. NOT currently
38+
/// emitted: existing PlotJuggler plugins reject any frame with flags != 0, so
39+
/// heavy frames ship unflagged (flags == 0) and heaviness is conveyed
40+
/// server-side via FramePriority instead. Reserved here for a future
41+
/// capability-negotiated rollout (see docs/API.md).
42+
static constexpr uint32_t kFrameFlagHeavy = 0x1;
43+
44+
/// Default per-message byte threshold at or above which a topic's message is
45+
/// isolated into its own "heavy" size-class frame instead of being aggregated
46+
/// with light topics (see docs/API.md). Chosen comfortably below the 1 MiB
47+
/// socket high-watermark and well above typical scalar/odom/tf frames. A
48+
/// threshold of 0 disables splitting (single aggregated frame, legacy behavior).
49+
static constexpr size_t kDefaultHeavyFrameThresholdBytes = 256 * 1024; // 256 KiB
50+
3651
/// Schema encoding identifier for ROS2 message definitions
3752
inline constexpr const char* kSchemaEncodingRos2Msg = "ros2msg";
3853

@@ -49,6 +64,7 @@ inline constexpr const char* kServerCapabilities[] = {
4964
"latched_replay", // retained samples replayed after subscribe/resume
5065
"topics_changed", // pushed topic advertisement (subscribe_topic_updates)
5166
"per_topic_rate_limit", // subscribe entries accept {name, max_rate_hz}
67+
"size_class_frames", // large topics isolated into own frames (header flag bit0 = heavy)
5268
};
5369

5470
} // namespace pj_bridge

0 commit comments

Comments
 (0)