Skip to content

Commit d039978

Browse files
authored
Merge pull request #538 from dgarske/broker_features
wolfMQTT broker: ordering, persistence, offline queue, AES-GCM at rest
2 parents cba7014 + 7c2fe8a commit d039978

13 files changed

Lines changed: 5059 additions & 133 deletions

File tree

.github/workflows/broker-check.yml

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,27 @@ jobs:
6363
cflags: ""
6464
wolfmqtt_opts: "--enable-v5 --enable-broker --enable-max-qos=0"
6565
skip_broker_test: "yes"
66+
- name: "Broker v5 (ordering / Receive Maximum)"
67+
cflags: ""
68+
wolfmqtt_opts: "--enable-broker --enable-v5"
69+
- name: "Broker v5 strict-serial (inflight=1)"
70+
cflags: "-DBROKER_MAX_INFLIGHT_PER_SUB=1"
71+
wolfmqtt_opts: "--enable-broker --enable-v5"
72+
- name: "Broker v5 static memory"
73+
cflags: "-DWOLFMQTT_STATIC_MEMORY"
74+
wolfmqtt_opts: "--enable-broker --enable-v5"
75+
- name: "Broker with persist"
76+
cflags: ""
77+
wolfmqtt_opts: "--enable-broker --enable-v5 --enable-broker-persist"
78+
- name: "Broker with persist + TLS"
79+
cflags: ""
80+
wolfmqtt_opts: "--enable-broker --enable-v5 --enable-broker-persist --enable-tls"
81+
- name: "Broker with persist + AES-GCM encryption"
82+
cflags: "-DWOLFMQTT_BROKER_PERSIST_ENCRYPT_DEV_KEY"
83+
wolfmqtt_opts: "--enable-broker --enable-v5 --enable-broker-persist --enable-broker-persist-encrypt"
84+
- name: "Broker with persist + static memory"
85+
cflags: "-DWOLFMQTT_STATIC_MEMORY"
86+
wolfmqtt_opts: "--enable-broker --enable-v5 --enable-broker-persist"
6687

6788
steps:
6889
- name: Install dependencies

.github/workflows/macos-check.yml

Lines changed: 33 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -84,5 +84,36 @@ jobs:
8484
- name: Show logs on failure
8585
if: failure() || cancelled()
8686
run: |
87-
cat test-suite.log
88-
cat scripts/*.log
87+
# Copy broker.test tmp dirs ($TMPDIR/tmp.* on macOS, e.g.
88+
# /var/folders/.../T/tmp.XXXXXX) into the workspace so the
89+
# next step can upload them as an artifact. Globbing the
90+
# real /var/folders path directly trips over unreadable
91+
# macOS LaunchServices files.
92+
mkdir -p ci-logs
93+
[ -f test-suite.log ] && cp test-suite.log ci-logs/ || true
94+
cp scripts/*.log ci-logs/ 2>/dev/null || true
95+
for d in "${TMPDIR%/}"/tmp.* /tmp/tmp.*; do
96+
[ -d "$d" ] || continue
97+
base=$(basename "$d")
98+
mkdir -p "ci-logs/$base"
99+
cp "$d"/*.log "ci-logs/$base/" 2>/dev/null || true
100+
done
101+
echo "=== test-suite.log ==="
102+
cat test-suite.log 2>/dev/null || true
103+
echo "=== scripts/*.log ==="
104+
cat scripts/*.log 2>/dev/null || true
105+
echo "=== ci-logs/tmp.*/*.log (broker.test per-test logs) ==="
106+
for f in ci-logs/tmp.*/*.log; do
107+
[ -f "$f" ] || continue
108+
echo "--- $f ---"
109+
cat "$f"
110+
done
111+
112+
- name: Upload broker.test logs on failure
113+
if: failure() || cancelled()
114+
uses: actions/upload-artifact@v4
115+
with:
116+
name: broker-test-logs-macos
117+
path: ci-logs/
118+
if-no-files-found: ignore
119+
retention-days: 7

BROKER.md

Lines changed: 152 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,152 @@
1+
# wolfMQTT Broker
2+
3+
wolfMQTT includes a lightweight MQTT broker suitable for embedded and resource-constrained environments. It serves both MQTT v3.1.1 and v5.0 clients, with optional TLS via wolfSSL, optional WebSocket transport, and optional encrypted persistence. The broker uses non-blocking sockets driven by a single `select()` loop, so it runs without threads.
4+
5+
## Features
6+
7+
* QoS 0, QoS 1, and QoS 2 publish/subscribe (full QoS 2 flow with PUBREC/PUBREL/PUBCOMP)
8+
* Retained messages
9+
* Last Will and Testament (LWT), including v5 Will Delay Interval
10+
* Wildcard subscriptions (`+` and `#`)
11+
* Username/password authentication
12+
* MQTT v5 ordering and Receive Maximum (per-subscriber inflight shaping)
13+
* TLS support (requires wolfSSL with `--enable-tls`)
14+
* WebSocket / secure WebSocket transport (requires libwebsockets; see the WebSocket section of the main [README.md](README.md))
15+
* Clean session handling with subscription persistence
16+
* Keep-alive monitoring with automatic client disconnect
17+
* Unique client ID enforcement (existing session takeover)
18+
* Optional on-disk persistence of sessions, subscriptions, retained messages, and offline queues, with optional AES-GCM encryption-at-rest
19+
* Static memory mode (`WOLFMQTT_STATIC_MEMORY`) for zero-malloc operation
20+
21+
## Quick start
22+
23+
With autotools:
24+
25+
```sh
26+
./configure --enable-broker
27+
make
28+
./src/mqtt_broker -p 1883
29+
```
30+
31+
With CMake:
32+
33+
```sh
34+
cmake .. -DWOLFMQTT_BROKER=yes
35+
cmake --build .
36+
```
37+
38+
For TLS:
39+
40+
```sh
41+
./configure --enable-broker --enable-tls
42+
make
43+
./src/mqtt_broker -p 8883 -t -A ca-cert.pem -K server-key.pem -c server-cert.pem
44+
```
45+
46+
Run `./src/mqtt_broker -h` to see the options compiled into your build.
47+
48+
## Command-line options
49+
50+
```
51+
usage: mqtt_broker [-p port] [-v level] [-u user] [-P pass]
52+
[-t] [-s port] [-V ver] [-c cert] [-K key] [-A ca]
53+
[-w port] [-D dir] [-E source]
54+
```
55+
56+
| Option | Available when | Description |
57+
|---|---|---|
58+
| `-p <port>` | always | Plain (non-TLS) port (default: 1883) |
59+
| `-v <level>` | always | Log level: 1=error, 2=info (default), 3=debug |
60+
| `-u <user>` | auth build | Username for authentication |
61+
| `-P <pass>` | auth build | Password for authentication |
62+
| `-t` | TLS build | Enable the TLS listener |
63+
| `-s <port>` | TLS build | TLS port (default: 8883) |
64+
| `-V <ver>` | TLS build | TLS version: 12=TLS 1.2, 13=TLS 1.3 (default: auto) |
65+
| `-c <file>` | TLS build | Server certificate file (PEM) |
66+
| `-K <file>` | TLS build | Server private key file (PEM) |
67+
| `-A <file>` | TLS build | CA certificate for mutual TLS (PEM) |
68+
| `-w <port>` | WebSocket build | WebSocket listen port (enables WebSocket) |
69+
| `-D <dir>` | persist build | Persistent storage directory (enables persistence; default `/var/lib/wolfmqtt`) |
70+
| `-E <source>` | encrypt + dev-key build | Encryption key source. Only `dev` is recognized, selecting the development hard-coded key. NOT FOR PRODUCTION. |
71+
72+
## Build options
73+
74+
All broker features are enabled by default and can be disabled at build time to reduce code and memory footprint on constrained platforms.
75+
76+
| Feature | Autotools | CMake | Define |
77+
|---|---|---|---|
78+
| Broker support | `--enable-broker` | `-DWOLFMQTT_BROKER=yes` | `WOLFMQTT_BROKER` |
79+
| Retained messages | `--disable-broker-retained` | `-DWOLFMQTT_BROKER_RETAINED=no` | `WOLFMQTT_BROKER_NO_RETAINED` |
80+
| Last Will and Testament | `--disable-broker-will` | `-DWOLFMQTT_BROKER_WILL=no` | `WOLFMQTT_BROKER_NO_WILL` |
81+
| Wildcard subscriptions | `--disable-broker-wildcards` | `-DWOLFMQTT_BROKER_WILDCARDS=no` | `WOLFMQTT_BROKER_NO_WILDCARDS` |
82+
| Authentication | `--disable-broker-auth` | `-DWOLFMQTT_BROKER_AUTH=no` | `WOLFMQTT_BROKER_NO_AUTH` |
83+
| Logging | `--disable-broker-log` | `-DWOLFMQTT_BROKER_LOG=no` | `WOLFMQTT_BROKER_NO_LOG` |
84+
| Plain-text listener | `--disable-broker-insecure` | `-DWOLFMQTT_BROKER_INSECURE=no` | `WOLFMQTT_BROKER_NO_INSECURE` |
85+
86+
The maximum QoS the broker negotiates is capped by `--enable-max-qos=<0,1,2>` (default 2). Setting it to 1 or 0 compiles out the QoS 2 state machine and shrinks the broker.
87+
88+
## Static memory tuning
89+
90+
When built with `WOLFMQTT_STATIC_MEMORY`, the broker uses fixed-size arrays instead of dynamic allocation. The limits below can be overridden via CFLAGS at build time.
91+
92+
| Macro | Default | Description |
93+
|---|---|---|
94+
| `BROKER_MAX_CLIENTS` | 8 | Maximum concurrent client connections |
95+
| `BROKER_MAX_SUBS` | 32 | Maximum total subscriptions across all clients |
96+
| `BROKER_MAX_RETAINED` | 16 | Maximum retained messages |
97+
| `BROKER_MAX_CLIENT_ID_LEN` | 64 | Maximum client ID length |
98+
| `BROKER_MAX_USERNAME_LEN` | 64 | Maximum username length |
99+
| `BROKER_MAX_PASSWORD_LEN` | 64 | Maximum password length |
100+
| `BROKER_MAX_FILTER_LEN` | 128 | Maximum subscription filter length |
101+
| `BROKER_MAX_TOPIC_LEN` | 128 | Maximum topic name length |
102+
| `BROKER_MAX_PAYLOAD_LEN` | 4096 | Maximum retained message payload |
103+
| `BROKER_MAX_WILL_PAYLOAD_LEN` | 256 | Maximum LWT payload |
104+
| `BROKER_MAX_PENDING_WILLS` | 4 | Maximum queued pending wills |
105+
| `BROKER_MAX_INBOUND_QOS2` | 16 | Concurrent inbound QoS 2 packet IDs per client |
106+
| `BROKER_RX_BUF_SZ` | 4096 | Per-client receive buffer size |
107+
| `BROKER_TX_BUF_SZ` | 4096 | Per-client transmit buffer size |
108+
| `BROKER_TIMEOUT_MS` | 1000 | `select()` timeout |
109+
| `BROKER_LISTEN_BACKLOG` | 128 | Listen queue depth |
110+
111+
With dynamic memory the per-subscriber inflight window is derived at runtime, bounded by `BROKER_MIN_INFLIGHT_PER_SUB` (default 8) and `BROKER_MAX_INFLIGHT_PER_SUB`. Define `BROKER_MAX_INFLIGHT_PER_SUB=1` to force strict serial delivery (one inflight QoS 1/2 message per subscriber).
112+
113+
## Persistence
114+
115+
Build with `--enable-broker-persist` to persist sessions, subscriptions, retained messages, and offline queues across restarts. The persistence layer is hook-based: a default POSIX backend stores records as files under the directory given with `-D` (default `/var/lib/wolfmqtt`). Embedded targets can supply their own storage backend through `MqttBroker_SetPersistHooks()`.
116+
117+
| Macro | Default | Description |
118+
|---|---|---|
119+
| `BROKER_MAX_PERSIST_SESSIONS` | 64 | Persistent sessions retained across restarts |
120+
| `BROKER_MAX_OFFLINE_MSGS_PER_SUB` | 32 | Offline queue depth per session |
121+
| `WOLFMQTT_BROKER_PERSIST_SCHEMA_VER` | 3 | On-disk record schema version |
122+
123+
### Encryption at rest
124+
125+
Add `--enable-broker-persist-encrypt` (requires `--enable-broker-persist`) to wrap persisted records with wolfCrypt AES-GCM. The key is provided by a `derive_key` callback that real deployments install via `MqttBroker_SetPersistHooks()` before starting the broker.
126+
127+
For development and CI only, the CLI can link a fixed-pattern `derive_key` hook so the AES-GCM round-trip can be exercised without external key management. This is NOT a configure option -- define the macro through CFLAGS:
128+
129+
```sh
130+
CFLAGS="-DWOLFMQTT_BROKER_PERSIST_ENCRYPT_DEV_KEY" \
131+
./configure --enable-broker --enable-broker-persist --enable-broker-persist-encrypt
132+
make
133+
./src/mqtt_broker -p 1883 -D ./state -E dev
134+
```
135+
136+
The dev key is a trivially-recoverable hard-coded pattern. Never define `WOLFMQTT_BROKER_PERSIST_ENCRYPT_DEV_KEY` in a production build, and never pass `-E dev` in production -- doing so substitutes the fixed key for real key management. Production builds omit the macro entirely, so the `-E` option and the dev hook are not present in the binary.
137+
138+
## Testing
139+
140+
The repository ships an end-to-end broker test harness:
141+
142+
```sh
143+
./scripts/broker.test
144+
```
145+
146+
It builds the client examples (`examples/pub-sub/mqtt-pub`, `examples/pub-sub/mqtt-sub`) and `mosquitto`-based checks against the wolfMQTT broker, covering QoS flows, retained messages, wildcards, persistence round-trips, and AES-GCM encryption (when the dev-key hook is linked). Tests that depend on features not present in the current build are reported as `SKIP`.
147+
148+
The CONNECT-handler unit test (`tests/test_broker_connect`) is part of `make check` and exercises the broker packet path with a mock network layer.
149+
150+
## Limitations
151+
152+
The wolfMQTT broker targets embedded and edge use cases. It is intentionally smaller in scope than full-featured server brokers such as Mosquitto or EMQX: there is no clustering, no bridging, no plugin/ACL framework, and no dynamic configuration reload. For large-scale or feature-rich deployments use a dedicated server broker; for a small, auditable, optionally-TLS broker that runs without threads or a heap, wolfMQTT is a good fit.

README.md

Lines changed: 2 additions & 62 deletions
Original file line numberDiff line numberDiff line change
@@ -418,75 +418,15 @@ is added to `make check`, and all other tests are disabled.
418418

419419
## Broker
420420

421-
wolfMQTT includes a lightweight MQTT broker implementation suitable for embedded and resource-constrained environments. It supports both MQTT v3.1.1 and v5.0 clients, with optional TLS via wolfSSL.
422-
423-
### Features
424-
425-
* QoS 0, QoS 1, and QoS 2 publish/subscribe (full QoS 2 flow with PUBREC/PUBREL/PUBCOMP)
426-
* Retained messages
427-
* Last Will and Testament (LWT) with v5 Will Delay Interval
428-
* Wildcard subscriptions (`+` and `#`)
429-
* Username/password authentication
430-
* TLS support (requires wolfSSL with `--enable-tls`)
431-
* Clean session handling with subscription persistence
432-
* Keep-alive monitoring with automatic client disconnect
433-
* Unique client ID enforcement (existing session takeover)
434-
* Static memory mode (`WOLFMQTT_STATIC_MEMORY`) for zero-malloc operation
435-
436-
### Building
437-
438-
With autotools:
421+
wolfMQTT includes a lightweight MQTT broker suitable for embedded and resource-constrained environments. It serves both MQTT v3.1.1 and v5.0 clients, with optional TLS, WebSocket transport, and encrypted persistence.
439422

440423
```
441424
./configure --enable-broker
442425
make
443-
```
444-
445-
With CMake:
446-
447-
```
448-
cmake .. -DWOLFMQTT_BROKER=yes
449-
cmake --build .
450-
```
451-
452-
### Running
453-
454-
```
455426
./src/mqtt_broker -p 1883
456427
```
457428

458-
For TLS:
459-
460-
```
461-
./src/mqtt_broker -p 8883 -t -A ca-cert.pem -K server-key.pem -c server-cert.pem
462-
```
463-
464-
Run `./src/mqtt_broker -h` to see all available options.
465-
466-
### Feature Build Options
467-
468-
All broker features are enabled by default. Individual features can be disabled at build time to reduce code and memory footprint on constrained platforms.
469-
470-
| Feature | Autotools | CMake | Define |
471-
|---|---|---|---|
472-
| Retained messages | `--disable-broker-retained` | `-DWOLFMQTT_BROKER_RETAINED=no` | `WOLFMQTT_BROKER_NO_RETAINED` |
473-
| Last Will and Testament | `--disable-broker-will` | `-DWOLFMQTT_BROKER_WILL=no` | `WOLFMQTT_BROKER_NO_WILL` |
474-
| Wildcard subscriptions | `--disable-broker-wildcards` | `-DWOLFMQTT_BROKER_WILDCARDS=no` | `WOLFMQTT_BROKER_NO_WILDCARDS` |
475-
| Authentication | `--disable-broker-auth` | `-DWOLFMQTT_BROKER_AUTH=no` | `WOLFMQTT_BROKER_NO_AUTH` |
476-
477-
### Static Memory Mode
478-
479-
When built with `WOLFMQTT_STATIC_MEMORY`, the broker uses fixed-size arrays instead of dynamic allocation. Buffer sizes and limits can be tuned via compile-time macros:
480-
481-
| Macro | Default | Description |
482-
|---|---|---|
483-
| `BROKER_MAX_CLIENTS` | 8 | Maximum concurrent client connections |
484-
| `BROKER_MAX_SUBS` | 32 | Maximum total subscriptions |
485-
| `BROKER_MAX_RETAINED` | 16 | Maximum retained messages |
486-
| `BROKER_RX_BUF_SZ` | 4096 | Per-client receive buffer size |
487-
| `BROKER_TX_BUF_SZ` | 4096 | Per-client transmit buffer size |
488-
| `BROKER_MAX_PAYLOAD_LEN` | 4096 | Maximum retained message payload |
489-
| `BROKER_MAX_WILL_PAYLOAD_LEN` | 256 | Maximum LWT payload size |
429+
See [BROKER.md](BROKER.md) for the full feature list, command-line options, build options, static-memory tuning, and persistence/encryption details.
490430

491431
## WebSocket Support
492432

configure.ac

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -498,6 +498,46 @@ then
498498
AM_CFLAGS="$AM_CFLAGS -DWOLFMQTT_BROKER_NO_INSECURE"
499499
fi
500500

501+
# Broker persistent storage (sessions, subs, retained, offline queue).
502+
# Opt-in; off by default. Adds the hook-based persistence layer plus a
503+
# default POSIX backend.
504+
AC_ARG_ENABLE([broker-persist],
505+
[AS_HELP_STRING([--enable-broker-persist],[Enable broker persistent storage via callback hooks (default: disabled)])],
506+
[ ENABLED_BROKER_PERSIST=$enableval ],
507+
[ ENABLED_BROKER_PERSIST=no ]
508+
)
509+
if test "x$ENABLED_BROKER_PERSIST" = "xyes"
510+
then
511+
if test "x$ENABLED_BROKER" != "xyes"
512+
then
513+
AC_MSG_ERROR([--enable-broker-persist requires --enable-broker])
514+
fi
515+
AM_CFLAGS="$AM_CFLAGS -DWOLFMQTT_BROKER_PERSIST"
516+
fi
517+
518+
# Optional encryption-at-rest for persisted records using wolfCrypt AES-GCM.
519+
# Requires --enable-broker-persist; off by default.
520+
AC_ARG_ENABLE([broker-persist-encrypt],
521+
[AS_HELP_STRING([--enable-broker-persist-encrypt],[Encrypt persisted records with AES-GCM (default: disabled, requires --enable-broker-persist)])],
522+
[ ENABLED_BROKER_PERSIST_ENCRYPT=$enableval ],
523+
[ ENABLED_BROKER_PERSIST_ENCRYPT=no ]
524+
)
525+
if test "x$ENABLED_BROKER_PERSIST_ENCRYPT" = "xyes"
526+
then
527+
if test "x$ENABLED_BROKER_PERSIST" != "xyes"
528+
then
529+
AC_MSG_ERROR([--enable-broker-persist-encrypt requires --enable-broker-persist])
530+
fi
531+
AM_CFLAGS="$AM_CFLAGS -DWOLFMQTT_BROKER_PERSIST_ENCRYPT"
532+
fi
533+
534+
# Note: the development-only fixed-pattern derive_key hook for the CLI
535+
# broker (required to use "-E dev" on encrypt builds) is not a configure
536+
# option. Define WOLFMQTT_BROKER_PERSIST_ENCRYPT_DEV_KEY via CFLAGS to
537+
# link it. Never enable it in a production build - the resulting binary
538+
# contains a trivially-recoverable AES-GCM key generator. Real
539+
# deployments install a derive_key hook via MqttBroker_SetPersistHooks.
540+
501541

502542
AM_CONDITIONAL([HAVE_LIBWOLFSSL], [test "x$ENABLED_TLS" = "xyes"])
503543
AM_CONDITIONAL([HAVE_LIBCURL], [test "x$ENABLED_CURL" = "xyes"])

examples/pub-sub/mqtt-sub.c

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -527,6 +527,13 @@ int sub_client(MQTTCtx *mqttCtx)
527527
break;
528528
}
529529
}
530+
#ifdef WOLFMQTT_NONBLOCK
531+
else if (rc == MQTT_CODE_CONTINUE) {
532+
/* Non-blocking: no data yet, keep polling. mqtt_check_timeout()
533+
* above will convert this to MQTT_CODE_ERROR_TIMEOUT after the
534+
* inactivity window, which drives the keep-alive ping branch. */
535+
}
536+
#endif
530537
else if (rc != MQTT_CODE_SUCCESS) {
531538
/* There was an error */
532539
PRINTF("MQTT Message Wait: %s (%d)",

0 commit comments

Comments
 (0)