Skip to content

Commit 871ffbd

Browse files
dgarskedgarske
committed
Add optional AES-GCM encryption at rest for persisted broker records
1 parent 9634343 commit 871ffbd

4 files changed

Lines changed: 634 additions & 18 deletions

File tree

‎.github/workflows/broker-check.yml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,12 @@ jobs:
6262
- name: "Broker with persist + TLS"
6363
cflags: ""
6464
wolfmqtt_opts: "--enable-broker --enable-v5 --enable-broker-persist --enable-tls"
65+
- name: "Broker with persist + AES-GCM encryption"
66+
cflags: ""
67+
wolfmqtt_opts: "--enable-broker --enable-v5 --enable-broker-persist --enable-broker-persist-encrypt"
68+
- name: "Broker with persist + static memory"
69+
cflags: "-DWOLFMQTT_STATIC_MEMORY"
70+
wolfmqtt_opts: "--enable-broker --enable-v5 --enable-broker-persist"
6571

6672
steps:
6773
- name: Install dependencies

‎scripts/broker.test‎

Lines changed: 288 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -87,8 +87,6 @@ start_broker() {
8787
wait $broker_pid 2>/dev/null || true
8888
broker_pid=$no_pid
8989
fi
90-
generate_port
91-
local tls_args=""
9290
local has_tls=0
9391
local has_s_port=0
9492
for arg in "$@"; do
@@ -98,13 +96,30 @@ start_broker() {
9896
has_s_port=1
9997
fi
10098
done
101-
if [ "$has_tls" -eq 1 ] && [ "$has_s_port" -eq 0 ]; then
102-
tls_args="-s $port"
103-
fi
104-
broker_log="${TMP_DIR}/broker_p${port}.log"
105-
./$broker_bin "$@" -p $port $tls_args >"$broker_log" 2>&1 &
106-
broker_pid=$!
107-
check_broker
99+
# Retry a few times to absorb random-port collisions with
100+
# unrelated processes on this host (the broker binds without
101+
# SO_REUSEADDR, so a previous PID lingering in TIME_WAIT on the
102+
# same port also lands us here).
103+
local attempt=0
104+
while [ $attempt -lt 5 ]; do
105+
generate_port
106+
local tls_args=""
107+
if [ "$has_tls" -eq 1 ] && [ "$has_s_port" -eq 0 ]; then
108+
tls_args="-s $port"
109+
fi
110+
broker_log="${TMP_DIR}/broker_p${port}.log"
111+
./$broker_bin "$@" -p $port $tls_args >"$broker_log" 2>&1 &
112+
broker_pid=$!
113+
if check_broker; then
114+
return 0
115+
fi
116+
kill $broker_pid 2>/dev/null
117+
wait $broker_pid 2>/dev/null || true
118+
broker_pid=$no_pid
119+
attempt=$((attempt + 1))
120+
done
121+
echo "WARNING: broker failed to start after 5 attempts" >&2
122+
return 1
108123
}
109124

110125
# Start broker with dual ports (plain + TLS).
@@ -1210,12 +1225,21 @@ fi # has_persist + has_retained
12101225
# --- Test 28: Persist schema-mismatch wipe-and-restart ---
12111226
# Corrupt a persisted record by stomping its schema_ver bytes. The
12121227
# broker must log "schema mismatch", overwrite META, and start cleanly.
1228+
# Skipped in encrypted-persist builds: an attacker-supplied plaintext
1229+
# bogus record gets rejected by the decrypt stage before the schema
1230+
# check, so the wipe path is exercised via the plaintext build matrix
1231+
# entry instead.
12131232
echo ""
12141233
echo "--- Test 28: Persist schema mismatch ---"
1234+
t28_persist_encrypt=no
1235+
echo "$broker_features" | grep -q " persist-encrypt" && \
1236+
t28_persist_encrypt=yes
12151237
if [ "$skip_plain" = "yes" ]; then
12161238
echo "SKIP: Persist schema mismatch (plain listener disabled)"
12171239
elif [ "$has_persist" = "no" ]; then
12181240
echo "SKIP: Persist schema mismatch (built without --enable-broker-persist)"
1241+
elif [ "$t28_persist_encrypt" = "yes" ]; then
1242+
echo "SKIP: Persist schema mismatch (encrypted build - covered by plaintext matrix entry)"
12191243
else
12201244
T28_DIR="${TMP_DIR}/persist_t28"
12211245
mkdir -p "$T28_DIR/1"
@@ -1247,6 +1271,261 @@ else
12471271
fi
12481272
fi # has_persist
12491273

1274+
# --- Test 29: Offline queue across reconnect (same broker) ---
1275+
# A persistent sub disconnects, publisher sends QoS 1 messages, sub
1276+
# reconnects and receives the queue. Exercises the orphan-session pool
1277+
# without going through a broker restart.
1278+
echo ""
1279+
echo "--- Test 29: Offline queue across reconnect ---"
1280+
if [ "$skip_plain" = "yes" ]; then
1281+
echo "SKIP: Offline queue (plain listener disabled)"
1282+
elif [ "$has_persist" = "no" ]; then
1283+
echo "SKIP: Offline queue (built without --enable-broker-persist)"
1284+
else
1285+
T29_DIR="${TMP_DIR}/persist_t29"
1286+
mkdir -p "$T29_DIR"
1287+
if [ $broker_pid != $no_pid ]; then
1288+
kill $broker_pid 2>/dev/null
1289+
wait $broker_pid 2>/dev/null || true
1290+
broker_pid=$no_pid
1291+
fi
1292+
generate_port
1293+
broker_log="${TMP_DIR}/t29_broker.log"
1294+
./$broker_bin -p $port -D "$T29_DIR" >"$broker_log" 2>&1 &
1295+
broker_pid=$!
1296+
check_broker
1297+
# 1. Persistent sub connects, subscribes; SIGKILL to disconnect
1298+
# abruptly (graceful exit sends UNSUBSCRIBE which would tear down
1299+
# the subscription and defeat the orphan path).
1300+
rm -f "${TMP_DIR}/t29_first.ready"
1301+
./$sub_bin -T -h 127.0.0.1 -p $port -n "test/offlineq" -q 1 \
1302+
-i "t29_sub" -s \
1303+
-R "${TMP_DIR}/t29_first.ready" \
1304+
>"${TMP_DIR}/t29_first.log" 2>&1 &
1305+
T29_FIRST_PID=$!
1306+
TEST_PIDS+=($T29_FIRST_PID)
1307+
wait_for_file "${TMP_DIR}/t29_first.ready" 5
1308+
kill -9 $T29_FIRST_PID 2>/dev/null
1309+
wait $T29_FIRST_PID 2>/dev/null || true
1310+
TEST_PIDS=()
1311+
sleep 0.5
1312+
# 2. Publish 3 QoS 1 messages while disconnected.
1313+
for t29_i in 1 2 3; do
1314+
./$pub_bin -T -h 127.0.0.1 -p $port -n "test/offlineq" -q 1 \
1315+
-m "off_${t29_i}" -i "t29_pub_${t29_i}" \
1316+
>>"${TMP_DIR}/t29_pub.log" 2>&1
1317+
done
1318+
sleep 0.5
1319+
# 3. Reconnect with same client_id + clean_session=0; receive backlog.
1320+
timeout 8 ./$sub_bin -T -h 127.0.0.1 -p $port -n "test/offlineq" -q 1 \
1321+
-i "t29_sub" -s -x -C 3 -R "${TMP_DIR}/t29_sub.ready" \
1322+
>"${TMP_DIR}/t29_sub.log" 2>&1 &
1323+
T29_PID=$!
1324+
TEST_PIDS+=($T29_PID)
1325+
wait_for_file "${TMP_DIR}/t29_sub.ready" 5
1326+
sleep 1
1327+
kill $T29_PID 2>/dev/null
1328+
wait $T29_PID 2>/dev/null || true
1329+
TEST_PIDS=()
1330+
T29_RECV=$(grep -oE 'off_[0-9]+' "${TMP_DIR}/t29_sub.log" 2>/dev/null \
1331+
| wc -l)
1332+
if [ "$T29_RECV" -ge 3 ]; then
1333+
echo "PASS: Offline queue across reconnect (received=$T29_RECV)"
1334+
else
1335+
echo "FAIL: Offline queue across reconnect (received=$T29_RECV expected>=3)"
1336+
FAIL=1
1337+
fi
1338+
fi # has_persist (t29)
1339+
1340+
# --- Test 30: Offline queue across broker restart ---
1341+
# Same as Test 29 but stops the broker between publish and reconnect to
1342+
# confirm NS_OUTQ records replay on restart.
1343+
echo ""
1344+
echo "--- Test 30: Offline queue across broker restart ---"
1345+
if [ "$skip_plain" = "yes" ]; then
1346+
echo "SKIP: Offline queue restart (plain listener disabled)"
1347+
elif [ "$has_persist" = "no" ]; then
1348+
echo "SKIP: Offline queue restart (built without --enable-broker-persist)"
1349+
else
1350+
T30_DIR="${TMP_DIR}/persist_t30"
1351+
mkdir -p "$T30_DIR"
1352+
if [ $broker_pid != $no_pid ]; then
1353+
kill $broker_pid 2>/dev/null
1354+
wait $broker_pid 2>/dev/null || true
1355+
broker_pid=$no_pid
1356+
fi
1357+
generate_port
1358+
broker_log="${TMP_DIR}/t30_broker1.log"
1359+
./$broker_bin -p $port -D "$T30_DIR" >"$broker_log" 2>&1 &
1360+
broker_pid=$!
1361+
check_broker
1362+
rm -f "${TMP_DIR}/t30_first.ready"
1363+
./$sub_bin -T -h 127.0.0.1 -p $port -n "test/restartq" -q 1 \
1364+
-i "t30_sub" -s \
1365+
-R "${TMP_DIR}/t30_first.ready" \
1366+
>"${TMP_DIR}/t30_first.log" 2>&1 &
1367+
T30_FIRST_PID=$!
1368+
TEST_PIDS+=($T30_FIRST_PID)
1369+
wait_for_file "${TMP_DIR}/t30_first.ready" 5
1370+
kill -9 $T30_FIRST_PID 2>/dev/null
1371+
wait $T30_FIRST_PID 2>/dev/null || true
1372+
TEST_PIDS=()
1373+
sleep 0.5
1374+
for t30_i in 1 2 3; do
1375+
./$pub_bin -T -h 127.0.0.1 -p $port -n "test/restartq" -q 1 \
1376+
-m "rst_${t30_i}" -i "t30_pub_${t30_i}" \
1377+
>>"${TMP_DIR}/t30_pub.log" 2>&1
1378+
done
1379+
sleep 0.5
1380+
# Restart broker against same directory
1381+
kill $broker_pid 2>/dev/null
1382+
wait $broker_pid 2>/dev/null || true
1383+
broker_pid=$no_pid
1384+
broker_log="${TMP_DIR}/t30_broker2.log"
1385+
./$broker_bin -p $port -D "$T30_DIR" >"$broker_log" 2>&1 &
1386+
broker_pid=$!
1387+
check_broker
1388+
T30_REPLAY=no
1389+
grep -q "persist restore outq loaded=3" "$broker_log" 2>/dev/null \
1390+
&& T30_REPLAY=yes
1391+
timeout 8 ./$sub_bin -T -h 127.0.0.1 -p $port -n "test/restartq" -q 1 \
1392+
-i "t30_sub" -s -x -C 3 -R "${TMP_DIR}/t30_sub.ready" \
1393+
>"${TMP_DIR}/t30_sub.log" 2>&1 &
1394+
T30_PID=$!
1395+
TEST_PIDS+=($T30_PID)
1396+
wait_for_file "${TMP_DIR}/t30_sub.ready" 5
1397+
sleep 1
1398+
kill $T30_PID 2>/dev/null
1399+
wait $T30_PID 2>/dev/null || true
1400+
TEST_PIDS=()
1401+
T30_RECV=$(grep -oE 'rst_[0-9]+' "${TMP_DIR}/t30_sub.log" 2>/dev/null \
1402+
| wc -l)
1403+
if [ "$T30_REPLAY" = "yes" ] && [ "$T30_RECV" -ge 3 ]; then
1404+
echo "PASS: Offline queue across restart (replay=$T30_REPLAY recv=$T30_RECV)"
1405+
else
1406+
echo "FAIL: Offline queue across restart (replay=$T30_REPLAY recv=$T30_RECV)"
1407+
FAIL=1
1408+
fi
1409+
fi # has_persist (t30)
1410+
1411+
# --- Test 31: AES-GCM encrypted records at rest ---
1412+
# Only meaningful when the broker is built with
1413+
# --enable-broker-persist-encrypt. Records on disk must not contain the
1414+
# plaintext payload string; round-trip must still work end-to-end.
1415+
echo ""
1416+
echo "--- Test 31: AES-GCM encrypted persist round-trip ---"
1417+
has_persist_encrypt=no
1418+
echo "$broker_features" | grep -q " persist-encrypt" && \
1419+
has_persist_encrypt=yes
1420+
if [ "$skip_plain" = "yes" ]; then
1421+
echo "SKIP: AES-GCM persist (plain listener disabled)"
1422+
elif [ "$has_persist_encrypt" = "no" ]; then
1423+
echo "SKIP: AES-GCM persist (built without --enable-broker-persist-encrypt)"
1424+
elif [ "$has_retained" = "no" ]; then
1425+
echo "SKIP: AES-GCM persist (retained support not built)"
1426+
else
1427+
T31_DIR="${TMP_DIR}/persist_t31"
1428+
mkdir -p "$T31_DIR"
1429+
if [ $broker_pid != $no_pid ]; then
1430+
kill $broker_pid 2>/dev/null
1431+
wait $broker_pid 2>/dev/null || true
1432+
broker_pid=$no_pid
1433+
fi
1434+
generate_port
1435+
broker_log="${TMP_DIR}/t31_broker1.log"
1436+
./$broker_bin -p $port -D "$T31_DIR" >"$broker_log" 2>&1 &
1437+
broker_pid=$!
1438+
check_broker
1439+
./$pub_bin -T -h 127.0.0.1 -p $port -n "test/secret_t31" \
1440+
-m "t31_secret_payload" -r >"${TMP_DIR}/t31_pub.log" 2>&1
1441+
sleep 0.3
1442+
# On-disk record must NOT contain the plaintext string.
1443+
T31_LEAK=no
1444+
if grep -ql "t31_secret_payload" "$T31_DIR"/4/*.bin 2>/dev/null; then
1445+
T31_LEAK=yes
1446+
fi
1447+
# Restart broker - decryption + retained replay should still work.
1448+
kill $broker_pid 2>/dev/null
1449+
wait $broker_pid 2>/dev/null || true
1450+
broker_pid=$no_pid
1451+
broker_log="${TMP_DIR}/t31_broker2.log"
1452+
./$broker_bin -p $port -D "$T31_DIR" >"$broker_log" 2>&1 &
1453+
broker_pid=$!
1454+
check_broker
1455+
rm -f "${TMP_DIR}/t31_sub.ready"
1456+
timeout 5 ./$sub_bin -T -h 127.0.0.1 -p $port -n "test/secret_t31" \
1457+
-i "t31_sub" -R "${TMP_DIR}/t31_sub.ready" \
1458+
>"${TMP_DIR}/t31_sub.log" 2>&1 &
1459+
T31_PID=$!
1460+
TEST_PIDS+=($T31_PID)
1461+
wait_for_file "${TMP_DIR}/t31_sub.ready" 3
1462+
sleep 0.5
1463+
kill $T31_PID 2>/dev/null
1464+
wait $T31_PID 2>/dev/null || true
1465+
TEST_PIDS=()
1466+
T31_RECV=no
1467+
grep -q "t31_secret_payload" "${TMP_DIR}/t31_sub.log" 2>/dev/null \
1468+
&& T31_RECV=yes
1469+
if [ "$T31_LEAK" = "no" ] && [ "$T31_RECV" = "yes" ]; then
1470+
echo "PASS: AES-GCM persist (no-leak + decrypt round-trip)"
1471+
else
1472+
echo "FAIL: AES-GCM persist (leak=$T31_LEAK recv=$T31_RECV)"
1473+
FAIL=1
1474+
fi
1475+
fi # has_persist_encrypt
1476+
1477+
# --- Test 32: Schema-mismatch wipe deletes every stale file ---
1478+
# Test 28 already verifies that a single bogus META makes the broker
1479+
# overwrite META, but doesn't check that OTHER namespaces get cleaned.
1480+
# This test plants stray files in every namespace and confirms they all
1481+
# disappear after the broker comes up.
1482+
echo ""
1483+
echo "--- Test 32: Schema mismatch wipes every namespace ---"
1484+
t32_persist_encrypt=no
1485+
echo "$broker_features" | grep -q " persist-encrypt" && \
1486+
t32_persist_encrypt=yes
1487+
if [ "$skip_plain" = "yes" ]; then
1488+
echo "SKIP: Schema wipe full (plain listener disabled)"
1489+
elif [ "$has_persist" = "no" ]; then
1490+
echo "SKIP: Schema wipe full (built without --enable-broker-persist)"
1491+
elif [ "$t32_persist_encrypt" = "yes" ]; then
1492+
echo "SKIP: Schema wipe full (encrypted build - exercised by plaintext matrix entry)"
1493+
else
1494+
T32_DIR="${TMP_DIR}/persist_t32"
1495+
rm -rf "$T32_DIR"
1496+
mkdir -p "$T32_DIR/1" "$T32_DIR/2" "$T32_DIR/3" "$T32_DIR/4" "$T32_DIR/5"
1497+
# Bogus META with wrong schema version (0xFFFF).
1498+
printf 'WMQB\xff\xff\x00\x01\x00\x00\x00\x04zzzz' > "$T32_DIR/1/00.bin"
1499+
# Plant a stray file in each other namespace.
1500+
echo "stray-session" > "$T32_DIR/2/aaaa.bin"
1501+
echo "stray-subs" > "$T32_DIR/3/bbbb.bin"
1502+
echo "stray-ret" > "$T32_DIR/4/cccc.bin"
1503+
echo "stray-outq" > "$T32_DIR/5/dddd.bin"
1504+
T32_BEFORE=$(find "$T32_DIR" -type f -name '*.bin' | wc -l)
1505+
if [ $broker_pid != $no_pid ]; then
1506+
kill $broker_pid 2>/dev/null
1507+
wait $broker_pid 2>/dev/null || true
1508+
broker_pid=$no_pid
1509+
fi
1510+
generate_port
1511+
broker_log="${TMP_DIR}/t32_broker.log"
1512+
./$broker_bin -p $port -D "$T32_DIR" >"$broker_log" 2>&1 &
1513+
broker_pid=$!
1514+
check_broker
1515+
# After wipe-and-restart only the fresh META record should remain.
1516+
T32_AFTER=$(find "$T32_DIR" -type f -name '*.bin' | wc -l)
1517+
T32_WIPED=no
1518+
grep -q "persist wipe deleted" "$broker_log" 2>/dev/null \
1519+
&& T32_WIPED=yes
1520+
if [ "$T32_BEFORE" -eq 5 ] && [ "$T32_AFTER" -eq 1 ] && \
1521+
[ "$T32_WIPED" = "yes" ]; then
1522+
echo "PASS: Schema wipe (before=$T32_BEFORE after=$T32_AFTER)"
1523+
else
1524+
echo "FAIL: Schema wipe (before=$T32_BEFORE after=$T32_AFTER wiped=$T32_WIPED)"
1525+
FAIL=1
1526+
fi
1527+
fi # has_persist (t32)
1528+
12501529
# --- WebSocket Tests ---
12511530
ws_client_bin="examples/websocket/websocket_client"
12521531
has_websocket=no

‎src/mqtt_broker.c‎

Lines changed: 40 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5732,6 +5732,29 @@ static void broker_signal_handler(int signo)
57325732
}
57335733
#endif
57345734

5735+
#if defined(WOLFMQTT_BROKER_PERSIST) && \
5736+
defined(WOLFMQTT_BROKER_PERSIST_ENCRYPT)
5737+
/* Development-only derive_key hook. Returns a fixed 32-byte key so the
5738+
* CLI can exercise the AES-GCM persistence round-trip without external
5739+
* key management. Real deployments override this via
5740+
* MqttBroker_SetPersistHooks before MqttBroker_Start. */
5741+
static int wolfmqtt_broker_dev_derive_key(void* ctx, byte* out_key,
5742+
word32 key_len)
5743+
{
5744+
word32 i;
5745+
(void)ctx;
5746+
if (out_key == NULL || key_len < 32) {
5747+
return MQTT_CODE_ERROR_BAD_ARG;
5748+
}
5749+
/* Fixed pattern. Operators must replace this with a real key
5750+
* derivation before relying on confidentiality. */
5751+
for (i = 0; i < key_len; i++) {
5752+
out_key[i] = (byte)(0xA0 + (i & 0x0F));
5753+
}
5754+
return 0;
5755+
}
5756+
#endif
5757+
57355758
int wolfmqtt_broker(int argc, char** argv)
57365759
{
57375760
int rc;
@@ -5839,8 +5862,24 @@ int wolfmqtt_broker(int argc, char** argv)
58395862
return rc;
58405863
}
58415864
persist_initialized = 1;
5865+
#ifdef WOLFMQTT_BROKER_PERSIST_ENCRYPT
5866+
/* Install a development-only derive_key hook. Production
5867+
* deployments override MqttBrokerPersistHooks.derive_key with
5868+
* their own KMS / secure-element / file-based key source. The
5869+
* fixed 0xAA...0xAA key here lets the CLI exercise the
5870+
* encrypt path end-to-end for smoke tests; a deployment that
5871+
* still uses this in real life would be detectable by any
5872+
* adversary with read access. */
5873+
persist_hooks.derive_key = wolfmqtt_broker_dev_derive_key;
5874+
#endif
58425875
(void)MqttBroker_SetPersistHooks(&broker, &persist_hooks);
5843-
PRINTF("broker: persist enabled dir=%s", persist_dir);
5876+
PRINTF("broker: persist enabled dir=%s%s", persist_dir,
5877+
#ifdef WOLFMQTT_BROKER_PERSIST_ENCRYPT
5878+
" (encrypted, DEV-KEY)"
5879+
#else
5880+
""
5881+
#endif
5882+
);
58445883
}
58455884
#endif
58465885

0 commit comments

Comments
 (0)