Skip to content

Commit 335de9c

Browse files
hyp3rdhyp3rd
committed
ci(test): add gitleaks config and overhaul cluster regression test
- Add `.gitleaksconfig.toml` extending the default gitleaks ruleset with a global allowlist for config and test shell files; wire it into the `gitleaks.yml` workflow via `GITLEAKS_CONFIG`. - Completely rewrite `scripts/tests/10-test-cluster-api.sh` to be a proper regression suite for the Phase D cluster bugs: - Replaces raw `curl` one-liners with reusable helper functions (`put_value`, `expect_value`, `expect_404`, `delete_key`) that assert both HTTP status codes and response body fields. - Collects all failures before exiting (non-short-circuit) so operators get a full report in one run. - Adds configurable `PORTS`, `WRITE_PORT`, and `DELETE_PORT` env vars for flexible local/CI overrides. - Phases cover: cluster propagation, wire-encoding fidelity for non-owner GETs, and cross-node DELETE propagation.
1 parent be6b9d9 commit 335de9c

3 files changed

Lines changed: 233 additions & 24 deletions

File tree

‎.github/workflows/gitleaks.yml‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ permissions:
55
on:
66
pull_request:
77
push:
8-
branches: [main]
8+
branches: [ main ]
99
workflow_dispatch:
1010
schedule:
1111
# run once a day at 4 AM
@@ -21,3 +21,4 @@ jobs:
2121
- uses: gitleaks/gitleaks-action@v2
2222
env:
2323
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
24+
GITLEAKS_CONFIG: .gitleaksconfig.toml

‎.gitleaksconfig.toml‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
# Title for the gitleaks configuration file.
2+
title = "Hypercache Gitleaks configuration"
3+
4+
# You have basically two options for your custom configuration:
5+
#
6+
# 1. define your own configuration, default rules do not apply
7+
#
8+
# use e.g., the default configuration as starting point:
9+
# https://github.com/gitleaks/gitleaks/blob/master/config/gitleaks.toml
10+
#
11+
# 2. extend a configuration, the rules are overwritten or extended
12+
#
13+
# When you extend a configuration the extended rules take precedence over the
14+
# default rules. I.e., if there are duplicate rules in both the extended
15+
# configuration and the default configuration the extended rules or
16+
# attributes of them will override the default rules.
17+
# Another thing to know with extending configurations is you can chain
18+
# together multiple configuration files to a depth of 2. Allowlist arrays are
19+
# appended and can contain duplicates.
20+
21+
# useDefault and path can NOT be used at the same time. Choose one.
22+
[extend]
23+
# useDefault will extend the default gitleaks config built in to the binary
24+
# the latest version is located at:
25+
# https://github.com/gitleaks/gitleaks/blob/master/config/gitleaks.toml
26+
useDefault = true
27+
28+
29+
# ⚠️ In v8.25.0 `[allowlist]` was replaced with `[[allowlists]]`.
30+
#
31+
# Global allowlists have a higher order of precedence than rule-specific allowlists.
32+
# If a commit listed in the `commits` field below is encountered then that commit will be skipped and no
33+
# secrets will be detected for said commit. The same logic applies for regexes and paths.
34+
[[allowlists]]
35+
description = "global allow list"
36+
paths = [
37+
'''gitleaks\.toml''',
38+
'''scripts/tests/._\.sh$'''
39+
]

‎scripts/tests/10-test-cluster-api.sh‎

Lines changed: 192 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -1,38 +1,207 @@
11
#!/usr/bin/env bash
2+
# End-to-end regression test against a running 5-node hypercache
3+
# cluster (docker-compose.cluster.yml). Asserts the three behaviors
4+
# that broke during initial Phase D and were fixed in the follow-up:
5+
#
6+
# 1. Cluster propagation: a value written to one node is visible
7+
# from every node, regardless of ring ownership.
8+
# 2. Wire-encoding fidelity: non-owner GETs (which forward through
9+
# the dist HTTP transport) return the original bytes, not a
10+
# base64 echo.
11+
# 3. Cross-node DELETE: a delete issued on any node propagates to
12+
# the primary so every node serves 404 afterward.
13+
#
14+
# Run after `docker compose -f docker-compose.cluster.yml up --build`.
15+
# Exit code 0 means every assertion passed; non-zero means at least
16+
# one mismatch — see the failing line for which.
17+
#
18+
# Usage:
19+
# ./scripts/tests/10-test-cluster-api.sh
20+
# PORTS="8081 8082" ./scripts/tests/10-test-cluster-api.sh # custom subset
221

322
set -euo pipefail
423

5-
echo "=== PUT to hypercache-1 ==="
6-
curl -sS -H "Authorization: Bearer dev-token" -X PUT --data 'world' 'http://localhost:8081/v1/cache/greeting' -w '\n status %{http_code}\n'
24+
readonly TOKEN="${HYPERCACHE_TOKEN:-dev-token}"
25+
readonly PORTS="${PORTS:-8081 8082 8083 8084 8085}"
26+
readonly WRITE_PORT="${WRITE_PORT:-8081}"
27+
readonly DELETE_PORT="${DELETE_PORT:-8083}"
728

29+
# Tracks failures so the script can report all of them, not just the
30+
# first — operators get one full report rather than discover-and-rerun.
31+
fail_count=0
32+
33+
# log_fail prints an assertion failure in red (when a TTY is attached)
34+
# and bumps the failure counter. Centralized so every assertion uses
35+
# the same shape.
36+
log_fail() {
37+
local msg="$1"
38+
39+
if [[ -t 1 ]]; then
40+
printf '\033[31mFAIL\033[0m %s\n' "$msg"
41+
else
42+
printf 'FAIL %s\n' "$msg"
43+
fi
44+
45+
fail_count=$((fail_count + 1))
46+
}
47+
48+
log_ok() {
49+
local msg="$1"
50+
51+
if [[ -t 1 ]]; then
52+
printf '\033[32m OK \033[0m %s\n' "$msg"
53+
else
54+
printf ' OK %s\n' "$msg"
55+
fi
56+
}
57+
58+
# put_value writes `$3` to /v1/cache/$2 on port $1 and asserts the
59+
# response status is 200 and the body's `stored` field is true.
60+
put_value() {
61+
local port="$1"
62+
local key="$2"
63+
local value="$3"
64+
65+
local status
66+
67+
status=$(curl -sS -o /tmp/hyp-put.body -w '%{http_code}' \
68+
-H "Authorization: Bearer $TOKEN" \
69+
-X PUT --data "$value" \
70+
"http://localhost:$port/v1/cache/$key")
71+
72+
if [[ "$status" != "200" ]]; then
73+
log_fail "PUT $key on :$port returned status $status (want 200); body: $(cat /tmp/hyp-put.body)"
74+
return 1
75+
fi
76+
77+
if ! grep -q '"stored":true' /tmp/hyp-put.body; then
78+
log_fail "PUT $key on :$port did not echo stored=true; body: $(cat /tmp/hyp-put.body)"
79+
return 1
80+
fi
81+
82+
log_ok "PUT $key on :$port"
83+
return 0
84+
}
85+
86+
# expect_value asserts GET /v1/cache/$key on port $port returns the
87+
# given value with status 200. Used for both writer-node reads and
88+
# non-owner reads — the assertion is the same.
89+
expect_value() {
90+
local port="$1"
91+
local key="$2"
92+
local want="$3"
93+
94+
local status
95+
96+
status=$(curl -sS -o /tmp/hyp-get.body -w '%{http_code}' \
97+
-H "Authorization: Bearer $TOKEN" \
98+
"http://localhost:$port/v1/cache/$key")
99+
100+
if [[ "$status" != "200" ]]; then
101+
log_fail "GET $key on :$port: status=$status (want 200); body: $(cat /tmp/hyp-get.body)"
102+
return 1
103+
fi
104+
105+
local got
106+
got=$(cat /tmp/hyp-get.body)
107+
if [[ "$got" != "$want" ]]; then
108+
log_fail "GET $key on :$port: got '$got' (want '$want')"
109+
return 1
110+
fi
111+
112+
log_ok "GET $key on :$port == '$want'"
113+
return 0
114+
}
115+
116+
# expect_404 asserts GET returns 404 with the canonical NOT_FOUND
117+
# JSON shape — used after the delete propagation tests.
118+
expect_404() {
119+
local port="$1"
120+
local key="$2"
121+
122+
local status
123+
124+
status=$(curl -sS -o /tmp/hyp-get.body -w '%{http_code}' \
125+
-H "Authorization: Bearer $TOKEN" \
126+
"http://localhost:$port/v1/cache/$key")
127+
128+
if [[ "$status" != "404" ]]; then
129+
log_fail "GET $key on :$port after delete: status=$status (want 404); body: $(cat /tmp/hyp-get.body)"
130+
return 1
131+
fi
132+
133+
if ! grep -q '"code":"NOT_FOUND"' /tmp/hyp-get.body; then
134+
log_fail "GET $key on :$port: 404 but missing NOT_FOUND code; body: $(cat /tmp/hyp-get.body)"
135+
return 1
136+
fi
137+
138+
log_ok "GET $key on :$port returned 404 NOT_FOUND"
139+
return 0
140+
}
141+
142+
# delete_key issues DELETE on the given port and asserts a 200 +
143+
# deleted=true response.
144+
delete_key() {
145+
local port="$1"
146+
local key="$2"
147+
148+
local status
149+
150+
status=$(curl -sS -o /tmp/hyp-del.body -w '%{http_code}' \
151+
-H "Authorization: Bearer $TOKEN" \
152+
-X DELETE \
153+
"http://localhost:$port/v1/cache/$key")
154+
155+
if [[ "$status" != "200" ]]; then
156+
log_fail "DELETE $key on :$port returned status $status; body: $(cat /tmp/hyp-del.body)"
157+
return 1
158+
fi
159+
160+
if ! grep -q '"deleted":true' /tmp/hyp-del.body; then
161+
log_fail "DELETE $key on :$port did not echo deleted=true; body: $(cat /tmp/hyp-del.body)"
162+
return 1
163+
fi
164+
165+
log_ok "DELETE $key on :$port"
166+
return 0
167+
}
168+
169+
echo "=== Phase 1: byte-value propagation (PUT 'world' on :$WRITE_PORT) ==="
170+
put_value "$WRITE_PORT" greeting world || true
8171
sleep 1
9-
echo ""
10-
echo "=== GET via every node (all should be world) ==="
11-
for port in 8081 8082 8083 8084 8085; do
12-
printf "node@%s -> " "$port"
13-
curl -H "Authorization: Bearer dev-token" "http://localhost:$port/v1/cache/greeting" -w ' [%{http_code}]'
14-
echo ""
172+
for port in $PORTS; do
173+
expect_value "$port" greeting world || true
15174
done
16175

17176
echo ""
18-
echo "=== PUT a JSON-y value to hypercache-2 ==="
19-
curl -sS -H "Authorization: Bearer dev-token" -X PUT --data 'plain string with spaces' 'http://localhost:8082/v1/cache/sentence' -w '\n status %{http_code}\n'
20-
177+
echo "=== Phase 2: text-value propagation (PUT spaces on :8082) ==="
178+
put_value 8082 sentence "plain string with spaces" || true
21179
sleep 1
22-
echo ""
23-
echo "=== GET sentence via every node ==="
24-
for port in 8081 8082 8083 8084 8085; do
25-
printf "node@%s -> " "$port"
26-
curl -sS -H "Authorization: Bearer dev-token" "http://localhost:$port/v1/cache/sentence" -w ' [%{http_code}]'
27-
echo ""
180+
for port in $PORTS; do
181+
expect_value "$port" sentence "plain string with spaces" || true
28182
done
29183

30184
echo ""
31-
echo "=== DELETE from hypercache-3, then GET from all ==="
32-
curl -sS -H "Authorization: Bearer dev-token" -X DELETE 'http://localhost:8083/v1/cache/greeting' -w '\n status %{http_code}\n'
185+
echo "=== Phase 3: cross-node DELETE (DELETE on :$DELETE_PORT, expect 404 cluster-wide) ==="
186+
delete_key "$DELETE_PORT" greeting || true
33187
sleep 1
34-
for port in 8081 8082 8083 8084 8085; do
35-
printf "node@%s -> " "$port"
36-
curl -sS -H "Authorization: Bearer dev-token" "http://localhost:$port/v1/cache/greeting" -w ' [%{http_code}]'
37-
echo ""
188+
for port in $PORTS; do
189+
expect_404 "$port" greeting || true
38190
done
191+
192+
echo ""
193+
if [[ "$fail_count" -gt 0 ]]; then
194+
if [[ -t 1 ]]; then
195+
printf '\033[31m=== %d assertion(s) failed ===\033[0m\n' "$fail_count"
196+
else
197+
printf '=== %d assertion(s) failed ===\n' "$fail_count"
198+
fi
199+
200+
exit 1
201+
fi
202+
203+
if [[ -t 1 ]]; then
204+
printf '\033[32m=== all assertions passed ===\033[0m\n'
205+
else
206+
printf '=== all assertions passed ===\n'
207+
fi

0 commit comments

Comments
 (0)