Skip to content

Commit dfe3e0c

Browse files
acmoreacmore
committed
test(e2e): cover stale sync health recovery
1 parent 7a4f362 commit dfe3e0c

5 files changed

Lines changed: 243 additions & 1 deletion

File tree

‎.github/workflows/e2e-kind.yml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,9 @@ jobs:
4242
- name: Run multi-session test
4343
run: bash scripts/e2e_kind_multi_session.sh
4444

45+
- name: Run sync health test
46+
run: bash scripts/e2e_kind_sync_health.sh
47+
4548
- name: Install Kubeflow Training Operator
4649
run: |
4750
kubectl apply --server-side -k "https://github.com/kubeflow/training-operator.git/manifests/overlays/standalone?ref=v1.8.1"

‎internal/cli/sync_health_test.go‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -241,3 +241,23 @@ func TestSyncHealthLoopRestoreFailureStillRetries(t *testing.T) {
241241
t.Fatalf("expected exactly 1 reconnection message, got %q", buf.String())
242242
}
243243
}
244+
245+
func TestSyncHealthLoopConfigFromEnv(t *testing.T) {
246+
t.Setenv("OKDEV_SYNC_HEALTH_CHECK_INTERVAL", "2s")
247+
t.Setenv("OKDEV_SYNC_HEALTH_CHECK_MAX_INTERVAL", "7s")
248+
249+
cfg := syncHealthLoopConfigFromEnv(syncHealthLoopConfig{
250+
interval: time.Minute,
251+
quickRetries: 5,
252+
backoffFactor: 2,
253+
maxInterval: 10 * time.Minute,
254+
maxRetries: 15,
255+
})
256+
257+
if cfg.interval != 2*time.Second {
258+
t.Fatalf("interval = %s, want 2s", cfg.interval)
259+
}
260+
if cfg.maxInterval != 7*time.Second {
261+
t.Fatalf("maxInterval = %s, want 7s", cfg.maxInterval)
262+
}
263+
}

‎internal/cli/syncthing.go‎

Lines changed: 24 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -251,11 +251,34 @@ var defaultSyncHealthLoopConfig = syncHealthLoopConfig{
251251
maxRetries: syncHealthCheckMaxRetries,
252252
}
253253

254+
func syncHealthLoopConfigFromEnv(cfg syncHealthLoopConfig) syncHealthLoopConfig {
255+
if d, ok := durationFromEnv("OKDEV_SYNC_HEALTH_CHECK_INTERVAL"); ok {
256+
cfg.interval = d
257+
}
258+
if d, ok := durationFromEnv("OKDEV_SYNC_HEALTH_CHECK_MAX_INTERVAL"); ok {
259+
cfg.maxInterval = d
260+
}
261+
return cfg
262+
}
263+
264+
func durationFromEnv(name string) (time.Duration, bool) {
265+
value := strings.TrimSpace(os.Getenv(name))
266+
if value == "" {
267+
return 0, false
268+
}
269+
d, err := time.ParseDuration(value)
270+
if err != nil || d <= 0 {
271+
slog.Debug("ignoring invalid duration environment override", "name", name, "value", value)
272+
return 0, false
273+
}
274+
return d, true
275+
}
276+
254277
// runSyncHealthLoop monitors Syncthing peer connectivity and restores the
255278
// port-forward when the peer is disconnected. It returns when a signal is
256279
// received on sigCh.
257280
func runSyncHealthLoop(sigCh <-chan os.Signal, errOut io.Writer, checker syncHealthChecker) {
258-
runSyncHealthLoopWithConfig(sigCh, errOut, checker, defaultSyncHealthLoopConfig)
281+
runSyncHealthLoopWithConfig(sigCh, errOut, checker, syncHealthLoopConfigFromEnv(defaultSyncHealthLoopConfig))
259282
}
260283

261284
func runSyncHealthLoopWithConfig(sigCh <-chan os.Signal, errOut io.Writer, checker syncHealthChecker, cfg syncHealthLoopConfig) {

‎scripts/e2e_kind_sync_health.sh‎

Lines changed: 191 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,191 @@
1+
#!/usr/bin/env bash
2+
set -euo pipefail
3+
4+
. "$(dirname "$0")/e2e_lib.sh"
5+
6+
OKDEV_BIN="${OKDEV_BIN:-$(pwd)/bin/okdev}"
7+
SIDECAR_IMAGE="${SIDECAR_IMAGE:-okdev-sidecar:v0.0.0-e2e}"
8+
SESSION_NAME="${SESSION_NAME:-e2e-sync-health}"
9+
NAMESPACE="${NAMESPACE:-default}"
10+
WORKDIR="$(make_workdir)"
11+
HOME_DIR="${HOME_DIR:-$WORKDIR/home}"
12+
CFG_PATH="$WORKDIR/.okdev/okdev.yaml"
13+
SYNC_DIR="$WORKDIR/workspace"
14+
ORIG_HOME="${HOME}"
15+
ORIG_KUBECONFIG="${KUBECONFIG:-}"
16+
KUBECONFIG_PATH="$HOME_DIR/.kube/config"
17+
18+
mkdir -p "$HOME_DIR" "$SYNC_DIR" "$HOME_DIR/.kube"
19+
if [[ -n "$ORIG_KUBECONFIG" ]]; then
20+
cp "$ORIG_KUBECONFIG" "$KUBECONFIG_PATH"
21+
elif [[ -f "$ORIG_HOME/.kube/config" ]]; then
22+
cp "$ORIG_HOME/.kube/config" "$KUBECONFIG_PATH"
23+
else
24+
echo "kubeconfig not found" >&2
25+
exit 1
26+
fi
27+
export HOME="$HOME_DIR"
28+
export KUBECONFIG="$KUBECONFIG_PATH"
29+
export OKDEV_SYNC_HEALTH_CHECK_INTERVAL="${OKDEV_SYNC_HEALTH_CHECK_INTERVAL:-2s}"
30+
export OKDEV_SYNC_HEALTH_CHECK_MAX_INTERVAL="${OKDEV_SYNC_HEALTH_CHECK_MAX_INTERVAL:-2s}"
31+
32+
cleanup() {
33+
status=$?
34+
if [[ "$status" -ne 0 ]]; then
35+
echo "--- local okdev logs ---"
36+
ls -R "$HOME_DIR/.okdev" 2>/dev/null || true
37+
echo "--- session sync log ---"
38+
cat "$HOME_DIR/.okdev/sessions/${SESSION_NAME}/syncthing/local.log" 2>/dev/null || true
39+
fi
40+
"$OKDEV_BIN" --config "$CFG_PATH" --session "$SESSION_NAME" down --yes >/dev/null 2>&1 || true
41+
return "$status"
42+
}
43+
trap cleanup EXIT
44+
45+
echo "Scaffolding sync health config via okdev init"
46+
cd "$WORKDIR"
47+
"$OKDEV_BIN" init \
48+
--workload pod \
49+
--yes \
50+
--name e2e-sync-health \
51+
--namespace "$NAMESPACE" \
52+
--dev-image ubuntu:22.04 \
53+
--sidecar-image "$SIDECAR_IMAGE" \
54+
--ssh-user root \
55+
--sync-local "$SYNC_DIR" \
56+
--sync-remote /workspace
57+
58+
if [[ ! -f "$CFG_PATH" ]]; then
59+
CFG_PATH="$WORKDIR/.okdev.yaml"
60+
fi
61+
if [[ ! -f "$CFG_PATH" ]]; then
62+
echo "ERROR: okdev init did not write a config file" >&2
63+
exit 1
64+
fi
65+
66+
replace_all_in_file "$CFG_PATH" 'persistentSession: true' 'persistentSession: false'
67+
insert_after_line_once "$CFG_PATH" ' ssh:' ' persistentSession: false'
68+
69+
echo "Starting sync health e2e session"
70+
echo "before stale injection" >"$SYNC_DIR/health.txt"
71+
"$OKDEV_BIN" --config "$CFG_PATH" --session "$SESSION_NAME" up --wait-timeout 5m
72+
73+
SYNC_HOME="$HOME_DIR/.okdev/sessions/${SESSION_NAME}/syncthing"
74+
LOG_PATH="$SYNC_HOME/local.log"
75+
76+
echo "Waiting for sync health to become active"
77+
for i in $(seq 1 20); do
78+
STATUS_OUTPUT=$("$OKDEV_BIN" --config "$CFG_PATH" --session "$SESSION_NAME" status --details)
79+
if [[ "$STATUS_OUTPUT" == *"health: active"* ]]; then
80+
break
81+
fi
82+
if [[ "$i" -eq 20 ]]; then
83+
echo "ERROR: expected status --details to report active sync" >&2
84+
echo "$STATUS_OUTPUT" >&2
85+
exit 1
86+
fi
87+
sleep 2
88+
done
89+
90+
echo "Injecting stale local Syncthing peer address"
91+
python3 - "$SYNC_HOME" <<'PY'
92+
import http.client
93+
import json
94+
import pathlib
95+
import sys
96+
import time
97+
import urllib.parse
98+
import xml.etree.ElementTree as ET
99+
100+
home = pathlib.Path(sys.argv[1])
101+
endpoint = home.joinpath("endpoint").read_text().strip()
102+
parsed = urllib.parse.urlparse(endpoint)
103+
cfg_path = home / "config.xml"
104+
root = ET.parse(cfg_path).getroot()
105+
api_key = root.findtext("./gui/apikey")
106+
if not api_key:
107+
raise SystemExit("local Syncthing API key not found")
108+
109+
110+
def request(method, path, body=None):
111+
conn = http.client.HTTPConnection(parsed.hostname, parsed.port, timeout=5)
112+
headers = {"X-API-Key": api_key}
113+
if body is not None:
114+
body = json.dumps(body).encode()
115+
headers["Content-Type"] = "application/json"
116+
conn.request(method, path, body=body, headers=headers)
117+
resp = conn.getresponse()
118+
data = resp.read()
119+
conn.close()
120+
if resp.status >= 300:
121+
raise SystemExit(f"{method} {path} failed: {resp.status} {data.decode(errors='replace')}")
122+
return data
123+
124+
125+
config = json.loads(request("GET", "/rest/config"))
126+
devices = config.get("devices", [])
127+
if len(devices) != 1:
128+
raise SystemExit(f"expected one remote device, got {len(devices)}")
129+
devices[0]["addresses"] = ["tcp://127.0.0.1:1"]
130+
request("PUT", "/rest/config", config)
131+
request("POST", "/rest/system/restart")
132+
133+
deadline = time.time() + 30
134+
while time.time() < deadline:
135+
try:
136+
request("GET", "/rest/system/status")
137+
break
138+
except Exception:
139+
time.sleep(0.5)
140+
else:
141+
raise SystemExit("local Syncthing API did not recover after restart")
142+
PY
143+
144+
echo "Waiting for sync health loop to restore peer connection"
145+
for i in $(seq 1 40); do
146+
if grep -q "sync: reconnected to peer" "$LOG_PATH"; then
147+
echo "Sync health restored the peer connection on attempt $i"
148+
break
149+
fi
150+
if [[ "$i" -eq 40 ]]; then
151+
echo "ERROR: expected sync health loop to restore stale peer connection" >&2
152+
exit 1
153+
fi
154+
sleep 1
155+
done
156+
157+
echo "Verifying sync remains active after health restoration"
158+
for i in $(seq 1 20); do
159+
STATUS_OUTPUT=$("$OKDEV_BIN" --config "$CFG_PATH" --session "$SESSION_NAME" status --details)
160+
if [[ "$STATUS_OUTPUT" == *"health: active"* ]]; then
161+
break
162+
fi
163+
if [[ "$i" -eq 20 ]]; then
164+
echo "ERROR: expected sync health active after restoration" >&2
165+
echo "$STATUS_OUTPUT" >&2
166+
exit 1
167+
fi
168+
sleep 2
169+
done
170+
171+
echo "Verifying post-restoration file sync"
172+
echo "after health restore" >"$SYNC_DIR/health-after.txt"
173+
SYNC_OK=false
174+
for i in $(seq 1 30); do
175+
REMOTE_CONTENT=$("$OKDEV_BIN" --config "$CFG_PATH" --session "$SESSION_NAME" exec --no-tty --cmd 'if [ -f /workspace/health-after.txt ]; then cat /workspace/health-after.txt; fi' || true)
176+
if [[ "$REMOTE_CONTENT" == "after health restore" ]]; then
177+
SYNC_OK=true
178+
break
179+
fi
180+
sleep 2
181+
done
182+
if [[ "$SYNC_OK" != "true" ]]; then
183+
echo "ERROR: expected post-restoration file to sync to remote workspace" >&2
184+
exit 1
185+
fi
186+
187+
echo "Testing explicit okdev down"
188+
"$OKDEV_BIN" --config "$CFG_PATH" --session "$SESSION_NAME" down --yes
189+
assert_no_local_sync_processes "$SESSION_NAME" "$SYNC_HOME"
190+
191+
echo "Sync health e2e completed"

‎scripts/e2e_local_kind.sh‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ CLUSTER_NAME="${CLUSTER_NAME:-okdev-e2e}"
88
OKDEV_BIN="${OKDEV_BIN:-$ROOT_DIR/bin/okdev}"
99
SIDECAR_IMAGE="${SIDECAR_IMAGE:-okdev-sidecar:v0.0.0-e2e}"
1010
RUN_PYTORCHJOB="${RUN_PYTORCHJOB:-0}"
11+
RUN_SYNC_HEALTH="${RUN_SYNC_HEALTH:-1}"
1112
RUN_LARGE_REPO="${RUN_LARGE_REPO:-0}"
1213
LARGE_REPO_PATH="${LARGE_REPO_PATH:-}"
1314
LARGE_REPO_URL="${LARGE_REPO_URL:-}"
@@ -63,6 +64,10 @@ bash scripts/e2e_kind_deployment.sh
6364
bash scripts/e2e_kind_job.sh
6465
bash scripts/e2e_kind_multi_session.sh
6566

67+
if [[ "$RUN_SYNC_HEALTH" == "1" ]]; then
68+
bash scripts/e2e_kind_sync_health.sh
69+
fi
70+
6671
if [[ "$RUN_PYTORCHJOB" == "1" ]]; then
6772
echo "Installing Kubeflow Training Operator"
6873
kubectl apply --server-side -k "https://github.com/kubeflow/training-operator.git/manifests/overlays/standalone?ref=v1.8.1"

0 commit comments

Comments
 (0)