Skip to content

Commit cb7148b

Browse files
jareddellittjareddellitt
authored
chore(PLA-2206): add exponential backoff with jitter to Phoenix socket reconnection (#862)
1 parent 35d745e commit cb7148b

5 files changed

Lines changed: 238 additions & 2 deletions

File tree

‎.changeset/socket-reconnect-backoff.md‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
---
2+
"@knocklabs/client": patch
3+
---
4+
5+
Add exponential backoff with jitter to Phoenix socket reconnection to prevent thundering herd on outages

‎packages/client/src/api.ts‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,8 @@ import axios, { AxiosError, AxiosInstance, AxiosRequestConfig } from "axios";
22
import axiosRetry from "axios-retry";
33
import { Socket } from "phoenix";
44

5+
import { exponentialBackoffFullJitter } from "./helpers";
6+
57
type ApiClientOptions = {
68
host: string;
79
apiKey: string;
@@ -53,6 +55,18 @@ class ApiClient {
5355
api_key: this.apiKey,
5456
branch_slug: this.branch,
5557
},
58+
reconnectAfterMs: (tries: number) => {
59+
return exponentialBackoffFullJitter(tries, {
60+
baseDelayMs: 1000,
61+
maxDelayMs: 30_000,
62+
});
63+
},
64+
rejoinAfterMs: (tries: number) => {
65+
return exponentialBackoffFullJitter(tries, {
66+
baseDelayMs: 1000,
67+
maxDelayMs: 60_000,
68+
});
69+
},
5670
});
5771
}
5872

‎packages/client/src/helpers.ts‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,3 +4,42 @@ const uuidRegex =
44
export function isValidUuid(uuid: string) {
55
return uuidRegex.test(uuid);
66
}
7+
8+
/**
9+
* Exponential backoff with full jitter and a minimum delay floor.
10+
*
11+
* - Uses exponential growth capped at maxDelayMs
12+
* - Applies full jitter to spread retries uniformly across the window
13+
* - Enforces a minimum delay to avoid tight retry loops
14+
*
15+
* Example (baseDelayMs = 1000):
16+
* Try 1: 250ms – 1,000ms
17+
* Try 2: 250ms – 2,000ms
18+
* Try 3: 250ms – 4,000ms
19+
* Try 4: 250ms – 8,000ms
20+
* Try 5+: 250ms – maxDelayMs
21+
*/
22+
export function exponentialBackoffFullJitter(
23+
tries: number,
24+
{
25+
baseDelayMs,
26+
maxDelayMs,
27+
minDelayMs = 250,
28+
}: {
29+
baseDelayMs: number;
30+
maxDelayMs: number;
31+
minDelayMs?: number;
32+
},
33+
): number {
34+
const exponentialDelay = Math.min(
35+
maxDelayMs,
36+
baseDelayMs * Math.pow(2, Math.max(0, tries - 1)),
37+
);
38+
39+
if (exponentialDelay <= minDelayMs) {
40+
return minDelayMs;
41+
}
42+
43+
const jitterRange = exponentialDelay - minDelayMs;
44+
return minDelayMs + Math.floor(Math.random() * jitterRange);
45+
}

‎packages/client/test/api.test.ts‎

Lines changed: 97 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
import { afterEach, beforeEach, describe, expect, test, vi } from "vitest";
2+
import { Socket } from "phoenix";
23

34
import packageJson from "../package.json";
45
import ApiClient from "../src/api";
@@ -609,6 +610,102 @@ describe("API Client", () => {
609610
(global as GlobalWithWindow).window = originalWindow;
610611
});
611612

613+
test("configures socket with reconnectAfterMs and rejoinAfterMs", () => {
614+
const originalWindow = (global as GlobalWithWindow).window;
615+
(global as GlobalWithWindow).window = {};
616+
617+
vi.mocked(Socket).mockClear();
618+
619+
new ApiClient({
620+
host: "https://api.knock.app",
621+
apiKey: "pk_test_12345",
622+
userToken: "user_token_456",
623+
});
624+
625+
const socketOpts = vi.mocked(Socket).mock.calls[0]![1] as Record<
626+
string,
627+
unknown
628+
>;
629+
expect(typeof socketOpts.reconnectAfterMs).toBe("function");
630+
expect(typeof socketOpts.rejoinAfterMs).toBe("function");
631+
632+
(global as GlobalWithWindow).window = originalWindow;
633+
});
634+
635+
test("reconnectAfterMs returns values within expected bounds", () => {
636+
const originalWindow = (global as GlobalWithWindow).window;
637+
(global as GlobalWithWindow).window = {};
638+
639+
vi.mocked(Socket).mockClear();
640+
641+
new ApiClient({
642+
host: "https://api.knock.app",
643+
apiKey: "pk_test_12345",
644+
userToken: "user_token_456",
645+
});
646+
647+
const socketOpts = vi.mocked(Socket).mock.calls[0]![1] as Record<
648+
string,
649+
unknown
650+
>;
651+
const reconnectAfterMs = socketOpts.reconnectAfterMs as (
652+
tries: number,
653+
) => number;
654+
655+
// Call it many times to verify the range holds
656+
for (let i = 0; i < 50; i++) {
657+
const delay = reconnectAfterMs(1);
658+
expect(delay).toBeGreaterThanOrEqual(250);
659+
expect(delay).toBeLessThanOrEqual(1000);
660+
}
661+
662+
// At high tries, should be capped at 30_000
663+
for (let i = 0; i < 50; i++) {
664+
const delay = reconnectAfterMs(100);
665+
expect(delay).toBeGreaterThanOrEqual(250);
666+
expect(delay).toBeLessThanOrEqual(30_000);
667+
}
668+
669+
(global as GlobalWithWindow).window = originalWindow;
670+
});
671+
672+
test("rejoinAfterMs returns values within expected bounds", () => {
673+
const originalWindow = (global as GlobalWithWindow).window;
674+
(global as GlobalWithWindow).window = {};
675+
676+
vi.mocked(Socket).mockClear();
677+
678+
new ApiClient({
679+
host: "https://api.knock.app",
680+
apiKey: "pk_test_12345",
681+
userToken: "user_token_456",
682+
});
683+
684+
const socketOpts = vi.mocked(Socket).mock.calls[0]![1] as Record<
685+
string,
686+
unknown
687+
>;
688+
const rejoinAfterMs = socketOpts.rejoinAfterMs as (
689+
tries: number,
690+
) => number;
691+
692+
// Call it many times to verify the range holds
693+
for (let i = 0; i < 50; i++) {
694+
const delay = rejoinAfterMs(1);
695+
expect(delay).toBeGreaterThanOrEqual(250);
696+
expect(delay).toBeLessThanOrEqual(1000);
697+
}
698+
699+
// At high tries, should be capped at 60_000
700+
for (let i = 0; i < 50; i++) {
701+
const delay = rejoinAfterMs(100);
702+
expect(delay).toBeGreaterThanOrEqual(250);
703+
expect(delay).toBeLessThanOrEqual(60_000);
704+
}
705+
706+
(global as GlobalWithWindow).window = originalWindow;
707+
});
708+
612709
test("gracefully handles missing WebSocket in server environment", () => {
613710
// Store original window value
614711
const originalWindow = (global as GlobalWithWindow).window;

‎packages/client/test/helpers.test.ts‎

Lines changed: 83 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
1-
import { describe, expect, test } from "vitest";
1+
import { afterEach, beforeEach, describe, expect, test, vi } from "vitest";
22

3-
import { isValidUuid } from "../src/helpers";
3+
import { exponentialBackoffFullJitter, isValidUuid } from "../src/helpers";
44

55
/**
66
* Helper Functions Test Suite
@@ -113,4 +113,85 @@ describe("Helper Functions", () => {
113113
expect(helpers.isValidUuid).toBe(isValidUuid);
114114
});
115115
});
116+
117+
describe("exponentialBackoffFullJitter", () => {
118+
beforeEach(() => {
119+
vi.spyOn(Math, "random");
120+
});
121+
122+
afterEach(() => {
123+
vi.restoreAllMocks();
124+
});
125+
126+
test("returns minDelayMs when random is 0", () => {
127+
vi.mocked(Math.random).mockReturnValue(0);
128+
129+
expect(
130+
exponentialBackoffFullJitter(1, {
131+
baseDelayMs: 1000,
132+
maxDelayMs: 30_000,
133+
}),
134+
).toBe(250);
135+
});
136+
137+
test("approaches the exponential ceiling when random approaches 1", () => {
138+
vi.mocked(Math.random).mockReturnValue(0.999);
139+
140+
// try 1 ceiling = 1000, result = 250 + floor(0.999 * 750) = 999
141+
expect(
142+
exponentialBackoffFullJitter(1, {
143+
baseDelayMs: 1000,
144+
maxDelayMs: 30_000,
145+
}),
146+
).toBe(999);
147+
});
148+
149+
test("grows exponentially across successive tries", () => {
150+
vi.mocked(Math.random).mockReturnValue(0.999);
151+
152+
const opts = { baseDelayMs: 1000, maxDelayMs: 60_000 };
153+
const results = [1, 2, 3, 4].map((t) =>
154+
exponentialBackoffFullJitter(t, opts),
155+
);
156+
157+
// Each result should roughly double the previous
158+
for (let i = 1; i < results.length; i++) {
159+
expect(results[i]).toBeGreaterThan(results[i - 1]!);
160+
expect(results[i]).toBeCloseTo(results[i - 1]! * 2, -2);
161+
}
162+
});
163+
164+
test("never exceeds maxDelayMs", () => {
165+
vi.mocked(Math.random).mockReturnValue(0.999);
166+
167+
expect(
168+
exponentialBackoffFullJitter(100, {
169+
baseDelayMs: 1000,
170+
maxDelayMs: 30_000,
171+
}),
172+
).toBeLessThanOrEqual(30_000);
173+
});
174+
175+
test("respects a custom minDelayMs", () => {
176+
vi.mocked(Math.random).mockReturnValue(0);
177+
178+
expect(
179+
exponentialBackoffFullJitter(1, {
180+
baseDelayMs: 1000,
181+
maxDelayMs: 30_000,
182+
minDelayMs: 500,
183+
}),
184+
).toBe(500);
185+
});
186+
187+
test("clamps to minDelayMs when exponential ceiling is below it", () => {
188+
// baseDelayMs 100, try 1 → ceiling = 100 < default minDelayMs 250
189+
expect(
190+
exponentialBackoffFullJitter(1, {
191+
baseDelayMs: 100,
192+
maxDelayMs: 30_000,
193+
}),
194+
).toBe(250);
195+
});
196+
})
116197
});

0 commit comments

Comments
 (0)