Skip to content

Commit 3da5d7c

Browse files
santigimenosantigimeno
committed
test: add gRPC retry policy tests
Add comprehensive test suite to verify gRPC retry policy behavior. Extend test infrastructure with fault injection capabilities to simulate transient failures and network issues. Test infrastructure enhancements: - Inject transient failures with configurable status codes and counts - Inject delays to simulate network latency - Track retry attempts via grpc-previous-rpc-attempts metadata - Clear all injected faults between tests Test coverage: - Successful retry after single transient UNAVAILABLE failure - Failure after exhausting max retries (5 attempts) on persistent errors - Verification of retry attempt counts via metadata headers - Multiple service types (ExportInfo, ExportMetricsCmd, ExportSpans) The fault injection system validates that the gRPC client properly retries on transient failures and respects retry limits.
1 parent 79d04ef commit 3da5d7c

7 files changed

Lines changed: 517 additions & 39 deletions

File tree

‎test/agents/test-grpc-retry-policies.mjs‎

Lines changed: 366 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,366 @@
1+
// Flags: --expose-internals
2+
3+
import { mustCall, mustSucceed } from '../common/index.mjs';
4+
import { GRPCServer, TestClient } from '../common/nsolid-grpc-agent/index.js';
5+
6+
import assert from 'node:assert';
7+
8+
const services = [
9+
{ name: 'ExportInfo', trigger: (client, s, id) => s.info(id), event: 'info' },
10+
{ name: 'ExportMetricsCmd', trigger: (client, s, id) => s.metrics(id), event: 'metrics_cmd' },
11+
{ name: 'ExportSpans', trigger: (client, s, id) => client.trace('http'), event: 'spans' },
12+
];
13+
14+
const tests = [];
15+
16+
tests.push({
17+
name: 'should retry on transient UNAVAILABLE and succeed',
18+
test: async (getEnv) => {
19+
return new Promise((resolve) => {
20+
const grpcServer = new GRPCServer();
21+
grpcServer.start(mustSucceed(async (port) => {
22+
const env = getEnv(port);
23+
const client = new TestClient([], { env });
24+
const agentId = await client.id();
25+
26+
let completed = 0;
27+
const total = services.length;
28+
29+
for (const svc of services) {
30+
// Inject failure for the first call
31+
grpcServer.injectFailure(svc.name, 'UNAVAILABLE', 1);
32+
33+
if (svc.name === 'ExportSpans') {
34+
grpcServer.on(svc.event, mustCall(async (data) => {
35+
const attemptsHeader = data.metadata['grpc-previous-rpc-attempts'];
36+
assert(attemptsHeader && attemptsHeader[0] === '1', `Should have retried once for ${svc.name}, got ${attemptsHeader}`);
37+
completed++;
38+
if (completed === total) {
39+
grpcServer.clearFaults();
40+
await client.shutdown(0);
41+
grpcServer.close();
42+
resolve();
43+
}
44+
}));
45+
await client.trace('http');
46+
continue;
47+
}
48+
49+
svc.trigger(client, grpcServer, agentId).then(async ({ data }) => {
50+
const attemptsHeader = data.metadata['grpc-previous-rpc-attempts'];
51+
assert(attemptsHeader && attemptsHeader[0] === '1', `Should have retried once for ${svc.name}, got ${attemptsHeader}`);
52+
completed++;
53+
if (completed === total) {
54+
grpcServer.clearFaults();
55+
await client.shutdown(0);
56+
grpcServer.close();
57+
resolve();
58+
}
59+
}).then(mustCall());
60+
}
61+
}));
62+
});
63+
},
64+
});
65+
66+
tests.push({
67+
name: 'should fail after max retries (5) on persistent UNAVAILABLE',
68+
test: async (getEnv) => {
69+
return new Promise((resolve) => {
70+
const grpcServer = new GRPCServer();
71+
grpcServer.start(mustSucceed(async (port) => {
72+
const env = getEnv(port);
73+
const client = new TestClient([], { env });
74+
const agentId = await client.id();
75+
76+
let completed = 0;
77+
const total = services.length;
78+
79+
for (const svc of services) {
80+
// Inject failures for more than 5 attempts
81+
grpcServer.injectFailure(svc.name, 'UNAVAILABLE', 20);
82+
83+
if (svc.name === 'ExportSpans') {
84+
const timeout = new Promise((resolveTimeout) => setTimeout(() => resolveTimeout('timeout'), 10000));
85+
const success = new Promise((resolve) => grpcServer.on(svc.event, () => resolve('success')));
86+
Promise.race([success, timeout]).then(async (result) => {
87+
if (result === 'timeout') {
88+
// Good, failed as expected due to exhausted retries
89+
completed++;
90+
if (completed === total) {
91+
grpcServer.clearFaults();
92+
await client.shutdown(0);
93+
grpcServer.close();
94+
resolve();
95+
}
96+
} else {
97+
throw new Error(`Should have failed after retries for ${svc.name}`);
98+
}
99+
}).then(mustCall());
100+
}
101+
102+
const triggerPromise = svc.trigger(client, grpcServer, agentId);
103+
if (svc.name !== 'ExportSpans') {
104+
const timeout = new Promise((resolveTimeout) => setTimeout(() => resolveTimeout('timeout'), 20000));
105+
Promise.race([triggerPromise, timeout]).then(async (result) => {
106+
if (result === 'timeout') {
107+
// Good, failed as expected due to exhausted retries
108+
completed++;
109+
if (completed === total) {
110+
grpcServer.clearFaults();
111+
await client.shutdown(0);
112+
grpcServer.close();
113+
resolve();
114+
}
115+
} else {
116+
throw new Error(`Should have failed after retries for ${svc.name}`);
117+
}
118+
}).then(mustCall());
119+
}
120+
}
121+
}));
122+
});
123+
},
124+
});
125+
126+
tests.push({
127+
name: 'should not retry on non-retryable DEADLINE_EXCEEDED',
128+
test: async (getEnv) => {
129+
return new Promise((resolve) => {
130+
const grpcServer = new GRPCServer();
131+
grpcServer.start(mustSucceed(async (port) => {
132+
const env = getEnv(port);
133+
const client = new TestClient([], { env });
134+
const agentId = await client.id();
135+
136+
let completed = 0;
137+
const total = services.length;
138+
139+
for (const svc of services) {
140+
grpcServer.injectFailure(svc.name, 'DEADLINE_EXCEEDED', 1);
141+
142+
if (svc.name === 'ExportSpans') {
143+
const timeout = new Promise((resolveTimeout) => setTimeout(() => resolveTimeout('timeout'), 1000));
144+
const success = new Promise((resolve) => grpcServer.on(svc.event, () => resolve('success')));
145+
Promise.race([success, timeout]).then(async (result) => {
146+
if (result === 'timeout') {
147+
// Good, failed immediately without retries
148+
completed++;
149+
if (completed === total) {
150+
await client.shutdown(0);
151+
grpcServer.clearFaults();
152+
grpcServer.close();
153+
resolve();
154+
}
155+
} else {
156+
await client.shutdown(0);
157+
throw new Error(`Should have failed immediately for ${svc.name}`);
158+
}
159+
}).then(mustCall());
160+
}
161+
162+
const triggerPromise = svc.trigger(client, grpcServer, agentId);
163+
if (svc.name !== 'ExportSpans') {
164+
const timeout = new Promise((resolveTimeout) => setTimeout(() => resolveTimeout('timeout'), 1000));
165+
Promise.race([triggerPromise, timeout]).then(async (result) => {
166+
if (result === 'timeout') {
167+
// Good, failed immediately without retries
168+
completed++;
169+
if (completed === total) {
170+
await client.shutdown(0);
171+
grpcServer.clearFaults();
172+
grpcServer.close();
173+
resolve();
174+
}
175+
} else {
176+
await client.shutdown(0);
177+
throw new Error(`Should have failed immediately for ${svc.name}`);
178+
}
179+
}).then(mustCall());
180+
}
181+
}
182+
}));
183+
});
184+
},
185+
});
186+
187+
tests.push({
188+
name: 'should enforce custom deadline from NSOLID_GRPC_DEADLINE',
189+
test: async (getEnv) => {
190+
return new Promise((resolve) => {
191+
const grpcServer = new GRPCServer();
192+
grpcServer.start(mustSucceed(async (port) => {
193+
// Inject 2s delay on services
194+
for (const svc of services) {
195+
grpcServer.injectDelay(svc.name, 2000);
196+
}
197+
198+
const env = { ...getEnv(port), NSOLID_GRPC_DEADLINE: '1' };
199+
const client = new TestClient([], { env });
200+
const agentId = await client.id();
201+
202+
let completed = 0;
203+
const total = services.length;
204+
205+
for (const svc of services) {
206+
if (svc.name === 'ExportSpans') {
207+
const timeout = new Promise((resolveTimeout) => setTimeout(() => resolveTimeout('timeout'), 2000));
208+
const success = new Promise((resolve) => grpcServer.on(svc.event, () => resolve('success')));
209+
Promise.race([success, timeout]).then(async (result) => {
210+
if (result === 'timeout') {
211+
// Good, timed out due to deadline
212+
completed++;
213+
if (completed === total) {
214+
await client.shutdown(0);
215+
grpcServer.clearFaults();
216+
grpcServer.close();
217+
resolve();
218+
}
219+
} else {
220+
throw new Error(`Should have timed out for ${svc.name}`);
221+
}
222+
}).then(mustCall());
223+
continue;
224+
}
225+
226+
const triggerPromise = svc.trigger(client, grpcServer, agentId);
227+
if (svc.name !== 'ExportSpans') {
228+
const timeout = new Promise((resolveTimeout) => setTimeout(() => resolveTimeout('timeout'), 2000));
229+
Promise.race([triggerPromise, timeout]).then(async (result) => {
230+
if (result === 'timeout') {
231+
// Good, timed out due to deadline
232+
completed++;
233+
if (completed === total) {
234+
await client.shutdown(0);
235+
grpcServer.clearFaults();
236+
grpcServer.close();
237+
resolve();
238+
}
239+
} else {
240+
await client.shutdown(0);
241+
throw new Error(`Should have timed out for ${svc.name}`);
242+
}
243+
}).then(mustCall());
244+
}
245+
}
246+
}));
247+
});
248+
},
249+
});
250+
251+
tests.push({
252+
name: 'should use default 10s deadline when env not set',
253+
test: async (getEnv) => {
254+
return new Promise((resolve) => {
255+
const grpcServer = new GRPCServer();
256+
grpcServer.start(mustSucceed(async (port) => {
257+
// Inject short delay, should succeed
258+
for (const svc of services) {
259+
grpcServer.injectDelay(svc.name, 500);
260+
}
261+
262+
const env = getEnv(port);
263+
const client = new TestClient([], { env });
264+
const agentId = await client.id();
265+
266+
let completed = 0;
267+
const total = services.length;
268+
269+
for (const svc of services) {
270+
if (svc.name === 'ExportSpans') {
271+
grpcServer.on(svc.event, mustCall(async () => {
272+
completed++;
273+
if (completed === total) {
274+
await client.shutdown(0);
275+
grpcServer.clearFaults();
276+
grpcServer.close();
277+
resolve();
278+
}
279+
}));
280+
281+
await client.trace('http');
282+
continue;
283+
}
284+
285+
const promise = svc.trigger(client, grpcServer, agentId);
286+
promise.then(async () => {
287+
completed++;
288+
if (completed === total) {
289+
await client.shutdown(0);
290+
grpcServer.clearFaults();
291+
grpcServer.close();
292+
resolve();
293+
}
294+
}).then(mustCall());
295+
}
296+
}));
297+
});
298+
},
299+
});
300+
301+
tests.push({
302+
name: 'should handle invalid NSOLID_GRPC_DEADLINE gracefully',
303+
test: async (getEnv) => {
304+
return new Promise((resolve) => {
305+
const grpcServer = new GRPCServer();
306+
grpcServer.start(mustSucceed(async (port) => {
307+
// Inject delay, should succeed with default 10s
308+
for (const svc of services) {
309+
grpcServer.injectDelay(svc.name, 1000);
310+
}
311+
312+
const env = { ...getEnv(port), NSOLID_GRPC_DEADLINE: 'invalid' };
313+
const client = new TestClient([], { env });
314+
const agentId = await client.id();
315+
316+
let completed = 0;
317+
const total = services.length;
318+
319+
for (const svc of services) {
320+
if (svc.name === 'ExportSpans') {
321+
grpcServer.on(svc.event, mustCall(async () => {
322+
completed++;
323+
if (completed === total) {
324+
await client.shutdown(0);
325+
grpcServer.clearFaults();
326+
grpcServer.close();
327+
resolve();
328+
}
329+
}));
330+
}
331+
332+
const promise = svc.trigger(client, grpcServer, agentId);
333+
if (svc.name !== 'ExportSpans') {
334+
promise.then(async () => {
335+
completed++;
336+
if (completed === total) {
337+
await client.shutdown(0);
338+
grpcServer.clearFaults();
339+
grpcServer.close();
340+
resolve();
341+
}
342+
}).then(mustCall());
343+
}
344+
}
345+
}));
346+
});
347+
},
348+
});
349+
350+
const testConfigs = [
351+
{
352+
getEnv: (port) => ({
353+
NODE_DEBUG_NATIVE: 'nsolid_grpc_agent',
354+
NSOLID_GRPC: `localhost:${port}`,
355+
NSOLID_GRPC_INSECURE: 1,
356+
NSOLID_TRACING_ENABLED: 1,
357+
}),
358+
},
359+
];
360+
361+
for (const testConfig of testConfigs) {
362+
for (const { name, test } of tests) {
363+
console.log(`[retry-policies] ${name}`);
364+
await test(testConfig.getEnv);
365+
}
366+
}

‎test/agents/test-grpc-tracing.mjs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -204,7 +204,7 @@ tests.push({
204204
if (phase === 'done')
205205
return;
206206

207-
mergeResourceSpans(spans, resourceSpans);
207+
mergeResourceSpans(spans.request, resourceSpans);
208208

209209
if (phase === 'initial' &&
210210
resourceSpans.length === 1 &&
@@ -273,7 +273,7 @@ tests.push({
273273
if (phase === 'done')
274274
return;
275275

276-
mergeResourceSpans(spans, resourceSpans);
276+
mergeResourceSpans(spans.request, resourceSpans);
277277

278278
if (phase === 'initial' &&
279279
resourceSpans.length === 1 &&

0 commit comments

Comments
 (0)