Skip to content

Commit 44ae0b3

Browse files
santigimenosantigimeno
committed
test: add gRPC retry policy tests
Add comprehensive test suite to verify gRPC retry policy behavior. Extend test infrastructure with fault injection capabilities to simulate transient failures and network issues. Test infrastructure enhancements: - Inject transient failures with configurable status codes and counts - Inject delays to simulate network latency - Track retry attempts via grpc-previous-rpc-attempts metadata - Clear all injected faults between tests Test coverage: - Successful retry after single transient UNAVAILABLE failure - Failure after exhausting max retries (5 attempts) on persistent errors - Verification of retry attempt counts via metadata headers - Multiple service types (ExportInfo, ExportMetricsCmd, ExportSpans) The fault injection system validates that the gRPC client properly retries on transient failures and respects retry limits.
1 parent 5757d67 commit 44ae0b3

7 files changed

Lines changed: 519 additions & 40 deletions

File tree

‎test/agents/test-grpc-retry-policies.mjs‎

Lines changed: 366 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,366 @@
1+
// Flags: --expose-internals
2+
3+
import { mustCall, mustSucceed } from '../common/index.mjs';
4+
import { GRPCServer, TestClient } from '../common/nsolid-grpc-agent/index.js';
5+
6+
import assert from 'node:assert';
7+
8+
const services = [
9+
{ name: 'ExportInfo', trigger: (client, s, id) => s.info(id), event: 'info' },
10+
{ name: 'ExportMetricsCmd', trigger: (client, s, id) => s.metrics(id), event: 'metrics_cmd' },
11+
{ name: 'ExportSpans', trigger: (client, s, id) => client.trace('http'), event: 'spans' },
12+
];
13+
14+
const tests = [];
15+
16+
tests.push({
17+
name: 'should retry on transient UNAVAILABLE and succeed',
18+
test: async (getEnv) => {
19+
return new Promise((resolve) => {
20+
const grpcServer = new GRPCServer();
21+
grpcServer.start(mustSucceed(async (port) => {
22+
const env = getEnv(port);
23+
const client = new TestClient([], { env });
24+
const agentId = await client.id();
25+
26+
let completed = 0;
27+
const total = services.length;
28+
29+
for (const svc of services) {
30+
// Inject failure for the first call
31+
grpcServer.injectFailure(svc.name, 'UNAVAILABLE', 1);
32+
33+
if (svc.name === 'ExportSpans') {
34+
grpcServer.on(svc.event, mustCall(async (data) => {
35+
const attemptsHeader = data.metadata['grpc-previous-rpc-attempts'];
36+
assert(attemptsHeader && attemptsHeader[0] === '1', `Should have retried once for ${svc.name}, got ${attemptsHeader}`);
37+
completed++;
38+
if (completed === total) {
39+
grpcServer.clearFaults();
40+
await client.shutdown(0);
41+
grpcServer.close();
42+
resolve();
43+
}
44+
}));
45+
await client.trace('http');
46+
continue;
47+
}
48+
49+
svc.trigger(client, grpcServer, agentId).then(mustCall(async ({ data }) => {
50+
const attemptsHeader = data.metadata['grpc-previous-rpc-attempts'];
51+
assert(attemptsHeader && attemptsHeader[0] === '1', `Should have retried once for ${svc.name}, got ${attemptsHeader}`);
52+
completed++;
53+
if (completed === total) {
54+
grpcServer.clearFaults();
55+
await client.shutdown(0);
56+
grpcServer.close();
57+
resolve();
58+
}
59+
}));
60+
}
61+
}));
62+
});
63+
},
64+
});
65+
66+
tests.push({
67+
name: 'should fail after max retries (5) on persistent UNAVAILABLE',
68+
test: async (getEnv) => {
69+
return new Promise((resolve) => {
70+
const grpcServer = new GRPCServer();
71+
grpcServer.start(mustSucceed(async (port) => {
72+
const env = getEnv(port);
73+
const client = new TestClient([], { env });
74+
const agentId = await client.id();
75+
76+
let completed = 0;
77+
const total = services.length;
78+
79+
for (const svc of services) {
80+
// Inject failures for more than 5 attempts
81+
grpcServer.injectFailure(svc.name, 'UNAVAILABLE', 20);
82+
83+
if (svc.name === 'ExportSpans') {
84+
const timeout = new Promise((resolveTimeout) => setTimeout(() => resolveTimeout('timeout'), 10000));
85+
const success = new Promise((resolve) => grpcServer.on(svc.event, () => resolve('success')));
86+
Promise.race([success, timeout]).then(mustCall(async (result) => {
87+
if (result === 'timeout') {
88+
// Good, failed as expected due to exhausted retries
89+
completed++;
90+
if (completed === total) {
91+
grpcServer.clearFaults();
92+
await client.shutdown(0);
93+
grpcServer.close();
94+
resolve();
95+
}
96+
} else {
97+
throw new Error(`Should have failed after retries for ${svc.name}`);
98+
}
99+
}));
100+
}
101+
102+
const triggerPromise = svc.trigger(client, grpcServer, agentId);
103+
if (svc.name !== 'ExportSpans') {
104+
const timeout = new Promise((resolveTimeout) => setTimeout(() => resolveTimeout('timeout'), 20000));
105+
Promise.race([triggerPromise, timeout]).then(mustCall(async (result) => {
106+
if (result === 'timeout') {
107+
// Good, failed as expected due to exhausted retries
108+
completed++;
109+
if (completed === total) {
110+
grpcServer.clearFaults();
111+
await client.shutdown(0);
112+
grpcServer.close();
113+
resolve();
114+
}
115+
} else {
116+
throw new Error(`Should have failed after retries for ${svc.name}`);
117+
}
118+
}));
119+
}
120+
}
121+
}));
122+
});
123+
},
124+
});
125+
126+
tests.push({
127+
name: 'should not retry on non-retryable DEADLINE_EXCEEDED',
128+
test: async (getEnv) => {
129+
return new Promise((resolve) => {
130+
const grpcServer = new GRPCServer();
131+
grpcServer.start(mustSucceed(async (port) => {
132+
const env = getEnv(port);
133+
const client = new TestClient([], { env });
134+
const agentId = await client.id();
135+
136+
let completed = 0;
137+
const total = services.length;
138+
139+
for (const svc of services) {
140+
grpcServer.injectFailure(svc.name, 'DEADLINE_EXCEEDED', 1);
141+
142+
if (svc.name === 'ExportSpans') {
143+
const timeout = new Promise((resolveTimeout) => setTimeout(() => resolveTimeout('timeout'), 1000));
144+
const success = new Promise((resolve) => grpcServer.on(svc.event, () => resolve('success')));
145+
Promise.race([success, timeout]).then(mustCall(async (result) => {
146+
if (result === 'timeout') {
147+
// Good, failed immediately without retries
148+
completed++;
149+
if (completed === total) {
150+
await client.shutdown(0);
151+
grpcServer.clearFaults();
152+
grpcServer.close();
153+
resolve();
154+
}
155+
} else {
156+
await client.shutdown(0);
157+
throw new Error(`Should have failed immediately for ${svc.name}`);
158+
}
159+
}));
160+
}
161+
162+
const triggerPromise = svc.trigger(client, grpcServer, agentId);
163+
if (svc.name !== 'ExportSpans') {
164+
const timeout = new Promise((resolveTimeout) => setTimeout(() => resolveTimeout('timeout'), 1000));
165+
Promise.race([triggerPromise, timeout]).then(mustCall(async (result) => {
166+
if (result === 'timeout') {
167+
// Good, failed immediately without retries
168+
completed++;
169+
if (completed === total) {
170+
await client.shutdown(0);
171+
grpcServer.clearFaults();
172+
grpcServer.close();
173+
resolve();
174+
}
175+
} else {
176+
await client.shutdown(0);
177+
throw new Error(`Should have failed immediately for ${svc.name}`);
178+
}
179+
}));
180+
}
181+
}
182+
}));
183+
});
184+
},
185+
});
186+
187+
tests.push({
188+
name: 'should enforce custom deadline from NSOLID_GRPC_DEADLINE',
189+
test: async (getEnv) => {
190+
return new Promise((resolve) => {
191+
const grpcServer = new GRPCServer();
192+
grpcServer.start(mustSucceed(async (port) => {
193+
// Inject 2s delay on services
194+
for (const svc of services) {
195+
grpcServer.injectDelay(svc.name, 2000);
196+
}
197+
198+
const env = { ...getEnv(port), NSOLID_GRPC_DEADLINE: '1' };
199+
const client = new TestClient([], { env });
200+
const agentId = await client.id();
201+
202+
let completed = 0;
203+
const total = services.length;
204+
205+
for (const svc of services) {
206+
if (svc.name === 'ExportSpans') {
207+
const timeout = new Promise((resolveTimeout) => setTimeout(() => resolveTimeout('timeout'), 2000));
208+
const success = new Promise((resolve) => grpcServer.on(svc.event, () => resolve('success')));
209+
Promise.race([success, timeout]).then(mustCall(async (result) => {
210+
if (result === 'timeout') {
211+
// Good, timed out due to deadline
212+
completed++;
213+
if (completed === total) {
214+
await client.shutdown(0);
215+
grpcServer.clearFaults();
216+
grpcServer.close();
217+
resolve();
218+
}
219+
} else {
220+
throw new Error(`Should have timed out for ${svc.name}`);
221+
}
222+
}));
223+
continue;
224+
}
225+
226+
const triggerPromise = svc.trigger(client, grpcServer, agentId);
227+
if (svc.name !== 'ExportSpans') {
228+
const timeout = new Promise((resolveTimeout) => setTimeout(() => resolveTimeout('timeout'), 2000));
229+
Promise.race([triggerPromise, timeout]).then(mustCall(async (result) => {
230+
if (result === 'timeout') {
231+
// Good, timed out due to deadline
232+
completed++;
233+
if (completed === total) {
234+
await client.shutdown(0);
235+
grpcServer.clearFaults();
236+
grpcServer.close();
237+
resolve();
238+
}
239+
} else {
240+
await client.shutdown(0);
241+
throw new Error(`Should have timed out for ${svc.name}`);
242+
}
243+
}));
244+
}
245+
}
246+
}));
247+
});
248+
},
249+
});
250+
251+
tests.push({
252+
name: 'should use default 10s deadline when env not set',
253+
test: async (getEnv) => {
254+
return new Promise((resolve) => {
255+
const grpcServer = new GRPCServer();
256+
grpcServer.start(mustSucceed(async (port) => {
257+
// Inject short delay, should succeed
258+
for (const svc of services) {
259+
grpcServer.injectDelay(svc.name, 500);
260+
}
261+
262+
const env = getEnv(port);
263+
const client = new TestClient([], { env });
264+
const agentId = await client.id();
265+
266+
let completed = 0;
267+
const total = services.length;
268+
269+
for (const svc of services) {
270+
if (svc.name === 'ExportSpans') {
271+
grpcServer.on(svc.event, mustCall(async () => {
272+
completed++;
273+
if (completed === total) {
274+
await client.shutdown(0);
275+
grpcServer.clearFaults();
276+
grpcServer.close();
277+
resolve();
278+
}
279+
}));
280+
281+
await client.trace('http');
282+
continue;
283+
}
284+
285+
const promise = svc.trigger(client, grpcServer, agentId);
286+
promise.then(mustCall(async () => {
287+
completed++;
288+
if (completed === total) {
289+
await client.shutdown(0);
290+
grpcServer.clearFaults();
291+
grpcServer.close();
292+
resolve();
293+
}
294+
}));
295+
}
296+
}));
297+
});
298+
},
299+
});
300+
301+
tests.push({
302+
name: 'should handle invalid NSOLID_GRPC_DEADLINE gracefully',
303+
test: async (getEnv) => {
304+
return new Promise((resolve) => {
305+
const grpcServer = new GRPCServer();
306+
grpcServer.start(mustSucceed(async (port) => {
307+
// Inject delay, should succeed with default 10s
308+
for (const svc of services) {
309+
grpcServer.injectDelay(svc.name, 1000);
310+
}
311+
312+
const env = { ...getEnv(port), NSOLID_GRPC_DEADLINE: 'invalid' };
313+
const client = new TestClient([], { env });
314+
const agentId = await client.id();
315+
316+
let completed = 0;
317+
const total = services.length;
318+
319+
for (const svc of services) {
320+
if (svc.name === 'ExportSpans') {
321+
grpcServer.on(svc.event, mustCall(async () => {
322+
completed++;
323+
if (completed === total) {
324+
await client.shutdown(0);
325+
grpcServer.clearFaults();
326+
grpcServer.close();
327+
resolve();
328+
}
329+
}));
330+
}
331+
332+
const promise = svc.trigger(client, grpcServer, agentId);
333+
if (svc.name !== 'ExportSpans') {
334+
promise.then(mustCall(async () => {
335+
completed++;
336+
if (completed === total) {
337+
await client.shutdown(0);
338+
grpcServer.clearFaults();
339+
grpcServer.close();
340+
resolve();
341+
}
342+
}));
343+
}
344+
}
345+
}));
346+
});
347+
},
348+
});
349+
350+
const testConfigs = [
351+
{
352+
getEnv: (port) => ({
353+
NODE_DEBUG_NATIVE: 'nsolid_grpc_agent',
354+
NSOLID_GRPC: `localhost:${port}`,
355+
NSOLID_GRPC_INSECURE: 1,
356+
NSOLID_TRACING_ENABLED: 1,
357+
}),
358+
},
359+
];
360+
361+
for (const testConfig of testConfigs) {
362+
for (const { name, test } of tests) {
363+
console.log(`[retry-policies] ${name}`);
364+
await test(testConfig.getEnv);
365+
}
366+
}

‎test/agents/test-grpc-tracing.mjs‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ const {
1616

1717
function checkResource(resource) {
1818
validateArray(resource.attributes, 'resource.attributes');
19-
assert.strictEqual(resource.attributes.length, 5);
19+
assert.strictEqual(resource.attributes.length, 6);
2020
}
2121

2222
// traceId: {
@@ -207,7 +207,7 @@ tests.push({
207207
if (phase === 'done')
208208
return;
209209

210-
mergeResourceSpans(spans, resourceSpans);
210+
mergeResourceSpans(spans.request, resourceSpans);
211211

212212
if (phase === 'initial' &&
213213
resourceSpans.length === 1 &&
@@ -275,7 +275,7 @@ tests.push({
275275
if (phase === 'done')
276276
return;
277277

278-
mergeResourceSpans(spans, resourceSpans);
278+
mergeResourceSpans(spans.request, resourceSpans);
279279

280280
if (phase === 'initial' &&
281281
resourceSpans.length === 1 &&

0 commit comments

Comments
 (0)