Skip to content

Commit a20677d

Browse files
davidje13davidje13
committed
Automatically retry transient server errors if an executable is running
1 parent 1216111 commit a20677d

6 files changed

Lines changed: 178 additions & 16 deletions

File tree

‎src/bin/ServerManager.mts‎

Lines changed: 32 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ import { findCause, HTTPError, WebListener, type ListenOptions } from '../index.
55
import type { ConfigServer, ConfigServerOptions, ConfigBackgroundTask } from './config/types.mts';
66
import { buildRouter } from './buildRouter.mts';
77
import type { Logger, AddColour } from './log.mts';
8+
import { TransientError } from './TransientError.mts';
89

910
export class ServerManager {
1011
declare private _started: boolean;
@@ -14,6 +15,7 @@ export class ServerManager {
1415
declare private readonly _colour: AddColour;
1516
declare private readonly _servers: Map<number, ServerState>;
1617
declare private _backgroundTasks: Set<BackgroundTaskState>;
18+
declare private _autoRetry: NodeJS.Timeout | undefined;
1719

1820
constructor(log: Logger, colour: AddColour) {
1921
this._started = false;
@@ -25,26 +27,35 @@ export class ServerManager {
2527
this._backgroundTasks = new Set();
2628
}
2729

28-
async set(servers: ConfigServer[], backgroundTasks: ConfigBackgroundTask[]) {
30+
async set(
31+
servers: ConfigServer[],
32+
backgroundTasks: ConfigBackgroundTask[],
33+
errorHandler: (error: unknown) => void,
34+
) {
2935
if (this._building || this._stopping) {
3036
return;
3137
}
38+
clearTimeout(this._autoRetry);
39+
this._autoRetry = undefined;
3240
try {
3341
this._building = true;
3442
const preTasks: (() => Promise<void>)[] = [];
3543
const tasks: (() => Promise<void>)[] = [];
3644

45+
let canRetry = false;
3746
const previousBackgroundTasks = this._backgroundTasks;
3847
this._backgroundTasks = new Set();
3948
tasksLoop: for (let i = 0; i < backgroundTasks.length; ++i) {
4049
const taskConfig = backgroundTasks[i]!;
4150
for (const taskState of previousBackgroundTasks) {
4251
if (backgroundTaskConfigEqual(taskState.config, taskConfig)) {
52+
canRetry ||= taskState.isRunning();
4353
this._backgroundTasks.add(taskState);
4454
previousBackgroundTasks.delete(taskState);
4555
continue tasksLoop;
4656
}
4757
}
58+
canRetry = true;
4859
tasks.push(async () => {
4960
this._backgroundTasks.add(await this._startBackgroundTask(taskConfig));
5061
});
@@ -83,14 +94,20 @@ export class ServerManager {
8394
}
8495
}
8596

97+
let doRetry = false;
8698
const runTasks = async (tasks: (() => Promise<void>)[]) => {
8799
let failures: unknown[] = [];
88100
await Promise.all(
89101
tasks.map(async (task) => {
90102
try {
91103
await task();
92-
} catch (err) {
93-
failures.push(err);
104+
} catch (error: unknown) {
105+
if (canRetry && !this._stopping && error instanceof TransientError) {
106+
this._log(1, `${error.message} (retrying)`);
107+
doRetry = true;
108+
} else {
109+
failures.push(error);
110+
}
94111
}
95112
}),
96113
);
@@ -105,11 +122,15 @@ export class ServerManager {
105122

106123
if (this._stopping) {
107124
this._shutdown();
125+
} else if (doRetry) {
126+
this._autoRetry = setTimeout(() => this.set(servers, backgroundTasks, errorHandler), 1000);
108127
} else if (this._servers.size) {
109128
this._log(1, 'all servers ready');
110129
} else {
111130
this._log(1, 'no servers configured');
112131
}
132+
} catch (error: unknown) {
133+
errorHandler(error);
113134
} finally {
114135
this._building = false;
115136
}
@@ -122,6 +143,7 @@ export class ServerManager {
122143

123144
return new Promise((resolve, reject) => {
124145
this._log(2, `${name} ${this._colour('2', 'starting')}`);
146+
let running = true;
125147
const proc = spawn(config.command, config.arguments, {
126148
cwd: config.cwd,
127149
env: { ...process.env, TERM: '', COLORTERM: '', NO_COLOR: '1', ...config.environment },
@@ -143,6 +165,7 @@ export class ServerManager {
143165
}
144166
proc.addListener('error', (err) => {
145167
this._log(0, `${name} startup failed: ${err.message}`);
168+
running = false;
146169
reject(err);
147170
});
148171
proc.addListener('exit', (code, signal) => {
@@ -151,10 +174,12 @@ export class ServerManager {
151174
} else {
152175
this._log(2, `${name} closed ${this._colour('2', `(exit signal ${signal})`)}`);
153176
}
177+
running = false;
154178
});
155179
proc.addListener('spawn', () =>
156180
resolve({
157181
config,
182+
isRunning: () => running,
158183
close: () =>
159184
new Promise((closeResolve) => {
160185
if (proc.signalCode !== null || proc.exitCode !== null) {
@@ -272,6 +297,8 @@ export class ServerManager {
272297
if (this._stopping) {
273298
return;
274299
}
300+
clearTimeout(this._autoRetry);
301+
this._autoRetry = undefined;
275302
this._stopping = true;
276303
if (!this._building) {
277304
this._shutdown();
@@ -328,7 +355,8 @@ interface ServerState {
328355

329356
interface BackgroundTaskState {
330357
config: ConfigBackgroundTask;
331-
close: () => Promise<void>;
358+
close(): Promise<void>;
359+
isRunning(): boolean;
332360
}
333361

334362
function backgroundTaskConfigEqual(a: ConfigBackgroundTask, b: ConfigBackgroundTask) {

‎src/bin/ServerManager.test.mts‎

Lines changed: 123 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,7 @@
1+
import { join } from 'node:path';
2+
import { mkdir } from 'node:fs/promises';
13
import { findAvailablePort } from '../test-helpers/findAvailablePort.mts';
4+
import { makeTestTempDir } from '../test-helpers/makeFileStructure.mts';
25
import type { ConfigServer } from './config/types.mts';
36
import type { AddColour } from './log.mts';
47
import { ServerManager } from './ServerManager.mts';
@@ -11,7 +14,7 @@ describe('ServerManager', () => {
1114
const logs: string[] = [];
1215
const manager = new ServerManager((_, msg) => logs.push(msg), NO_COLOUR);
1316
try {
14-
await manager.set([fixtureServer(port, 'content')], []);
17+
await manager.set([fixtureServer(port, 'content')], [], () => fail());
1518
expect(logs).equals([
1619
`http://localhost:${port} starting`,
1720
`http://localhost:${port} ready`,
@@ -34,7 +37,11 @@ describe('ServerManager', () => {
3437
const logs: string[] = [];
3538
const manager = new ServerManager((_, msg) => logs.push(msg), NO_COLOUR);
3639
try {
37-
await manager.set([fixtureServer(port1, 'content 1'), fixtureServer(port2, 'content 2')], []);
40+
await manager.set(
41+
[fixtureServer(port1, 'content 1'), fixtureServer(port2, 'content 2')],
42+
[],
43+
() => fail(),
44+
);
3845
expect(logs[logs.length - 1]).equals('all servers ready');
3946

4047
const res1 = await fetch(`http://localhost:${port1}`);
@@ -53,7 +60,11 @@ describe('ServerManager', () => {
5360
const logs: string[] = [];
5461
const manager = new ServerManager((_, msg) => logs.push(msg), NO_COLOUR);
5562
try {
56-
await manager.set([fixtureServer(port, 'content 1'), fixtureServer(port, 'content 2')], []);
63+
await manager.set(
64+
[fixtureServer(port, 'content 1'), fixtureServer(port, 'content 2')],
65+
[],
66+
() => fail(),
67+
);
5768
expect(logs).equals([
5869
`skipping servers[1] because port ${port} has already been defined`,
5970
`http://localhost:${port} starting`,
@@ -72,7 +83,11 @@ describe('ServerManager', () => {
7283
const logs: string[] = [];
7384
const manager = new ServerManager((_, msg) => logs.push(msg), NO_COLOUR);
7485
try {
75-
await manager.set([fixtureServer(0, 'content 1'), fixtureServer(65536, 'content 2')], []);
86+
await manager.set(
87+
[fixtureServer(0, 'content 1'), fixtureServer(65536, 'content 2')],
88+
[],
89+
() => fail(),
90+
);
7691
expect(logs).equals([
7792
'servers[0] must have a specific port from 1 to 65535',
7893
'servers[1] must have a specific port from 1 to 65535',
@@ -89,7 +104,7 @@ describe('ServerManager', () => {
89104
const logs: string[] = [];
90105
const manager = new ServerManager((_, msg) => logs.push(msg), NO_COLOUR);
91106
try {
92-
await manager.set([fixtureServer(port, 'content')], []);
107+
await manager.set([fixtureServer(port, 'content')], [], () => fail());
93108
expect(logs).equals([
94109
`http://localhost:${port} starting`,
95110
`http://localhost:${port} ready`,
@@ -100,7 +115,7 @@ describe('ServerManager', () => {
100115
expect(await res1.text()).equals('content');
101116

102117
logs.length = 0;
103-
await manager.set([fixtureServer(port, 'updated content')], []);
118+
await manager.set([fixtureServer(port, 'updated content')], [], () => fail());
104119
expect(logs).equals([`http://localhost:${port} updated`, 'all servers ready']);
105120

106121
const res2 = await fetch(`http://localhost:${port}`);
@@ -116,7 +131,7 @@ describe('ServerManager', () => {
116131
const logs: string[] = [];
117132
const manager = new ServerManager((_, msg) => logs.push(msg), NO_COLOUR);
118133
try {
119-
await manager.set([fixtureServer(port, 'content')], []);
134+
await manager.set([fixtureServer(port, 'content')], [], () => fail());
120135
expect(logs).equals([
121136
`http://localhost:${port} starting`,
122137
`http://localhost:${port} ready`,
@@ -135,6 +150,7 @@ describe('ServerManager', () => {
135150
},
136151
],
137152
[],
153+
() => fail(),
138154
);
139155
expect(logs).equals([
140156
`http://localhost:${port} restarting (step 1: shutdown)`,
@@ -161,6 +177,7 @@ describe('ServerManager', () => {
161177
await manager.set(
162178
[fixtureServer(port1, 'content 1'), fixtureServer(port2, 'content 2')],
163179
[],
180+
() => fail(),
164181
);
165182
manager.shutdown();
166183
await expect.poll(() => logs[logs.length - 1], equals('shutdown complete'), {
@@ -180,7 +197,11 @@ describe('ServerManager', () => {
180197
const logs: string[] = [];
181198
const manager = new ServerManager((_, msg) => logs.push(msg), NO_COLOUR);
182199
try {
183-
manager.set([fixtureServer(port1, 'content 1'), fixtureServer(port2, 'content 2')], []);
200+
manager.set(
201+
[fixtureServer(port1, 'content 1'), fixtureServer(port2, 'content 2')],
202+
[],
203+
() => fail(),
204+
);
184205
manager.shutdown();
185206
await expect.poll(() => logs[logs.length - 1], equals('shutdown complete'), {
186207
timeout: 500,
@@ -193,6 +214,100 @@ describe('ServerManager', () => {
193214
}
194215
});
195216
});
217+
218+
const TEST_DIR = makeTestTempDir('sm-');
219+
220+
it(
221+
'retries transient errors if an executable is running',
222+
{ timeout: 3000 },
223+
async ({ getTyped }) => {
224+
const port = await findAvailablePort();
225+
226+
const logs: string[] = [];
227+
const manager = new ServerManager((_, msg) => logs.push(msg), NO_COLOUR);
228+
try {
229+
await manager.set(
230+
[
231+
{
232+
port,
233+
host: 'localhost',
234+
mount: [
235+
{ type: 'files', path: '/', dir: join(getTyped(TEST_DIR), 'sub'), options: {} },
236+
],
237+
options: DEFAULT_SERVER_OPTIONS,
238+
},
239+
],
240+
[
241+
{
242+
command: 'true',
243+
arguments: [],
244+
cwd: '.',
245+
environment: {},
246+
options: { killSignal: 'SIGTERM', displayStdout: true, displayStderr: true },
247+
},
248+
],
249+
() => fail(),
250+
);
251+
expect(logs).contains(matches(/directory to serve not found/));
252+
expect(logs).not(contains(`http://localhost:${port} ready`));
253+
expect(logs).not(contains('all servers ready'));
254+
255+
await mkdir(join(getTyped(TEST_DIR), 'sub'));
256+
257+
await expect.poll(() => logs, contains('all servers ready'));
258+
expect(logs).contains(`http://localhost:${port} ready`);
259+
} finally {
260+
manager.shutdown();
261+
}
262+
},
263+
);
264+
265+
it(
266+
'does not retry transient errors if no executables are running',
267+
{ timeout: 3000 },
268+
async ({ getTyped }) => {
269+
const port = await findAvailablePort();
270+
271+
const logs: string[] = [];
272+
const manager = new ServerManager((_, msg) => logs.push(msg), NO_COLOUR);
273+
try {
274+
let errorCaptor = (_: unknown) => {};
275+
const awaitError = new Promise<unknown>((resolve) => {
276+
errorCaptor = resolve;
277+
});
278+
await manager.set(
279+
[
280+
{
281+
port,
282+
host: 'localhost',
283+
mount: [
284+
{ type: 'files', path: '/', dir: join(getTyped(TEST_DIR), 'nope'), options: {} },
285+
],
286+
options: DEFAULT_SERVER_OPTIONS,
287+
},
288+
],
289+
[
290+
{
291+
command: 'true',
292+
arguments: [],
293+
cwd: '.',
294+
environment: {},
295+
options: { killSignal: 'SIGTERM', displayStdout: true, displayStderr: true },
296+
},
297+
],
298+
errorCaptor,
299+
);
300+
const capturedError = await awaitError;
301+
expect(capturedError).isInstanceOf(Error);
302+
expect((capturedError as Error).message).contains('directory to serve not found');
303+
expect(logs).contains(matches(/directory to serve not found/));
304+
expect(logs).not(contains(`http://localhost:${port} ready`));
305+
expect(logs).not(contains('all servers ready'));
306+
} finally {
307+
manager.shutdown();
308+
}
309+
},
310+
);
196311
});
197312

198313
const NO_COLOUR: AddColour = (_, message) => message;

‎src/bin/TransientError.mts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
export class TransientError extends Error {}

‎src/bin/buildRouter.mts‎

Lines changed: 20 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ import type { ConfigMount } from './config/types.mts';
1717
import { dependencies } from './modules/dependencies.mts';
1818
import { Mapper, nginxTokenise } from './nginx.mts';
1919
import { render } from './template.mts';
20+
import { TransientError } from './TransientError.mts';
2021

2122
export interface LogInfo {
2223
method: string;
@@ -50,7 +51,15 @@ export async function buildRouter(mount: ConfigMount[], log: (info: LogInfo) =>
5051
item.options.negotiation && item.options.negotiation.length > 0
5152
? new Negotiator(item.options.negotiation)
5253
: undefined;
53-
router.mount(item.path, await fileServer(item.dir, { ...item.options, negotiator }));
54+
try {
55+
router.mount(item.path, await fileServer(item.dir, { ...item.options, negotiator }));
56+
} catch (error: unknown) {
57+
if (error && typeof error === 'object' && 'code' in error && error.code === 'ENOENT') {
58+
throw new TransientError(`directory to serve not found at ${item.dir}`);
59+
} else {
60+
throw error;
61+
}
62+
}
5463
}
5564
break;
5665
case 'proxy':
@@ -103,7 +112,16 @@ export async function buildRouter(mount: ConfigMount[], log: (info: LogInfo) =>
103112
case 'redirect-map':
104113
const mapper = new Mapper(item.options.caseSensitive);
105114
if (typeof item.mapping === 'string') {
106-
const content = await readFile(item.mapping, 'utf-8');
115+
let content: string;
116+
try {
117+
content = await readFile(item.mapping, 'utf-8');
118+
} catch (error: unknown) {
119+
if (error && typeof error === 'object' && 'code' in error && error.code === 'ENOENT') {
120+
throw new TransientError(`redirect-map file not found at ${item.mapping}`);
121+
} else {
122+
throw error;
123+
}
124+
}
107125
tokens: for (const statement of nginxTokenise(content)) {
108126
const key = statement[0]!;
109127
if (key.literal) {

0 commit comments

Comments
 (0)