Skip to content

Commit bf6e0bc

Browse files
carlalexandercarlalexander
committed
fix: restart php-fpm and retry once after fastcgi connect failure
1 parent 471d578 commit bf6e0bc

2 files changed

Lines changed: 180 additions & 19 deletions

File tree

‎src/FastCgi/PhpFpmProcess.php‎

Lines changed: 57 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -98,28 +98,31 @@ public static function createForConfig(Logger $logger, string $configPath = self
9898
*/
9999
public function handle(ProvidesRequestData $request, int $timeoutMs): ProvidesResponseData
100100
{
101-
try {
102-
$response = $this->client->handle($request, $timeoutMs);
103-
} catch (ConnectException $exception) {
104-
throw new PhpFpmProcessException('Unable to connect to PHP-FPM FastCGI socket');
105-
} catch (ReadFailedException $exception) {
106-
throw new PhpFpmProcessException('PHP-FPM process crashed unexpectedly');
107-
} catch (TimedoutException $exception) {
108-
$message = sprintf('PHP-FPM request timed out after %dms', $timeoutMs);
101+
$hasRetriedAfterConnectException = false;
109102

110-
$this->logger->info($message);
103+
while (true) {
104+
try {
105+
return $this->handleRequest($request, $timeoutMs);
106+
} catch (ConnectException $exception) {
107+
$message = 'Unable to connect to PHP-FPM FastCGI socket';
111108

112-
$this->restart();
109+
if (true === $hasRetriedAfterConnectException) {
110+
$this->logger->info(sprintf('%s after retry', $message));
113111

114-
throw new PhpFpmTimeoutException($message);
115-
}
112+
throw new PhpFpmProcessException($message);
113+
}
116114

117-
// This also triggers "updateStatus" inside the Symfony process which will make it output the logs from PHP-FPM.
118-
if (!$this->process->isRunning()) {
119-
throw new PhpFpmProcessException('PHP-FPM has stopped unexpectedly');
120-
}
115+
$this->logger->info(sprintf('%s, restarting process and retrying request', $message));
121116

122-
return $response;
117+
$this->restart();
118+
119+
$hasRetriedAfterConnectException = true;
120+
} catch (PhpFpmTimeoutException $exception) {
121+
$this->restartAfterTimeout();
122+
123+
throw $exception;
124+
}
125+
}
123126
}
124127

125128
/**
@@ -166,6 +169,31 @@ public function stop(): void
166169
$this->process->stop();
167170
}
168171

172+
/**
173+
* Handle the request with the PHP-FPM process.
174+
*/
175+
private function handleRequest(ProvidesRequestData $request, int $timeoutMs): ProvidesResponseData
176+
{
177+
try {
178+
$response = $this->client->handle($request, $timeoutMs);
179+
} catch (ReadFailedException $exception) {
180+
throw new PhpFpmProcessException('PHP-FPM process crashed unexpectedly');
181+
} catch (TimedoutException $exception) {
182+
$message = sprintf('PHP-FPM request timed out after %dms', $timeoutMs);
183+
184+
$this->logger->info($message);
185+
186+
throw new PhpFpmTimeoutException($message);
187+
}
188+
189+
// This also triggers "updateStatus" inside the Symfony process which will make it output the logs from PHP-FPM.
190+
if (!$this->process->isRunning()) {
191+
throw new PhpFpmProcessException('PHP-FPM has stopped unexpectedly');
192+
}
193+
194+
return $response;
195+
}
196+
169197
/**
170198
* Checks if the PHP-FPM process is started.
171199
*/
@@ -200,6 +228,18 @@ private function restart(): void
200228
$this->start();
201229
}
202230

231+
/**
232+
* Attempt to restart PHP-FPM after a timeout without changing the request outcome.
233+
*/
234+
private function restartAfterTimeout(): void
235+
{
236+
try {
237+
$this->restart();
238+
} catch (\Throwable $exception) {
239+
$this->logger->info(sprintf('Failed to restart PHP-FPM process after timeout: %s', $exception->getMessage()));
240+
}
241+
}
242+
203243
/**
204244
* Wait for the given callback to finish.
205245
*/

‎tests/Unit/FastCgi/PhpFpmProcessTest.php‎

Lines changed: 123 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -82,6 +82,60 @@ public function testHandle(): void
8282
}
8383

8484
public function testHandleWithConnectException(): void
85+
{
86+
$client = $this->getFastCgiServerClientMock();
87+
$logger = $this->getLoggerMock();
88+
$process = $this->getProcessMock();
89+
$request = $this->getProvidesRequestDataMock();
90+
$response = $this->getProvidesResponseDataMock();
91+
$calls = 0;
92+
93+
$this->getFunctionMock($this->getNamespace(PhpFpmProcess::class), 'file_exists')
94+
->expects($this->once())
95+
->with('/tmp/.ymir/php-fpm.sock')
96+
->willReturn(false);
97+
$this->getFunctionMock($this->getNamespace(PhpFpmProcess::class), 'unlink')
98+
->expects($this->never());
99+
100+
$client->expects($this->exactly(2))
101+
->method('handle')
102+
->willReturnCallback(function ($receivedRequest, $receivedTimeoutMs) use (&$calls, $request, $response) {
103+
++$calls;
104+
105+
$this->assertSame($request, $receivedRequest);
106+
$this->assertSame(1000, $receivedTimeoutMs);
107+
108+
if (1 === $calls) {
109+
throw new ConnectException('connection refused');
110+
}
111+
112+
return $response;
113+
});
114+
115+
$logger->expects($this->exactly(2))
116+
->method('info')
117+
->withConsecutive(
118+
['Unable to connect to PHP-FPM FastCGI socket, restarting process and retrying request'],
119+
['Restarting PHP-FPM process']
120+
);
121+
122+
$process->method('isRunning')
123+
->willReturn(true);
124+
125+
$phpFpmProcess = $this->getMockBuilder(PhpFpmProcess::class)
126+
->setConstructorArgs([$client, $logger, $process])
127+
->setMethods(['start', 'stop'])
128+
->getMock();
129+
130+
$phpFpmProcess->expects($this->once())
131+
->method('stop');
132+
$phpFpmProcess->expects($this->once())
133+
->method('start');
134+
135+
$this->assertSame($response, $phpFpmProcess->handle($request, 1000));
136+
}
137+
138+
public function testHandleWithConnectExceptionAfterRetry(): void
85139
{
86140
$this->expectException(PhpFpmProcessException::class);
87141
$this->expectExceptionMessage('Unable to connect to PHP-FPM FastCGI socket');
@@ -91,12 +145,35 @@ public function testHandleWithConnectException(): void
91145
$process = $this->getProcessMock();
92146
$request = $this->getProvidesRequestDataMock();
93147

94-
$client->expects($this->once())
148+
$this->getFunctionMock($this->getNamespace(PhpFpmProcess::class), 'file_exists')
149+
->expects($this->once())
150+
->with('/tmp/.ymir/php-fpm.sock')
151+
->willReturn(false);
152+
$this->getFunctionMock($this->getNamespace(PhpFpmProcess::class), 'unlink')
153+
->expects($this->never());
154+
155+
$client->expects($this->exactly(2))
95156
->method('handle')
96157
->with($this->identicalTo($request), 1000)
97158
->willThrowException(new ConnectException('connection refused'));
98159

99-
$phpFpmProcess = new PhpFpmProcess($client, $logger, $process);
160+
$logger->expects($this->exactly(3))
161+
->method('info')
162+
->withConsecutive(
163+
['Unable to connect to PHP-FPM FastCGI socket, restarting process and retrying request'],
164+
['Restarting PHP-FPM process'],
165+
['Unable to connect to PHP-FPM FastCGI socket after retry']
166+
);
167+
168+
$phpFpmProcess = $this->getMockBuilder(PhpFpmProcess::class)
169+
->setConstructorArgs([$client, $logger, $process])
170+
->setMethods(['start', 'stop'])
171+
->getMock();
172+
173+
$phpFpmProcess->expects($this->once())
174+
->method('stop');
175+
$phpFpmProcess->expects($this->once())
176+
->method('start');
100177

101178
$phpFpmProcess->handle($request, 1000);
102179
}
@@ -189,6 +266,50 @@ public function testHandleWithTimeout(): void
189266
$phpFpmProcess->handle($request, 1000);
190267
}
191268

269+
public function testHandleWithTimeoutAndRestartFailure(): void
270+
{
271+
$this->expectException(PhpFpmTimeoutException::class);
272+
$this->expectExceptionMessage('PHP-FPM request timed out after 1000ms');
273+
274+
$client = $this->getFastCgiServerClientMock();
275+
$logger = $this->getLoggerMock();
276+
$process = $this->getProcessMock();
277+
$request = $this->getProvidesRequestDataMock();
278+
279+
$this->getFunctionMock($this->getNamespace(PhpFpmProcess::class), 'file_exists')
280+
->expects($this->once())
281+
->with('/tmp/.ymir/php-fpm.sock')
282+
->willReturn(false);
283+
$this->getFunctionMock($this->getNamespace(PhpFpmProcess::class), 'unlink')
284+
->expects($this->never());
285+
286+
$client->expects($this->once())
287+
->method('handle')
288+
->with($this->identicalTo($request), 1000)
289+
->willThrowException(new \hollodotme\FastCGI\Exceptions\TimedoutException());
290+
291+
$logger->expects($this->exactly(3))
292+
->method('info')
293+
->withConsecutive(
294+
['PHP-FPM request timed out after 1000ms'],
295+
['Restarting PHP-FPM process'],
296+
['Failed to restart PHP-FPM process after timeout: PHP-FPM process failed to start']
297+
);
298+
299+
$phpFpmProcess = $this->getMockBuilder(PhpFpmProcess::class)
300+
->setConstructorArgs([$client, $logger, $process])
301+
->setMethods(['start', 'stop'])
302+
->getMock();
303+
304+
$phpFpmProcess->expects($this->once())
305+
->method('stop');
306+
$phpFpmProcess->expects($this->once())
307+
->method('start')
308+
->willThrowException(new PhpFpmProcessException('PHP-FPM process failed to start'));
309+
310+
$phpFpmProcess->handle($request, 1000);
311+
}
312+
192313
public function testHandleWithTimeoutWithoutSocketFile(): void
193314
{
194315
$this->expectException(PhpFpmTimeoutException::class);

0 commit comments

Comments
 (0)