Skip to content

Commit aa266e1

Browse files
RichardAndersonRichardAnderson
authored
[4.x] Worker QoL (#1147)
* restart workers + qol * resync + restart + env * pre-commit * qol * coderabit fixes * lint
1 parent 829c726 commit aa266e1

42 files changed

Lines changed: 2093 additions & 96 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎app/Actions/Site/UpdateSiteWorkerEnvironment.php‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
<?php
2+
3+
namespace App\Actions\Site;
4+
5+
use App\Actions\Worker\UpdateWorkerEnvironment;
6+
use App\Actions\Worker\WorkerEnvironmentUpdateResult;
7+
use App\Exceptions\SSHError;
8+
use App\Models\Site;
9+
use App\SiteTypes\AbstractProxiedSiteType;
10+
use Illuminate\Support\Facades\Validator;
11+
12+
class UpdateSiteWorkerEnvironment
13+
{
14+
/**
15+
* @param array<string, mixed> $input
16+
*
17+
* @throws SSHError
18+
*/
19+
public function update(Site $site, array $input): WorkerEnvironmentUpdateResult
20+
{
21+
$type = $site->type();
22+
$worker = $type instanceof AbstractProxiedSiteType ? $type->bootstrapWorker() : null;
23+
24+
if ($worker !== null) {
25+
return app(UpdateWorkerEnvironment::class)->update($worker, $input);
26+
}
27+
28+
$validated = Validator::make($input, UpdateWorkerEnvironment::rules())->validate();
29+
30+
$site->worker_environment = UpdateWorkerEnvironment::processVariables(
31+
$validated['variables'],
32+
$site->worker_environment,
33+
);
34+
$site->save();
35+
36+
return WorkerEnvironmentUpdateResult::PreFirstDeploy;
37+
}
38+
}

‎app/Actions/Worker/CreateWorker.php‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,9 @@ public function create(Server $server, array $input, ?Site $site = null): Worker
3737
'auto_start' => $input['auto_start'] ? 1 : 0,
3838
'auto_restart' => $input['auto_restart'] ? 1 : 0,
3939
'numprocs' => $input['numprocs'],
40-
'environment' => $input['environment'] ?? null,
40+
'environment' => isset($input['environment'])
41+
? UpdateWorkerEnvironment::processVariables($input['environment'], null)
42+
: null,
4143
'status' => WorkerStatus::CREATING,
4244
]);
4345
$worker->save();
@@ -83,6 +85,13 @@ private function validate(Server $server, array $input, ?Site $site = null): voi
8385
'numeric',
8486
'min:1',
8587
],
88+
'environment' => [
89+
'sometimes',
90+
'nullable',
91+
'array',
92+
'max:100',
93+
],
94+
...UpdateWorkerEnvironment::nestedRules('environment'),
8695
];
8796

8897
// Add site_id validation if provided in input

‎app/Actions/Worker/RestartAllWorkers.php‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
<?php
2+
3+
namespace App\Actions\Worker;
4+
5+
use App\Enums\WorkerStatus;
6+
use App\Jobs\Worker\RestartAllJob;
7+
use App\Models\Server;
8+
use App\Models\Site;
9+
use App\Models\Worker;
10+
11+
class RestartAllWorkers
12+
{
13+
public function restart(Server $server, ?Site $site = null): void
14+
{
15+
$server->workers()
16+
->when($site, fn ($query) => $query->where('site_id', $site->id))
17+
->whereNotIn('status', [WorkerStatus::CREATING, WorkerStatus::DELETING])
18+
->get()
19+
->each(function (Worker $worker): void {
20+
$worker->status = WorkerStatus::RESTARTING;
21+
$worker->error = null;
22+
$worker->save();
23+
});
24+
25+
dispatch(new RestartAllJob($server, $site))->onQueue('ssh');
26+
}
27+
}

‎app/Actions/Worker/SyncWorkerStatuses.php‎

Lines changed: 128 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,128 @@
1+
<?php
2+
3+
namespace App\Actions\Worker;
4+
5+
use App\Actions\Site\BroadcastSiteUpdate;
6+
use App\Enums\WorkerStatus;
7+
use App\Models\Server;
8+
use App\Models\Service;
9+
use App\Models\Site;
10+
use App\Models\Worker;
11+
use App\Services\ProcessManager\ProcessManager;
12+
use App\Traits\HandlesWorkerFailure;
13+
14+
class SyncWorkerStatuses
15+
{
16+
use HandlesWorkerFailure;
17+
18+
private const LOG_TYPE = 'sync-worker-statuses-failed';
19+
20+
public function sync(Server $server, ?Site $site = null): int
21+
{
22+
/** @var Service $service */
23+
$service = $server->processManager();
24+
/** @var ProcessManager $handler */
25+
$handler = $service->handler();
26+
27+
$workers = $server->workers()
28+
->when($site, fn ($query) => $query->where('site_id', $site->id))
29+
->whereNotIn('status', [WorkerStatus::CREATING, WorkerStatus::DELETING])
30+
->get();
31+
32+
if ($workers->isEmpty()) {
33+
return 0;
34+
}
35+
36+
$statuses = $handler->statuses();
37+
38+
$changed = $workers->filter(fn (Worker $worker): bool => $this->settle($worker, $statuses[$worker->id] ?? []));
39+
40+
$changed->loadMissing('site')
41+
->pluck('site')
42+
->filter()
43+
->unique('id')
44+
->each(fn (Site $workerSite) => app(BroadcastSiteUpdate::class)->broadcast($workerSite));
45+
46+
return $changed->count();
47+
}
48+
49+
/**
50+
* @param array<string, array{state: string, description: string}> $processes
51+
*/
52+
private function settle(Worker $worker, array $processes): bool
53+
{
54+
[$status, $error] = $this->target($processes);
55+
56+
if ($worker->status === $status && $worker->error === $error) {
57+
return false;
58+
}
59+
60+
if ($status === WorkerStatus::FAILED) {
61+
$this->failWorker($worker, $error, self::LOG_TYPE, (string) $error);
62+
63+
return true;
64+
}
65+
66+
$worker->status = $status;
67+
$worker->error = null;
68+
$worker->save();
69+
70+
$this->broadcastWorkerUpdate($worker);
71+
72+
return true;
73+
}
74+
75+
/**
76+
* @param array<string, array{state: string, description: string}> $processes
77+
* @return array{0: WorkerStatus, 1: ?string}
78+
*/
79+
private function target(array $processes): array
80+
{
81+
if ($processes === []) {
82+
return [WorkerStatus::FAILED, 'Process not found in supervisor'];
83+
}
84+
85+
$worst = WorkerStatus::RUNNING;
86+
$errors = [];
87+
88+
foreach ($processes as $process => $info) {
89+
$status = $this->mapState($info['state']);
90+
91+
if ($status === WorkerStatus::FAILED) {
92+
$errors[] = trim("{$process}: {$info['state']} {$info['description']}");
93+
}
94+
95+
if ($this->severity($status) > $this->severity($worst)) {
96+
$worst = $status;
97+
}
98+
}
99+
100+
if ($worst === WorkerStatus::FAILED) {
101+
return [WorkerStatus::FAILED, mb_substr(implode("\n", $errors), 0, 500)];
102+
}
103+
104+
return [$worst, null];
105+
}
106+
107+
private function mapState(string $state): WorkerStatus
108+
{
109+
return match ($state) {
110+
'RUNNING' => WorkerStatus::RUNNING,
111+
'STARTING' => WorkerStatus::STARTING,
112+
'STOPPING' => WorkerStatus::STOPPING,
113+
'STOPPED', 'EXITED' => WorkerStatus::STOPPED,
114+
default => WorkerStatus::FAILED,
115+
};
116+
}
117+
118+
private function severity(WorkerStatus $status): int
119+
{
120+
return match ($status) {
121+
WorkerStatus::FAILED => 4,
122+
WorkerStatus::STOPPED => 3,
123+
WorkerStatus::STOPPING => 2,
124+
WorkerStatus::STARTING => 1,
125+
default => 0,
126+
};
127+
}
128+
}

‎app/Actions/Worker/UpdateWorkerEnvironment.php‎

Lines changed: 88 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,88 @@
1+
<?php
2+
3+
namespace App\Actions\Worker;
4+
5+
use App\Exceptions\SSHError;
6+
use App\Helpers\EnvParser;
7+
use App\Models\Worker;
8+
use App\Services\ProcessManager\ProcessManager;
9+
use Illuminate\Support\Facades\Validator;
10+
use Illuminate\Validation\ValidationException;
11+
12+
class UpdateWorkerEnvironment
13+
{
14+
/**
15+
* @param array<string, mixed> $input
16+
*
17+
* @throws SSHError
18+
* @throws ValidationException
19+
*/
20+
public function update(Worker $worker, array $input): WorkerEnvironmentUpdateResult
21+
{
22+
$validated = Validator::make($input, [
23+
...self::rules(),
24+
'restart' => ['sometimes', 'boolean'],
25+
])->validate();
26+
27+
$worker->environment = self::processVariables($validated['variables'], $worker->environment);
28+
$worker->save();
29+
30+
/** @var ProcessManager $processManager */
31+
$processManager = $worker->server->processManager()->handler();
32+
$processManager->writeConfig($worker);
33+
34+
if ($validated['restart'] ?? false) {
35+
app(ManageWorker::class)->restart($worker);
36+
37+
return WorkerEnvironmentUpdateResult::Restarting;
38+
}
39+
40+
return WorkerEnvironmentUpdateResult::PendingRestart;
41+
}
42+
43+
/**
44+
* @return array<string, array<int, string>>
45+
*/
46+
public static function rules(string $attribute = 'variables'): array
47+
{
48+
return [
49+
$attribute => ['present', 'array', 'max:100'],
50+
...self::nestedRules($attribute),
51+
];
52+
}
53+
54+
/**
55+
* @return array<string, array<int, string>>
56+
*/
57+
public static function nestedRules(string $attribute = 'variables'): array
58+
{
59+
return [
60+
"{$attribute}.*.key" => ['required', 'string', 'max:255', 'regex:/^[A-Za-z_][A-Za-z0-9_]*$/', 'distinct'],
61+
"{$attribute}.*.value" => ['present', 'nullable', 'string', 'max:10000', 'regex:/\A[^\x00-\x1F\x7F"]*\z/'],
62+
"{$attribute}.*.is_secret" => ['required', 'boolean'],
63+
];
64+
}
65+
66+
/**
67+
* @param array<int, array<string, mixed>> $incoming
68+
* @param ?array<int, array{key: string, value: string, is_secret: bool}> $stored
69+
* @return array<int, array{key: string, value: string, is_secret: bool}>
70+
*/
71+
public static function processVariables(array $incoming, ?array $stored): array
72+
{
73+
$normalized = array_map(fn (array $variable): array => [
74+
'key' => (string) $variable['key'],
75+
'value' => (string) ($variable['value'] ?? ''),
76+
'is_secret' => (bool) ($variable['is_secret'] ?? false),
77+
], $incoming);
78+
79+
return EnvParser::mergeWithStored($normalized, $stored);
80+
}
81+
}
82+
83+
enum WorkerEnvironmentUpdateResult
84+
{
85+
case PreFirstDeploy;
86+
case PendingRestart;
87+
case Restarting;
88+
}

‎app/Http/Controllers/API/WorkerController.php‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,8 @@
77
use App\Actions\Worker\EditWorker;
88
use App\Actions\Worker\GetWorkerLogs;
99
use App\Actions\Worker\ManageWorker;
10+
use App\Actions\Worker\RestartAllWorkers;
11+
use App\Actions\Worker\SyncWorkerStatuses;
1012
use App\Http\Controllers\Controller;
1113
use App\Http\Resources\WorkerResource;
1214
use App\Models\Project;
@@ -23,6 +25,7 @@
2325
use Spatie\RouteAttributes\Attributes\Post;
2426
use Spatie\RouteAttributes\Attributes\Prefix;
2527
use Spatie\RouteAttributes\Attributes\Put;
28+
use Spatie\RouteAttributes\Attributes\WhereNumber;
2629

2730
#[Prefix('api/projects/{project}/servers/{server}')]
2831
#[Middleware(['auth:sanctum', 'can-see-project'])]
@@ -76,7 +79,38 @@ public function siteShow(Project $project, Server $server, Site $site, Worker $w
7679
return new WorkerResource($worker);
7780
}
7881

82+
#[Post('/workers/resync/{site?}', name: 'api.projects.servers.workers.resync', middleware: 'ability:write')]
83+
#[WhereNumber('site')]
84+
public function resync(Project $project, Server $server, ?Site $site = null): JsonResponse
85+
{
86+
$this->authorize('update', [$project, $server]);
87+
88+
$this->validateRoute($project, $server, $site);
89+
90+
$count = app(SyncWorkerStatuses::class)->sync($server, $site);
91+
92+
return response()->json([
93+
'synced' => $count,
94+
]);
95+
}
96+
97+
#[Post('/workers/restart-all/{site?}', name: 'api.projects.servers.workers.restart-all', middleware: 'ability:write')]
98+
#[WhereNumber('site')]
99+
public function restartAll(Project $project, Server $server, ?Site $site = null): JsonResponse
100+
{
101+
$this->authorize('update', [$project, $server]);
102+
103+
$this->validateRoute($project, $server, $site);
104+
105+
app(RestartAllWorkers::class)->restart($server, $site);
106+
107+
return response()->json([
108+
'message' => 'Workers are being restarted.',
109+
], 202);
110+
}
111+
79112
#[Post('/workers/{site?}', name: 'api.projects.servers.workers.create', middleware: 'ability:write')]
113+
#[WhereNumber('site')]
80114
public function create(Request $request, Project $project, Server $server, ?Site $site = null): WorkerResource
81115
{
82116
$this->authorize('create', [$project, $server, $site]);

0 commit comments

Comments
 (0)