[4.x] Worker QoL

app/Actions/Site/UpdateSiteWorkerEnvironment.php

@@ -0,0 +1,38 @@
+<?php
+
+namespace App\Actions\Site;
+
+use App\Actions\Worker\UpdateWorkerEnvironment;
+use App\Actions\Worker\WorkerEnvironmentUpdateResult;
+use App\Exceptions\SSHError;
+use App\Models\Site;
+use App\SiteTypes\AbstractProxiedSiteType;
+use Illuminate\Support\Facades\Validator;
+
+class UpdateSiteWorkerEnvironment
+{
+    /**
+     * @param  array<string, mixed>  $input
+     *
+     * @throws SSHError
+     */
+    public function update(Site $site, array $input): WorkerEnvironmentUpdateResult
+    {
+        $type = $site->type();
+        $worker = $type instanceof AbstractProxiedSiteType ? $type->bootstrapWorker() : null;
+
+        if ($worker !== null) {
+            return app(UpdateWorkerEnvironment::class)->update($worker, $input);
+        }
+
+        $validated = Validator::make($input, UpdateWorkerEnvironment::rules())->validate();
+
+        $site->worker_environment = UpdateWorkerEnvironment::processVariables(
+            $validated['variables'],
+            $site->worker_environment,
+        );
+        $site->save();
+
+        return WorkerEnvironmentUpdateResult::PreFirstDeploy;
+    }
+}

app/Actions/Worker/CreateWorker.php

@@ -37,7 +37,9 @@ public function create(Server $server, array $input, ?Site $site = null): Worker
             'auto_start' => $input['auto_start'] ? 1 : 0,
             'auto_restart' => $input['auto_restart'] ? 1 : 0,
             'numprocs' => $input['numprocs'],
-            'environment' => $input['environment'] ?? null,
+            'environment' => isset($input['environment'])
+                ? UpdateWorkerEnvironment::processVariables($input['environment'], null)
+                : null,
             'status' => WorkerStatus::CREATING,
         ]);
         $worker->save();
@@ -83,6 +85,13 @@ private function validate(Server $server, array $input, ?Site $site = null): voi
                 'numeric',
                 'min:1',
             ],
+            'environment' => [
+                'sometimes',
+                'nullable',
+                'array',
+                'max:100',
+            ],
+            ...UpdateWorkerEnvironment::nestedRules('environment'),
         ];
 
         // Add site_id validation if provided in input

app/Actions/Worker/RestartAllWorkers.php

@@ -0,0 +1,27 @@
+<?php
+
+namespace App\Actions\Worker;
+
+use App\Enums\WorkerStatus;
+use App\Jobs\Worker\RestartAllJob;
+use App\Models\Server;
+use App\Models\Site;
+use App\Models\Worker;
+
+class RestartAllWorkers
+{
+    public function restart(Server $server, ?Site $site = null): void
+    {
+        $server->workers()
+            ->when($site, fn ($query) => $query->where('site_id', $site->id))
+            ->whereNotIn('status', [WorkerStatus::CREATING, WorkerStatus::DELETING])
+            ->get()
+            ->each(function (Worker $worker): void {
+                $worker->status = WorkerStatus::RESTARTING;
+                $worker->error = null;
+                $worker->save();
+            });
+
+        dispatch(new RestartAllJob($server, $site))->onQueue('ssh');
+    }
+}

app/Actions/Worker/SyncWorkerStatuses.php

@@ -0,0 +1,128 @@
+<?php
+
+namespace App\Actions\Worker;
+
+use App\Actions\Site\BroadcastSiteUpdate;
+use App\Enums\WorkerStatus;
+use App\Models\Server;
+use App\Models\Service;
+use App\Models\Site;
+use App\Models\Worker;
+use App\Services\ProcessManager\ProcessManager;
+use App\Traits\HandlesWorkerFailure;
+
+class SyncWorkerStatuses
+{
+    use HandlesWorkerFailure;
+
+    private const LOG_TYPE = 'sync-worker-statuses-failed';
+
+    public function sync(Server $server, ?Site $site = null): int
+    {
+        /** @var Service $service */
+        $service = $server->processManager();
+        /** @var ProcessManager $handler */
+        $handler = $service->handler();
+
+        $workers = $server->workers()
+            ->when($site, fn ($query) => $query->where('site_id', $site->id))
+            ->whereNotIn('status', [WorkerStatus::CREATING, WorkerStatus::DELETING])
+            ->get();
+
+        if ($workers->isEmpty()) {
+            return 0;
+        }
+
+        $statuses = $handler->statuses();
+
+        $changed = $workers->filter(fn (Worker $worker): bool => $this->settle($worker, $statuses[$worker->id] ?? []));
+
+        $changed->loadMissing('site')
+            ->pluck('site')
+            ->filter()
+            ->unique('id')
+            ->each(fn (Site $workerSite) => app(BroadcastSiteUpdate::class)->broadcast($workerSite));
+
+        return $changed->count();
+    }
+
+    /**
+     * @param  array<string, array{state: string, description: string}>  $processes
+     */
+    private function settle(Worker $worker, array $processes): bool
+    {
+        [$status, $error] = $this->target($processes);
+
+        if ($worker->status === $status && $worker->error === $error) {
+            return false;
+        }
+
+        if ($status === WorkerStatus::FAILED) {
+            $this->failWorker($worker, $error, self::LOG_TYPE, (string) $error);
+
+            return true;
+        }
+
+        $worker->status = $status;
+        $worker->error = null;
+        $worker->save();
+
+        $this->broadcastWorkerUpdate($worker);
+
+        return true;
+    }
+
+    /**
+     * @param  array<string, array{state: string, description: string}>  $processes
+     * @return array{0: WorkerStatus, 1: ?string}
+     */
+    private function target(array $processes): array
+    {
+        if ($processes === []) {
+            return [WorkerStatus::FAILED, 'Process not found in supervisor'];
+        }
+
+        $worst = WorkerStatus::RUNNING;
+        $errors = [];
+
+        foreach ($processes as $process => $info) {
+            $status = $this->mapState($info['state']);
+
+            if ($status === WorkerStatus::FAILED) {
+                $errors[] = trim("{$process}: {$info['state']} {$info['description']}");
+            }
+
+            if ($this->severity($status) > $this->severity($worst)) {
+                $worst = $status;
+            }
+        }
+
+        if ($worst === WorkerStatus::FAILED) {
+            return [WorkerStatus::FAILED, mb_substr(implode("\n", $errors), 0, 500)];
+        }
+
+        return [$worst, null];
+    }
+
+    private function mapState(string $state): WorkerStatus
+    {
+        return match ($state) {
+            'RUNNING' => WorkerStatus::RUNNING,
+            'STARTING' => WorkerStatus::STARTING,
+            'STOPPING' => WorkerStatus::STOPPING,
+            'STOPPED', 'EXITED' => WorkerStatus::STOPPED,
+            default => WorkerStatus::FAILED,
+        };
+    }
+
+    private function severity(WorkerStatus $status): int
+    {
+        return match ($status) {
+            WorkerStatus::FAILED => 4,
+            WorkerStatus::STOPPED => 3,
+            WorkerStatus::STOPPING => 2,
+            WorkerStatus::STARTING => 1,
+            default => 0,
+        };
+    }
+}

app/Actions/Worker/UpdateWorkerEnvironment.php

@@ -0,0 +1,88 @@
+<?php
+
+namespace App\Actions\Worker;
+
+use App\Exceptions\SSHError;
+use App\Helpers\EnvParser;
+use App\Models\Worker;
+use App\Services\ProcessManager\ProcessManager;
+use Illuminate\Support\Facades\Validator;
+use Illuminate\Validation\ValidationException;
+
+class UpdateWorkerEnvironment
+{
+    /**
+     * @param  array<string, mixed>  $input
+     *
+     * @throws SSHError
+     * @throws ValidationException
+     */
+    public function update(Worker $worker, array $input): WorkerEnvironmentUpdateResult
+    {
+        $validated = Validator::make($input, [
+            ...self::rules(),
+            'restart' => ['sometimes', 'boolean'],
+        ])->validate();
+
+        $worker->environment = self::processVariables($validated['variables'], $worker->environment);
+        $worker->save();
+
+        /** @var ProcessManager $processManager */
+        $processManager = $worker->server->processManager()->handler();
+        $processManager->writeConfig($worker);
+
+        if ($validated['restart'] ?? false) {
+            app(ManageWorker::class)->restart($worker);
+
+            return WorkerEnvironmentUpdateResult::Restarting;
+        }
+
+        return WorkerEnvironmentUpdateResult::PendingRestart;
+    }
+
+    /**
+     * @return array<string, array<int, string>>
+     */
+    public static function rules(string $attribute = 'variables'): array
+    {
+        return [
+            $attribute => ['present', 'array', 'max:100'],
+            ...self::nestedRules($attribute),
+        ];
+    }
+
+    /**
+     * @return array<string, array<int, string>>
+     */
+    public static function nestedRules(string $attribute = 'variables'): array
+    {
+        return [
+            "{$attribute}.*.key" => ['required', 'string', 'max:255', 'regex:/^[A-Za-z_][A-Za-z0-9_]*$/', 'distinct'],
+            "{$attribute}.*.value" => ['present', 'nullable', 'string', 'max:10000', 'regex:/\A[^\x00-\x1F\x7F"]*\z/'],
+            "{$attribute}.*.is_secret" => ['required', 'boolean'],
+        ];
+    }
+
+    /**
+     * @param  array<int, array<string, mixed>>  $incoming
+     * @param  ?array<int, array{key: string, value: string, is_secret: bool}>  $stored
+     * @return array<int, array{key: string, value: string, is_secret: bool}>
+     */
+    public static function processVariables(array $incoming, ?array $stored): array
+    {
+        $normalized = array_map(fn (array $variable): array => [
+            'key' => (string) $variable['key'],
+            'value' => (string) ($variable['value'] ?? ''),
+            'is_secret' => (bool) ($variable['is_secret'] ?? false),
+        ], $incoming);
+
+        return EnvParser::mergeWithStored($normalized, $stored);
+    }
+}
+
+enum WorkerEnvironmentUpdateResult
+{
+    case PreFirstDeploy;
+    case PendingRestart;
+    case Restarting;
+}

app/Http/Controllers/API/WorkerController.php

@@ -7,6 +7,8 @@
 use App\Actions\Worker\EditWorker;
 use App\Actions\Worker\GetWorkerLogs;
 use App\Actions\Worker\ManageWorker;
+use App\Actions\Worker\RestartAllWorkers;
+use App\Actions\Worker\SyncWorkerStatuses;
 use App\Http\Controllers\Controller;
 use App\Http\Resources\WorkerResource;
 use App\Models\Project;
@@ -23,6 +25,7 @@
 use Spatie\RouteAttributes\Attributes\Post;
 use Spatie\RouteAttributes\Attributes\Prefix;
 use Spatie\RouteAttributes\Attributes\Put;
+use Spatie\RouteAttributes\Attributes\WhereNumber;
 
 #[Prefix('api/projects/{project}/servers/{server}')]
 #[Middleware(['auth:sanctum', 'can-see-project'])]
@@ -76,7 +79,38 @@ public function siteShow(Project $project, Server $server, Site $site, Worker $w
         return new WorkerResource($worker);
     }
 
+    #[Post('/workers/resync/{site?}', name: 'api.projects.servers.workers.resync', middleware: 'ability:write')]
+    #[WhereNumber('site')]
+    public function resync(Project $project, Server $server, ?Site $site = null): JsonResponse
+    {
+        $this->authorize('update', [$project, $server]);
+
+        $this->validateRoute($project, $server, $site);
+
+        $count = app(SyncWorkerStatuses::class)->sync($server, $site);
+
+        return response()->json([
+            'synced' => $count,
+        ]);
+    }
+
+    #[Post('/workers/restart-all/{site?}', name: 'api.projects.servers.workers.restart-all', middleware: 'ability:write')]
+    #[WhereNumber('site')]
+    public function restartAll(Project $project, Server $server, ?Site $site = null): JsonResponse
+    {
+        $this->authorize('update', [$project, $server]);
+
+        $this->validateRoute($project, $server, $site);
+
+        app(RestartAllWorkers::class)->restart($server, $site);
+
+        return response()->json([
+            'message' => 'Workers are being restarted.',
+        ], 202);
+    }
+
     #[Post('/workers/{site?}', name: 'api.projects.servers.workers.create', middleware: 'ability:write')]
+    #[WhereNumber('site')]
     public function create(Request $request, Project $project, Server $server, ?Site $site = null): WorkerResource
     {
         $this->authorize('create', [$project, $server, $site]);