Stage restore downloads on the data volume, not the OS disk (#344)

marcpope · marcpope · commit a20a263855d5 · 2026-07-15T14:17:27.000-04:00
Restore-to-download extracted the full selection to /tmp — the (often
small) root filesystem. A large download filled it, which starved the
server-side borg serve of temp space and crashed every running backup
with 'No space left on device' self-test failures, while the actual
backup disk sat idle.

Downloads (and the S3 server-restore staging) now stage under
/var/bbs/tmp on the volume sized for backup data, with /tmp as a
fallback when that isn't writable. A pre-flight check estimates the
selection's uncompressed size (catalog sum for partial selections, the
archive's original_size for full ones) and refuses with a clear
message when the staging disk can't hold it, instead of dying
mid-extract. Cleanup now survives the browser disconnecting mid-stream
(ignore_user_abort + shutdown handler), and the scheduler sweeps
staging directories older than 24h hourly as a crash safety net.
diff --git a/bin/bbs-ssh-helper b/bin/bbs-ssh-helper
@@ -191,9 +191,10 @@ case "$ACTION" in
             exit 1
         fi
 
-        # Safety: tmpdir must be under /tmp
-        if [[ ! "$TMPDIR" =~ ^/tmp/ ]]; then
-            echo "Error: tmpdir must be under /tmp/"
+        # Safety: staging must be under the data volume's tmp (preferred —
+        # sized for backup data, #344) or legacy /tmp
+        if [[ ! "$TMPDIR" =~ ^(/var/bbs/tmp|/tmp)/ ]]; then
+            echo "Error: tmpdir must be under /var/bbs/tmp/ or /tmp/"
             exit 1
         fi
 
@@ -1163,8 +1164,8 @@ case "$ACTION" in
             echo "Error: filename must match bbs-backup-*.tar.gz"
             exit 1
         fi
-        if [[ ! "$DEST_DIR" =~ ^/tmp/bbs-restore-[0-9a-f]+$ ]]; then
-            echo "Error: dest_dir must be a /tmp/bbs-restore-* staging directory"
+        if [[ ! "$DEST_DIR" =~ ^(/var/bbs/tmp|/tmp)/bbs-restore-[0-9a-f]+$ ]]; then
+            echo "Error: dest_dir must be a bbs-restore-* staging directory under /var/bbs/tmp or /tmp"
             exit 1
         fi
 
@@ -1205,8 +1206,8 @@ case "$ACTION" in
             echo "Usage: bbs-ssh-helper server-restore <backup_file>"
             exit 1
         fi
-        if [[ ! "$BACKUP_FILE" =~ ^/tmp/bbs-restore-[0-9a-f]+/bbs-backup-[A-Za-z0-9._-]+\.tar\.gz$ ]]; then
-            echo "Error: backup file must be inside a /tmp/bbs-restore-* staging directory"
+        if [[ ! "$BACKUP_FILE" =~ ^(/var/bbs/tmp|/tmp)/bbs-restore-[0-9a-f]+/bbs-backup-[A-Za-z0-9._-]+\.tar\.gz$ ]]; then
+            echo "Error: backup file must be inside a bbs-restore-* staging directory under /var/bbs/tmp or /tmp"
             exit 1
         fi
         if [ ! -f "$BACKUP_FILE" ]; then
@@ -1382,9 +1383,9 @@ SSHEOF
             exit 1
         fi
 
-        # Safety: tmpdir must be under /tmp
-        if [[ ! "$TMPDIR" =~ ^/tmp/bbs-download- ]]; then
-            echo "Error: tmpdir must be under /tmp/bbs-download-*"
+        # Safety: must be a download staging dir on the data volume (or legacy /tmp)
+        if [[ ! "$TMPDIR" =~ ^(/var/bbs/tmp|/tmp)/bbs-download- ]]; then
+            echo "Error: tmpdir must be under /var/bbs/tmp/bbs-download-* or /tmp/bbs-download-*"
             exit 1
         fi
 
diff --git a/scheduler.php b/scheduler.php
@@ -2075,6 +2075,16 @@
     }
 }
 
+// Step 4c: Sweep stale download/restore staging directories (hourly).
+// Normally cleaned by the request that created them, but a PHP crash or
+// container restart mid-download can strand multi-GB extractions (#344).
+if ((int) date('i') === 0) {
+    foreach (['/var/bbs/tmp', '/tmp'] as $stagingBase) {
+        if (!is_dir($stagingBase)) continue;
+        exec('find ' . escapeshellarg($stagingBase) . ' -maxdepth 1 \\( -name "bbs-download-*" -o -name "bbs-restore-*" \\) -mmin +1440 -exec rm -rf {} + 2>/dev/null');
+    }
+}
+
 // Step 5: Bootstrap size for any local repo whose size_bytes is still 0
 // (fresh install, newly added repo, or legacy migration). Runs every minute
 // but only touches disks once per repo, since the UPDATE makes size_bytes > 0.
diff --git a/src/Controllers/ClientController.php b/src/Controllers/ClientController.php
@@ -1374,6 +1374,56 @@ public function restoreMongoSubmit(int $id): void
      * POST /clients/{id}/download
      * Extracts selected paths from a borg archive on the server and streams as tar.gz.
      */
+    /**
+     * Estimate the uncompressed size of a download selection so the space
+     * pre-flight can refuse before extraction (#344). Full archive: the
+     * stored original_size. Selection: catalog sum over the selected path
+     * prefixes (normalized to the catalog's leading-slash form, nested
+     * selections deduped). Returns null when no estimate is possible.
+     */
+    private function estimateDownloadBytes(int $agentId, int $archiveId, array $selectedFiles, int $archiveOriginalSize): ?int
+    {
+        if (empty($selectedFiles)) {
+            return $archiveOriginalSize > 0 ? $archiveOriginalSize : null;
+        }
+        try {
+            $ch = \BBS\Core\ClickHouse::getInstance();
+            if (!$ch->isAvailable()) {
+                return null;
+            }
+            $paths = array_values(array_filter(array_map(
+                fn($p) => '/' . trim((string) $p, '/'),
+                $selectedFiles
+            ), fn($p) => $p !== '/'));
+            $roots = array_filter($paths, function ($p) use ($paths) {
+                foreach ($paths as $other) {
+                    if ($other !== $p && str_starts_with($p, $other . '/')) return false;
+                }
+                return true;
+            });
+
+            $conds = [];
+            $params = [$agentId, $archiveId];
+            foreach ($roots as $p) {
+                $conds[] = '(path = ? OR startsWith(path, ?))';
+                $params[] = $p;
+                $params[] = $p . '/';
+            }
+            if (!$conds) {
+                return null;
+            }
+            $row = $ch->fetchOne(
+                "SELECT sum(file_size) AS bytes FROM file_catalog
+                 WHERE agent_id = ? AND archive_id = ? AND (" . implode(' OR ', $conds) . ")",
+                $params
+            );
+            $bytes = (int) ($row['bytes'] ?? 0);
+            return $bytes > 0 ? $bytes : null;
+        } catch (\Exception $e) {
+            return null;
+        }
+    }
+
     public function download(int $id): void
     {
         $this->requireAuth();
@@ -1428,12 +1478,53 @@ public function download(int $id): void
         $localPath = $isRemoteSsh ? $archive['repo_path'] : \BBS\Services\BorgCommandBuilder::getLocalRepoPath($repo);
         $env = \BBS\Services\BorgCommandBuilder::buildEnv($repo, false);
 
-        // Create temp directory for extraction
-        $tmpDir = '/tmp/bbs-download-' . bin2hex(random_bytes(8));
+        // Stage the extraction under the data volume, NOT the OS disk (#344):
+        // /tmp lives on the (often small) root filesystem — a large download
+        // filled it and starved borg-serve of temp space, crashing every
+        // running backup. /var/bbs is the volume actually sized for backups.
+        $stagingBase = '/var/bbs/tmp';
+        if (!is_dir($stagingBase) && !@mkdir($stagingBase, 0770, true)) {
+            $stagingBase = sys_get_temp_dir(); // fall back to /tmp if the volume is unwritable
+        }
+
+        // Pre-flight: refuse up front when the staging disk can't hold the
+        // selection, instead of dying mid-extract with a full disk.
+        $neededBytes = $this->estimateDownloadBytes($id, $archive_id, $selectedFiles, (int) $archive['original_size']);
+        $freeBytes = (float) @disk_free_space($stagingBase);
+        if ($neededBytes !== null && $freeBytes > 0 && $neededBytes * 1.05 > $freeBytes) {
+            $this->flash('danger', sprintf(
+                'Not enough space to prepare this download: it needs about %s, but only %s is free on the server (%s). Download a smaller selection or free up space.',
+                \BBS\Services\ServerStats::formatBytes((int) $neededBytes),
+                \BBS\Services\ServerStats::formatBytes((int) $freeBytes),
+                $stagingBase
+            ));
+            $this->redirect("/clients/{$id}?tab=restore");
+            return;
+        }
+
+        $tmpDir = $stagingBase . '/bbs-download-' . bin2hex(random_bytes(8));
         mkdir($tmpDir, 0700, true);
 
         $remoteSshKeyFile = null; // Track temp SSH key for cleanup
 
+        // The user closing the browser mid-download kills this request after
+        // the extract has already landed on disk — make sure the staging dir
+        // is removed no matter how the request ends.
+        ignore_user_abort(true);
+        register_shutdown_function(function () use ($tmpDir, &$remoteSshKeyFile) {
+            if ($remoteSshKeyFile && file_exists($remoteSshKeyFile)) {
+                @unlink($remoteSshKeyFile);
+            }
+            if (is_dir($tmpDir)) {
+                exec('rm -rf ' . escapeshellarg($tmpDir) . ' 2>/dev/null');
+                if (is_dir($tmpDir)) {
+                    // Files may be owned by the repo's bbs-* user — remove via helper perms fix + retry
+                    exec('sudo /usr/local/bin/bbs-ssh-helper fix-download-perms ' . escapeshellarg($tmpDir) . ' 2>/dev/null');
+                    exec('rm -rf ' . escapeshellarg($tmpDir) . ' 2>/dev/null');
+                }
+            }
+        });
+
         try {
             // Build borg extract args: repo::archive + selected paths
             $borgArgs = [$localPath . '::' . $archive['archive_name']];
diff --git a/src/Controllers/StorageLocationController.php b/src/Controllers/StorageLocationController.php
@@ -351,8 +351,12 @@ public function restoreS3Backup(): void
             return;
         }
 
-        // Create temp directory
-        $tmpDir = '/tmp/bbs-restore-' . bin2hex(random_bytes(8));
+        // Stage under the data volume, not the OS disk (#344)
+        $stagingBase = '/var/bbs/tmp';
+        if (!is_dir($stagingBase) && !@mkdir($stagingBase, 0770, true)) {
+            $stagingBase = '/tmp';
+        }
+        $tmpDir = $stagingBase . '/bbs-restore-' . bin2hex(random_bytes(8));
         mkdir($tmpDir, 0700, true);
 
         $helper = '/usr/local/bin/bbs-ssh-helper';
