feat(flash-backup): streaming download + save-to-server, with compression & progress

.gitignore

@@ -116,3 +116,4 @@ emhttp/plugins/pnpm-lock.yaml
 /etc/rc.d/rc0.d
 /etc/rc.d/rc6.d/K10flash-backup
 /etc/rc.d/rc6.d/K20unraid-api
+.gstack/

emhttp/plugins/dynamix/BootInfo.page

@@ -16,36 +16,23 @@ Tag="usb"
 ?>
 <?$tpmEnabled = !empty($var['flashGUID']) && strpos($var['flashGUID'], '01-') !== 0 && !empty($var['tpmGUID']) ? true : false;?>
 <script>
-function cleanUp(zip) {
-  if (document.hasFocus()) {
-    $('input[value="_(Creating boot device backup)_..."]').val("_(Boot device backup)_").prop('disabled',false);
-    $('div.spinner').hide('slow');
-    $('#pleaseWait').hide('slow');
-    $.post('/webGui/include/Download.php',{cmd:'unlink',file:zip});
+function backup() {
+  var level = $('#backupLevel').val();
+  if ($('#backupMode').val() === 'save') {
+    // Background job tracked by the shared task tray (openPlugin -> TaskQueue):
+    // it shows in the tray, opens the standard progress modal, and keeps running
+    // when the modal is closed. flash_backup_save publishes progress on the
+    // standard 'plugins' task channel and exits 0/non-zero for done/error.
+    openPlugin('flash_backup_save '+level, "_(Boot device backup)_", '', '', 1);
   } else {
-    setTimeout(function(){cleanUp(zip);},2000);
+    // Stream straight to the browser; the browser's own download indicator shows
+    // progress. A hidden iframe (not window.location) keeps us on this page.
+    $('#fbFrame').remove();
+    $('<iframe>',{id:'fbFrame',src:'/webGui/include/FlashBackup.php?level='+level}).hide().appendTo('body');
+    swal({title:"_(Boot device backup)_",text:"_(Your download will begin shortly)_",type:'info',html:true,animation:'none',timer:4000,showConfirmButton:false});
   }
 }
-function backup() {
-  $('input[value="_(Boot device backup)_"]').val('_(Creating boot device backup)_...').prop('disabled',true);
-  $('div.spinner').show('slow');
-  $('#pleaseWait').show('slow');
-  $.post('/webGui/include/Download.php',{cmd:'backup'},function(zip) {
-    if (zip) {
-      location = '/'+zip;
-      setTimeout(function(){cleanUp(zip);},6000);
-    } else {
-      var $btn = $('input[value="_(Creating boot device backup)_..."]');
-      $btn.val("_(Boot device backup)_").prop('disabled',false);
-      $('div.spinner').hide('slow');
-      $('#pleaseWait').hide('slow');
-      swal({title:"_(Creation error)_",text:"_(Insufficient free disk space available)_",type:'error',html:true,confirmButtonText:"_(Ok)_"});
-    }
-  });
-}
 </script>
-<div id="pleaseWait" style="display:none;text-align:center;margin-bottom:24px"><span class="red-text strong">_(Please wait)_... _(creating boot device backup zip file (this may take several minutes))_</span></div>
-
 _(Active GUID)_:
 : <?if (!empty($var['flashGUID'])):?>
     <?=htmlspecialchars($var['flashGUID'], ENT_QUOTES, 'UTF-8');?>
@@ -81,8 +68,48 @@ _(TPM GUID)_:
 
 :flash_backup_help:
 
+_(Backup mode)_:
+: <select id="backupMode">
+    <option value="download" selected>_(Download to your computer)_</option>
+    <option value="save">_(Save to the server)_</option>
+  </select>
+
+_(Compression level)_:
+: <select id="backupLevel">
+    <option value="0">_(None - fastest)_</option>
+    <option value="1">_(Low)_</option>
+    <option value="6" selected>_(Normal)_</option>
+    <option value="9">_(Maximum - smallest)_</option>
+  </select>
+
 &nbsp;
 : <span class="inline-block">
     <input type="button" value="_(Boot device backup)_" onclick="backup()">
     <input type="button" value="_(Done)_" onclick="done()">
   </span>
+
+<?
+// List boot backups already saved on the server (save-to-server mode keeps them
+// on a pool/array/boot mount). Build a single inline value string so it stays in
+// the definition-list value column like the rows above (a multi-line block here
+// breaks out of the markdown definition list and renders full-width).
+$fb_saved = [];
+foreach (array_merge(glob("/mnt/*/*-boot-backup-*.zip"), glob("/mnt/user/*/*-boot-backup-*.zip"), glob("/boot/*-boot-backup-*.zip")) as $fb_f) {
+  $fb_r = realpath($fb_f);
+  if ($fb_r && is_file($fb_r)) $fb_saved[$fb_r] = filesize($fb_r);
+}
+krsort($fb_saved);
+$fb_rows = [];
+foreach ($fb_saved as $fb_path => $fb_size) {
+  $fb_bn = basename($fb_path); $fb_dir = dirname($fb_path);
+  $fb_rows[] = "<code>".htmlspecialchars($fb_path, ENT_QUOTES, 'UTF-8')."</code> (".number_format($fb_size/1073741824, 2)." GB) &nbsp;"
+    ."<a href='/webGui/include/FlashBackup.php?serve=".urlencode($fb_bn)."'><i class='fa fa-download'></i> "._('Download')."</a>"
+    ." &nbsp;&middot;&nbsp; <a href='/Main/Browse?dir=".urlencode($fb_dir)."'><i class='fa fa-folder-open'></i> "._('File Manager')."</a>";
+}
+$fb_html = implode('<br>', $fb_rows);
+?>
+<?if ($fb_html):?>
+_(Saved backups on this server)_:
+: <?=$fb_html?>
+
+<?endif;?>

emhttp/plugins/dynamix/include/Download.php

@@ -49,8 +49,5 @@ function validpath($file) {
   if ($backup = readlink("$docroot/$file")) unlink($backup);
   @unlink("$docroot/$file");
   break;
-case 'backup':
-  echo exec("$docroot/webGui/scripts/flash_backup");
-  break;
 }
 ?>

emhttp/plugins/dynamix/include/FlashBackup.php

@@ -0,0 +1,68 @@
+<?PHP
+/* Copyright 2005-2025, Lime Technology
+ * Copyright 2012-2023, Bergware International.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License version 2,
+ * as published by the Free Software Foundation.
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ */
+?>
+<?
+/* Boot-device (flash) backup download endpoint.
+ *
+ *   ?level=0..9   stream a freshly-built zip of the boot device straight to the
+ *                 browser as it is built (nothing staged on disk/RAM). The
+ *                 browser's own download indicator shows progress.
+ *   ?serve=<name> stream an already-saved backup file (the save-to-server
+ *                 Download button / notification link). Strictly validated.
+ *
+ * `X-Accel-Buffering: no` tells nginx to stream the response instead of spooling
+ * it to a temp file. nginx requires a logged-in session to reach this endpoint.
+ */
+$docroot ??= ($_SERVER['DOCUMENT_ROOT'] ?: '/usr/local/emhttp');
+
+// "serve" mode: stream an already-saved backup. Strictly validated - only a file
+// whose name matches the flash-backup pattern AND that resolves onto a
+// pool/array/boot mount - so it can't be coerced into an arbitrary file read.
+if (isset($_GET['serve'])) {
+  $name = basename($_GET['serve']);
+  if (!preg_match('/-boot-backup-[0-9-]+\.zip$/', $name)) { http_response_code(404); exit; }
+  $found = null;
+  foreach (array_merge(glob("/mnt/*/$name"), glob("/mnt/user/*/$name"), ["/boot/$name"]) as $cand) {
+    $rp = realpath($cand);
+    if ($rp && is_file($rp) && basename($rp) === $name &&
+        (strncmp($rp, '/mnt/', 5) === 0 || strncmp($rp, '/boot/', 6) === 0)) { $found = $rp; break; }
+  }
+  if (!$found) { http_response_code(404); exit; }
+  set_time_limit(0);
+  header('Content-Type: application/zip');
+  header('Content-Disposition: attachment; filename="'.$name.'"');
+  header('Content-Length: '.filesize($found));
+  header('X-Accel-Buffering: no');
+  while (ob_get_level()) ob_end_clean();
+  readfile($found);
+  exit;
+}
+
+// Stream a freshly-built backup. Named server-version-date.zip.
+$var = (array)@parse_ini_file('/var/local/emhttp/var.ini');
+$level = isset($_GET['level']) && is_numeric($_GET['level']) ? max(0, min(9, (int)$_GET['level'])) : 6;
+$server = isset($var['NAME']) ? str_replace(' ', '_', strtolower($var['NAME'])) : 'tower';
+$osVersion = $var['version'] ?? 'unknown';
+$name = "$server-v$osVersion-boot-backup-".date('Ymd-Hi').".zip";
+
+set_time_limit(0);
+ignore_user_abort(false); // let zip die (SIGPIPE) if the user cancels the download
+
+header('Content-Type: application/zip');
+header('Content-Disposition: attachment; filename="'.$name.'"');
+header('X-Accel-Buffering: no');
+header('Cache-Control: no-store');
+while (ob_get_level()) ob_end_clean();
+
+passthru(escapeshellarg("$docroot/webGui/scripts/flash_backup").' '.(int)$level.' 2>/dev/null');
+exit;
+?>

emhttp/plugins/dynamix/scripts/flash_backup

@@ -1,6 +1,6 @@
 #!/usr/bin/php -q
 <?PHP
-/* Copyright 2005-2023, Lime Technology
+/* Copyright 2005-2025, Lime Technology
  * Copyright 2012-2023, Bergware International.
  *
  * This program is free software; you can redistribute it and/or
@@ -12,38 +12,50 @@
  */
 ?>
 <?
-$docroot ??= ($_SERVER['DOCUMENT_ROOT'] ?: '/usr/local/emhttp');
-require_once "$docroot/webGui/include/Wrappers.php";
+/* Stream a zip backup of the boot device (USB flash) to stdout.
+ *
+ * Usage: flash_backup [compression-level] > backup.zip   (stream the archive)
+ *        flash_backup size                                (print total bytes)
+ *   compression-level  0 (store, fastest) .. 9 (smallest, slowest), default 6
+ *
+ * The archive is written straight to stdout so it can be streamed directly to a
+ * browser (see include/FlashBackup.php) without ever staging a file on disk or
+ * in RAM - this avoids any size limit, works with the array stopped, and needs
+ * no cleanup. zip runs under nice + best-effort-low ionice to stay gentle on
+ * CPU and disk. The prior-release dirs (previous/prev) and desktop junk are
+ * excluded so the archive stays small.
+ *
+ * The 'size' mode prints the total byte size of the same included set; the
+ * endpoint uses it as the denominator for a download progress bar.
+ */
 
-$var = (array)@parse_ini_file('/var/local/emhttp/var.ini');
-$dir = ['system','appdata','isos','domains'];
-$out = ['prev','previous'];
+$sizeOnly = isset($argv[1]) && $argv[1] === 'size';
+// Clamp the requested compression level to a valid zip range (default 6).
+$level = isset($argv[1]) && is_numeric($argv[1]) ? max(0, min(9, (int)$argv[1])) : 6;
 
-$server = isset($var['NAME']) ? str_replace(' ','_',strtolower($var['NAME'])) : 'tower';
-$mydate = date('Ymd-Hi');
-$osVersion = _var($var,'version','_unknown');
-$backup = "$server-v$osVersion-boot-backup-$mydate.zip";
+// Top-level entries to exclude. 'previous'/'prev' hold the entire prior OS
+// release (often >1GB); the rest is desktop junk created when the flash is
+// mounted on a Mac/PC.
+$out = ['prev','previous','System Volume Information','.Trash-0','.Trashes','.Spotlight-V100','.fseventsd','.TemporaryItems'];
 
-$used = exec("df /boot|awk 'END{print $3}'") * 1.5;
-$free = exec("df /|awk 'END{print $4}'");
-if ($free > $used) $zip = "/$backup"; else {
-  foreach ($dir as $share) {
-    if (!is_dir("/mnt/user/$share")) continue;
-    $free = exec("df /mnt/user/$share|awk 'END{print $4}'");
-    if ($free > $used) {$zip = "/mnt/user/$share/$backup"; break;}
-  }
+chdir('/boot');
+$items = [];
+foreach (glob('*', GLOB_NOSORT) as $entry) {
+  if (in_array($entry, $out)) continue;
+  if (preg_match('/-boot-backup-.*\.zip$/', $entry)) continue; // stray old backups
+  $items[] = $entry;
 }
-if (isset($zip)) {
-  chdir("/boot");
-  foreach (glob("*",GLOB_NOSORT+GLOB_ONLYDIR) as $folder) {
-    if (in_array($folder,$out)) continue;
-    exec("zip -qr ".escapeshellarg($zip)." ".escapeshellarg($folder));
-  }
-  foreach (glob("*",GLOB_NOSORT) as $file) {
-    if (is_dir($file)) continue;
-    exec("zip -q ".escapeshellarg($zip)." ".escapeshellarg($file));
-  }
-  symlink($zip,"$docroot/$backup");
-  echo $backup;
+if (!$items) exit(1);
+$args = implode(' ', array_map('escapeshellarg', $items));
+
+// 'size' mode: print the total apparent byte size of the included set and exit.
+if ($sizeOnly) {
+  echo (int)trim(exec("du -sbc $args 2>/dev/null | awk 'END{print $1}'"));
+  exit(0);
 }
+
+// Stream the archive to stdout. zip writes a valid streamable archive (data
+// descriptors) when its output is not seekable.
+passthru("nice -n 19 ionice -c2 -n7 zip -$level -qr - $args 2>/dev/null", $rc);
+exit($rc);
 ?>