fix(software): retry truncated/empty downloads and report real checksum algorithm

Software downloads failed the whole workflow on a transient CDN/network
hiccup: a 200-with-empty-body, a truncated copy, or a mid-stream reset
wrote a corrupt file that the checksum verifier correctly rejected, but
with no retry the entire run aborted.

- Download now rejects zero-byte bodies and, when Content-Length is
  advertised, fails on a byte-count mismatch instead of persisting a
  truncated file.
- Add Downloader.DownloadAndVerify, which retries download+verify with
  capped exponential backoff (default 3 attempts), deleting the bad file
  between attempts; both download errors and checksum mismatches are
  retryable. The three base_installer call sites now use it.
- Thread the real algorithm through the checksum helper so mismatch
  errors report e.g. 'sha256' instead of the misleading 'unknown'.

Closes #786

Signed-off-by: alex-au <alex.w.aus@gmail.com>

Author: alex-au

pkg/software/base_installer.go

@@ -201,14 +201,8 @@ func (b *baseInstaller) downloadAndVerifyArchive(archive ArchiveDetail) error {
 		}
 	}
 
-	// Download the archive
-	err = b.downloader.Download(downloadURL, destinationFile)
-	if err != nil {
-		return err
-	}
-
-	// Verify the downloaded archive's checksum
-	err = VerifyChecksum(destinationFile, checksum.Value, checksum.Algorithm)
+	// Download the archive and verify its checksum, retrying transient corruption
+	err = b.downloader.DownloadAndVerify(downloadURL, destinationFile, checksum.Value, checksum.Algorithm)
 	if err != nil {
 		return err
 	}
@@ -285,14 +279,8 @@ func (b *baseInstaller) downloadAndVerifyBinary(binary BinaryDetail) error {
 		}
 	}
 
-	// Download the binary
-	err = b.downloader.Download(downloadURL, destinationFile)
-	if err != nil {
-		return err
-	}
-
-	// Verify the downloaded binary's checksum
-	err = VerifyChecksum(destinationFile, checksum.Value, checksum.Algorithm)
+	// Download the binary and verify its checksum, retrying transient corruption
+	err = b.downloader.DownloadAndVerify(downloadURL, destinationFile, checksum.Value, checksum.Algorithm)
 	if err != nil {
 		return err
 	}
@@ -328,14 +316,8 @@ func (b *baseInstaller) downloadConfigs() error {
 			}
 		}
 
-		// Download the config file
-		err = b.downloader.Download(config.URL, configFile)
-		if err != nil {
-			return err
-		}
-
-		// Verify the downloaded config file's checksum
-		err = VerifyChecksum(configFile, config.Value, config.Algorithm)
+		// Download the config file and verify its checksum, retrying transient corruption
+		err = b.downloader.DownloadAndVerify(config.URL, configFile, config.Value, config.Algorithm)
 		if err != nil {
 			return err
 		}

pkg/software/downloader.go

@@ -20,13 +20,22 @@ import (
 	"github.com/hashgraph/solo-weaver/pkg/sanity"
 )
 
+// Default retry settings for DownloadAndVerify.
+const (
+	defaultMaxAttempts = 3               // initial attempt + 2 retries
+	defaultRetryDelay  = 1 * time.Second // base delay; grows exponentially per attempt
+	maxRetryDelay      = 30 * time.Second
+)
+
 // Downloader is responsible for downloading a software package and checking its integrity.
 type Downloader struct {
 	client         *http.Client
 	timeout        time.Duration
-	basePath       string   // Base directory for validating download/extraction paths
-	allowedDomains []string // List of allowed domains for SSRF protection
-	insecureTLS    bool     // Skip TLS certificate verification (for local dev with self-signed certs)
+	basePath       string        // Base directory for validating download/extraction paths
+	allowedDomains []string      // List of allowed domains for SSRF protection
+	insecureTLS    bool          // Skip TLS certificate verification (for local dev with self-signed certs)
+	maxAttempts    int           // Number of download+verify attempts before giving up
+	retryDelay     time.Duration // Base backoff delay between attempts (0 disables sleeping)
 }
 
 // DownloaderOption is a function that configures a Downloader
@@ -63,6 +72,25 @@ func WithHTTPClient(client *http.Client) DownloaderOption {
 	}
 }
 
+// WithMaxAttempts sets the number of download+verify attempts before giving up.
+// Values below 1 are clamped to 1.
+func WithMaxAttempts(attempts int) DownloaderOption {
+	return func(d *Downloader) {
+		if attempts < 1 {
+			attempts = 1
+		}
+		d.maxAttempts = attempts
+	}
+}
+
+// WithRetryDelay sets the base backoff delay between download attempts.
+// A zero delay disables sleeping (useful for tests).
+func WithRetryDelay(delay time.Duration) DownloaderOption {
+	return func(d *Downloader) {
+		d.retryDelay = delay
+	}
+}
+
 // WithInsecureTLS skips TLS certificate verification.
 // WARNING: Only use this for local development with self-signed certificates!
 // This option is ignored in release builds for security.
@@ -97,6 +125,8 @@ func NewDownloader(opts ...DownloaderOption) *Downloader {
 		basePath:       models.Paths().HomeDir,
 		allowedDomains: sanity.AllowedDomains(),
 		insecureTLS:    false,
+		maxAttempts:    defaultMaxAttempts,
+		retryDelay:     defaultRetryDelay,
 	}
 
 	// Apply options
@@ -160,14 +190,105 @@ func (fd *Downloader) Download(url, destination string) error {
 	}
 	defer out.Close()
 
-	_, err = io.Copy(out, resp.Body)
+	written, err := io.Copy(out, resp.Body)
 	if err != nil {
+		_ = os.Remove(cleanDest)
 		return NewDownloadError(err, url, 0)
 	}
 
+	// Reject a zero-byte body: a 200-with-empty-response is a corrupt download,
+	// not a valid file. This is treated as a retryable download error.
+	if written == 0 {
+		_ = os.Remove(cleanDest)
+		return NewDownloadError(errorx.ExternalError.New("downloaded file is empty (0 bytes written)"), url, 0)
+	}
+
+	// When the response advertises a Content-Length, fail fast on a size mismatch
+	// (a truncated copy or mid-stream reset) rather than persisting a partial file.
+	// ContentLength is -1 when unknown (e.g. chunked transfer), so only check when >= 0.
+	if resp.ContentLength >= 0 && written != resp.ContentLength {
+		_ = os.Remove(cleanDest)
+		return NewDownloadError(
+			errorx.ExternalError.New("download truncated: wrote %d of %d advertised bytes", written, resp.ContentLength),
+			url, 0)
+	}
+
 	return nil
 }
 
+// DownloadAndVerify downloads a file from url to destination and verifies its
+// checksum, retrying transient corruption. Both download errors (network reset,
+// empty/truncated body) and a subsequent checksum mismatch are treated as
+// retryable: on failure the bad file is removed and the download is re-attempted
+// with capped exponential backoff, up to maxAttempts. The last error is returned
+// once attempts are exhausted.
+func (fd *Downloader) DownloadAndVerify(url, destination, expectedValue, algorithm string) error {
+	attempts := fd.maxAttempts
+	if attempts < 1 {
+		attempts = 1
+	}
+
+	// Sanitize the destination once so the download, verification, and cleanup all
+	// operate on the exact same path (Download sanitizes internally, so a caller
+	// passing a relative/unclean path would otherwise write to one path and verify
+	// or remove another).
+	cleanDest, err := sanity.ValidatePathWithinBase(fd.basePath, destination)
+	if err != nil {
+		return NewDownloadError(err, url, 0)
+	}
+
+	// Fail fast on deterministic errors that no retry can fix: an invalid/unsafe URL
+	// and an unsupported checksum algorithm will fail identically on every attempt,
+	// so retrying them only adds backoff delay and log noise.
+	if err := sanity.ValidateURL(url, &sanity.ValidateURLOptions{AllowedDomains: fd.allowedDomains}); err != nil {
+		return NewInvalidURLError(err, url)
+	}
+	if !IsSupportedAlgorithm(algorithm) {
+		return NewChecksumError(cleanDest, algorithm, expectedValue, "")
+	}
+
+	var lastErr error
+	for attempt := 1; attempt <= attempts; attempt++ {
+		lastErr = fd.Download(url, cleanDest)
+		if lastErr == nil {
+			lastErr = VerifyChecksum(cleanDest, expectedValue, algorithm)
+			if lastErr == nil {
+				return nil
+			}
+		}
+
+		// Remove the corrupt/partial file so a bad download never persists and the
+		// next attempt (or a later run) starts clean.
+		_ = os.Remove(cleanDest)
+
+		if attempt < attempts {
+			logx.As().Warn().
+				Str("url", url).
+				Int("attempt", attempt).
+				Int("maxAttempts", attempts).
+				Err(lastErr).
+				Msg("Download or checksum verification failed; retrying after backoff")
+			fd.sleepBackoff(attempt)
+		}
+	}
+
+	return lastErr
+}
+
+// sleepBackoff sleeps for a capped exponential delay before the next attempt.
+// A zero base retryDelay disables sleeping.
+func (fd *Downloader) sleepBackoff(attempt int) {
+	if fd.retryDelay <= 0 {
+		return
+	}
+
+	delay := fd.retryDelay << (attempt - 1)
+	if delay > maxRetryDelay || delay < fd.retryDelay {
+		delay = maxRetryDelay
+	}
+	time.Sleep(delay)
+}
+
 // ExtractTarGz extracts a tar.gz file
 func (fd *Downloader) Extract(compressedFilePath string, destDir string) error {
 	// Validate and sanitize the compressed file path

pkg/software/downloader_test.go

@@ -5,11 +5,13 @@ package software
 import (
 	"archive/tar"
 	"compress/gzip"
+	"crypto/sha256"
 	"fmt"
 	"net/http"
 	"net/http/httptest"
 	"os"
 	"path/filepath"
+	"sync/atomic"
 	"testing"
 	"time"
 
@@ -106,6 +108,155 @@ func Test_Downloader_Timeout(t *testing.T) {
 	require.True(t, errorx.IsOfType(err, DownloadError), "Error should be of type DownloadError")
 }
 
+func Test_Downloader_Download_ZeroByte(t *testing.T) {
+	// Server returns 200 with an empty body — a corrupt download.
+	server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusOK)
+	}))
+	defer server.Close()
+
+	tmpFile, err := os.CreateTemp("", "test_zerobyte_*.txt")
+	require.NoError(t, err, "Failed to create temp file")
+	_ = tmpFile.Close()
+	defer func() { _ = os.Remove(tmpFile.Name()) }()
+
+	downloader := NewDownloader(
+		WithHTTPClient(server.Client()),
+		WithBasePath(filepath.Dir(tmpFile.Name())),
+		WithAllowedDomains([]string{"localhost", "127.0.0.1"}),
+	)
+
+	err = downloader.Download(server.URL, tmpFile.Name())
+	require.Error(t, err, "Download should reject a zero-byte body")
+	require.True(t, errorx.IsOfType(err, DownloadError), "Error should be of type DownloadError")
+}
+
+func Test_Downloader_Download_ContentLengthMismatch(t *testing.T) {
+	// Server advertises a larger Content-Length than the body it actually writes,
+	// simulating a truncated/reset transfer.
+	server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Length", "1024")
+		w.WriteHeader(http.StatusOK)
+		_, _ = w.Write([]byte("short body"))
+	}))
+	defer server.Close()
+
+	tmpFile, err := os.CreateTemp("", "test_truncated_*.txt")
+	require.NoError(t, err, "Failed to create temp file")
+	_ = tmpFile.Close()
+	defer func() { _ = os.Remove(tmpFile.Name()) }()
+
+	downloader := NewDownloader(
+		WithHTTPClient(server.Client()),
+		WithBasePath(filepath.Dir(tmpFile.Name())),
+		WithAllowedDomains([]string{"localhost", "127.0.0.1"}),
+	)
+
+	err = downloader.Download(server.URL, tmpFile.Name())
+	require.Error(t, err, "Download should reject a Content-Length mismatch")
+	require.True(t, errorx.IsOfType(err, DownloadError), "Error should be of type DownloadError")
+}
+
+func Test_Downloader_DownloadAndVerify_RetriesThenSucceeds(t *testing.T) {
+	goodContent := "the real payload"
+	goodSum := fmt.Sprintf("%x", sha256.Sum256([]byte(goodContent)))
+
+	var calls int32
+	server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		n := atomic.AddInt32(&calls, 1)
+		w.WriteHeader(http.StatusOK)
+		if n == 1 {
+			// First attempt: empty body (transient corruption).
+			return
+		}
+		_, _ = w.Write([]byte(goodContent))
+	}))
+	defer server.Close()
+
+	tmpFile, err := os.CreateTemp("", "test_retry_ok_*.txt")
+	require.NoError(t, err, "Failed to create temp file")
+	_ = tmpFile.Close()
+	defer func() { _ = os.Remove(tmpFile.Name()) }()
+
+	downloader := NewDownloader(
+		WithHTTPClient(server.Client()),
+		WithBasePath(filepath.Dir(tmpFile.Name())),
+		WithAllowedDomains([]string{"localhost", "127.0.0.1"}),
+		WithRetryDelay(0), // no sleeping in tests
+	)
+
+	err = downloader.DownloadAndVerify(server.URL, tmpFile.Name(), goodSum, "sha256")
+	require.NoError(t, err, "DownloadAndVerify should recover from a transient empty download")
+	require.Equal(t, int32(2), atomic.LoadInt32(&calls), "Expected exactly one retry")
+
+	content, err := os.ReadFile(tmpFile.Name())
+	require.NoError(t, err, "Failed to read downloaded file")
+	require.Equal(t, goodContent, string(content), "Downloaded content mismatch")
+}
+
+func Test_Downloader_DownloadAndVerify_ExhaustsAttempts(t *testing.T) {
+	// Server always returns a non-empty body that never matches the expected checksum.
+	var calls int32
+	server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		atomic.AddInt32(&calls, 1)
+		w.WriteHeader(http.StatusOK)
+		_, _ = w.Write([]byte("persistently wrong content"))
+	}))
+	defer server.Close()
+
+	tmpFile, err := os.CreateTemp("", "test_retry_fail_*.txt")
+	require.NoError(t, err, "Failed to create temp file")
+	_ = tmpFile.Close()
+	defer func() { _ = os.Remove(tmpFile.Name()) }()
+
+	downloader := NewDownloader(
+		WithHTTPClient(server.Client()),
+		WithBasePath(filepath.Dir(tmpFile.Name())),
+		WithAllowedDomains([]string{"localhost", "127.0.0.1"}),
+		WithMaxAttempts(3),
+		WithRetryDelay(0),
+	)
+
+	expectedSum := fmt.Sprintf("%x", sha256.Sum256([]byte("the content we wanted")))
+	err = downloader.DownloadAndVerify(server.URL, tmpFile.Name(), expectedSum, "sha256")
+	require.Error(t, err, "DownloadAndVerify should fail after exhausting attempts")
+	require.True(t, errorx.IsOfType(err, ChecksumError), "Final error should be the checksum mismatch")
+	require.Equal(t, int32(3), atomic.LoadInt32(&calls), "Expected exactly maxAttempts attempts")
+
+	// The corrupt file must not persist after the retries are exhausted.
+	_, statErr := os.Stat(tmpFile.Name())
+	require.True(t, os.IsNotExist(statErr), "Corrupt download should be removed")
+}
+
+func Test_Downloader_DownloadAndVerify_FailsFastOnUnsupportedAlgorithm(t *testing.T) {
+	// An unsupported algorithm can never verify, so DownloadAndVerify must fail
+	// before ever contacting the server or entering the retry loop.
+	var calls int32
+	server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		atomic.AddInt32(&calls, 1)
+		w.WriteHeader(http.StatusOK)
+		_, _ = w.Write([]byte("content"))
+	}))
+	defer server.Close()
+
+	tmpFile, err := os.CreateTemp("", "test_failfast_*.txt")
+	require.NoError(t, err, "Failed to create temp file")
+	_ = tmpFile.Close()
+	defer func() { _ = os.Remove(tmpFile.Name()) }()
+
+	downloader := NewDownloader(
+		WithHTTPClient(server.Client()),
+		WithBasePath(filepath.Dir(tmpFile.Name())),
+		WithAllowedDomains([]string{"localhost", "127.0.0.1"}),
+		WithRetryDelay(0),
+	)
+
+	err = downloader.DownloadAndVerify(server.URL, tmpFile.Name(), "abc123", "sha1")
+	require.Error(t, err, "DownloadAndVerify should reject an unsupported algorithm")
+	require.True(t, errorx.IsOfType(err, ChecksumError), "Error should be of type ChecksumError")
+	require.Equal(t, int32(0), atomic.LoadInt32(&calls), "No download should be attempted for an unsupported algorithm")
+}
+
 func Test_Downloader_Extract(t *testing.T) {
 	// Create a temporary directory for test files
 	tempDir, err := os.MkdirTemp("", "test_extract_*")