Fix large APK archive fallback

Author: besendorf

acquisition/acquisition.go

@@ -11,6 +11,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"os"
 	"strings"
 	"time"
 
@@ -197,13 +198,15 @@ func (a *Acquisition) StreamAPKToZip(remotePath, zipPath string, processFunc fun
 	buffer, err := a.StreamingPuller.PullToBuffer(remotePath)
 	if err != nil {
 		if errors.Is(err, ErrStreamingBufferMemoryLimit) && processFunc == nil {
-			log.Debugf("APK %s exceeded streaming buffer limit; streaming directly to archive", remotePath)
-			writer, err := a.ZipWriter.CreateFile(zipPath)
+			log.Debugf("APK %s exceeded streaming buffer limit; staging it before archiving", remotePath)
+			tempPath, err := a.StreamingPuller.PullToTempFile(remotePath)
 			if err != nil {
-				return fmt.Errorf("failed to create zip entry for APK %q: %v", remotePath, err)
+				return fmt.Errorf("failed to pull APK %q to a temporary file: %w", remotePath, err)
 			}
-			if err := a.StreamingPuller.PullToWriter(remotePath, writer); err != nil {
-				return fmt.Errorf("failed to stream APK %q to zip: %v", remotePath, err)
+			defer os.Remove(tempPath)
+
+			if err := a.ZipWriter.CreateFileFromPath(zipPath, tempPath); err != nil {
+				return fmt.Errorf("failed to add APK %q to zip: %w", remotePath, err)
 			}
 			return nil
 		}

acquisition/streaming_buffer.go

@@ -10,6 +10,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"os"
 	"os/exec"
 	"strings"
 )
@@ -132,6 +133,28 @@ func (sp *StreamingPuller) PullToWriter(remotePath string, writer io.Writer) err
 	return nil
 }
 
+// PullToTempFile pulls a file from the device into a temporary file and
+// returns its path. The caller is responsible for removing the file.
+func (sp *StreamingPuller) PullToTempFile(remotePath string) (string, error) {
+	tempFile, err := os.CreateTemp("", "androidqf-pull-*")
+	if err != nil {
+		return "", fmt.Errorf("failed to create temporary file: %w", err)
+	}
+	tempPath := tempFile.Name()
+
+	if err := sp.PullToWriter(remotePath, tempFile); err != nil {
+		_ = tempFile.Close()
+		_ = os.Remove(tempPath)
+		return "", fmt.Errorf("failed to pull to temporary file: %w", err)
+	}
+	if err := tempFile.Close(); err != nil {
+		_ = os.Remove(tempPath)
+		return "", fmt.Errorf("failed to close temporary file: %w", err)
+	}
+
+	return tempPath, nil
+}
+
 // BackupToBuffer creates a backup directly into memory buffer using exec-out
 func (sp *StreamingPuller) BackupToBuffer(arg string) (*StreamingBuffer, error) {
 	if arg == "" {

acquisition/streaming_buffer_test.go

@@ -4,6 +4,7 @@ import (
 	"errors"
 	"os"
 	"path/filepath"
+	"runtime"
 	"testing"
 )
 
@@ -27,6 +28,10 @@ func TestDefaultStreamingPullerMemoryLimit(t *testing.T) {
 }
 
 func TestPullToBufferPreservesMemoryLimitError(t *testing.T) {
+	if runtime.GOOS == "windows" {
+		t.Skip("uses a POSIX shell script as a fake adb executable")
+	}
+
 	fakeADB := filepath.Join(t.TempDir(), "adb")
 	if err := os.WriteFile(fakeADB, []byte("#!/bin/sh\nhead -c 1048577 /dev/zero\n"), 0o700); err != nil {
 		t.Fatalf("WriteFile(fake adb) error = %v", err)

modules/packages.go

@@ -208,40 +208,33 @@ func (p *Packages) processAPKStreaming(packageName string, packageFile *adb.Pack
 }
 
 func (p *Packages) processLargeEncryptedAPK(packageFile *adb.PackageFile, zipPath string, acq *acquisition.Acquisition, usedZipPaths map[string]struct{}) (string, error) {
-	log.Debugf("APK %s exceeded streaming buffer limit; streaming directly to encrypted archive without certificate check", packageFile.Path)
+	log.Debugf("APK %s exceeded streaming buffer limit; staging it before adding it to the encrypted archive without certificate check", packageFile.Path)
 
 	packageFile.CertificateError = "Skipped certificate check: APK exceeds streaming buffer limit"
 	packageFile.VerifiedCertificate = false
 
-	zipPath = reserveUniqueZipPath(zipPath, usedZipPaths)
-	writer, err := acq.ZipWriter.CreateFile(zipPath)
+	tempPath, err := acq.StreamingPuller.PullToTempFile(packageFile.Path)
 	if err != nil {
-		return "", fmt.Errorf("failed to create zip entry for APK: %v", err)
+		return "", fmt.Errorf("failed to pull APK to temporary file: %w", err)
 	}
-	if err := acq.StreamingPuller.PullToWriter(packageFile.Path, writer); err != nil {
-		return "", fmt.Errorf("failed to stream APK to archive: %v", err)
+	defer os.Remove(tempPath)
+
+	zipPath = reserveUniqueZipPath(zipPath, usedZipPaths)
+	if err := acq.ZipWriter.CreateFileFromPath(zipPath, tempPath); err != nil {
+		return "", fmt.Errorf("failed to add APK to archive: %w", err)
 	}
 	return zipPath, nil
 }
 
 func (p *Packages) processLargeAPKFromTemp(packageFile *adb.PackageFile, keepOption, zipPath string, acq *acquisition.Acquisition, usedZipPaths map[string]struct{}) (string, bool, error) {
 	log.Debugf("APK %s exceeded streaming buffer limit; using temporary file for certificate check", packageFile.Path)
 
-	tempFile, err := os.CreateTemp("", "androidqf-apk-*.apk")
+	tempPath, err := acq.StreamingPuller.PullToTempFile(packageFile.Path)
 	if err != nil {
-		return "", false, fmt.Errorf("failed to create temporary APK file: %v", err)
+		return "", false, fmt.Errorf("failed to pull APK to temporary file: %w", err)
 	}
-	tempPath := tempFile.Name()
 	defer os.Remove(tempPath)
 
-	if err := acq.StreamingPuller.PullToWriter(packageFile.Path, tempFile); err != nil {
-		tempFile.Close()
-		return "", false, fmt.Errorf("failed to pull APK to temporary file: %v", err)
-	}
-	if err := tempFile.Close(); err != nil {
-		return "", false, fmt.Errorf("failed to close temporary APK file: %v", err)
-	}
-
 	shouldSkip, err := p.processCertificateFromPath(packageFile, keepOption, tempPath)
 	if err != nil {
 		return "", false, fmt.Errorf("certificate processing failed: %v", err)