imagemounter: fetch DDI from GitHub and improve mount reliability

ios/imagemounter/imagedownloader.go

@@ -122,6 +122,15 @@ func MatchAvailable(version string) string {
 }
 
 func Download17Plus(baseDir string, version *semver.Version) (string, error) {
+	restoreDir, err := downloadPersonalizedDDI(baseDir)
+	if err != nil {
+		log.Warnf("GitHub DDI download failed: %v, falling back to deviceboxhq.com", err)
+		return download17PlusDevicebox(baseDir, version)
+	}
+	return restoreDir, nil
+}
+
+func download17PlusDevicebox(baseDir string, version *semver.Version) (string, error) {
 	downloadUrl := fmt.Sprintf("%s%s%s", devicebox, xcode15_4_ddi, ".zip")
 	log.Infof("device iOS version: %s, getting developer image: %s", version.String(), downloadUrl)
 

ios/imagemounter/imagedownloader_github.go

@@ -0,0 +1,154 @@
+package imagemounter
+
+import (
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"path/filepath"
+	"time"
+
+	log "github.com/sirupsen/logrus"
+)
+
+const (
+	githubOwner     = "doronz88"
+	githubRepo      = "DeveloperDiskImage"
+	githubBranch    = "main"
+	personalizedDDI = "PersonalizedImages/Xcode_iOS_DDI_Personalized"
+	latestDDI       = "17E5179g"
+)
+
+var githubHTTPClient = &http.Client{Timeout: 2 * time.Minute}
+
+func downloadPersonalizedDDI(baseDir string) (string, error) {
+	cacheDir := filepath.Join(baseDir, "ddi-latest")
+	restoreDir := filepath.Join(cacheDir, "Restore")
+	manifestPath := filepath.Join(restoreDir, "BuildManifest.plist")
+
+	if cached, err := isDDICacheValid(manifestPath); err == nil && cached {
+		log.Infof("using already downloaded DDI (build %s) from: %s", latestDDI, restoreDir)
+		return restoreDir, nil
+	}
+
+	log.Infof("downloading personalized DDI (build %s) from GitHub", latestDDI)
+
+	if err := os.MkdirAll(restoreDir, 0o755); err != nil {
+		return "", err
+	}
+
+	manifestBytes, err := downloadGitHubFile(personalizedDDI + "/BuildManifest.plist")
+	if err != nil {
+		return "", fmt.Errorf("downloadPersonalizedDDI: failed to download BuildManifest.plist: %w", err)
+	}
+	if err := os.WriteFile(manifestPath, manifestBytes, 0o644); err != nil {
+		return "", fmt.Errorf("downloadPersonalizedDDI: failed to write BuildManifest.plist: %w", err)
+	}
+
+	manifest, err := loadBuildManifest(manifestPath)
+	if err != nil {
+		return "", fmt.Errorf("downloadPersonalizedDDI: failed to parse BuildManifest.plist: %w", err)
+	}
+
+	dmgPath, trustPath := getFilePaths(manifest)
+	if dmgPath == "" || trustPath == "" {
+		return "", fmt.Errorf("downloadPersonalizedDDI: could not determine DMG/trustcache paths from manifest")
+	}
+
+	dmgFullPath := filepath.Join(restoreDir, dmgPath)
+	trustFullPath := filepath.Join(restoreDir, trustPath)
+
+	if err := downloadGitHubFileToDisk(personalizedDDI+"/Image.dmg", dmgFullPath); err != nil {
+		return "", fmt.Errorf("downloadPersonalizedDDI: failed to download Image.dmg: %w", err)
+	}
+	if err := downloadGitHubFileToDisk(personalizedDDI+"/Image.dmg.trustcache", trustFullPath); err != nil {
+		return "", fmt.Errorf("downloadPersonalizedDDI: failed to download Image.dmg.trustcache: %w", err)
+	}
+
+	log.Infof("successfully downloaded DDI to: %s", restoreDir)
+	return restoreDir, nil
+}
+
+func getFilePaths(manifest buildManifest) (dmgPath string, trustPath string) {
+	if len(manifest.BuildIdentities) == 0 {
+		return "", ""
+	}
+	identity := manifest.BuildIdentities[0]
+	for _, i := range manifest.BuildIdentities {
+		if i.dmgPath() != "" {
+			identity = i
+			break
+		}
+	}
+	return identity.dmgPath(), identity.trustCachePath()
+}
+
+func isDDICacheValid(manifestPath string) (bool, error) {
+	if _, err := os.Stat(manifestPath); os.IsNotExist(err) {
+		return false, nil
+	}
+	manifest, err := loadBuildManifest(manifestPath)
+	if err != nil {
+		return false, nil
+	}
+	if manifest.ProductBuildVersion != latestDDI {
+		log.Infof("cached DDI build is %q, need %q — will re-download",
+			manifest.ProductBuildVersion, latestDDI)
+		return false, nil
+	}
+	dmgPath, trustPath := getFilePaths(manifest)
+	if dmgPath == "" || trustPath == "" {
+		return false, nil
+	}
+	restoreDir := filepath.Dir(manifestPath)
+	if _, err := os.Stat(filepath.Join(restoreDir, dmgPath)); os.IsNotExist(err) {
+		return false, nil
+	}
+	if _, err := os.Stat(filepath.Join(restoreDir, trustPath)); os.IsNotExist(err) {
+		return false, nil
+	}
+	return true, nil
+}
+
+func downloadGitHubFile(path string) ([]byte, error) {
+	url := fmt.Sprintf("https://raw.githubusercontent.com/%s/%s/%s/%s",
+		githubOwner, githubRepo, githubBranch, path)
+	resp, err := githubHTTPClient.Get(url)
+	if err != nil {
+		return nil, fmt.Errorf("downloadGitHubFile: request failed: %w", err)
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("downloadGitHubFile: unexpected status %d for %s", resp.StatusCode, url)
+	}
+	data, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return nil, fmt.Errorf("downloadGitHubFile: failed to read response: %w", err)
+	}
+	return data, nil
+}
+
+func downloadGitHubFileToDisk(path string, destPath string) error {
+	url := fmt.Sprintf("https://raw.githubusercontent.com/%s/%s/%s/%s",
+		githubOwner, githubRepo, githubBranch, path)
+	resp, err := githubHTTPClient.Get(url)
+	if err != nil {
+		return fmt.Errorf("downloadGitHubFileToDisk: request failed: %w", err)
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		return fmt.Errorf("downloadGitHubFileToDisk: unexpected status %d for %s", resp.StatusCode, url)
+	}
+	if err := os.MkdirAll(filepath.Dir(destPath), 0o755); err != nil {
+		return err
+	}
+	f, err := os.Create(destPath)
+	if err != nil {
+		return fmt.Errorf("downloadGitHubFileToDisk: failed to create file: %w", err)
+	}
+	defer f.Close()
+	if _, err := io.Copy(f, resp.Body); err != nil {
+		return fmt.Errorf("downloadGitHubFileToDisk: failed to write file: %w", err)
+	}
+	return nil
+}

ios/imagemounter/personalized_image.go

@@ -10,7 +10,8 @@ import (
 )
 
 type buildManifest struct {
-	BuildIdentities []buildIdentity
+	ProductBuildVersion string          `plist:"ProductBuildVersion"`
+	BuildIdentities     []buildIdentity
 }
 
 func loadBuildManifest(p string) (buildManifest, error) {
@@ -37,25 +38,23 @@ func (m buildManifest) findIdentity(identifiers personalizationIdentifiers) (bui
 	return buildIdentity{}, fmt.Errorf("findIdentity: failed to find identity for ApBoardId 0x%x and ApChipId 0x%x", identifiers.BoardId, identifiers.ChipID)
 }
 
-type buildIdentity struct {
-	BoardID  string `plist:"ApBoardID"`
-	ChipID   string `plist:"ApChipID"`
-	Manifest struct {
-		LoadableTrustCache struct {
-			Digest []byte
-			Info   struct {
-				Path string
-			}
-		}
-		PersonalizedDmg struct {
-			Digest []byte
-			Info   struct {
-				Path string
-			}
-		} `plist:"PersonalizedDMG"`
+type manifestEntry struct {
+	Digest  []byte
+	Trusted bool `plist:"Trusted"`
+	EPRO    bool `plist:"EPRO"`
+	ESEC    bool `plist:"ESEC"`
+	Name    string
+	Info    struct {
+		Path string
 	}
 }
 
+type buildIdentity struct {
+	BoardID  string                   `plist:"ApBoardID"`
+	ChipID   string                   `plist:"ApChipID"`
+	Manifest map[string]manifestEntry
+}
+
 func (b buildIdentity) ApBoardID() int {
 	return hexToInt(b.BoardID)
 }
@@ -64,6 +63,23 @@ func (b buildIdentity) ApChipID() int {
 	return hexToInt(b.ChipID)
 }
 
+func (b buildIdentity) dmgPath() string {
+	if entry, ok := b.Manifest["PersonalizedDMG"]; ok {
+		return entry.Info.Path
+	}
+	if entry, ok := b.Manifest["PersonalizedDmg"]; ok {
+		return entry.Info.Path
+	}
+	return ""
+}
+
+func (b buildIdentity) trustCachePath() string {
+	if entry, ok := b.Manifest["LoadableTrustCache"]; ok {
+		return entry.Info.Path
+	}
+	return ""
+}
+
 type personalizationIdentifiers struct {
 	BoardId               int
 	ChipID                int

ios/imagemounter/personalized_image_mounter.go

@@ -1,6 +1,7 @@
 package imagemounter
 
 import (
+	"crypto/sha512"
 	"fmt"
 	"io"
 	"os"
@@ -22,6 +23,7 @@ type PersonalizedDeveloperDiskImageMounter struct {
 	version    *semver.Version
 	tss        tssClient
 	ecid       uint64
+	entry      ios.DeviceEntry
 }
 
 // NewPersonalizedDeveloperDiskImageMounter creates a PersonalizedDeveloperDiskImageMounter for the device entry
@@ -46,6 +48,7 @@ func NewPersonalizedDeveloperDiskImageMounter(entry ios.DeviceEntry, version *se
 		version:    version,
 		tss:        newTssClient(),
 		ecid:       ecid,
+		entry:      entry,
 	}, nil
 }
 
@@ -62,8 +65,9 @@ func (p PersonalizedDeveloperDiskImageMounter) ListImages() ([][]byte, error) {
 // MountImage mounts the personalized developer disk image present at imagePath.
 // imagePath needs to point to the 'Restore' directory of the personalized developer disk image.
 //
-// MountImage gets device identifiers and a nonce from the device first, which needs to be signed by Apple
-// and after that the developer disk image is sent to the device with this signature to be able to mount it.
+// MountImage first tries to reuse an existing device-side manifest via QueryPersonalizationManifest
+// to avoid unnecessary Apple TSS requests on re-mounts. If no manifest exists, it falls back
+// to querying a nonce and getting a new signature from Apple's TSS server.
 func (p PersonalizedDeveloperDiskImageMounter) MountImage(imagePath string) error {
 	manifest, err := loadBuildManifest(path.Join(imagePath, "BuildManifest.plist"))
 	if err != nil {
@@ -74,24 +78,40 @@ func (p PersonalizedDeveloperDiskImageMounter) MountImage(imagePath string) erro
 	if err != nil {
 		return fmt.Errorf("MountImage: failed to query personalization identifiers: %w", err)
 	}
-	nonce, err := p.queryPersonalizedImageNonce()
-	if err != nil {
-		return fmt.Errorf("MountImage: failed to get nonce: %w", err)
-	}
 
 	identity, err := manifest.findIdentity(identifiers)
 	if err != nil {
 		return fmt.Errorf("MountImage: could not find identity for identifiers %+v: %w", identifiers, err)
 	}
 
-	signature, err := p.tss.getSignature(identity, identifiers, nonce, p.ecid)
+	dmgPath := path.Join(imagePath, identity.dmgPath())
+
+	signature, err := p.queryPersonalizationManifest(dmgPath)
 	if err != nil {
-		return fmt.Errorf("MountImage: failed to get signature from Apple: %w", err)
-	}
+		log.Info("no existing device-side manifest, requesting new signature from Apple TSS")
+
+		p, err = p.closeAndReconnect()
+		if err != nil {
+			return fmt.Errorf("MountImage: failed to reconnect after manifest query: %w", err)
+		}
 
-	dmgPath := path.Join(imagePath, identity.Manifest.PersonalizedDmg.Info.Path)
+		nonce, err := p.queryPersonalizedImageNonce()
+		if err != nil {
+			return fmt.Errorf("MountImage: failed to get nonce: %w", err)
+		}
+
+		signature, err = p.tss.getSignature(identity, identifiers, nonce, p.ecid)
+		if err != nil {
+			return fmt.Errorf("MountImage: failed to get signature from Apple: %w", err)
+		}
+	} else {
+		log.Info("reusing existing device-side manifest, skipping Apple TSS")
+	}
 
 	imageSize, err := getFileSize(dmgPath)
+	if err != nil {
+		return fmt.Errorf("MountImage: %w", err)
+	}
 
 	err = sendUploadRequest(p.plistRw, "Personalized", signature, imageSize)
 	if err != nil {
@@ -112,7 +132,7 @@ func (p PersonalizedDeveloperDiskImageMounter) MountImage(imagePath string) erro
 		return err
 	}
 
-	trustCache, err := os.ReadFile(path.Join(imagePath, identity.Manifest.LoadableTrustCache.Info.Path))
+	trustCache, err := os.ReadFile(path.Join(imagePath, identity.trustCachePath()))
 	if err != nil {
 		return fmt.Errorf("MountImage: could not load trust-cache. %w", err)
 	}
@@ -142,6 +162,59 @@ func (p PersonalizedDeveloperDiskImageMounter) UnmountImage() error {
 	return nil
 }
 
+func (p PersonalizedDeveloperDiskImageMounter) queryPersonalizationManifest(dmgPath string) ([]byte, error) {
+	f, err := os.Open(dmgPath)
+	if err != nil {
+		return nil, fmt.Errorf("queryPersonalizationManifest: failed to open DMG: %w", err)
+	}
+	defer f.Close()
+
+	h := sha512.New384()
+	if _, err := io.Copy(h, f); err != nil {
+		return nil, fmt.Errorf("queryPersonalizationManifest: failed to hash DMG: %w", err)
+	}
+	digest := h.Sum(nil)
+
+	err = p.plistRw.Write(map[string]interface{}{
+		"Command":               "QueryPersonalizationManifest",
+		"PersonalizedImageType": "DeveloperDiskImage",
+		"ImageType":             "DeveloperDiskImage",
+		"ImageSignature":        digest,
+	})
+	if err != nil {
+		return nil, fmt.Errorf("queryPersonalizationManifest: failed to write command: %w", err)
+	}
+
+	var resp map[string]interface{}
+	err = p.plistRw.Read(&resp)
+	if err != nil {
+		return nil, fmt.Errorf("queryPersonalizationManifest: failed to read response: %w", err)
+	}
+
+	if sig, ok := resp["ImageSignature"].([]byte); ok {
+		return sig, nil
+	}
+	return nil, fmt.Errorf("queryPersonalizationManifest: no ImageSignature in response %+v", resp)
+}
+
+func (p PersonalizedDeveloperDiskImageMounter) closeAndReconnect() (PersonalizedDeveloperDiskImageMounter, error) {
+	p.deviceConn.Close()
+
+	deviceConn, err := ios.ConnectToService(p.entry, serviceName)
+	if err != nil {
+		return p, fmt.Errorf("closeAndReconnect: failed to reconnect to %s: %w", serviceName, err)
+	}
+
+	return PersonalizedDeveloperDiskImageMounter{
+		deviceConn: deviceConn,
+		plistRw:    ios.NewPlistCodecReadWriter(deviceConn.Reader(), deviceConn.Writer()),
+		version:    p.version,
+		tss:        p.tss,
+		ecid:       p.ecid,
+		entry:      p.entry,
+	}, nil
+}
+
 func (p PersonalizedDeveloperDiskImageMounter) queryPersonalizedImageNonce() ([]byte, error) {
 	err := p.plistRw.Write(map[string]interface{}{
 		"Command":               "QueryNonce",