fix(download): add 7z extraction support and expand Python manifest (#161)

Author: CalvinAllen

.github/workflows/integration-test-migrate-python-windows-system.yml

@@ -21,6 +21,13 @@ jobs:
           go-version: '1.23'
           cache: true
 
+      # Install a Python version that's available in our manifest for Windows
+      # python-build-standalone only has Windows builds starting from 3.10.14
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+
       - name: Build dtvem
         shell: bash
         run: |

go.mod

@@ -3,6 +3,7 @@ module github.com/dtvem/dtvem
 go 1.23
 
 require (
+	github.com/bodgit/sevenzip v1.6.1
 	github.com/briandowns/spinner v1.23.2
 	github.com/charmbracelet/lipgloss v1.1.0
 	github.com/fatih/color v1.18.0
@@ -13,19 +14,29 @@ require (
 )
 
 require (
+	github.com/andybalholm/brotli v1.1.1 // indirect
 	github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect
+	github.com/bodgit/plumbing v1.3.0 // indirect
+	github.com/bodgit/windows v1.0.1 // indirect
 	github.com/charmbracelet/colorprofile v0.2.3-0.20250311203215-f60798e515dc // indirect
 	github.com/charmbracelet/x/ansi v0.8.0 // indirect
 	github.com/charmbracelet/x/cellbuf v0.0.13-0.20250311204145-2c3ea96c31dd // indirect
 	github.com/charmbracelet/x/term v0.2.1 // indirect
+	github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
+	github.com/klauspost/compress v1.17.11 // indirect
 	github.com/lucasb-eyer/go-colorful v1.2.0 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
 	github.com/mattn/go-runewidth v0.0.16 // indirect
 	github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect
+	github.com/pierrec/lz4/v4 v4.1.22 // indirect
 	github.com/rivo/uniseg v0.4.7 // indirect
+	github.com/spf13/afero v1.11.0 // indirect
 	github.com/spf13/pflag v1.0.10 // indirect
+	github.com/ulikunitz/xz v0.5.12 // indirect
 	github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect
+	go4.org v0.0.0-20200411211856-f5505b9728dd // indirect
 	golang.org/x/term v0.28.0 // indirect
+	golang.org/x/text v0.21.0 // indirect
 )

go.sum

@@ -2,12 +2,15 @@ package main
 
 import (
 	"fmt"
+	"io"
+	"net/http"
 	"regexp"
 	"strings"
 )
 
 const (
 	pythonReleasesURL = "https://api.github.com/repos/astral-sh/python-build-standalone/releases"
+	pythonOrgFTPURL   = "https://www.python.org/ftp/python/"
 	pythonSchemaURL   = "https://raw.githubusercontent.com/dtvem/dtvem/main/schemas/manifest.schema.json"
 )
 
@@ -25,56 +28,71 @@ var pythonPlatformMap = map[string]string{
 // Regex to parse asset names like: cpython-3.13.1+20251209-x86_64-unknown-linux-gnu-install_only.tar.gz
 var pythonAssetRegex = regexp.MustCompile(`^cpython-(\d+\.\d+\.\d+)\+\d+-(.+)-install_only\.tar\.gz$`)
 
-func generatePythonManifest(outputDir string) error {
-	fmt.Println("Generating Python manifest...")
+// Regex to parse python.org version directories
+var pythonOrgVersionRegex = regexp.MustCompile(`href="(\d+\.\d+\.\d+)/"`)
 
-	// Fetch releases from GitHub API
-	releases, err := fetchPythonReleases()
-	if err != nil {
-		return fmt.Errorf("failed to fetch releases: %w", err)
-	}
+// Regex to parse python.org embeddable package names
+var pythonOrgEmbedRegex = regexp.MustCompile(`href="(python-(\d+\.\d+\.\d+)-embed-(amd64|arm64)\.zip)"`)
 
-	fmt.Printf("Found %d releases\n", len(releases))
+func generatePythonManifest(outputDir string) error {
+	fmt.Println("Generating Python manifest...")
 
 	manifest := &Manifest{
 		Schema:   pythonSchemaURL,
 		Version:  1,
 		Versions: make(map[string]map[string]*Download),
 	}
 
-	// Process each release
-	for _, release := range releases {
-		fmt.Printf("Processing release %s (%d assets)...\n", release.TagName, len(release.Assets))
+	// First, fetch from python-build-standalone (primary source for Linux/macOS, newer Windows)
+	if err := fetchPythonBuildStandalone(manifest); err != nil {
+		fmt.Printf("Warning: failed to fetch python-build-standalone: %v\n", err)
+	}
 
+	// Then, fetch from python.org for Windows (fills gaps for older versions)
+	if err := fetchPythonOrg(manifest); err != nil {
+		fmt.Printf("Warning: failed to fetch python.org: %v\n", err)
+	}
+
+	fmt.Printf("Generated manifest with %d versions\n", len(manifest.Versions))
+
+	return writeManifest(manifest, outputDir, "python.json")
+}
+
+// fetchPythonBuildStandalone fetches releases from astral-sh/python-build-standalone
+func fetchPythonBuildStandalone(manifest *Manifest) error {
+	fmt.Println("Fetching from python-build-standalone...")
+
+	releases, err := fetchPythonReleases()
+	if err != nil {
+		return fmt.Errorf("failed to fetch releases: %w", err)
+	}
+
+	fmt.Printf("Found %d releases from python-build-standalone\n", len(releases))
+
+	for _, release := range releases {
 		for _, asset := range release.Assets {
-			// Parse asset name to extract version and platform
 			matches := pythonAssetRegex.FindStringSubmatch(asset.Name)
 			if matches == nil {
 				continue
 			}
 
-			version := matches[1]     // e.g., "3.13.1"
-			pbsPlatform := matches[2] // e.g., "x86_64-unknown-linux-gnu"
+			version := matches[1]
+			pbsPlatform := matches[2]
 
-			// Map to our platform key
 			platform, ok := pythonPlatformMap[pbsPlatform]
 			if !ok {
 				continue
 			}
 
-			// Extract SHA256 from digest if available (format: "sha256:abc123...")
 			sha256 := ""
 			if strings.HasPrefix(asset.Digest, "sha256:") {
 				sha256 = strings.TrimPrefix(asset.Digest, "sha256:")
 			}
 
-			// Initialize version map if needed
 			if manifest.Versions[version] == nil {
 				manifest.Versions[version] = make(map[string]*Download)
 			}
 
-			// Only add if we don't already have this version/platform
-			// (prefer newer releases which come first from the API)
 			if manifest.Versions[version][platform] == nil {
 				manifest.Versions[version][platform] = &Download{
 					URL:    asset.BrowserDownloadURL,
@@ -84,9 +102,128 @@ func generatePythonManifest(outputDir string) error {
 		}
 	}
 
-	fmt.Printf("Generated manifest with %d versions\n", len(manifest.Versions))
+	return nil
+}
 
-	return writeManifest(manifest, outputDir, "python.json")
+// fetchPythonOrg fetches Windows embeddable packages from python.org
+func fetchPythonOrg(manifest *Manifest) error {
+	fmt.Println("Fetching Windows packages from python.org...")
+
+	// Get list of versions from python.org FTP
+	versions, err := listPythonOrgVersions()
+	if err != nil {
+		return fmt.Errorf("failed to list versions: %w", err)
+	}
+
+	fmt.Printf("Found %d versions on python.org\n", len(versions))
+
+	addedCount := 0
+	for _, version := range versions {
+		// Skip if we already have Windows builds for this version
+		if manifest.Versions[version] != nil && manifest.Versions[version]["windows-amd64"] != nil {
+			continue
+		}
+
+		// Check for embeddable packages
+		packages, err := listPythonOrgPackages(version)
+		if err != nil {
+			continue // Skip versions without packages
+		}
+
+		for _, pkg := range packages {
+			if manifest.Versions[version] == nil {
+				manifest.Versions[version] = make(map[string]*Download)
+			}
+
+			// Only add if we don't already have this platform
+			if manifest.Versions[version][pkg.Platform] == nil {
+				manifest.Versions[version][pkg.Platform] = &Download{
+					URL:    pkg.URL,
+					SHA256: "", // python.org doesn't provide easy SHA256 access
+				}
+				addedCount++
+			}
+		}
+	}
+
+	fmt.Printf("Added %d Windows packages from python.org\n", addedCount)
+	return nil
+}
+
+// listPythonOrgVersions lists available Python versions from python.org FTP
+func listPythonOrgVersions() ([]string, error) {
+	resp, err := http.Get(pythonOrgFTPURL)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return nil, err
+	}
+
+	matches := pythonOrgVersionRegex.FindAllStringSubmatch(string(body), -1)
+	versions := make([]string, 0, len(matches))
+	for _, m := range matches {
+		version := m[1]
+		// Only include Python 3.x versions (skip 2.x)
+		if strings.HasPrefix(version, "3.") {
+			versions = append(versions, version)
+		}
+	}
+
+	return versions, nil
+}
+
+// pythonOrgPackage represents a package from python.org
+type pythonOrgPackage struct {
+	URL      string
+	Platform string
+}
+
+// listPythonOrgPackages lists embeddable packages for a specific version
+func listPythonOrgPackages(version string) ([]pythonOrgPackage, error) {
+	url := pythonOrgFTPURL + version + "/"
+	resp, err := http.Get(url)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != 200 {
+		return nil, fmt.Errorf("HTTP %d", resp.StatusCode)
+	}
+
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return nil, err
+	}
+
+	matches := pythonOrgEmbedRegex.FindAllStringSubmatch(string(body), -1)
+	packages := make([]pythonOrgPackage, 0, len(matches))
+
+	for _, m := range matches {
+		filename := m[1]
+		arch := m[3]
+
+		platform := ""
+		switch arch {
+		case "amd64":
+			platform = "windows-amd64"
+		case "arm64":
+			platform = "windows-arm64"
+		default:
+			continue
+		}
+
+		packages = append(packages, pythonOrgPackage{
+			URL:      url + filename,
+			Platform: platform,
+		})
+	}
+
+	return packages, nil
 }
 
 func fetchPythonReleases() ([]githubRelease, error) {