Fixed the issue with downloader doesn't work with redirects #101 (#137)

* Fixed the issue with downloader doesn't work with redirects #101

* Coderabbit was mad

* Configurable max reddirects!

* Fix off-by-one error in redirect limit.

Author: NaysKutzu

config/config.go

@@ -93,6 +93,12 @@ type ApiConfiguration struct {
 	// servers.
 	DisableRemoteDownload bool `json:"-" yaml:"disable_remote_download"`
 
+	// RemoteDownload contains configuration for server remote download functionality.
+	RemoteDownload struct {
+		// MaxRedirects controls how many HTTP redirects are followed when performing a remote download.
+		MaxRedirects int `default:"10" json:"max_redirects" yaml:"max_redirects"`
+	} `json:"remote_download" yaml:"remote_download"`
+
 	// The maximum size for files uploaded through the Panel in MiB.
 	UploadLimit int64 `default:"100" json:"upload_limit" yaml:"upload_limit"`
 

router/downloader/downloader.go

@@ -17,6 +17,7 @@ import (
 	"github.com/goccy/go-json"
 	"github.com/google/uuid"
 
+	"github.com/pelican-dev/wings/config"
 	"github.com/pelican-dev/wings/server"
 )
 
@@ -101,6 +102,8 @@ const (
 	ErrDownloadFailed     = errors.Sentinel("downloader: download request failed")
 )
 
+const defaultMaxRedirects = 10
+
 type Counter struct {
 	total   int
 	onWrite func(total int)
@@ -184,17 +187,67 @@ func (dl *Download) Execute() error {
 	// At this point we have verified the destination is not within the local network, so we can
 	// now make a request to that URL and pull down the file, saving it to the server's data
 	// directory.
-	req, err := http.NewRequestWithContext(ctx, http.MethodGet, dl.req.URL.String(), nil)
-	if err != nil {
-		return errors.WrapIf(err, "downloader: failed to create request")
+	currentURL := dl.req.URL
+	if currentURL == nil {
+		return errors.New("downloader: download request url is nil")
 	}
 
-	req.Header.Set("User-Agent", "Pelican Panel (https://pelican.dev)")
-	res, err := client.Do(req)
-	if err != nil {
-		return ErrDownloadFailed
+	visited := make(map[string]struct{})
+	var res *http.Response
+	var finalURL *url.URL
+
+	maxRedirects := maxRedirectAttempts()
+	for redirects := 0; redirects < maxRedirects; redirects++ {
+		urlStr := currentURL.String()
+		if _, seen := visited[urlStr]; seen {
+			return errors.New("downloader: detected redirect loop")
+		}
+		visited[urlStr] = struct{}{}
+
+		req, err := http.NewRequestWithContext(ctx, http.MethodGet, urlStr, nil)
+		if err != nil {
+			return errors.WrapIf(err, "downloader: failed to create request")
+		}
+
+		req.Header.Set("User-Agent", "Pelican Panel (https://pelican.dev)")
+		res, err = client.Do(req)
+		if err != nil {
+			return errors.WrapIf(err, "downloader: failed to perform request")
+		}
+
+		if res.StatusCode >= http.StatusMultipleChoices && res.StatusCode < http.StatusBadRequest {
+			location := res.Header.Get("Location")
+			res.Body.Close()
+			if location == "" {
+				return errors.New("downloader: redirect response missing location header")
+			}
+
+			nextURL, err := currentURL.Parse(location)
+			if err != nil {
+				return errors.WrapIf(err, "downloader: invalid redirect location")
+			}
+			if nextURL.Scheme != "http" && nextURL.Scheme != "https" {
+				return errors.New("downloader: redirect to unsupported scheme")
+			}
+
+			currentURL = nextURL
+			finalURL = nextURL
+			continue
+		}
+
+		finalURL = currentURL
+		break
+	}
+
+	if res == nil {
+		return errors.New("downloader: exceeded maximum redirect attempts")
 	}
 	defer res.Body.Close()
+
+	if res.StatusCode >= http.StatusMultipleChoices && res.StatusCode < http.StatusBadRequest {
+		return errors.New("downloader: exceeded maximum redirect attempts")
+	}
+
 	if res.StatusCode != http.StatusOK {
 		return errors.New("downloader: got bad response status from endpoint: " + res.Status)
 	}
@@ -219,7 +272,11 @@ func (dl *Download) Execute() error {
 		if dl.req.FileName != "" {
 			dl.path = dl.req.FileName
 		} else {
-			parts := strings.Split(dl.req.URL.Path, "/")
+			pathSource := dl.req.URL
+			if finalURL != nil {
+				pathSource = finalURL
+			}
+			parts := strings.Split(pathSource.Path, "/")
 			dl.path = parts[len(parts)-1]
 		}
 	}
@@ -335,3 +392,13 @@ func mustParseCIDR(ip string) *net.IPNet {
 	}
 	return block
 }
+
+func maxRedirectAttempts() int {
+	cfg := config.Get()
+	if cfg != nil {
+		if v := cfg.Api.RemoteDownload.MaxRedirects; v > 0 {
+			return v
+		}
+	}
+	return defaultMaxRedirects
+}