feat(s3): support direct transfer redirects (#2598)

Redirect S3 GET object requests to direct links when the underlying storage can provide one and proxying is not required.

Redirect eligible S3 PUT object requests to HttpDirect single PUT upload URLs with 307, while falling back to the existing gofakes3 stream path for unsupported uploads.

Allow OneDrive Sharelink direct upload info to use simple content PUT URLs for files within Microsoft's single-upload limit.

Co-authored-by: syscc <syscc@users.noreply.github.com>
Co-authored-by: Codex <codex@openai.com>

Author: syscc

drivers/onedrive_sharelink/driver.go

@@ -33,6 +33,7 @@ const (
 	headerTTL          = 25 * time.Minute
 	driveTokenTTL      = 20 * time.Minute
 	directLinkTTL      = 20 * time.Minute
+	simpleUploadLimit  = 250 * 1024 * 1024
 	uploadSessionChunk = 10 * 1024 * 1024
 )
 
@@ -302,7 +303,7 @@ func (d *OnedriveSharelink) createUploadInfo(ctx context.Context, path string, f
 	if err != nil {
 		return nil, err
 	}
-	if fileSize == 0 {
+	if fileSize >= 0 && fileSize <= simpleUploadLimit {
 		return &model.HttpDirectUploadInfo{
 			UploadURL: injectAccessToken(d.drivePathAPIURL(path)+"/content", token),
 			Method:    http.MethodPut,

server/s3/redirect.go

@@ -0,0 +1,169 @@
+// Credits: https://pkg.go.dev/github.com/rclone/rclone@v1.65.2/cmd/serve/s3
+// Package s3 implements a fake s3 server for openlist
+package s3
+
+import (
+	"context"
+	"net"
+	"net/http"
+	"path"
+	"strings"
+
+	"github.com/OpenListTeam/OpenList/v4/internal/conf"
+	"github.com/OpenListTeam/OpenList/v4/internal/fs"
+	"github.com/OpenListTeam/OpenList/v4/internal/model"
+	"github.com/OpenListTeam/OpenList/v4/internal/op"
+	"github.com/OpenListTeam/OpenList/v4/server/common"
+	"github.com/itsHenry35/gofakes3/signature"
+)
+
+func redirectHandler(next http.Handler, authPairs map[string]string) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if u, ok := directObjectURL(r, authPairs); ok {
+			w.Header().Set("Location", u)
+			w.WriteHeader(http.StatusFound)
+			return
+		}
+		if u, ok := directUploadURL(r, authPairs); ok {
+			w.Header().Set("Location", u)
+			w.Header().Set("Cache-Control", "no-store")
+			w.WriteHeader(http.StatusTemporaryRedirect)
+			return
+		}
+		next.ServeHTTP(w, r)
+	})
+}
+
+func directObjectURL(r *http.Request, authPairs map[string]string) (string, bool) {
+	if r.Method != http.MethodGet {
+		return "", false
+	}
+	if hasNonObjectQuery(r) || !s3RequestAuthorized(r, authPairs) {
+		return "", false
+	}
+	bucketName, objectName, ok := parseObjectPath(r.URL.Path)
+	if !ok {
+		return "", false
+	}
+	bucket, err := getBucketByName(bucketName)
+	if err != nil {
+		return "", false
+	}
+	reqPath := path.Join(bucket.Path, objectName)
+	meta, _ := op.GetNearestMeta(reqPath)
+	ctx := context.WithValue(r.Context(), conf.MetaKey, meta)
+	storage, err := fs.GetStorage(reqPath, &fs.GetStoragesArgs{})
+	if err != nil || common.ShouldProxy(storage, path.Base(reqPath)) {
+		return "", false
+	}
+	link, file, err := fs.Link(ctx, reqPath, model.LinkArgs{
+		IP:       clientIP(r),
+		Header:   r.Header,
+		Redirect: true,
+	})
+	if err != nil {
+		return "", false
+	}
+	defer link.Close()
+	if file == nil || file.IsDir() || link.URL == "" || link.RangeReader != nil {
+		return "", false
+	}
+	return link.URL, true
+}
+
+func directUploadURL(r *http.Request, authPairs map[string]string) (string, bool) {
+	if r.Method != http.MethodPut || r.ContentLength < 0 {
+		return "", false
+	}
+	if hasNonObjectQuery(r) || !s3RequestAuthorized(r, authPairs) {
+		return "", false
+	}
+	if r.Header.Get("X-Amz-Copy-Source") != "" ||
+		r.Header.Get("X-Amz-Content-Sha256") == "STREAMING-AWS4-HMAC-SHA256-PAYLOAD" {
+		return "", false
+	}
+	bucketName, objectName, ok := parseObjectPath(r.URL.Path)
+	if !ok || strings.HasSuffix(objectName, "/") {
+		return "", false
+	}
+	bucket, err := getBucketByName(bucketName)
+	if err != nil {
+		return "", false
+	}
+	reqPath := path.Join(bucket.Path, objectName)
+	storage, dstDirActualPath, err := op.GetStorageAndActualPath(path.Dir(reqPath))
+	if err != nil || storage.Config().NoUpload {
+		return "", false
+	}
+	info, err := op.GetDirectUploadInfo(r.Context(), "HttpDirect", storage, dstDirActualPath, path.Base(reqPath), r.ContentLength)
+	if err != nil {
+		return "", false
+	}
+	httpInfo, ok := asHTTPDirectUploadInfo(info)
+	if !ok {
+		return "", false
+	}
+	method := httpInfo.Method
+	if method == "" {
+		method = http.MethodPut
+	}
+	if !strings.EqualFold(method, http.MethodPut) || httpInfo.UploadURL == "" ||
+		httpInfo.ChunkSize > 0 || len(httpInfo.Headers) > 0 {
+		return "", false
+	}
+	return httpInfo.UploadURL, true
+}
+
+func asHTTPDirectUploadInfo(info any) (*model.HttpDirectUploadInfo, bool) {
+	switch v := info.(type) {
+	case *model.HttpDirectUploadInfo:
+		return v, v != nil
+	case model.HttpDirectUploadInfo:
+		return &v, true
+	default:
+		return nil, false
+	}
+}
+
+func parseObjectPath(rawPath string) (bucket, object string, ok bool) {
+	parts := strings.SplitN(strings.TrimPrefix(rawPath, "/"), "/", 2)
+	if len(parts) != 2 || parts[0] == "" || parts[1] == "" {
+		return "", "", false
+	}
+	return parts[0], parts[1], true
+}
+
+func hasNonObjectQuery(r *http.Request) bool {
+	query := r.URL.Query()
+	for key := range query {
+		lower := strings.ToLower(key)
+		if strings.HasPrefix(lower, "x-amz-") ||
+			lower == "awsaccesskeyid" ||
+			lower == "signature" ||
+			lower == "expires" ||
+			lower == "x-id" {
+			continue
+		}
+		return true
+	}
+	return false
+}
+
+func s3RequestAuthorized(r *http.Request, authPairs map[string]string) bool {
+	if len(authPairs) == 0 {
+		return true
+	}
+	result := signature.V4SignVerify(r)
+	if result == signature.ErrUnsupportAlgorithm {
+		result = signature.V2SignVerify(r)
+	}
+	return result == signature.ErrNone
+}
+
+func clientIP(r *http.Request) string {
+	host, _, err := net.SplitHostPort(r.RemoteAddr)
+	if err != nil {
+		return r.RemoteAddr
+	}
+	return host
+}

server/s3/redirect_test.go

@@ -0,0 +1,74 @@
+package s3
+
+import (
+	"net/http/httptest"
+	"testing"
+
+	"github.com/OpenListTeam/OpenList/v4/internal/model"
+)
+
+func TestParseObjectPath(t *testing.T) {
+	tests := []struct {
+		name       string
+		path       string
+		wantBucket string
+		wantObject string
+		wantOK     bool
+	}{
+		{name: "object", path: "/bucket/path/to/file.txt", wantBucket: "bucket", wantObject: "path/to/file.txt", wantOK: true},
+		{name: "missing object", path: "/bucket", wantOK: false},
+		{name: "empty bucket", path: "//file.txt", wantOK: false},
+		{name: "empty object", path: "/bucket/", wantOK: false},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			bucket, object, ok := parseObjectPath(tt.path)
+			if ok != tt.wantOK || bucket != tt.wantBucket || object != tt.wantObject {
+				t.Fatalf("parseObjectPath(%q) = (%q, %q, %v), want (%q, %q, %v)",
+					tt.path, bucket, object, ok, tt.wantBucket, tt.wantObject, tt.wantOK)
+			}
+		})
+	}
+}
+
+func TestHasNonObjectQuery(t *testing.T) {
+	tests := []struct {
+		name string
+		raw  string
+		want bool
+	}{
+		{name: "no query", raw: "/bucket/object", want: false},
+		{name: "aws auth query", raw: "/bucket/object?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Signature=abc", want: false},
+		{name: "legacy auth query", raw: "/bucket/object?AWSAccessKeyId=ak&Signature=sig&Expires=1", want: false},
+		{name: "sdk operation marker", raw: "/bucket/object?x-id=GetObject", want: false},
+		{name: "list query", raw: "/bucket/object?list-type=2", want: true},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			req := httptest.NewRequest("GET", tt.raw, nil)
+			if got := hasNonObjectQuery(req); got != tt.want {
+				t.Fatalf("hasNonObjectQuery(%q) = %v, want %v", tt.raw, got, tt.want)
+			}
+		})
+	}
+}
+
+func TestAsHTTPDirectUploadInfo(t *testing.T) {
+	info := model.HttpDirectUploadInfo{UploadURL: "https://example.com/upload"}
+	got, ok := asHTTPDirectUploadInfo(info)
+	if !ok || got == nil || got.UploadURL != info.UploadURL {
+		t.Fatalf("asHTTPDirectUploadInfo(value) = (%v, %v), want upload info", got, ok)
+	}
+
+	got, ok = asHTTPDirectUploadInfo(&info)
+	if !ok || got == nil || got.UploadURL != info.UploadURL {
+		t.Fatalf("asHTTPDirectUploadInfo(pointer) = (%v, %v), want upload info", got, ok)
+	}
+
+	got, ok = asHTTPDirectUploadInfo(struct{}{})
+	if ok || got != nil {
+		t.Fatalf("asHTTPDirectUploadInfo(unsupported) = (%v, %v), want nil false", got, ok)
+	}
+}

server/s3/server.go

@@ -13,15 +13,16 @@ import (
 // Make a new S3 Server to serve the remote
 func NewServer(ctx context.Context) (h http.Handler, err error) {
 	var newLogger logger
+	authPairs := authlistResolver()
 	faker := gofakes3.New(
 		newBackend(),
 		// gofakes3.WithHostBucket(!opt.pathBucketMode),
 		gofakes3.WithLogger(newLogger),
 		gofakes3.WithRequestID(rand.Uint64()),
 		gofakes3.WithoutVersioning(),
-		gofakes3.WithV4Auth(authlistResolver()),
+		gofakes3.WithV4Auth(authPairs),
 		gofakes3.WithIntegrityCheck(true), // Check Content-MD5 if supplied
 	)
 
-	return faker.Server(), nil
+	return redirectHandler(faker.Server(), authPairs), nil
 }