update cleanfilepath and use global env variables (#618)

Co-authored-by: Oskar Hurst <oskar.a.hurst@usace.army.mil>

Author: oskarhurst

api/handlers/media.go

@@ -4,7 +4,7 @@ import (
 	"context"
 	"net/http"
 	"net/url"
-	"path/filepath"
+	"path"
 	"strings"
 
 	"github.com/aws/aws-sdk-go-v2/aws"
@@ -14,29 +14,30 @@ import (
 
 func cleanFilepath(rawPath string) (string, error) {
 	p, err := url.PathUnescape(rawPath)
-	// Replace /api with /
-	p = strings.Replace(p, "/api", "/", 1)
 	if err != nil {
 		return "", err
 	}
-	return filepath.Clean("/" + p), nil
+	// The download route is mounted under the /api group; strip that prefix to
+	// recover the S3 key.
+	p = strings.TrimPrefix(p, "/api")
+	// S3 keys are exact, literal strings under the aws-sdk-go-v2 client: a leading
+	// slash makes "/cumulus/..." a different key than the stored "cumulus/..." and
+	// real S3 returns NoSuchKey (the v1 SDK and MinIO normalize it, hiding this).
+	// path.Clean still resolves any ".." for traversal safety.
+	return strings.TrimPrefix(path.Clean("/"+p), "/"), nil
 }
 
-func ServeMedia(awsCfg *aws.Config, bucket *string, endpoint string, forcePathStyle, disableSSL bool) echo.HandlerFunc {
+func ServeMedia(awsCfg *aws.Config, bucket *string, forcePathStyle bool) echo.HandlerFunc {
 	return func(c echo.Context) error {
 		path, err := cleanFilepath(c.Request().RequestURI)
 		if err != nil {
 			return c.String(http.StatusBadRequest, err.Error())
 		}
 
+		// Endpoint/scheme come from the environment (AWS_ENDPOINT_URL_S3) via
+		// awsCfg; only path-style addressing still needs to be set per-client.
 		_client := s3.NewFromConfig(*awsCfg, func(o *s3.Options) {
-			o.UsePathStyle = forcePathStyle // was a.WithS3ForcePathStyle(...)
-			if endpoint != "" {             // was: if cfg.AWSS3Endpoint != "" { a.WithEndpoint(...) }
-				o.BaseEndpoint = aws.String(endpoint)
-			}
-			if disableSSL { // was a.WithDisableSSL(...)
-				o.EndpointOptions.DisableHTTPS = true
-			}
+			o.UsePathStyle = forcePathStyle
 		})
 		output, err := _client.GetObject(context.Background(), &s3.GetObjectInput{Bucket: bucket, Key: aws.String(path)})
 		if err != nil {

api/handlers/productfile_cog.go

@@ -59,7 +59,7 @@ func ListProductfilesCOG(db *pgxpool.Pool) echo.HandlerFunc {
 // Content-Range/Accept-Ranges so a GDAL /vsicurl/ client can read tiles directly
 // (no full download). HEAD returns size + range support for /vsicurl/ probing.
 // Every request is authenticated (private route) and logged for metering.
-func StreamProductfileCOG(db *pgxpool.Pool, awsCfg *aws.Config, endpoint string, forcePathStyle, disableSSL bool) echo.HandlerFunc {
+func StreamProductfileCOG(db *pgxpool.Pool, awsCfg *aws.Config, forcePathStyle bool) echo.HandlerFunc {
 	return func(c echo.Context) error {
 		pfID, err := uuid.Parse(c.Param("productfile_id"))
 		if err != nil {
@@ -72,14 +72,10 @@ func StreamProductfileCOG(db *pgxpool.Pool, awsCfg *aws.Config, endpoint string,
 		}
 
 		ctx := c.Request().Context()
+		// Endpoint/scheme come from the environment (AWS_ENDPOINT_URL_S3) via
+		// awsCfg; only path-style addressing still needs to be set per-client.
 		client := s3.NewFromConfig(*awsCfg, func(o *s3.Options) {
 			o.UsePathStyle = forcePathStyle
-			if endpoint != "" {
-				o.BaseEndpoint = aws.String(endpoint)
-			}
-			if disableSSL {
-				o.EndpointOptions.DisableHTTPS = true
-			}
 		})
 
 		// HEAD: metadata only (size + range support) for /vsicurl/ probing.

api/main.go

@@ -162,8 +162,8 @@ func main() {
 	public.GET("/products/:product_id/files", handlers.ListProductfiles(db))
 	// Direct, Range-capable COG access (authenticated + metered) for desktop clients
 	private.GET("/products/:product_id/cog-files", handlers.ListProductfilesCOG(db))
-	private.GET("/products/:product_id/cog/:productfile_id", handlers.StreamProductfileCOG(db, &cfg.AwsConfig, cfg.AWSS3Endpoint, cfg.AWSS3ForcePathStyle, cfg.AWSS3DisableSSL))
-	private.HEAD("/products/:product_id/cog/:productfile_id", handlers.StreamProductfileCOG(db, &cfg.AwsConfig, cfg.AWSS3Endpoint, cfg.AWSS3ForcePathStyle, cfg.AWSS3DisableSSL))
+	private.GET("/products/:product_id/cog/:productfile_id", handlers.StreamProductfileCOG(db, &cfg.AwsConfig, cfg.AWSS3ForcePathStyle))
+	private.HEAD("/products/:product_id/cog/:productfile_id", handlers.StreamProductfileCOG(db, &cfg.AwsConfig, cfg.AWSS3ForcePathStyle))
 
 	// Productfiles
 	private.POST("/productfiles", handlers.CreateProductfiles(db),
@@ -234,8 +234,7 @@ func main() {
 
 	// Serve Downloads
 	public.GET("/cumulus/download/*", handlers.ServeMedia(
-		&cfg.AwsConfig, &cfg.AWSS3Bucket,
-		cfg.AWSS3Endpoint, cfg.AWSS3ForcePathStyle, cfg.AWSS3DisableSSL,
+		&cfg.AwsConfig, &cfg.AWSS3Bucket, cfg.AWSS3ForcePathStyle,
 	))
 
 	// List Downloads

docker-compose.yml

@@ -165,6 +165,9 @@ services:
       - CUMULUS_AWS_S3_ENDPOINT=http://minio:9000
       - CUMULUS_AWS_S3_DISABLE_SSL=True
       - CUMULUS_AWS_S3_FORCE_PATH_STYLE=True
+      # Global (S3-scoped) endpoint picked up by aws-sdk-go-v2 LoadDefaultConfig.
+      # Unset in prod so the SDK uses real S3. http:// disables TLS for MinIO.
+      - AWS_ENDPOINT_URL_S3=http://minio:9000
       - PG_FEATURESERV_URL=http://featureserv:8080
     volumes:
       - ./api/:/go/src/app/:rw