Revamp CLI serve syntax

Dockerfile

@@ -46,8 +46,7 @@ LABEL org.opencontainers.image.source="https://github.com/readium/cli"
 ADD https://pagure.io/mailcap/raw/master/f/mime.types /etc/
 
 # Add demo EPUBs to the container by default
-# This will go away soon!
-ADD --chown=nonroot:nonroot https://readium-playground-files.storage.googleapis.com/demo/moby-dick.epub /srv/publications/
+# ADD --chown=nonroot:nonroot https://readium-playground-files.storage.googleapis.com/demo/moby-dick.epub /srv/publications/
 
 # Copy built Go binary
 COPY --from=builder "/app/readium" /opt/
@@ -57,4 +56,4 @@ EXPOSE 15080
 USER nonroot:nonroot
 
 ENTRYPOINT ["/opt/readium"]
-CMD ["serve", "/srv/publications", "--address", "0.0.0.0"]
+CMD ["serve", "-s", "http,https", "--address", "0.0.0.0"]

internal/cli/serve.go

@@ -2,12 +2,13 @@ package cli
 
 import (
 	"context"
-	"errors"
 	"fmt"
 	"log"
 	"net/http"
 	"os"
 	"path/filepath"
+	"slices"
+	"strings"
 	"time"
 
 	"log/slog"
@@ -17,9 +18,11 @@ import (
 	"github.com/aws/aws-sdk-go-v2/config"
 	"github.com/aws/aws-sdk-go-v2/credentials"
 	"github.com/aws/aws-sdk-go-v2/service/s3"
+	"github.com/pkg/errors"
 	"github.com/readium/cli/pkg/serve"
 	"github.com/readium/cli/pkg/serve/client"
 	"github.com/readium/go-toolkit/pkg/streamer"
+	"github.com/readium/go-toolkit/pkg/util/url"
 	"github.com/spf13/cobra"
 	"google.golang.org/api/option"
 )
@@ -30,6 +33,10 @@ var bindAddressFlag string
 
 var bindPortFlag uint16
 
+var schemeFlag []string
+
+var fileDirectoryFlag string
+
 // Cloud-related flags
 var s3EndpointFlag string
 var s3RegionFlag string
@@ -45,28 +52,29 @@ var remoteArchiveCacheCount uint32
 var remoteArchiveCacheAll uint32
 
 var serveCmd = &cobra.Command{
-	Use:   "serve <directory>",
+	Use:   "serve",
 	Short: "Start a local HTTP server, serving a specified directory of publications",
 	Long: `Start a local HTTP server, serving a specified directory of publications.
 
 This command will start an HTTP serve listening by default on 'localhost:15080',
-serving all compatible files (EPUB, PDF, CBZ, etc.) found in the directory
+serving all compatible files (EPUB, PDF, CBZ, etc.) available from the enabled
+access schemes (file, http, https, s3, gs, or a local path if file scheme is enabled)
 as Readium Web Publications. To get started, the manifest can be accessed from
 'http://localhost:15080/<filename in base64url encoding without padding>/manifest.json'.
 This file serves as the entry point and contains metadata and links to the rest
 of the files that can be accessed for the publication.
 
-For debugging purposes, the server also exposes a '/list.json' endpoint that
-returns a list of all the publications found in the directory along with their
-encoded paths. This will be replaced by an OPDS 2 feed in a future release.
+If local file access is enabled, the server also exposes a '/list.json' endpoint that, 
+for debugging purposes, returns a list of all the publications found in the directory
+along with their encoded paths. This will be replaced by an OPDS 2 feed (or similar)
+in a future release.
 
-Note: This server is not meant for production usage, and should not be exposed
-to the internet except for testing/debugging purposes.`,
+Note: Take caution before exposing this server on the internet. It does not
+implement any authentication, and may have more access to files than expected.`,
 	Args: func(cmd *cobra.Command, args []string) error {
-		if len(args) == 0 {
-			return errors.New("expects a directory path to serve publications from")
-		} else if len(args) > 1 {
-			return errors.New("accepts a directory path")
+		if len(args) > 0 {
+			// For users migrating from previous versions of the CLI
+			return errors.New("no arguments expected, base directory for local files is now set with a flag")
 		}
 		return nil
 	},
@@ -78,16 +86,37 @@ to the internet except for testing/debugging purposes.`,
 		// occurs.
 		cmd.SilenceUsage = true
 
-		path := filepath.Clean(args[0])
-		fi, err := os.Stat(path)
-		if err != nil {
-			if os.IsNotExist(err) {
-				return fmt.Errorf("given directory %s does not exist", path)
+		// Validate schemes
+		schemes := make([]url.Scheme, len(schemeFlag))
+		for i, v := range schemeFlag {
+			lowerScheme := url.Scheme(strings.ToLower(v)) // Accomodate for wrong capitalization
+			switch lowerScheme {
+			case url.SchemeFile, url.SchemeHTTP, url.SchemeHTTPS, url.SchemeS3, url.SchemeGS:
+				schemes[i] = lowerScheme
+			default:
+				return fmt.Errorf("invalid scheme %q, acceptable values: file, http, https, s3, gs", v)
 			}
-			return fmt.Errorf("failed to stat %s: %w", path, err)
 		}
-		if !fi.IsDir() {
-			return fmt.Errorf("given path %s is not a directory", path)
+
+		if fileDirectoryFlag != "" {
+			if !slices.Contains(schemes, url.SchemeFile) {
+				slog.Warn("local directory specified, but file scheme is not enabled")
+			}
+
+			path := filepath.Clean(fileDirectoryFlag)
+			fi, err := os.Stat(path)
+			if err != nil {
+				if os.IsNotExist(err) {
+					return fmt.Errorf("given directory %s does not exist", path)
+				}
+				return fmt.Errorf("failed to stat %s: %w", path, err)
+			}
+			if !fi.IsDir() {
+				return fmt.Errorf("given path %s is not a directory", path)
+			}
+			fileDirectoryFlag = path
+		} else if slices.Contains(schemes, url.SchemeFile) {
+			return fmt.Errorf("file scheme is enabled, but no local directory was specified with the --file-directory flag")
 		}
 
 		// Log level
@@ -98,51 +127,67 @@ to the internet except for testing/debugging purposes.`,
 		}
 
 		// Set up remote publication retrieval clients
-		remote := serve.Remote{}
+		remote := serve.Remote{
+			LocalDirectory: fileDirectoryFlag,
+		}
+
+		// 30 seconds to set up remote retrieval clients
+		ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+		defer cancel()
 
 		// S3
-		options := []func(*config.LoadOptions) error{
-			config.WithRegion(s3RegionFlag),
-			config.WithRequestChecksumCalculation(0),
-			config.WithResponseChecksumValidation(0),
-			// TODO: look into custom HTTP client, user-agent
-		}
-		if s3AccessKeyFlag != "" && s3SecretKeyFlag != "" {
-			options = append(options, config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(s3AccessKeyFlag, s3SecretKeyFlag, "")))
-		}
-		cfg, err := config.LoadDefaultConfig(context.Background(), options...)
-		if err != nil {
-			log.Fatal(err)
-		}
-		_, err = cfg.Credentials.Retrieve(context.Background())
-		if err == nil {
-			remote.S3 = s3.NewFromConfig(cfg, func(o *s3.Options) {
-				if s3EndpointFlag != "" {
-					o.BaseEndpoint = aws.String(s3EndpointFlag)
-				}
-				o.DisableLogOutputChecksumValidationSkipped = true // Non-AWS S3 tends not to support this and it causes logspam
-				o.UsePathStyle = s3UsePathStyleFlag
-			})
-		} else {
-			slog.Warn("S3 credentials retrieval failed, S3 support will be disabled", "error", err)
+		if slices.Contains(schemes, url.SchemeS3) {
+			options := []func(*config.LoadOptions) error{
+				config.WithRegion(s3RegionFlag),
+				config.WithRequestChecksumCalculation(0),
+				config.WithResponseChecksumValidation(0),
+				// TODO: look into custom HTTP client, user-agent
+			}
+			if s3AccessKeyFlag != "" && s3SecretKeyFlag != "" {
+				options = append(options, config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(s3AccessKeyFlag, s3SecretKeyFlag, "")))
+			}
+			cfg, err := config.LoadDefaultConfig(ctx, options...)
+			if err != nil {
+				log.Fatal(err)
+			}
+			_, err = cfg.Credentials.Retrieve(ctx)
+			if err == nil {
+				remote.S3 = s3.NewFromConfig(cfg, func(o *s3.Options) {
+					if s3EndpointFlag != "" {
+						o.BaseEndpoint = aws.String(s3EndpointFlag)
+					}
+					o.DisableLogOutputChecksumValidationSkipped = true // Non-AWS S3 tends not to support this and it causes logspam
+					o.UsePathStyle = s3UsePathStyleFlag
+				})
+			} else {
+				return fmt.Errorf("S3 credentials retrieval failed: %w", err)
+			}
+		} else if s3AccessKeyFlag != "" || s3SecretKeyFlag != "" || s3EndpointFlag != "" {
+			slog.Warn("S3-related flags are set, but S3 scheme is not enabled")
 		}
 
 		// GCS
-		opts := []option.ClientOption{
-			option.WithScopes(storage.ScopeReadOnly),
-			storage.WithJSONReads(),
-			// option.WithUserAgent(TODO),
-			// TODO: look into more efficient transport (HTTP client)
-		}
-		remote.GCS, err = storage.NewClient(context.Background(), opts...)
-		if err != nil {
-			slog.Warn("GCS client creation failed, GCS support will be disabled", "error", err)
+		var err error
+		if slices.Contains(schemes, url.SchemeGS) {
+			opts := []option.ClientOption{
+				option.WithScopes(storage.ScopeReadOnly),
+				storage.WithJSONReads(),
+				// option.WithUserAgent(TODO),
+				// TODO: look into more efficient transport (HTTP client)
+			}
+			remote.GCS, err = storage.NewClient(ctx, opts...)
+			if err != nil {
+				return fmt.Errorf("GCS client creation failed: %w", err)
+			}
 		}
 
+		// HTTP/HTTPS
 		remote.HTTP, err = client.NewHTTPClient(httpAuthorizationFlag)
 		if err != nil {
 			slog.Warn("HTTP client creation failed, HTTP support will be disabled", "error", err)
 		}
+		remote.HTTPEnabled = slices.Contains(schemes, url.SchemeHTTP)
+		remote.HTTPSEnabled = slices.Contains(schemes, url.SchemeHTTPS)
 
 		// Remote archive streaming tweaks
 		remote.Config.CacheCountThreshold = int64(remoteArchiveCacheCount)
@@ -153,7 +198,6 @@ to the internet except for testing/debugging purposes.`,
 		// Create server
 		pubServer := serve.NewServer(serve.ServerConfig{
 			Debug:             debugFlag,
-			BaseDirectory:     path,
 			JSONIndent:        indentFlag,
 			InferA11yMetadata: streamer.InferA11yMetadata(inferA11yFlag),
 		}, remote)
@@ -180,12 +224,15 @@ to the internet except for testing/debugging purposes.`,
 func init() {
 	rootCmd.AddCommand(serveCmd)
 
+	serveCmd.Flags().StringSliceVarP(&schemeFlag, "scheme", "s", []string{"file"}, "Scheme(s) to enable for accessing content. Acceptable values: file, http, https, s3, gs")
 	serveCmd.Flags().StringVarP(&bindAddressFlag, "address", "a", "localhost", "Address to bind the HTTP server to")
 	serveCmd.Flags().Uint16VarP(&bindPortFlag, "port", "p", 15080, "Port to bind the HTTP server to")
 	serveCmd.Flags().StringVarP(&indentFlag, "indent", "i", "", "Indentation used to pretty-print JSON files")
 	serveCmd.Flags().Var(&inferA11yFlag, "infer-a11y", "Infer accessibility metadata: no, merged, split")
 	serveCmd.Flags().BoolVarP(&debugFlag, "debug", "d", false, "Enable debug mode")
 
+	serveCmd.Flags().StringVar(&fileDirectoryFlag, "file-directory", "", "Local directory path to serve publications from")
+
 	serveCmd.Flags().StringVar(&s3EndpointFlag, "s3-endpoint", "", "Custom S3 endpoint URL")
 	serveCmd.Flags().StringVar(&s3RegionFlag, "s3-region", "auto", "S3 region")
 	serveCmd.Flags().StringVar(&s3AccessKeyFlag, "s3-access-key", "", "S3 access key")

pkg/serve/api.go

@@ -34,8 +34,15 @@ type demoListItem struct {
 	Path     string `json:"path"`
 }
 
+// TODO: replace with OPDS or something better
 func (s *Server) demoList(w http.ResponseWriter, req *http.Request) {
-	fi, err := os.ReadDir(s.config.BaseDirectory)
+	if s.remote.LocalDirectory == "" {
+		slog.Warn("demo publication list requested, but no local directory configured")
+		w.WriteHeader(404)
+		return
+	}
+
+	fi, err := os.ReadDir(s.remote.LocalDirectory)
 	if err != nil {
 		slog.Error("failed reading publications directory", "error", err)
 		w.WriteHeader(500)
@@ -72,8 +79,11 @@ func (s *Server) getPublication(ctx context.Context, filename string) (*pub.Publ
 			InferA11yMetadata: s.config.InferA11yMetadata,
 			HttpClient:        s.remote.HTTP,
 		}
+		if !s.remote.AcceptsScheme(u.Scheme()) {
+			return nil, remote, errors.New("unacceptable scheme " + u.Scheme().String())
+		}
 		if u.IsFile() {
-			path, err := url.FromFilepath(filepath.Join(s.config.BaseDirectory, path.Clean(u.Path())))
+			path, err := url.FromFilepath(filepath.Join(s.remote.LocalDirectory, path.Clean(u.Path())))
 			if err != nil {
 				return nil, remote, errors.Wrap(err, "failed creating URL from filepath")
 			}

pkg/serve/helpers.go

@@ -49,6 +49,10 @@ var compressableMimes = []string{
 	"application/font-woff",
 	"font/x-woff",
 	"application/vnd.ms-fontobject",
+	mediatype.OPF.String(),
+	mediatype.NCX.String(),
+	mediatype.XML.String(),
+	mediatype.JSON.String(),
 }
 
 func conformsToAsMimetype(conformsTo manifest.Profiles) mediatype.MediaType {

pkg/serve/server.go

@@ -10,18 +10,38 @@ import (
 	"github.com/readium/cli/pkg/serve/cache"
 	"github.com/readium/go-toolkit/pkg/archive"
 	"github.com/readium/go-toolkit/pkg/streamer"
+	"github.com/readium/go-toolkit/pkg/util/url"
 )
 
 type Remote struct {
-	S3     *s3.Client      // AWS S3-compatible storage
-	GCS    *storage.Client // Google Cloud Storage
-	HTTP   *http.Client    // HTTP-requested storage
-	Config archive.RemoteArchiveConfig
+	LocalDirectory string          // Local directory base path
+	S3             *s3.Client      // AWS S3-compatible storage
+	GCS            *storage.Client // Google Cloud Storage
+	HTTP           *http.Client    // HTTP-requested storage
+	HTTPEnabled    bool            // Whether HTTP is enabled
+	HTTPSEnabled   bool            // Whether HTTPS is enabled
+	Config         archive.RemoteArchiveConfig
+}
+
+func (r Remote) AcceptsScheme(scheme url.Scheme) bool {
+	switch scheme {
+	case url.SchemeFile:
+		return r.LocalDirectory != ""
+	case url.SchemeS3:
+		return r.S3 != nil
+	case url.SchemeGS:
+		return r.GCS != nil
+	case url.SchemeHTTP:
+		return r.HTTPEnabled && r.HTTP != nil
+	case url.SchemeHTTPS:
+		return r.HTTPSEnabled && r.HTTP != nil
+	default:
+		return false
+	}
 }
 
 type ServerConfig struct {
 	Debug             bool
-	BaseDirectory     string
 	JSONIndent        string
 	InferA11yMetadata streamer.InferA11yMetadata
 }