diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml deleted file mode 100644 index 0ecb437c..00000000 --- a/.github/workflows/release.yml +++ /dev/null @@ -1,77 +0,0 @@ -name: Release - -on: - push: - tags: - - '*' - -permissions: - contents: write - packages: write - -env: - IMAGE_NAME: rwp - -jobs: - release: - runs-on: [self-hosted, arm64] - steps: - - name: Harden Runner - uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0 - with: - egress-policy: audit - - - name: Checkout - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - fetch-depth: 0 - - run: git fetch --force --tags - - name: Set up Go - uses: actions/setup-go@f111f3307d8850f501ac008e886eec1fd1932a34 # v5.3.0 - with: - go-version: '>=1.23.0' - cache: false - - name: Build release - uses: goreleaser/goreleaser-action@90a3faa9d0182683851fbfa97ca1a2cb983bfca3 # v6.2.1 - with: - distribution: goreleaser - version: latest - args: release --clean - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - container: - runs-on: [self-hosted, arm64] - steps: - - name: Harden Runner - uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0 - with: - egress-policy: audit - - - name: Checkout - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - fetch-depth: 0 - - run: git fetch --force --tags - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2 # v3.10.0 - - name: Build and push Docker image - run: docker buildx build --platform=linux/amd64,linux/arm64,linux/arm/v7 . --file Dockerfile --tag $IMAGE_NAME --label "runnumber=${GITHUB_RUN_ID}" --build-arg NO_SNAPSHOT=true - - name: Log in to registry - run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin - - name: Push image - run: | - IMAGE_ID=ghcr.io/${{ github.repository_owner }}/$IMAGE_NAME - # This changes all uppercase characters to lowercase. - IMAGE_ID=$(echo $IMAGE_ID | tr '[A-Z]' '[a-z]') - # This strips the git ref prefix from the version. - VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,') - # This strips the "v" prefix from the tag name. - [[ "${{ github.ref }}" == "refs/tags/"* ]] && VERSION=$(echo $VERSION | sed -e 's/^v//') - # This uses the Docker `latest` tag convention. - [ "$VERSION" == "main" ] && VERSION=latest - echo IMAGE_ID=$IMAGE_ID - echo VERSION=$VERSION - docker buildx build --push \ - --tag $IMAGE_ID:$VERSION \ - --build-arg NO_SNAPSHOT=true \ - --platform linux/amd64,linux/arm64,linux/arm/v7 . diff --git a/.gitignore b/.gitignore index 6c60a66d..e60b3492 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,2 @@ publications/* -*.old -dist/ +*.old \ No newline at end of file diff --git a/.goreleaser.yaml b/.goreleaser.yaml deleted file mode 100644 index 159a7c1a..00000000 --- a/.goreleaser.yaml +++ /dev/null @@ -1,49 +0,0 @@ -# Make sure to check the documentation at https://goreleaser.com -version: 2 -before: - hooks: - - go mod tidy - - go generate ./... -gomod: - proxy: true -builds: - - main: ./cmd/rwp/ - env: - - CGO_ENABLED=0 - id: rwp - binary: rwp - goos: - - linux - - windows - - darwin - goarch: - - '386' - - amd64 - - arm - - arm64 - goarm: - - '7' - goamd64: - - v3 - ldflags: - - -s -w - -archives: - - formats: tar.gz - # this name template makes the OS and Arch compatible with the results of uname. - # Used to start with {{ .ProjectName }} - name_template: >- - rwp_ - {{- tolower .Os }}_ - {{- if eq .Arch "amd64" }}x86_64 - {{- else if eq .Arch "386" }}i386 - {{- else }}{{ .Arch }}{{ end }} - {{- if .Arm }}v{{ .Arm }}{{ end }} - # use zip for windows archives - format_overrides: - - goos: windows - formats: ['zip'] -checksum: - name_template: 'checksums.txt' -snapshot: - version_template: "{{ incpatch .Version }}-next" diff --git a/CHANGELOG.md b/CHANGELOG.md index e4aaf43a..5abf794a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,22 @@ All notable changes to this project will be documented in this file. ## [Unreleased] +### Removed + +- The `cmd` folder has been removed, along with the `rwp` command and its command-line utilities including the web server. Please use [the new Readium CLI repo](https://github.com/readium/cli) as a replacement. The docker build and executable releases are also now migrated to that repository, and the `rwp` verbiage is now `readium`. + +### Added + +- Remote streaming of publications is now supported. Sources include HTTP servers (capable of byte range requests), Amazon S3 and S3-compatible object storage, Google Cloud Storage (GCS) +- Added a new helper to transform fetchers and resources into interfaces compatible with Go's `fs.FS` and `fs.File` +- Add support for `hash` property in links, with a list of recognized algorithms and utility functions +- A new `analyzer` package has been added that supports image analysis. We're not 100% sure this will remain in the toolkit, it could migrate to the cli repository. + +### Changed + +- In order to support remote streaming, a lot of APIs have been altered to accept a `context.Context` as the first parameter, to provide implementors with the ability to e.g. cancel a request to fetch a resource. +- `ReadAsString`, `ReadAsJSON`, and `ReadAsXML` functions have been removed from `Resource` and are instead available as helper functions. + ## [0.8.1] - 2025-02-24 ### Changed diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index 90793f23..00000000 --- a/Dockerfile +++ /dev/null @@ -1,63 +0,0 @@ -FROM --platform=$BUILDPLATFORM golang:1-bookworm@sha256:b970e6d47c09fdd34179acef5c4fecaf6410f0b597a759733b3cbea04b4e604a AS builder -ARG BUILDARCH TARGETOS TARGETARCH -ARG NO_SNAPSHOT=false - -# Install GoReleaser -RUN wget --no-verbose "https://github.com/goreleaser/goreleaser/releases/download/v2.7.0/goreleaser_2.7.0_$BUILDARCH.deb" -RUN dpkg -i "goreleaser_2.7.0_$BUILDARCH.deb" - -# Create and change to the app directory. -WORKDIR /app - -# Retrieve application dependencies. -# This allows the container build to reuse cached dependencies. -# Expecting to copy go.mod and if present go.sum. -COPY go.* ./ -RUN go mod download - -# Copy local code to the container image. -COPY . ./ - -RUN git describe --tags --always - -# RUN git lfs pull && ls -alh publications - -# Run goreleaser -RUN --mount=type=cache,target=/root/.cache/go-build \ - --mount=type=cache,target=/go/pkg \ - GOOS=$TARGETOS GOARCH=$TARGETARCH GOAMD64=v3 GOARM=7 \ - goreleaser build --single-target --id rwp --skip=validate $(case "$NO_SNAPSHOT" in yes|true|1) ;; *) echo "--snapshot";; esac) --output ./rwp - -# Run tests -# FROM builder AS tester -# RUN go test ./... - -# Produces very small images -FROM gcr.io/distroless/static-debian12 AS packager - -# Extra metadata -LABEL org.opencontainers.image.source="https://github.com/readium/go-toolkit" - -# Add Fedora's mimetypes (pretty up-to-date and expansive) -# since the distroless container doesn't have any. Go uses -# this file as part of its mime package, and readium/go-toolkit -# has a mediatype package that falls back to Go's mime -# package to discover a file's mimetype when all else fails. -ADD https://pagure.io/mailcap/raw/master/f/mime.types /etc/ - -# Add two demo EPUBs to the container by default -ADD --chown=nonroot:nonroot https://readium-playground-files.storage.googleapis.com/demo/moby-dick.epub /srv/publications/ -ADD --chown=nonroot:nonroot https://readium-playground-files.storage.googleapis.com/demo/BellaOriginal3.epub /srv/publications/ -ADD --chown=nonroot:nonroot https://readium-playground-files.storage.googleapis.com/demo/coup002elin01_01.epub /srv/publications/ -ADD --chown=nonroot:nonroot https://readium-playground-files.storage.googleapis.com/demo/les_diaboliques.epub /srv/publications/ -ADD --chown=nonroot:nonroot https://readium-playground-files.storage.googleapis.com/demo/nathaniel-hawthorne_the-house-of-the-seven-gables_advanced.epub /srv/publications/ - -# Copy built Go binary -COPY --from=builder "/app/rwp" /opt/ - -EXPOSE 15080 - -USER nonroot:nonroot - -ENTRYPOINT ["/opt/rwp"] -CMD ["serve", "/srv/publications", "--address", "0.0.0.0"] \ No newline at end of file diff --git a/LICENSE.md b/LICENSE similarity index 100% rename from LICENSE.md rename to LICENSE diff --git a/Makefile b/Makefile deleted file mode 100644 index 77d60547..00000000 --- a/Makefile +++ /dev/null @@ -1,15 +0,0 @@ -help: - @echo "Usage: make \n\n\ - build\t\tBuild the \`rwp\` command-line utility in the current directory\n\ - install\tBuild and install the \`rwp\` command-line utility\n\ - " - -.PHONY: build -build: - (cd cmd/rwp; go build; mv rwp ../..) - -.PHONY: install -install: - (cd cmd/rwp; go install) - - diff --git a/README.md b/README.md index 2bddc6bf..76edb95d 100644 --- a/README.md +++ b/README.md @@ -1,63 +1,5 @@ # Readium Go Toolkit -More documentation coming soon! Things are changing too quickly right now. +**Looking for the `rwp` command-line tools? Checkout the new [Readium CLI](https://github.com/readium/cli) repo!** -For development, run `go run cmd/server/main.go` to start the server, which by default listens on `127.0.0.1:5080`. Check out the [example configuration file](https://github.com/readium/go-toolkit/blob/master/cmd/server/configs/config.local.toml.example) for configuration options. - -## Command line utility - -The `rwp` command provides utilities to parse and generate Web Publications. - -To install `rwp` in `~/go/bin`, run `make install`. Use `make build` to build the binary in the current directory. - -### Generating a Readium Web Publication Manifest - -The `rwp manifest` command will parse a publication file (such as EPUB, PDF, audiobook, etc.) and build a Readium Web Publication Manifest for it. The JSON manifest is -printed to stdout. - -Examples: - -* Print out a compact JSON RWPM. - ```sh - rwp manifest publication.epub - ``` -* Pretty-print a JSON RWPM using two-space indent. - ```sh - rwp manifest --indent " " publication.epub - ``` -* Extract the publication title with `jq`. - ```sh - rwp manifest publication.epub | jq -r .metadata.title - ``` - -#### Accessibility inference - -`rwp manifest` can infer additional accessibility metadata when they are missing, with the `--infer-a11y` flag. It takes one of the following arguments: - -| Option | Description | -|------------------|--------------------------------------------------------------------------------------------------------| -| `no` (*default*) | No accessibility metadata will be inferred. | -| `merged` | Accessibility metadata will be inferred and merged with the authored ones in `metadata.accessibility`. | -| `split` | Accessibility metadata will be inferred but stored separately in `metadata.inferredAccessibility`. | - -```sh -rwp manifest --infer-a11y=merged publication.epub | jq .metadata -``` - -##### Inferred metadata - -| Key | Value | Inferred? | -|-----|-------|-----------| -| `accessMode` | `auditory` | If the publication contains a reference to an audio or video resource (inspect `resources` and `readingOrder` in RWPM) | -| `accessMode` | `visual` | If the publications contains a reference to an image or a video resource (inspect `resources` and `readingOrder` in RWPM) | -| `accessModeSufficient` | `textual` | If the publication is partially or fully accessible (WCAG A or above)
Or if the publication does not contain any image, audio or video resource (inspect "resources" and "readingOrder" in RWPM)
Or if the only image available can be identified as a cover | -| `feature` | `displayTransformability` | If the publication is fully accessible (WCAG AA or above)
:warning: This property should only be inferred for reflowable EPUB files as it doesn't apply to other formats (FXL, PDF, audiobooks, CBZ/CBR). | -| `feature` | `printPageNumbers` | If the publications contains a page list (check for the presence of a `pageList` collection in RWPM) | -| `feature` | `tableOfContents` | If the publications contains a table of contents (check for the presence of a `toc` collection in RWPM) | -| `feature` | `MathML` | If the publication contains any resource with MathML (check for the presence of the `contains` property where the value is `mathml` in `readingOrder` or `resources` in RWPM) | -| `feature` | `synchronizedAudioText` | If the publication contains any reference to Media Overlays (TBD in RWPM) | - -### HTTP streaming of local publications - -`rwp serve` starts an HTTP server that serves EPUB, CBZ and other compatible formats from a given directory. -A log is printed to stdout. \ No newline at end of file +More documentation coming soon! \ No newline at end of file diff --git a/cmd/rwp/cmd/helpers/inference.go b/cmd/rwp/cmd/helpers/inference.go deleted file mode 100644 index 326757b0..00000000 --- a/cmd/rwp/cmd/helpers/inference.go +++ /dev/null @@ -1,43 +0,0 @@ -package helpers - -import ( - "errors" - - "github.com/readium/go-toolkit/pkg/streamer" -) - -type InferA11yMetadata streamer.InferA11yMetadata - -// String is used both by fmt.Print and by Cobra in help text -func (e *InferA11yMetadata) String() string { - if e == nil { - return "no" - } - switch *e { - case InferA11yMetadata(streamer.InferA11yMetadataMerged): - return "merged" - case InferA11yMetadata(streamer.InferA11yMetadataSplit): - return "split" - default: - return "no" - } -} - -func (e *InferA11yMetadata) Set(v string) error { - switch v { - case "no": - *e = InferA11yMetadata(streamer.InferA11yMetadataNo) - case "merged": - *e = InferA11yMetadata(streamer.InferA11yMetadataMerged) - case "split": - *e = InferA11yMetadata(streamer.InferA11yMetadataSplit) - default: - return errors.New(`must be one of "no", "merged", or "split"`) - } - return nil -} - -// Type is only used in help text. -func (e *InferA11yMetadata) Type() string { - return "string" -} diff --git a/cmd/rwp/cmd/helpers/inspector.go b/cmd/rwp/cmd/helpers/inspector.go deleted file mode 100644 index 4f82f9ef..00000000 --- a/cmd/rwp/cmd/helpers/inspector.go +++ /dev/null @@ -1,51 +0,0 @@ -package helpers - -import ( - "io/fs" - - "github.com/pkg/errors" - "github.com/readium/go-toolkit/pkg/analyzer" - "github.com/readium/go-toolkit/pkg/manifest" -) - -type ImageInspector struct { - Filesystem fs.FS - Algorithms []manifest.HashAlgorithm - err error -} - -func (n *ImageInspector) Error() error { - return n.err -} - -// TransformHREF implements ManifestTransformer -func (n *ImageInspector) TransformHREF(href manifest.HREF) manifest.HREF { - // Identity - return href -} - -// TransformLink implements ManifestTransformer -func (n *ImageInspector) TransformLink(link manifest.Link) manifest.Link { - if n.err != nil || link.MediaType == nil || !link.MediaType.IsBitmap() { - return link - } - - newLink, err := analyzer.Image(n.Filesystem, link, n.Algorithms) - if err != nil { - n.err = errors.Wrap(err, "failed inspecting image "+link.Href.String()) - return link - } - return *newLink -} - -// TransformManifest implements ManifestTransformer -func (n *ImageInspector) TransformManifest(manifest manifest.Manifest) manifest.Manifest { - // Identity - return manifest -} - -// TransformMetadata implements ManifestTransformer -func (n *ImageInspector) TransformMetadata(metadata manifest.Metadata) manifest.Metadata { - // Identity - return metadata -} diff --git a/cmd/rwp/cmd/manifest.go b/cmd/rwp/cmd/manifest.go deleted file mode 100644 index ef2fad26..00000000 --- a/cmd/rwp/cmd/manifest.go +++ /dev/null @@ -1,129 +0,0 @@ -package cmd - -import ( - "context" - "encoding/json" - "fmt" - "path/filepath" - - "github.com/pkg/errors" - "github.com/readium/go-toolkit/cmd/rwp/cmd/helpers" - "github.com/readium/go-toolkit/pkg/asset" - "github.com/readium/go-toolkit/pkg/fetcher" - "github.com/readium/go-toolkit/pkg/manifest" - "github.com/readium/go-toolkit/pkg/streamer" - "github.com/readium/go-toolkit/pkg/util/url" - "github.com/spf13/cobra" -) - -// Indentation used to pretty-print. -var indentFlag string - -// Infer accessibility metadata. -var inferA11yFlag helpers.InferA11yMetadata - -// Infer the number of pages from the generated position list. -var inferPageCountFlag bool - -/*var inferIgnoreImageHashesFlag []string - -var inferIgnoreImageDirectoryFlag string*/ - -var hash []string - -var inspectImagesFlag bool - -var manifestCmd = &cobra.Command{ - Use: "manifest ", - Short: "Generate a Readium Web Publication Manifest for a publication", - Long: `Generate a Readium Web Publication Manifest for a publication. - -This command will parse a publication file (such as EPUB, PDF, audiobook, etc.) -and build a Readium Web Publication Manifest for it. The JSON manifest is -printed to stdout. - -Examples: - Print out a compact JSON RWPM. - $ rwp manifest publication.epub - - Pretty-print a JSON RWPM using two-space indent. - $ rwp manifest --indent " " publication.epub - - Extract the publication title with ` + "`jq`" + `. - $ rwp manifest publication.epub | jq -r .metadata.title - `, - Args: func(cmd *cobra.Command, args []string) error { - if len(args) == 0 { - return errors.New("expects a path to the publication") - } else if len(args) > 1 { - return errors.New("accepts a single path to a publication") - } - return nil - }, - RunE: func(cmd *cobra.Command, args []string) error { - // By the time we reach this point, we know that the arguments were - // properly parsed, and we don't want to show the usage if an API error - // occurs. - cmd.SilenceUsage = true - - path, err := url.FromFilepath(filepath.Clean(args[0])) - if err != nil { - return fmt.Errorf("failed creating URL from filepath: %w", err) - } - pub, err := streamer.New(streamer.Config{ - InferA11yMetadata: streamer.InferA11yMetadata(inferA11yFlag), - InferPageCount: inferPageCountFlag, - }).Open( - context.TODO(), - asset.File(path), "", - ) - if err != nil { - return fmt.Errorf("failed opening %s: %w", path, err) - } - - if inspectImagesFlag { - hashAlgorithms := make([]manifest.HashAlgorithm, len(hash)) - for i, h := range hash { - hashAlgorithms[i] = manifest.HashAlgorithm(h) - } - inspector := &helpers.ImageInspector{ - Algorithms: hashAlgorithms, - Filesystem: fetcher.ToFS(context.TODO(), pub.Fetcher), - } - - // Inspect publication files and overwrite the links - pub.Manifest.ReadingOrder = pub.Manifest.ReadingOrder.Copy(inspector) - if inspector.Error() != nil { - return fmt.Errorf("failed inspecting images in reading order: %w", inspector.Error()) - } - pub.Manifest.Resources = pub.Manifest.Resources.Copy(inspector) - if inspector.Error() != nil { - return fmt.Errorf("failed inspecting images in resources: %w", inspector.Error()) - } - } - - var jsonBytes []byte - if indentFlag == "" { - jsonBytes, err = json.Marshal(pub.Manifest) - } else { - jsonBytes, err = json.MarshalIndent(pub.Manifest, "", indentFlag) - } - if err != nil { - return fmt.Errorf("failed rendering JSON for %s: %w", path, err) - } - - fmt.Println(string(jsonBytes)) - return err - }, -} - -func init() { - rootCmd.AddCommand(manifestCmd) - manifestCmd.Flags().StringVarP(&indentFlag, "indent", "i", "", "Indentation used to pretty-print") - manifestCmd.Flags().Var(&inferA11yFlag, "infer-a11y", "Infer accessibility metadata: no, merged, split") - manifestCmd.Flags().BoolVar(&inferPageCountFlag, "infer-page-count", false, "Infer the number of pages from the generated position list.") - manifestCmd.Flags().StringSliceVar(&hash, "hash", []string{string(manifest.HashAlgorithmSHA256), string(manifest.HashAlgorithmMD5)}, "Hashes to use when enhancing links, such as with image inspection. Note visual hashes are more computationally expensive. Acceptable values: sha256,md5,phash-dct,https://blurha.sh") - manifestCmd.Flags().BoolVar(&inspectImagesFlag, "inspect-images", false, "Inspect images in the manifest. Their links will be enhanced with size, width and height, and hashes") - // manifestCmd.Flags().StringSliceVar(&inferIgnoreImageHashesFlag, "infer-a11y-ignore-image-hashes", nil, "Ignore the given hashes when inferring textual accessibility. Hashes are in the format :, separated by commas.") - // manifestCmd.Flags().StringVar(&inferIgnoreImageDirectoryFlag, "infer-a11y-ignore-image-dir", "", "Ignore the images in a given directory when inferring textual accessibility.") -} diff --git a/cmd/rwp/cmd/root.go b/cmd/rwp/cmd/root.go deleted file mode 100644 index fb872c7f..00000000 --- a/cmd/rwp/cmd/root.go +++ /dev/null @@ -1,29 +0,0 @@ -package cmd - -import ( - "os" - - "github.com/readium/go-toolkit/pkg/util/version" - "github.com/spf13/cobra" -) - -// rootCmd represents the base command when called without any subcommands -var rootCmd = &cobra.Command{ - Use: "rwp", - Short: "Utilities for Readium Web Publications", - Version: version.Version, -} - -// Execute adds all child commands to the root command and sets flags appropriately. -// This is called by main.main(). It only needs to happen once to the rootCmd. -func Execute() { - err := rootCmd.Execute() - if err != nil { - // Error is already printed to stderr by Cobra. - os.Exit(1) - } -} - -func init() { - rootCmd.CompletionOptions.DisableDefaultCmd = true -} diff --git a/cmd/rwp/cmd/serve.go b/cmd/rwp/cmd/serve.go deleted file mode 100644 index d10be98e..00000000 --- a/cmd/rwp/cmd/serve.go +++ /dev/null @@ -1,197 +0,0 @@ -package cmd - -import ( - "context" - "errors" - "fmt" - "log" - "net/http" - "os" - "path/filepath" - "time" - - "log/slog" - - "cloud.google.com/go/storage" - "github.com/aws/aws-sdk-go-v2/aws" - "github.com/aws/aws-sdk-go-v2/config" - "github.com/aws/aws-sdk-go-v2/credentials" - "github.com/aws/aws-sdk-go-v2/service/s3" - "github.com/readium/go-toolkit/cmd/rwp/cmd/serve" - "github.com/readium/go-toolkit/cmd/rwp/cmd/serve/client" - "github.com/readium/go-toolkit/pkg/streamer" - "github.com/spf13/cobra" - "google.golang.org/api/option" -) - -var debugFlag bool - -var bindAddressFlag string - -var bindPortFlag uint16 - -// Cloud-related flags -var s3EndpointFlag string -var s3RegionFlag string -var s3AccessKeyFlag string -var s3SecretKeyFlag string - -var httpAuthorizationFlag string - -var remoteArchiveTimeoutFlag uint32 -var remoteArchiveCacheSize uint32 -var remoteArchiveCacheCount uint32 -var remoteArchiveCacheAll uint32 - -var serveCmd = &cobra.Command{ - Use: "serve ", - Short: "Start a local HTTP server, serving a specified directory of publications", - Long: `Start a local HTTP server, serving a specified directory of publications. - -This command will start an HTTP serve listening by default on 'localhost:15080', -serving all compatible files (EPUB, PDF, CBZ, etc.) found in the directory -as Readium Web Publications. To get started, the manifest can be accessed from -'http://localhost:15080//manifest.json'. -This file serves as the entry point and contains metadata and links to the rest -of the files that can be accessed for the publication. - -For debugging purposes, the server also exposes a '/list.json' endpoint that -returns a list of all the publications found in the directory along with their -encoded paths. This will be replaced by an OPDS 2 feed in a future release. - -Note: This server is not meant for production usage, and should not be exposed -to the internet except for testing/debugging purposes.`, - Args: func(cmd *cobra.Command, args []string) error { - if len(args) == 0 { - return errors.New("expects a directory path to serve publications from") - } else if len(args) > 1 { - return errors.New("accepts a directory path") - } - return nil - }, - - SuggestFor: []string{"server"}, - RunE: func(cmd *cobra.Command, args []string) error { - // By the time we reach this point, we know that the arguments were - // properly parsed, and we don't want to show the usage if an API error - // occurs. - cmd.SilenceUsage = true - - path := filepath.Clean(args[0]) - fi, err := os.Stat(path) - if err != nil { - if os.IsNotExist(err) { - return fmt.Errorf("given directory %s does not exist", path) - } - return fmt.Errorf("failed to stat %s: %w", path, err) - } - if !fi.IsDir() { - return fmt.Errorf("given path %s is not a directory", path) - } - - // Log level - if debugFlag { - slog.SetLogLoggerLevel(slog.LevelDebug) - } else { - slog.SetLogLoggerLevel(slog.LevelInfo) - } - - // Set up remote publication retrieval clients - remote := serve.Remote{} - - // S3 - options := []func(*config.LoadOptions) error{ - config.WithRegion(s3RegionFlag), - config.WithRequestChecksumCalculation(0), - config.WithResponseChecksumValidation(0), - // TODO: look into custom HTTP client, user-agent - } - if s3AccessKeyFlag != "" && s3SecretKeyFlag != "" { - options = append(options, config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(s3AccessKeyFlag, s3SecretKeyFlag, ""))) - } - cfg, err := config.LoadDefaultConfig(context.Background(), options...) - if err != nil { - log.Fatal(err) - } - _, err = cfg.Credentials.Retrieve(context.Background()) - if err == nil { - remote.S3 = s3.NewFromConfig(cfg, func(o *s3.Options) { - if s3EndpointFlag != "" { - o.BaseEndpoint = aws.String(s3EndpointFlag) - } - }) - } else { - slog.Warn("S3 credentials retrieval failed, S3 support will be disabled", "error", err) - } - - // GCS - opts := []option.ClientOption{ - option.WithScopes(storage.ScopeReadOnly), - storage.WithJSONReads(), - // option.WithUserAgent(TODO), - // TODO: look into more efficient transport (HTTP client) - } - remote.GCS, err = storage.NewClient(context.Background(), opts...) - if err != nil { - slog.Warn("GCS client creation failed, GCS support will be disabled", "error", err) - } - - remote.HTTP, err = client.NewHTTPClient(httpAuthorizationFlag) - if err != nil { - slog.Warn("HTTP client creation failed, HTTP support will be disabled", "error", err) - } - - // Remote archive streaming tweaks - remote.Config.CacheCountThreshold = int64(remoteArchiveCacheCount) - remote.Config.CacheSizeThreshold = int64(remoteArchiveCacheSize) - remote.Config.Timeout = time.Duration(remoteArchiveTimeoutFlag) * time.Second - remote.Config.CacheAllThreshold = int64(remoteArchiveCacheAll) - - // Create server - pubServer := serve.NewServer(serve.ServerConfig{ - Debug: debugFlag, - BaseDirectory: path, - JSONIndent: indentFlag, - InferA11yMetadata: streamer.InferA11yMetadata(inferA11yFlag), - }, remote) - - bind := fmt.Sprintf("%s:%d", bindAddressFlag, bindPortFlag) - httpServer := &http.Server{ - ReadTimeout: 10 * time.Second, - WriteTimeout: 10 * time.Second, - MaxHeaderBytes: 1 << 20, - Addr: bind, - Handler: pubServer.Routes(), - } - slog.Info("Starting HTTP server", "address", "http://"+httpServer.Addr) - if err := httpServer.ListenAndServe(); err != http.ErrServerClosed { - slog.Error("Server stopped", "error", err) - } else { - slog.Info("Goodbye!") - } - - return nil - }, -} - -func init() { - rootCmd.AddCommand(serveCmd) - - serveCmd.Flags().StringVarP(&bindAddressFlag, "address", "a", "localhost", "Address to bind the HTTP server to") - serveCmd.Flags().Uint16VarP(&bindPortFlag, "port", "p", 15080, "Port to bind the HTTP server to") - serveCmd.Flags().StringVarP(&indentFlag, "indent", "i", "", "Indentation used to pretty-print JSON files") - serveCmd.Flags().Var(&inferA11yFlag, "infer-a11y", "Infer accessibility metadata: no, merged, split") - serveCmd.Flags().BoolVarP(&debugFlag, "debug", "d", false, "Enable debug mode") - - serveCmd.Flags().StringVar(&s3EndpointFlag, "s3-endpoint", "", "Custom S3 endpoint URL") - serveCmd.Flags().StringVar(&s3RegionFlag, "s3-region", "auto", "S3 region") - serveCmd.Flags().StringVar(&s3AccessKeyFlag, "s3-access-key", "", "S3 access key") - serveCmd.Flags().StringVar(&s3SecretKeyFlag, "s3-secret-key", "", "S3 secret key") - - serveCmd.Flags().StringVar(&httpAuthorizationFlag, "http-authorization", "", "HTTP authorization header value (e.g. 'Bearer ' or 'Basic ')") - - serveCmd.Flags().Uint32Var(&remoteArchiveTimeoutFlag, "remote-archive-timeout", 60, "Timeout for remote archive requests (in seconds)") - serveCmd.Flags().Uint32Var(&remoteArchiveCacheSize, "remote-archive-cache-size", 1024*1024, "Max size of items in an archive that can be cached (in bytes)") - serveCmd.Flags().Uint32Var(&remoteArchiveCacheCount, "remote-archive-cache-count", 64, "Max number of items in an archive that can be cached") - serveCmd.Flags().Uint32Var(&remoteArchiveCacheAll, "remote-archive-cache-all", 1024*1024, "Archives this size or less (in bytes) will be cached in full") -} diff --git a/cmd/rwp/cmd/serve/api.go b/cmd/rwp/cmd/serve/api.go deleted file mode 100644 index cd5cacab..00000000 --- a/cmd/rwp/cmd/serve/api.go +++ /dev/null @@ -1,399 +0,0 @@ -package serve - -import ( - "bytes" - "context" - "encoding/base64" - "encoding/json" - "log/slog" - "net/http" - "os" - "path" - "path/filepath" - "slices" - "strconv" - "strings" - "syscall" - - "github.com/gorilla/mux" - httprange "github.com/gotd/contrib/http_range" - "github.com/pkg/errors" - "github.com/readium/go-toolkit/cmd/rwp/cmd/serve/cache" - "github.com/readium/go-toolkit/pkg/archive" - "github.com/readium/go-toolkit/pkg/asset" - "github.com/readium/go-toolkit/pkg/fetcher" - "github.com/readium/go-toolkit/pkg/manifest" - "github.com/readium/go-toolkit/pkg/pub" - "github.com/readium/go-toolkit/pkg/streamer" - "github.com/readium/go-toolkit/pkg/util/url" - "github.com/zeebo/xxh3" -) - -type demoListItem struct { - Filename string `json:"filename"` - Path string `json:"path"` -} - -func (s *Server) demoList(w http.ResponseWriter, req *http.Request) { - fi, err := os.ReadDir(s.config.BaseDirectory) - if err != nil { - slog.Error("failed reading publications directory", "error", err) - w.WriteHeader(500) - return - } - files := make([]demoListItem, len(fi)) - for i, f := range fi { - files[i] = demoListItem{ - Filename: f.Name(), - Path: base64.RawURLEncoding.EncodeToString([]byte(f.Name())), - } - } - enc := json.NewEncoder(w) - enc.SetIndent("", s.config.JSONIndent) - enc.Encode(files) -} - -func (s *Server) getPublication(ctx context.Context, filename string) (*pub.Publication, bool, error) { - fpath, err := base64.RawURLEncoding.DecodeString(filename) - if err != nil { - return nil, false, err - } - loc, err := url.URLFromString(string(fpath)) - if err != nil { - return nil, false, errors.Wrap(err, "failed creating URL from filepath") - } - u := url.BaseFile.Resolve(loc).(url.AbsoluteURL) // Turn relative filepaths into file:/// URLs - - dat, ok := s.lfu.Get(u.String()) - if !ok { - var pub *pub.Publication - var remote bool - config := streamer.Config{ - InferA11yMetadata: s.config.InferA11yMetadata, - HttpClient: s.remote.HTTP, - } - if u.IsFile() { - path, err := url.FromFilepath(filepath.Join(s.config.BaseDirectory, path.Clean(u.Path()))) - if err != nil { - return nil, remote, errors.Wrap(err, "failed creating URL from filepath") - } - - pub, err = streamer.New(config).Open(ctx, asset.File(path), "") - if err != nil { - return nil, remote, errors.Wrap(err, "failed opening "+path.String()) - } - } else { - switch u.Scheme() { - case url.SchemeS3: - remote = true - if s.remote.S3 == nil { - return nil, remote, errors.New("S3 client not configured") - } - config.ArchiveFactory = archive.NewS3ArchiveFactory(s.remote.S3, archive.NewDefaultRemoteArchiveConfig()) - pub, err = streamer.New(config).Open(ctx, asset.S3(s.remote.S3, u), "") - if err != nil { - return nil, remote, errors.Wrap(err, "failed opening "+u.String()) - } - case url.SchemeGS: - remote = true - if s.remote.GCS == nil { - return nil, remote, errors.New("GCS client not configured") - } - config.ArchiveFactory = archive.NewGCSArchiveFactory(s.remote.GCS, archive.NewDefaultRemoteArchiveConfig()) - pub, err = streamer.New(config).Open(ctx, asset.GCS(s.remote.GCS, u), "") - if err != nil { - return nil, remote, errors.Wrap(err, "failed opening "+u.String()) - } - case url.SchemeHTTP, url.SchemeHTTPS: - remote = true - if s.remote.HTTP == nil { - return nil, remote, errors.New("HTTP client not configured") - } - config.ArchiveFactory = archive.NewHTTPArchiveFactory(s.remote.HTTP, archive.NewDefaultRemoteArchiveConfig()) - pub, err = streamer.New(config).Open(ctx, asset.HTTP(s.remote.HTTP, u), "") - if err != nil { - return nil, remote, errors.Wrap(err, "failed opening "+u.String()) - } - default: - return nil, remote, errors.New("unsupported scheme " + u.Scheme().String()) - } - } - - // Cache the publication - encPub := cache.EncapsulatePublication(pub, remote) - s.lfu.Set(u.String(), encPub) - - return encPub.Publication, remote, nil - } - cp := dat.(*cache.CachedPublication) - return cp.Publication, cp.Remote, nil -} - -func (s *Server) getManifest(w http.ResponseWriter, req *http.Request) { - vars := mux.Vars(req) - filename := vars["path"] - - // Load the publication - publication, _, err := s.getPublication(req.Context(), filename) - if err != nil { - slog.Error("failed opening publication", "error", err) - w.WriteHeader(500) - if s.config.Debug { - w.Write([]byte(err.Error())) - } - return - } - - // Create "self" link in manifest - scheme := "http://" - if req.TLS != nil || req.Header.Get("X-Forwarded-Proto") == "https" { - // Note: this is never going to be 100% accurate behind proxies, - // but it's better than nothing for a dev server. - scheme = "https://" - } - rPath, _ := s.router.Get("manifest").URLPath("path", vars["path"]) - conformsTo := conformsToAsMimetype(publication.Manifest.Metadata.ConformsTo) - - selfUrl, err := url.AbsoluteURLFromString(scheme + req.Host + rPath.String()) - if err != nil { - slog.Error("failed creating self URL", "error", err) - w.WriteHeader(500) - if s.config.Debug { - w.Write([]byte(err.Error())) - } - return - } - - selfLink := &manifest.Link{ - Rels: manifest.Strings{"self"}, - MediaType: &conformsTo, - Href: manifest.NewHREF(selfUrl), - } - - // Marshal the manifest - j, err := json.Marshal(publication.Manifest.ToMap(selfLink)) - if err != nil { - slog.Error("failed marshalling manifest JSON", "error", err) - w.WriteHeader(500) - if s.config.Debug { - w.Write([]byte(err.Error())) - } - return - } - - // Indent JSON - var identJSON bytes.Buffer - if s.config.JSONIndent == "" { - _, err = identJSON.Write(j) - if err != nil { - slog.Error("failed writing manifest JSON to buffer", "error", err) - w.WriteHeader(500) - if s.config.Debug { - w.Write([]byte(err.Error())) - } - return - } - } else { - err = json.Indent(&identJSON, j, "", s.config.JSONIndent) - if err != nil { - slog.Error("failed indenting manifest JSON", "error", err) - w.WriteHeader(500) - if s.config.Debug { - w.Write([]byte(err.Error())) - } - return - } - } - - // Add headers - w.Header().Set("content-type", conformsTo.String()+"; charset=utf-8") - w.Header().Set("cache-control", "private, must-revalidate") - w.Header().Set("access-control-allow-origin", "*") // TODO: provide options? - - // Etag based on hash of the manifest bytes - etag := `"` + strconv.FormatUint(xxh3.Hash(identJSON.Bytes()), 36) + `"` - w.Header().Set("Etag", etag) - if match := req.Header.Get("If-None-Match"); match != "" { - if strings.Contains(match, etag) { - w.WriteHeader(http.StatusNotModified) - return - } - } - - // Write response body - _, err = identJSON.WriteTo(w) - if err != nil { - slog.Error("failed writing manifest JSON to response writer", "error", err) - w.WriteHeader(500) - if s.config.Debug { - w.Write([]byte(err.Error())) - } - return - } -} - -func (s *Server) getAsset(w http.ResponseWriter, r *http.Request) { - vars := mux.Vars(r) - filename := vars["path"] - - // Load the publication - publication, remote, err := s.getPublication(r.Context(), filename) - if err != nil { - slog.Error("failed opening publication", "error", err) - w.WriteHeader(500) - if s.config.Debug { - w.Write([]byte(err.Error())) - } - return - } - - // Parse asset path from mux vars - href, err := url.URLFromDecodedPath(path.Clean(vars["asset"])) - if err != nil { - slog.Error("failed parsing asset path as URL", "error", err) - w.WriteHeader(400) - if s.config.Debug { - w.Write([]byte(err.Error())) - } - return - } - rawHref := href.Raw() - rawHref.RawQuery = r.URL.Query().Encode() // Add the query parameters of the URL - href, _ = url.RelativeURLFromGo(rawHref) // Turn it back into a go-toolkit relative URL - - // Make sure the asset exists in the publication - link := publication.LinkWithHref(href) - if link == nil { - w.WriteHeader(http.StatusNotFound) - return - } - finalLink := *link - - // Expand templated links to include URL query parameters - if finalLink.Href.IsTemplated() { - finalLink.Href = manifest.NewHREF(finalLink.URL(nil, convertURLValuesToMap(r.URL.Query()))) - } - - // Get the asset from the publication - res := publication.Get(r.Context(), finalLink) - defer res.Close() - - // Get asset length in bytes - l, rerr := res.Length(r.Context()) - if rerr != nil { - w.WriteHeader(rerr.HTTPStatus()) - w.Write([]byte(rerr.Error())) - return - } - - // Patch mimetype where necessary - contentType := link.MediaType.String() - if sub, ok := mimeSubstitutions[contentType]; ok { - contentType = sub - } - if slices.Contains(utfCharsetNeeded, contentType) { - contentType += "; charset=utf-8" - } - w.Header().Set("content-type", contentType) - w.Header().Set("cache-control", "private, max-age=86400, immutable") - w.Header().Set("content-length", strconv.FormatInt(l, 10)) - w.Header().Set("access-control-allow-origin", "*") // TODO: provide options? - - var start, end int64 - // Range reading assets - rangeHeader := r.Header.Get("range") - if rangeHeader != "" { - rng, err := httprange.ParseRange(rangeHeader, l) - if err != nil { - slog.Error("failed parsing range header", "error", err) - w.WriteHeader(http.StatusLengthRequired) - return - } - if len(rng) > 1 { - slog.Error("no support for multiple read ranges") - w.WriteHeader(http.StatusNotImplemented) - return - } - if len(rng) > 0 { - w.Header().Set("content-range", rng[0].ContentRange(l)) - start = rng[0].Start - end = start + rng[0].Length - 1 - w.Header().Set("content-length", strconv.FormatInt(rng[0].Length, 10)) - } - } - if w.Header().Get("content-range") != "" { - w.WriteHeader(http.StatusPartialContent) - } - - cres, ok := res.(fetcher.CompressedResource) - normalResponse := func() { - if remote { - var bin []byte - bin, rerr = res.Read(r.Context(), start, end) - if rerr == nil { - _, err = w.Write(bin) - if err != nil { - rerr = fetcher.Other(err) - } - } - } else { - _, rerr = res.Stream(r.Context(), w, start, end) - } - } - if ok && cres.CompressedAs(archive.CompressionMethodDeflate) && start == 0 && end == 0 { - // Stream the asset in compressed format if supported by the user agent - if supportsEncoding(r, "deflate") { - headers := func() { - w.Header().Set("content-encoding", "deflate") - w.Header().Set("content-length", strconv.FormatInt(cres.CompressedLength(r.Context()), 10)) - } - if remote { - var bin []byte - bin, rerr = cres.ReadCompressed(r.Context()) - if rerr == nil { - headers() - _, err = w.Write(bin) - if err != nil { - rerr = fetcher.Other(err) - } - } - } else { - headers() - _, rerr = cres.StreamCompressed(r.Context(), w) - } - } else if supportsEncoding(r, "gzip") && l <= archive.GzipMaxLength { - headers := func() { - w.Header().Set("content-encoding", "gzip") - w.Header().Set("content-length", strconv.FormatInt(cres.CompressedLength(r.Context())+archive.GzipWrapperLength, 10)) - } - if remote { - var bin []byte - bin, rerr = cres.ReadCompressedGzip(r.Context()) - if rerr == nil { - headers() - _, err = w.Write(bin) - if err != nil { - rerr = fetcher.Other(err) - } - } - } else { - headers() - _, rerr = cres.StreamCompressedGzip(r.Context(), w) - } - } else { - normalResponse() - } - } else { - normalResponse() - } - - if rerr != nil { - if errors.Is(rerr.Cause, syscall.EPIPE) || errors.Is(rerr.Cause, syscall.ECONNRESET) { - // Ignore client errors - return - } - - slog.Error("failed streaming asset", "error", rerr.Error()) - } - -} diff --git a/cmd/rwp/cmd/serve/cache/local.go b/cmd/rwp/cmd/serve/cache/local.go deleted file mode 100644 index 0ddb8899..00000000 --- a/cmd/rwp/cmd/serve/cache/local.go +++ /dev/null @@ -1,90 +0,0 @@ -package cache - -// Originally from https://github.com/go-redis/cache/blob/v8.4.3/local.go -// Modified to store interface{} instead of []byte - -import ( - "sync" - "time" - - "github.com/vmihailenco/go-tinylfu" - "golang.org/x/exp/rand" -) - -type Evictable interface { - OnEvict() -} - -type LocalCache interface { - Set(key string, data Evictable) - Get(key string) (Evictable, bool) - Del(key string) -} - -type TinyLFU struct { - mu sync.Mutex - rand *rand.Rand - lfu *tinylfu.T - ttl time.Duration - offset time.Duration -} - -var _ LocalCache = (*TinyLFU)(nil) - -func NewTinyLFU(size int, ttl time.Duration) *TinyLFU { - const maxOffset = 10 * time.Second - - offset := ttl / 10 - if offset > maxOffset { - offset = maxOffset - } - - return &TinyLFU{ - rand: rand.New(rand.NewSource(uint64(time.Now().UnixNano()))), - lfu: tinylfu.New(size, 100000), - ttl: ttl, - offset: offset, - } -} - -func (c *TinyLFU) UseRandomizedTTL(offset time.Duration) { - c.offset = offset -} - -func (c *TinyLFU) Set(key string, b Evictable) { - c.mu.Lock() - defer c.mu.Unlock() - - ttl := c.ttl - if c.offset > 0 { - ttl += time.Duration(c.rand.Int63n(int64(c.offset))) - } - - c.lfu.Set(&tinylfu.Item{ - Key: key, - Value: b, - ExpireAt: time.Now().Add(ttl), - OnEvict: func() { - b.OnEvict() - }, - }) -} - -func (c *TinyLFU) Get(key string) (Evictable, bool) { - c.mu.Lock() - defer c.mu.Unlock() - - val, ok := c.lfu.Get(key) - if !ok { - return nil, false - } - - return val.(Evictable), true -} - -func (c *TinyLFU) Del(key string) { - c.mu.Lock() - defer c.mu.Unlock() - - c.lfu.Del(key) -} diff --git a/cmd/rwp/cmd/serve/cache/pubcache.go b/cmd/rwp/cmd/serve/cache/pubcache.go deleted file mode 100644 index 2c038739..00000000 --- a/cmd/rwp/cmd/serve/cache/pubcache.go +++ /dev/null @@ -1,22 +0,0 @@ -package cache - -import ( - "github.com/readium/go-toolkit/pkg/pub" -) - -// CachedPublication implements Evictable -type CachedPublication struct { - *pub.Publication - Remote bool -} - -func EncapsulatePublication(pub *pub.Publication, remote bool) *CachedPublication { - return &CachedPublication{pub, remote} -} - -func (cp *CachedPublication) OnEvict() { - // Cleanup - if cp.Publication != nil { - cp.Publication.Close() - } -} diff --git a/cmd/rwp/cmd/serve/client/http_auth.go b/cmd/rwp/cmd/serve/client/http_auth.go deleted file mode 100644 index 9ea8a422..00000000 --- a/cmd/rwp/cmd/serve/client/http_auth.go +++ /dev/null @@ -1,33 +0,0 @@ -package client - -import ( - "net/http" -) - -type authTransport struct { - Authorization string - Transport http.RoundTripper -} - -func (a *authTransport) RoundTrip(req *http.Request) (*http.Response, error) { - if a.Authorization == "" { - return a.transport().RoundTrip(req) - } - req2 := req.Clone(req.Context()) - req2.Header.Set("Authorization", a.Authorization) - return a.transport().RoundTrip(req2) -} - -func (a *authTransport) transport() http.RoundTripper { - if a.Transport != nil { - return a.Transport - } - return http.DefaultTransport -} - -func newAuthenticatedRoundTripper(auth string, transport *http.Transport) http.RoundTripper { - return &authTransport{ - Authorization: auth, - Transport: transport, - } -} diff --git a/cmd/rwp/cmd/serve/client/http_client.go b/cmd/rwp/cmd/serve/client/http_client.go deleted file mode 100644 index 7778a5b0..00000000 --- a/cmd/rwp/cmd/serve/client/http_client.go +++ /dev/null @@ -1,67 +0,0 @@ -package client - -import ( - "fmt" - "net" - "net/http" - "runtime" - "syscall" - "time" -) - -// Code below mostly from https://www.agwa.name/blog/post/preventing_server_side_request_forgery_in_golang - -func safeSocketControl(network string, address string, conn syscall.RawConn) error { - if !(network == "tcp4" || network == "tcp6") { - return fmt.Errorf("%s is not a safe network type", network) - } - - host, port, err := net.SplitHostPort(address) - if err != nil { - return fmt.Errorf("%s is not a valid host/port pair: %s", address, err) - } - - ipaddress := net.ParseIP(host) - if ipaddress == nil { - return fmt.Errorf("%s is not a valid IP address", host) - } - - if !isPublicIPAddress(ipaddress) { - return fmt.Errorf("%s is not a public IP address", ipaddress) - } - - if !(port == "80" || port == "443") { - return fmt.Errorf("%s is not a safe port number", port) - } - - return nil -} - -// Some of the below conf values from https://github.com/imgproxy/imgproxy/blob/master/transport/transport.go - -const ClientKeepAliveTimeout = 90 // Imgproxy default -var Workers = runtime.GOMAXPROCS(0) * 2 // Imgproxy default - -func NewHTTPClient(auth string) (*http.Client, error) { - safeDialer := &net.Dialer{ - Timeout: 30 * time.Second, - KeepAlive: 30 * time.Second, - DualStack: true, - Control: safeSocketControl, - } - - safeTransport := &http.Transport{ - Proxy: http.ProxyFromEnvironment, - DialContext: safeDialer.DialContext, - ForceAttemptHTTP2: true, - MaxIdleConns: 100, - MaxIdleConnsPerHost: Workers + 1, - IdleConnTimeout: time.Duration(ClientKeepAliveTimeout) * time.Second, - TLSHandshakeTimeout: 10 * time.Second, - ExpectContinueTimeout: 1 * time.Second, - } - - return &http.Client{ - Transport: newAuthenticatedRoundTripper(auth, safeTransport), - }, nil -} diff --git a/cmd/rwp/cmd/serve/client/ipaddress.go b/cmd/rwp/cmd/serve/client/ipaddress.go deleted file mode 100644 index 6dd40fb0..00000000 --- a/cmd/rwp/cmd/serve/client/ipaddress.go +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Written in 2019 by Andrew Ayer - * - * To the extent possible under law, the author(s) have dedicated all - * copyright and related and neighboring rights to this software to the - * public domain worldwide. This software is distributed without any - * warranty. - * - * You should have received a copy of the CC0 Public - * Domain Dedication along with this software. If not, see - * . - */ -package client - -import ( - "net" -) - -func ipv4Net(a, b, c, d byte, subnetPrefixLen int) net.IPNet { - return net.IPNet{net.IPv4(a, b, c, d), net.CIDRMask(96+subnetPrefixLen, 128)} -} - -var reservedIPv4Nets = []net.IPNet{ - ipv4Net(0, 0, 0, 0, 8), // Current network - ipv4Net(10, 0, 0, 0, 8), // Private - ipv4Net(100, 64, 0, 0, 10), // RFC6598 - ipv4Net(127, 0, 0, 0, 8), // Loopback - ipv4Net(169, 254, 0, 0, 16), // Link-local - ipv4Net(172, 16, 0, 0, 12), // Private - ipv4Net(192, 0, 0, 0, 24), // RFC6890 - ipv4Net(192, 0, 2, 0, 24), // Test, doc, examples - ipv4Net(192, 88, 99, 0, 24), // IPv6 to IPv4 relay - ipv4Net(192, 168, 0, 0, 16), // Private - ipv4Net(198, 18, 0, 0, 15), // Benchmarking tests - ipv4Net(198, 51, 100, 0, 24), // Test, doc, examples - ipv4Net(203, 0, 113, 0, 24), // Test, doc, examples - ipv4Net(224, 0, 0, 0, 4), // Multicast - ipv4Net(240, 0, 0, 0, 4), // Reserved (includes broadcast / 255.255.255.255) -} - -var globalUnicastIPv6Net = net.IPNet{net.IP{0x20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, net.CIDRMask(3, 128)} - -func isIPv6GlobalUnicast(address net.IP) bool { - return globalUnicastIPv6Net.Contains(address) -} - -func isIPv4Reserved(address net.IP) bool { - for _, reservedNet := range reservedIPv4Nets { - if reservedNet.Contains(address) { - return true - } - } - return false -} - -func isPublicIPAddress(address net.IP) bool { - if address.To4() != nil { - return !isIPv4Reserved(address) - } else { - return isIPv6GlobalUnicast(address) - } -} diff --git a/cmd/rwp/cmd/serve/helpers.go b/cmd/rwp/cmd/serve/helpers.go deleted file mode 100644 index f7a1c04e..00000000 --- a/cmd/rwp/cmd/serve/helpers.go +++ /dev/null @@ -1,102 +0,0 @@ -package serve - -import ( - "net/http" - "net/url" - "strings" - - "github.com/readium/go-toolkit/pkg/manifest" - "github.com/readium/go-toolkit/pkg/mediatype" -) - -var mimeSubstitutions = map[string]string{ - "application/vnd.ms-opentype": "font/otf", // Not just because it's sane, but because CF will compress it! -} - -var utfCharsetNeeded = []string{ - mediatype.ReadiumWebpubManifest.String(), - mediatype.ReadiumDivinaManifest.String(), - mediatype.ReadiumAudiobookManifest.String(), - mediatype.ReadiumPositionList.String(), - mediatype.ReadiumContentDocument.String(), - mediatype.ReadiumGuidedNavigationDocument.String(), -} - -var compressableMimes = []string{ - "application/javascript", - "application/x-javascript", - "image/x-icon", - "text/css", - "text/html", - "application/xhtml+xml", - mediatype.ReadiumWebpubManifest.String(), - mediatype.ReadiumDivinaManifest.String(), - mediatype.ReadiumPositionList.String(), - mediatype.ReadiumContentDocument.String(), - mediatype.ReadiumAudiobookManifest.String(), - "font/ttf", - "application/ttf", - "application/x-ttf", - "application/x-font-ttf", - "font/otf", - "application/otf", - "application/x-otf", - "application/vnd.ms-opentype", - "font/opentype", - "application/opentype", - "application/x-opentype", - "application/truetype", - "application/font-woff", - "font/x-woff", - "application/vnd.ms-fontobject", -} - -func conformsToAsMimetype(conformsTo manifest.Profiles) mediatype.MediaType { - mime := mediatype.ReadiumWebpubManifest - for _, profile := range conformsTo { - if profile == manifest.ProfileDivina { - mime = mediatype.ReadiumDivinaManifest - } else if profile == manifest.ProfileAudiobook { - mime = mediatype.ReadiumAudiobookManifest - } else { - continue - } - break - } - return mime -} - -func supportsEncoding(r *http.Request, encoding string) bool { - vv := r.Header.Values("Accept-Encoding") - for _, v := range vv { - for _, sv := range strings.Split(v, ",") { - coding := parseCoding(sv) - if coding == "" { - continue - } - if coding == encoding { - return true - } - } - } - return false -} - -func parseCoding(s string) (coding string) { - p := strings.IndexRune(s, ';') - if p == -1 { - p = len(s) - } - coding = strings.ToLower(strings.TrimSpace(s[:p])) - return -} - -func convertURLValuesToMap(values url.Values) map[string]string { - result := make(map[string]string) - for key, val := range values { - if len(val) > 0 { - result[key] = val[0] // Take the first value for each key - } - } - return result -} diff --git a/cmd/rwp/cmd/serve/router.go b/cmd/rwp/cmd/serve/router.go deleted file mode 100644 index e3b5fd19..00000000 --- a/cmd/rwp/cmd/serve/router.go +++ /dev/null @@ -1,47 +0,0 @@ -package serve - -import ( - "net/http" - "net/http/pprof" - - "github.com/CAFxX/httpcompression" - "github.com/gorilla/mux" -) - -func (s *Server) Routes() *mux.Router { - r := mux.NewRouter() - - r.HandleFunc("/health", func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(http.StatusOK) - w.Write([]byte("OK")) - }) - - if s.config.Debug { - r.HandleFunc("/debug/pprof/", pprof.Index) - r.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline) - r.HandleFunc("/debug/pprof/profile", pprof.Profile) - r.HandleFunc("/debug/pprof/symbol", pprof.Symbol) - r.HandleFunc("/debug/pprof/trace", pprof.Trace) - - r.Handle("/debug/pprof/allocs", pprof.Handler("allocs")) - r.Handle("/debug/pprof/block", pprof.Handler("block")) - r.Handle("/debug/pprof/goroutine", pprof.Handler("goroutine")) - r.Handle("/debug/pprof/heap", pprof.Handler("heap")) - r.Handle("/debug/pprof/mutex", pprof.Handler("mutex")) - r.Handle("/debug/pprof/threadcreate", pprof.Handler("threadcreate")) - } - - r.HandleFunc("/list.json", s.demoList).Name("demo_list") - - pub := r.PathPrefix("/{path}").Subrouter() - // TODO: publication loading middleware with pub.Use() - pub.Use(func(h http.Handler) http.Handler { - adapter, _ := httpcompression.DefaultAdapter(httpcompression.ContentTypes(compressableMimes, false)) - return adapter(h) - }) - pub.HandleFunc("/manifest.json", s.getManifest).Name("manifest") - pub.HandleFunc("/{asset:.*}", s.getAsset).Name("asset") - - s.router = r - return r -} diff --git a/cmd/rwp/cmd/serve/server.go b/cmd/rwp/cmd/serve/server.go deleted file mode 100644 index 00527e0b..00000000 --- a/cmd/rwp/cmd/serve/server.go +++ /dev/null @@ -1,45 +0,0 @@ -package serve - -import ( - "net/http" - "time" - - "cloud.google.com/go/storage" - "github.com/aws/aws-sdk-go-v2/service/s3" - "github.com/gorilla/mux" - "github.com/readium/go-toolkit/cmd/rwp/cmd/serve/cache" - "github.com/readium/go-toolkit/pkg/archive" - "github.com/readium/go-toolkit/pkg/streamer" -) - -type Remote struct { - S3 *s3.Client // AWS S3-compatible storage - GCS *storage.Client // Google Cloud Storage - HTTP *http.Client // HTTP-requested storage - Config archive.RemoteArchiveConfig -} - -type ServerConfig struct { - Debug bool - BaseDirectory string - JSONIndent string - InferA11yMetadata streamer.InferA11yMetadata -} - -type Server struct { - config ServerConfig - remote Remote - router *mux.Router - lfu *cache.TinyLFU -} - -const MaxCachedPublicationAmount = 10 -const MaxCachedPublicationTTL = time.Second * time.Duration(600) - -func NewServer(config ServerConfig, remote Remote) *Server { - return &Server{ - config: config, - remote: remote, - lfu: cache.NewTinyLFU(MaxCachedPublicationAmount, MaxCachedPublicationTTL), - } -} diff --git a/cmd/rwp/main.go b/cmd/rwp/main.go deleted file mode 100644 index 1c05f845..00000000 --- a/cmd/rwp/main.go +++ /dev/null @@ -1,22 +0,0 @@ -package main - -import ( - "os" - - "github.com/readium/go-toolkit/cmd/rwp/cmd" -) - -func main() { - // From the archive/zip docs: - // If any file inside the archive uses a non-local name - // (as defined by [filepath.IsLocal]) or a name containing backslashes - // and the GODEBUG environment variable contains `zipinsecurepath=0`, - // NewReader returns the reader with an [ErrInsecurePath] error. - if os.Getenv("GODEBUG") == "" { - os.Setenv("GODEBUG", "zipinsecurepath=0") - } else { - os.Setenv("GODEBUG", os.Getenv("GODEBUG")+",zipinsecurepath=0") - } - - cmd.Execute() -} diff --git a/go.mod b/go.mod index 502ae3eb..0a910c98 100644 --- a/go.mod +++ b/go.mod @@ -4,12 +4,9 @@ go 1.24.0 require ( cloud.google.com/go/storage v1.51.0 - github.com/CAFxX/httpcompression v0.0.9 github.com/agext/regexp v1.3.0 github.com/andybalholm/cascadia v1.3.3 github.com/aws/aws-sdk-go-v2 v1.36.3 - github.com/aws/aws-sdk-go-v2/config v1.29.14 - github.com/aws/aws-sdk-go-v2/credentials v1.17.67 github.com/aws/aws-sdk-go-v2/service/s3 v1.79.2 github.com/aws/smithy-go v1.22.3 github.com/azr/phash v0.2.0 @@ -17,18 +14,13 @@ require ( github.com/deckarep/golang-set v1.8.0 github.com/disintegration/imaging v1.6.2 github.com/go-viper/mapstructure/v2 v2.2.1 - github.com/gorilla/mux v1.8.1 - github.com/gotd/contrib v0.21.0 github.com/kettek/apng v0.0.0-20220823221153-ff692776a607 github.com/pdfcpu/pdfcpu v0.9.1 github.com/pkg/errors v0.9.1 github.com/readium/xmlquery v0.0.0-20230106230237-8f493145aef4 github.com/relvacode/iso8601 v1.6.0 - github.com/spf13/cobra v1.9.1 github.com/stretchr/testify v1.10.0 github.com/trimmer-io/go-xmp v1.0.0 - github.com/vmihailenco/go-tinylfu v0.2.2 - github.com/zeebo/xxh3 v1.0.2 go4.org v0.0.0-20230225012048-214862532bf5 golang.org/x/exp v0.0.0-20250408133849-7e4ce0ab07d0 golang.org/x/image v0.26.0 @@ -48,21 +40,15 @@ require ( github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.27.0 // indirect github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.51.0 // indirect github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.51.0 // indirect - github.com/andybalholm/brotli v1.1.1 // indirect github.com/antchfx/xpath v1.3.4 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.10 // indirect - github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.30 // indirect github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.34 // indirect github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.34 // indirect - github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 // indirect github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.34 // indirect github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.3 // indirect github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.7.0 // indirect github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.15 // indirect github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.15 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.25.3 // indirect - github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.1 // indirect - github.com/aws/aws-sdk-go-v2/service/sts v1.33.19 // indirect github.com/azr/gift v1.1.2 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/cncf/xds/go v0.0.0-20250326154945-ae57f3c0d45f // indirect @@ -79,12 +65,8 @@ require ( github.com/googleapis/gax-go/v2 v2.14.1 // indirect github.com/hhrutter/lzw v1.0.0 // indirect github.com/hhrutter/tiff v1.0.2 // indirect - github.com/inconshreveable/mousetrap v1.1.0 // indirect - github.com/klauspost/compress v1.18.0 // indirect - github.com/klauspost/cpuid/v2 v2.2.10 // indirect github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/spf13/pflag v1.0.6 // indirect go.opentelemetry.io/auto/sdk v1.1.0 // indirect go.opentelemetry.io/contrib/detectors/gcp v1.35.0 // indirect go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.60.0 // indirect diff --git a/go.sum b/go.sum index 77d7f30f..685b3cd8 100644 --- a/go.sum +++ b/go.sum @@ -39,8 +39,6 @@ cloud.google.com/go/trace v1.11.5/go.mod h1:TwblCcqNInriu5/qzaeYEIH7wzUcchSdeY2l dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/CAFxX/httpcompression v0.0.9 h1:0ue2X8dOLEpxTm8tt+OdHcgA+gbDge0OqFQWGKSqgrg= -github.com/CAFxX/httpcompression v0.0.9/go.mod h1:XX8oPZA+4IDcfZ0A71Hz0mZsv/YJOgYygkFhizVPilM= github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.27.0 h1:ErKg/3iS1AKcTkf3yixlZ54f9U1rljCkQyEXWUnIUxc= github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.27.0/go.mod h1:yAZHSGnqScoU556rBOVkwLze6WP5N+U11RHuWaGVxwY= github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.51.0 h1:fYE9p3esPxA/C0rQ0AHhP0drtPXDRhaWiwg1DPqO7IU= @@ -51,9 +49,6 @@ github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapp github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.51.0/go.mod h1:otE2jQekW/PqXk1Awf5lmfokJx4uwuqcj1ab5SpGeW0= github.com/agext/regexp v1.3.0 h1:6+9tp+S41TU48gFNV47bX+pp1q7WahGofw6JccmsCDs= github.com/agext/regexp v1.3.0/go.mod h1:6phv1gViOJXWcTfpxOi9VMS+MaSAo+SUDf7do3ur1HA= -github.com/andybalholm/brotli v1.0.5/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= -github.com/andybalholm/brotli v1.1.1 h1:PR2pgnyFznKEugtsUo0xLdDop5SKXd5Qf5ysW+7XdTA= -github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA= github.com/andybalholm/cascadia v1.3.3 h1:AG2YHrzJIm4BZ19iwJ/DAua6Btl3IwJX+VI4kktS1LM= github.com/andybalholm/cascadia v1.3.3/go.mod h1:xNd9bqTn98Ln4DwST8/nG+H0yuB8Hmgu1YHNnWw0GeA= github.com/antchfx/xpath v1.2.1/go.mod h1:i54GszH55fYfBmoZXapTHN8T8tkcHfRgLyVwwqzXNcs= @@ -63,18 +58,10 @@ github.com/aws/aws-sdk-go-v2 v1.36.3 h1:mJoei2CxPutQVxaATCzDUjcZEjVRdpsiiXi2o38y github.com/aws/aws-sdk-go-v2 v1.36.3/go.mod h1:LLXuLpgzEbD766Z5ECcRmi8AzSwfZItDtmABVkRLGzg= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.10 h1:zAybnyUQXIZ5mok5Jqwlf58/TFE7uvd3IAsa1aF9cXs= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.10/go.mod h1:qqvMj6gHLR/EXWZw4ZbqlPbQUyenf4h82UQUlKc+l14= -github.com/aws/aws-sdk-go-v2/config v1.29.14 h1:f+eEi/2cKCg9pqKBoAIwRGzVb70MRKqWX4dg1BDcSJM= -github.com/aws/aws-sdk-go-v2/config v1.29.14/go.mod h1:wVPHWcIFv3WO89w0rE10gzf17ZYy+UVS1Geq8Iei34g= -github.com/aws/aws-sdk-go-v2/credentials v1.17.67 h1:9KxtdcIA/5xPNQyZRgUSpYOE6j9Bc4+D7nZua0KGYOM= -github.com/aws/aws-sdk-go-v2/credentials v1.17.67/go.mod h1:p3C44m+cfnbv763s52gCqrjaqyPikj9Sg47kUVaNZQQ= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.30 h1:x793wxmUWVDhshP8WW2mlnXuFrO4cOd3HLBroh1paFw= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.30/go.mod h1:Jpne2tDnYiFascUEs2AWHJL9Yp7A5ZVy3TNyxaAjD6M= github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.34 h1:ZK5jHhnrioRkUNOc+hOgQKlUL5JeC3S6JgLxtQ+Rm0Q= github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.34/go.mod h1:p4VfIceZokChbA9FzMbRGz5OV+lekcVtHlPKEO0gSZY= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.34 h1:SZwFm17ZUNNg5Np0ioo/gq8Mn6u9w19Mri8DnJ15Jf0= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.34/go.mod h1:dFZsC0BLo346mvKQLWmoJxT+Sjp+qcVR1tRVHQGOH9Q= -github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 h1:bIqFDwgGXXN1Kpp99pDOdKMTTb5d2KyU5X/BZxjOkRo= -github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3/go.mod h1:H5O/EsxDWyU+LP/V8i5sm8cxoZgc2fdNR9bxlOFrQTo= github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.34 h1:ZNTqv4nIdE/DiBfUUfXcLZ/Spcuz+RjeziUtNJackkM= github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.34/go.mod h1:zf7Vcd1ViW7cPqYWEHLHJkS50X0JS2IKz9Cgaj6ugrs= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.3 h1:eAh2A4b5IzM/lum78bZ590jy36+d/aFLgKF/4Vd1xPE= @@ -87,12 +74,6 @@ github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.15 h1:moLQUoVq91Liq github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.15/go.mod h1:ZH34PJUc8ApjBIfgQCFvkWcUDBtl/WTD+uiYHjd8igA= github.com/aws/aws-sdk-go-v2/service/s3 v1.79.2 h1:tWUG+4wZqdMl/znThEk9tcCy8tTMxq8dW0JTgamohrY= github.com/aws/aws-sdk-go-v2/service/s3 v1.79.2/go.mod h1:U5SNqwhXB3Xe6F47kXvWihPl/ilGaEDe8HD/50Z9wxc= -github.com/aws/aws-sdk-go-v2/service/sso v1.25.3 h1:1Gw+9ajCV1jogloEv1RRnvfRFia2cL6c9cuKV2Ps+G8= -github.com/aws/aws-sdk-go-v2/service/sso v1.25.3/go.mod h1:qs4a9T5EMLl/Cajiw2TcbNt2UNo/Hqlyp+GiuG4CFDI= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.1 h1:hXmVKytPfTy5axZ+fYbR5d0cFmC3JvwLm5kM83luako= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.1/go.mod h1:MlYRNmYu/fGPoxBQVvBYr9nyr948aY/WLUvwBMBJubs= -github.com/aws/aws-sdk-go-v2/service/sts v1.33.19 h1:1XuUZ8mYJw9B6lzAkXhqHlJd/XvaX32evhproijJEZY= -github.com/aws/aws-sdk-go-v2/service/sts v1.33.19/go.mod h1:cQnB8CUnxbMU82JvlqjKR2HBOm3fe9pWorWBza6MBJ4= github.com/aws/smithy-go v1.22.3 h1:Z//5NuZCSW6R4PhQ93hShNbyBbn8BWCmCVCt+Q8Io5k= github.com/aws/smithy-go v1.22.3/go.mod h1:t1ufH5HMublsJYulve2RKmHDC15xu1f26kHCp/HgceI= github.com/azr/gift v1.1.2 h1:EbQ8/1QMtDfz5Beqg+RY5F21KbwGhE8aWSEbF1pp95A= @@ -102,7 +83,6 @@ github.com/azr/phash v0.2.0/go.mod h1:vUennaUN3i09UA33YxHpCR5l2CeENoCRB2Jo6pvWNf github.com/bbrks/go-blurhash v1.1.1 h1:uoXOxRPDca9zHYabUTwvS4KnY++KKUbwFo+Yxb8ME4M= github.com/bbrks/go-blurhash v1.1.1/go.mod h1:lkAsdyXp+EhARcUo85yS2G1o+Sh43I2ebF5togC4bAY= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= @@ -111,7 +91,6 @@ github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMn github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/xds/go v0.0.0-20250326154945-ae57f3c0d45f h1:C5bqEmzEPLsHm9Mv73lSE9e9bKV23aB1vxOsmZrkl3k= github.com/cncf/xds/go v0.0.0-20250326154945-ae57f3c0d45f/go.mod h1:W+zGtBO5Y1IgJhy4+A9GOqVhqLpfZi+vwmdNXUehLA8= -github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -156,8 +135,6 @@ github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5y github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= -github.com/google/brotli/go/cbrotli v0.0.0-20230829110029-ed738e842d2f h1:jopqB+UTSdJGEJT8tEqYyE29zN91fi2827oLET8tl7k= -github.com/google/brotli/go/cbrotli v0.0.0-20230829110029-ed738e842d2f/go.mod h1:nOPhAkwVliJdNTkj3gXpljmWhjc4wCaVqbMJcPKWP4s= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= @@ -185,10 +162,6 @@ github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+ github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/gax-go/v2 v2.14.1 h1:hb0FFeiPaQskmvakKu5EbCbpntQn48jyHuvrkurSS/Q= github.com/googleapis/gax-go/v2 v2.14.1/go.mod h1:Hb/NubMaVM88SrNkvl8X/o8XWwDJEPqouaLeN2IUxoA= -github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY= -github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ= -github.com/gotd/contrib v0.21.0 h1:4Fj05jnyBE84toXZl7mVTvt7f732n5uglvztyG6nTr4= -github.com/gotd/contrib v0.21.0/go.mod h1:ENoUh75IhHGxfz/puVJg8BU4ZF89yrL6Q47TyoNqFYo= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hhrutter/lzw v1.0.0 h1:laL89Llp86W3rRs83LvKbwYRx6INE8gDn0XNb1oXtm0= @@ -196,19 +169,11 @@ github.com/hhrutter/lzw v1.0.0/go.mod h1:2HC6DJSn/n6iAZfgM3Pg+cP1KxeWc3ezG8bBqW5 github.com/hhrutter/tiff v1.0.2 h1:7H3FQQpKu/i5WaSChoD1nnJbGx4MxU5TlNqqpxw55z8= github.com/hhrutter/tiff v1.0.2/go.mod h1:pcOeuK5loFUE7Y/WnzGw20YxUdnqjY1P0Jlcieb/cCw= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= -github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/kettek/apng v0.0.0-20220823221153-ff692776a607 h1:8tP9cdXzcGX2AvweVVG/lxbI7BSjWbNNUustwJ9dQVA= github.com/kettek/apng v0.0.0-20220823221153-ff692776a607/go.mod h1:x78/VRQYKuCftMWS0uK5e+F5RJ7S4gSlESRWI0Prl6Q= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= -github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= -github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= -github.com/klauspost/cpuid/v2 v2.2.10 h1:tBs3QSyvjDyFTq3uoc/9xFpCuOsJQFNPiAhYdw2skhE= -github.com/klauspost/cpuid/v2 v2.2.10/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= -github.com/klauspost/pgzip v1.2.6/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= @@ -220,8 +185,6 @@ github.com/matryer/is v1.2.0 h1:92UTHpy8CDwaJ08GqLDzhhuixiBUUD1p3AU6PHddz4A= github.com/matryer/is v1.2.0/go.mod h1:2fLPjFQM9rhQ15aVEtbuwhJinnOqrmgXPNdZsdwlWXA= github.com/pdfcpu/pdfcpu v0.9.1 h1:q8/KlBdHjkE7ZJU4ofhKG5Rjf7M6L324CVM6BMDySao= github.com/pdfcpu/pdfcpu v0.9.1/go.mod h1:fVfOloBzs2+W2VJCCbq60XIxc3yJHAZ0Gahv1oO0gyI= -github.com/pierrec/lz4/v4 v4.1.18 h1:xaKrnTkyoqfh1YItXl56+6KJNVYWlEEPuAQW9xsplYQ= -github.com/pierrec/lz4/v4 v4.1.18/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo= @@ -236,37 +199,15 @@ github.com/relvacode/iso8601 v1.6.0/go.mod h1:FlNp+jz+TXpyRqgmM7tnzHHzBnz776kmAH github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= -github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd h1:CmH9+J6ZSsIjUK3dcGsnCnO41eRBOnY12zwkn5qVwgc= github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd/go.mod h1:hPqNNc0+uJM6H+SuU8sEs5K5IQeKccPqeSjfgcKGgPk= -github.com/spf13/cobra v1.9.1 h1:CXSaggrXdbHK9CF+8ywj8Amf7PBRmPCOJugH954Nnlo= -github.com/spf13/cobra v1.9.1/go.mod h1:nDyEzZ8ogv936Cinf6g1RU9MRY64Ir93oCnqb9wxYW0= -github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o= -github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= -github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/trimmer-io/go-xmp v1.0.0 h1:zY8bolSga5kOjBAaHS6hrdxLgEoYuT875xTy0QDwZWs= github.com/trimmer-io/go-xmp v1.0.0/go.mod h1:Aaptr9sp1lLv7UnCAdQ+gSHZyY2miYaKmcNVj7HRBwA= -github.com/ulikunitz/xz v0.5.11/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= -github.com/valyala/gozstd v1.20.1 h1:xPnnnvjmaDDitMFfDxmQ4vpx0+3CdTg2o3lALvXTU/g= -github.com/valyala/gozstd v1.20.1/go.mod h1:y5Ew47GLlP37EkTB+B4s7r6A5rdaeB7ftbl9zoYiIPQ= -github.com/vmihailenco/go-tinylfu v0.2.2 h1:H1eiG6HM36iniK6+21n9LLpzx1G9R3DJa2UjUjbynsI= -github.com/vmihailenco/go-tinylfu v0.2.2/go.mod h1:CutYi2Q9puTxfcolkliPq4npPuofg9N9t8JVrjzwa3Q= -github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU= -github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= -github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= -github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0= -github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= @@ -521,7 +462,6 @@ gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=