diff --git a/.github/workflows/docker-s3-test.yml b/.github/workflows/docker-s3-test.yml new file mode 100644 index 00000000000..2affb3c1594 --- /dev/null +++ b/.github/workflows/docker-s3-test.yml @@ -0,0 +1,61 @@ +name: Build S3-test Docker image + +on: + push: + branches: + - feat/s3-vfs-backend + workflow_dispatch: + inputs: + tag: + description: "Image tag to publish under ghcr.io/" + required: false + default: "s3-test" + +permissions: + contents: read + packages: write + +jobs: + build-and-push: + runs-on: ubuntu-22.04 + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to GHCR + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Compute image metadata + id: meta + run: | + repo_lc="$(echo "${{ github.repository }}" | tr '[:upper:]' '[:lower:]')" + tag="${{ github.event.inputs.tag }}" + if [ -z "$tag" ]; then + tag="s3-test" + fi + tag_lc="$(echo "$tag" | tr '[:upper:]' '[:lower:]')" + sha_tag="${tag_lc}-${GITHUB_SHA::7}" + echo "image=ghcr.io/${repo_lc}" >> "$GITHUB_OUTPUT" + echo "tag=${tag_lc}" >> "$GITHUB_OUTPUT" + echo "sha_tag=${sha_tag}" >> "$GITHUB_OUTPUT" + + - name: Build and push + uses: docker/build-push-action@v6 + with: + context: . + file: scripts/docker/production/Dockerfile + push: true + tags: | + ${{ steps.meta.outputs.image }}:${{ steps.meta.outputs.tag }} + ${{ steps.meta.outputs.image }}:${{ steps.meta.outputs.sha_tag }} + build-args: | + VERSION_STRING=${{ steps.meta.outputs.tag }}-${{ github.sha }} + cache-from: type=gha + cache-to: type=gha,mode=max diff --git a/.github/workflows/go-tests.yml b/.github/workflows/go-tests.yml index 0985e46ddf2..fec5ca6ba5d 100644 --- a/.github/workflows/go-tests.yml +++ b/.github/workflows/go-tests.yml @@ -64,4 +64,4 @@ jobs: cache: true - name: Run tests - run: go test -p 1 -timeout 5m ./... + run: go test -p 1 -timeout 10m ./... diff --git a/cozy.example.yaml b/cozy.example.yaml index 8213d59bf6e..8dbfa3e7e58 100644 --- a/cozy.example.yaml +++ b/cozy.example.yaml @@ -67,6 +67,7 @@ fs: # url: file://localhost/var/lib/cozy # url: swift://openstack/?UserName={{ .Env.OS_USERNAME }}&Password={{ .Env.OS_PASSWORD }}&ProjectName={{ .Env.OS_PROJECT_NAME }}&UserDomainName={{ .Env.OS_USER_DOMAIN_NAME }}&Timeout={{ .Env.GOSWIFT_TIMEOUT }} + # url: s3://{{ .Env.S3_ENDPOINT }}?access_key={{ .Env.S3_ACCESS_KEY }}&secret_key={{ .Env.S3_SECRET_KEY }}®ion={{ .Env.S3_REGION }}&bucket_prefix=cozy&use_ssl=true # Swift FS can be used with advanced parameters to activate TLS properties. # For using swift with https, you must use the "swift+https" scheme. diff --git a/docs/s3.md b/docs/s3.md new file mode 100644 index 00000000000..c757f3b4455 --- /dev/null +++ b/docs/s3.md @@ -0,0 +1,248 @@ +[Table of contents](README.md#table-of-contents) + +# S3 Storage Backend + +cozy-stack supports S3-compatible object storage as a file system backend, +alongside the existing local filesystem (afero) and OpenStack Swift backends. +It has been designed to work with any S3-compatible provider (OVH, MinIO, +Scaleway, etc.) and does not depend on the AWS SDK. + +## Configuration + +The S3 backend is configured via the `fs.url` parameter using the `s3://` +scheme. All connection parameters are passed as query parameters: + +```yaml +fs: + url: s3://s3.rbx.io.cloud.ovh.net?access_key=ACCESS&secret_key=SECRET®ion=rbx&bucket_prefix=cozy&use_ssl=true +``` + +| Parameter | Description | Default | +|-----------------|--------------------------------------|---------| +| `access_key` | S3 access key ID | — | +| `secret_key` | S3 secret access key | — | +| `region` | S3 region | — | +| `bucket_prefix` | Prefix for all bucket names | `cozy` | +| `use_ssl` | Use HTTPS for S3 connections | `true` | + +The host part of the URL is the S3 endpoint (e.g. `s3.rbx.io.cloud.ovh.net` +for OVH, `localhost:9000` for MinIO). + +### Local development with MinIO + +This tutorial explains how to set up a local S3 backend using MinIO for +development and testing. + +**1. Start MinIO with Docker:** + +```bash +docker run -d --name minio \ + -p 9000:9000 \ + -p 9001:9001 \ + -e MINIO_ROOT_USER=minioadmin \ + -e MINIO_ROOT_PASSWORD=minioadmin \ + minio/minio server /data --console-address ":9001" +``` + +MinIO is now running: +- S3 API: `http://localhost:9000` +- Web console: `http://localhost:9001` (login: `minioadmin` / `minioadmin`) + +**2. Configure cozy-stack:** + +Buckets are created automatically at startup. No manual bucket creation +is needed. + +Edit your `~/.cozy/cozy.yaml`: + +```yaml +fs: + url: s3://localhost:9000?access_key=minioadmin&secret_key=minioadmin&bucket_prefix=cozy&use_ssl=false +``` + +**3. Build and start:** + +```bash +go build -o ~/go/bin/cozy-stack . +~/go/bin/cozy-stack serve +``` + +You should see in the logs: + +``` +Successfully connected to S3 endpoint localhost:9000 +``` + +**4. (Re)install your apps:** + +When switching from a different storage backend (e.g. `file://`), you need +to reinstall the apps so their assets are stored in S3: + +```bash +cozy-stack apps uninstall drive --domain your.domain.localhost:8080 +cozy-stack apps install drive --domain your.domain.localhost:8080 +cozy-stack apps uninstall home --domain your.domain.localhost:8080 +cozy-stack apps install home --domain your.domain.localhost:8080 +``` + +**5. Verify:** + +Check that objects appear in MinIO: + +```bash +docker exec minio mc ls --recursive local/cozy-apps-web/ +``` + +Upload a file via the Drive UI or the API: + +```bash +TOKEN=$(cozy-stack instances token-cli your.domain.localhost:8080 io.cozy.files) +curl -X POST \ + -H "Authorization: Bearer $TOKEN" \ + -H "Content-Type: text/plain" \ + "http://your.domain.localhost:8080/files/io.cozy.files.root-dir?Type=file&Name=test.txt" \ + -d "Hello S3!" +``` + +Verify the file is in MinIO: + +```bash +docker exec minio mc ls --recursive local/cozy-default/ +``` + +**6. Switching back to local filesystem:** + +Comment out the S3 URL in your config and restart cozy-stack: + +```yaml +fs: + # url: s3://localhost:9000?access_key=minioadmin&secret_key=minioadmin&bucket_prefix=cozy&use_ssl=false +``` + +Note: files uploaded to S3 won't be accessible when using the local +filesystem backend, and vice versa. Each backend has its own storage. + +## Bucket strategy + +### Design rationale + +Swift uses one container per instance (`cozy-v3-`). This doesn't +scale well for S3 where bucket creation can be limited (AWS limits to 100 +buckets per account by default, OVH to 100 as well). Instead, the S3 backend +uses a **shared bucket per organization** with **key prefixes per instance**. + +### Bucket naming + +Each bucket name is derived from the instance's `OrgID` field: + +``` +- +``` + +- If `OrgID` is empty, `"default"` is used as fallback +- The org ID is sanitized: lowercased, underscores/dots replaced by hyphens, + non-alphanumeric characters stripped, consecutive hyphens collapsed, + truncated to respect the 63-character S3 bucket name limit +- Examples: `cozy-default`, `cozy-acme-corp`, `cozy-org-12345` + +### Dedicated buckets for secondary storage + +In addition to the main VFS bucket, the S3 backend uses dedicated buckets for +other storage needs: + +| Bucket | Content | +|-------------------------------|-----------------------------------------| +| `-` | Main VFS data (files, versions) | +| `-apps-web` | Web application assets (drive, etc.) | +| `-apps-konnectors` | Konnector assets | +| `-assets` | Dynamic assets | +| `-previews` | PDF preview and icon cache | +| `-exports` | Instance export archives | + +Buckets are created automatically on first use. + +## Object key structure + +Within a bucket, each instance's data is isolated by a key prefix derived +from `DBPrefix()` (typically the instance domain or a CouchDB prefix). + +### VFS files + +``` +//// +``` + +The document ID (a 32-character UUID v7 hex string) is split into virtual +subfolders to avoid flat hierarchies: + +``` +cozy218def.../019d35b1-9dc3-78ec-994d-f5/44336/7f1b6/e0AbCdEfGh123456 + ^^^^^^^^^^^^^^^^^^^^^^ ^^^^^ ^^^^^ ^^^^^^^^^^^^^^^^ + first 22 chars 5 ch 5 ch 16-char internalID +``` + +This structure mirrors the Swift V3 layout (`MakeObjectNameV3`). + +### Thumbnails + +``` +/thumbs/- +``` + +Formats: `small`, `medium`, `large`. + +### Avatar + +``` +/avatar +``` + +## Memory consumption + +The S3 backend is designed to have comparable memory usage to Swift: + +| Scenario | Memory per upload | +|-----------------------------|-------------------| +| Known size, file < 5 GiB | ~32 KB (single PUT, stream) | +| Unknown size (rare) | ~5 MiB (multipart, PartSize=5MiB, NumThreads=1) | + +When `ByteSize` is known on the file document (the common case for drive +uploads), the backend passes the exact size to `PutObject`, which uses a +single PUT request that streams directly to S3 with minimal buffering — the +same behavior as Swift's `ObjectCreate`. + +Multipart upload is only used for files with unknown size or exceeding 5 GiB, +with `PartSize=5MiB` and `NumThreads=1` to limit memory. + +## Encryption at rest + +The S3 backend does not implement client-side encryption. Encryption should +be configured at the infrastructure level (S3 bucket default encryption / +SSE-S3), the same approach used for the Swift backend. + +## Differences from Swift + +| Aspect | Swift | S3 | +|--------------------|-------------------------------|----------------------------------------| +| Container/Bucket | One per instance | One per organization (shared) | +| Instance isolation | Container name | Key prefix within bucket | +| Delete instance | Delete entire container | Delete all objects with key prefix | +| File streaming | Native `io.WriteCloser` | `io.Pipe` + `PutObject` goroutine | +| Bulk delete | `BulkDelete` API | `RemoveObjects` channel API | +| Server-side copy | `ObjectCopy` | `CopyObject` (same endpoint only) | + +## Testing + +The VFS integration tests run against all three backends (afero, swift, s3) +using a table-driven approach. The S3 tests use +[testcontainers-go](https://testcontainers.com/guides/getting-started-with-testcontainers-for-go/) +with a MinIO container that is started automatically. + +```bash +# Run VFS tests (requires CouchDB + Docker) +COZY_COUCHDB_URL=http://admin:admin@localhost:5984/ \ + go test ./model/vfs/ -run TestVfs -v -count=1 -timeout 300s + +# Run naming unit tests (no external deps) +go test ./model/vfs/vfss3/ -run "TestSanitize|TestBucketName|TestMakeObjectKey|TestMakeDocID" -v +``` diff --git a/go.mod b/go.mod index 516ab6f28f7..2707bfd5a82 100644 --- a/go.mod +++ b/go.mod @@ -31,6 +31,7 @@ require ( github.com/justincampbell/bigduration v0.0.0-20160531141349-e45bf03c0666 github.com/labstack/echo/v4 v4.15.1 github.com/leonelquinteros/gotext v1.7.2 + github.com/minio/minio-go/v7 v7.0.99 github.com/mitchellh/mapstructure v1.5.0 github.com/mssola/user_agent v0.6.0 github.com/ncw/swift/v2 v2.0.3 @@ -103,6 +104,7 @@ require ( github.com/fatih/structs v1.1.0 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect + github.com/go-ini/ini v1.67.0 // indirect github.com/go-jose/go-jose/v4 v4.1.3 // indirect github.com/go-logr/logr v1.4.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect @@ -121,13 +123,17 @@ require ( github.com/imkira/go-interpol v1.1.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/jonas-p/go-shp v0.1.1 // indirect - github.com/klauspost/compress v1.18.0 // indirect + github.com/klauspost/compress v1.18.2 // indirect + github.com/klauspost/cpuid/v2 v2.2.11 // indirect + github.com/klauspost/crc32 v1.3.0 // indirect github.com/labstack/gommon v0.4.2 // indirect github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect github.com/magiconair/properties v1.8.10 // indirect github.com/mattn/go-colorable v0.1.14 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect + github.com/minio/crc64nvme v1.1.1 // indirect + github.com/minio/md5-simd v1.1.2 // indirect github.com/mitchellh/go-wordwrap v1.0.1 // indirect github.com/moby/docker-image-spec v1.3.1 // indirect github.com/moby/go-archive v0.1.0 // indirect @@ -142,6 +148,7 @@ require ( github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.1.1 // indirect github.com/pelletier/go-toml/v2 v2.2.2 // indirect + github.com/philhofer/fwd v1.2.0 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect @@ -149,6 +156,7 @@ require ( github.com/prometheus/client_model v0.6.2 // indirect github.com/prometheus/common v0.45.0 // indirect github.com/prometheus/procfs v0.12.0 // indirect + github.com/rs/xid v1.6.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/sagikazarmark/locafero v0.4.0 // indirect github.com/sagikazarmark/slog-shim v0.1.0 // indirect @@ -161,6 +169,7 @@ require ( github.com/spiffe/go-spiffe/v2 v2.6.0 // indirect github.com/stretchr/objx v0.5.2 // indirect github.com/subosito/gotenv v1.6.0 // indirect + github.com/tinylib/msgp v1.6.1 // indirect github.com/tklauser/go-sysconf v0.3.12 // indirect github.com/tklauser/numcpus v0.6.1 // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect @@ -185,6 +194,7 @@ require ( go.opentelemetry.io/otel/trace v1.39.0 // indirect go.uber.org/atomic v1.9.0 // indirect go.uber.org/multierr v1.9.0 // indirect + go.yaml.in/yaml/v3 v3.0.4 // indirect golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect golang.org/x/sys v0.41.0 // indirect golang.org/x/time v0.14.0 // indirect diff --git a/go.sum b/go.sum index a4aede5048a..5ce9490e682 100644 --- a/go.sum +++ b/go.sum @@ -144,6 +144,8 @@ github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyT github.com/garyburd/redigo v1.1.1-0.20170914051019-70e1b1943d4f/go.mod h1:NR3MbYisc3/PwhQ00EMzDiPmrwpPxAn5GI05/YaO1SY= github.com/gavv/httpexpect/v2 v2.16.0 h1:Ty2favARiTYTOkCRZGX7ojXXjGyNAIohM1lZ3vqaEwI= github.com/gavv/httpexpect/v2 v2.16.0/go.mod h1:uJLaO+hQ25ukBJtQi750PsztObHybNllN+t+MbbW8PY= +github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A= +github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= github.com/go-jose/go-jose/v4 v4.1.3 h1:CVLmWDhDVRa6Mi/IgCgaopNosCaHz7zrMeF9MlZRkrs= github.com/go-jose/go-jose/v4 v4.1.3/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= @@ -243,8 +245,13 @@ github.com/jonas-p/go-shp v0.1.1/go.mod h1:MRIhyxDQ6VVp0oYeD7yPGr5RSTNScUFKCDsI5 github.com/justincampbell/bigduration v0.0.0-20160531141349-e45bf03c0666 h1:abLciEiilfMf19Q1TFWDrp9j5z5one60dnnpvc6eabg= github.com/justincampbell/bigduration v0.0.0-20160531141349-e45bf03c0666/go.mod h1:xqGOmDZzLOG7+q/CgsbXv10g4tgPsbjhmAxyaTJMvis= github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= -github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= -github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= +github.com/klauspost/compress v1.18.2 h1:iiPHWW0YrcFgpBYhsA6D1+fqHssJscY/Tm/y2Uqnapk= +github.com/klauspost/compress v1.18.2/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4= +github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.2.11 h1:0OwqZRYI2rFrjS4kvkDnqJkKHdHaRnCm68/DY4OxRzU= +github.com/klauspost/cpuid/v2 v2.2.11/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= +github.com/klauspost/crc32 v1.3.0 h1:sSmTt3gUt81RP655XGZPElI0PelVTZ6YwCRnPSupoFM= +github.com/klauspost/crc32 v1.3.0/go.mod h1:D7kQaZhnkX/Y0tstFGf8VUzv2UofNGqCjnC3zdHB0Hw= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= @@ -272,6 +279,12 @@ github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWE github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 h1:jWpvCLoY8Z/e3VKvlsiIGKtc+UG6U5vzxaoagmhXfyg= github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0/go.mod h1:QUyp042oQthUoa9bqDv0ER0wrtXnBruoNd7aNjkbP+k= +github.com/minio/crc64nvme v1.1.1 h1:8dwx/Pz49suywbO+auHCBpCtlW1OfpcLN7wYgVR6wAI= +github.com/minio/crc64nvme v1.1.1/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg= +github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= +github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= +github.com/minio/minio-go/v7 v7.0.99 h1:2vH/byrwUkIpFQFOilvTfaUpvAX3fEFhEzO+DR3DlCE= +github.com/minio/minio-go/v7 v7.0.99/go.mod h1:EtGNKtlX20iL2yaYnxEigaIvj0G0GwSDnifnG8ClIdw= github.com/mitchellh/go-wordwrap v1.0.1 h1:TLuKupo69TCn6TQSyGxwI1EblZZEsQ0vMlAFQflz0v0= github.com/mitchellh/go-wordwrap v1.0.1/go.mod h1:R62XHJLzvMFRBbcrT7m7WgmE1eOyTSsCt+hzestvNj0= github.com/mitchellh/mapstructure v0.0.0-20170523030023-d0303fe80992/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= @@ -326,6 +339,8 @@ github.com/oschwald/maxminddb-golang v1.13.1/go.mod h1:K4pgV9N/GcK694KSTmVSDTODk github.com/pelletier/go-toml v1.0.1-0.20170904195809-1d6b12b7cb29/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= +github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM= +github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM= github.com/pkg/diff v0.0.0-20200914180035-5b29258ca4f7/go.mod h1:zO8QMzTeZd5cpnIkz/Gn6iK0jDfGicM1nynOkkPIl28= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -355,6 +370,8 @@ github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= +github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU= +github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6keLGt6kNQ= @@ -412,6 +429,8 @@ github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSW github.com/tailscale/depaware v0.0.0-20210622194025-720c4b409502/go.mod h1:p9lPsd+cx33L3H9nNoecRRxPssFKUwwI50I3pZ0yT+8= github.com/testcontainers/testcontainers-go v0.40.0 h1:pSdJYLOVgLE8YdUY2FHQ1Fxu+aMnb6JfVz1mxk7OeMU= github.com/testcontainers/testcontainers-go v0.40.0/go.mod h1:FSXV5KQtX2HAMlm7U3APNyLkkap35zNLxukw9oBi/MY= +github.com/tinylib/msgp v1.6.1 h1:ESRv8eL3u+DNHUoSAAQRE50Hm162zqAnBoGv9PzScPY= +github.com/tinylib/msgp v1.6.1/go.mod h1:RSp0LW9oSxFut3KzESt5Voq4GVWyS+PSulT77roAqEA= github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFAEVmqU= github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI= github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk= @@ -478,6 +497,8 @@ go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI= go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTVQ= +go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= golang.org/x/crypto v0.0.0-20170512130425-ab89591268e0/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= diff --git a/model/app/apps.go b/model/app/apps.go index 34c099a869e..6e4ecb4c74c 100644 --- a/model/app/apps.go +++ b/model/app/apps.go @@ -156,6 +156,10 @@ func Copier(appsType consts.AppType, inst *instance.Instance) appfs.Copier { return appfs.NewAferoCopier(baseFS) case config.SchemeSwift, config.SchemeSwiftSecure: return appfs.NewSwiftCopier(config.GetSwiftConnection(), appsType) + case config.SchemeS3: + client := config.GetS3Client() + bucket := appfs.S3AppsBucket(config.GetS3BucketPrefix(), appsType) + return appfs.NewS3Copier(client, bucket) default: panic(fmt.Sprintf("instance: unknown storage provider %s", fsURL.Scheme)) } @@ -175,6 +179,10 @@ func AppsFileServer(i *instance.Instance) appfs.FileServer { return appfs.NewAferoFileServer(baseFS, nil) case config.SchemeSwift, config.SchemeSwiftSecure: return appfs.NewSwiftFileServer(config.GetSwiftConnection(), consts.WebappType) + case config.SchemeS3: + client := config.GetS3Client() + bucket := appfs.S3AppsBucket(config.GetS3BucketPrefix(), consts.WebappType) + return appfs.NewS3FileServer(client, bucket) default: panic(fmt.Sprintf("instance: unknown storage provider %s", fsURL.Scheme)) } @@ -194,6 +202,10 @@ func KonnectorsFileServer(i *instance.Instance) appfs.FileServer { return appfs.NewAferoFileServer(baseFS, nil) case config.SchemeSwift, config.SchemeSwiftSecure: return appfs.NewSwiftFileServer(config.GetSwiftConnection(), consts.KonnectorType) + case config.SchemeS3: + client := config.GetS3Client() + bucket := appfs.S3AppsBucket(config.GetS3BucketPrefix(), consts.KonnectorType) + return appfs.NewS3FileServer(client, bucket) default: panic(fmt.Sprintf("instance: unknown storage provider %s", fsURL.Scheme)) } diff --git a/model/instance/instance.go b/model/instance/instance.go index 6159123f452..a11992fd87c 100644 --- a/model/instance/instance.go +++ b/model/instance/instance.go @@ -16,6 +16,7 @@ import ( "github.com/cozy/cozy-stack/model/permission" "github.com/cozy/cozy-stack/model/vfs" "github.com/cozy/cozy-stack/model/vfs/vfsafero" + "github.com/cozy/cozy-stack/model/vfs/vfss3" "github.com/cozy/cozy-stack/model/vfs/vfsswift" build "github.com/cozy/cozy-stack/pkg/config" "github.com/cozy/cozy-stack/pkg/config/config" @@ -196,6 +197,11 @@ func (i *Instance) GetContextName() string { return i.ContextName } +// GetOrgID returns the organization ID of the instance. +func (i *Instance) GetOrgID() string { + return i.OrgID +} + // SessionSecret returns the session secret. func (i *Instance) SessionSecret() []byte { // The prefix is here to invalidate all the sessions that were created on @@ -261,6 +267,8 @@ func (i *Instance) MakeVFS() error { default: err = ErrInvalidSwiftLayout } + case config.SchemeS3: + i.vfs, err = vfss3.New(i, index, disk, mutex) default: err = fmt.Errorf("instance: unknown storage provider %s", fsURL.Scheme) } @@ -285,6 +293,11 @@ func (i *Instance) AvatarFS() vfs.Avatarer { default: panic(ErrInvalidSwiftLayout) } + case config.SchemeS3: + client := config.GetS3Client() + bucket := vfss3.BucketName(i.GetOrgID(), config.GetS3BucketPrefix()) + keyPrefix := i.DBPrefix() + "/" + return vfss3.NewAvatarFs(client, bucket, keyPrefix) default: panic(fmt.Sprintf("instance: unknown storage provider %s", fsURL.Scheme)) } @@ -309,6 +322,11 @@ func (i *Instance) ThumbsFS() vfs.Thumbser { default: panic(ErrInvalidSwiftLayout) } + case config.SchemeS3: + client := config.GetS3Client() + bucket := vfss3.BucketName(i.GetOrgID(), config.GetS3BucketPrefix()) + keyPrefix := i.DBPrefix() + "/" + return vfss3.NewThumbsFs(client, bucket, keyPrefix) default: panic(fmt.Sprintf("instance: unknown storage provider %s", fsURL.Scheme)) } diff --git a/model/move/archiver.go b/model/move/archiver.go index be36ccb9b7e..03bb896ac3a 100644 --- a/model/move/archiver.go +++ b/model/move/archiver.go @@ -14,6 +14,7 @@ import ( "github.com/cozy/cozy-stack/pkg/config/config" "github.com/cozy/cozy-stack/pkg/crypto" multierror "github.com/hashicorp/go-multierror" + "github.com/minio/minio-go/v7" "github.com/ncw/swift/v2" "github.com/spf13/afero" ) @@ -45,6 +46,8 @@ func SystemArchiver() Archiver { return newAferoArchiver(fs) case config.SchemeSwift, config.SchemeSwiftSecure: return newSwiftArchiver() + case config.SchemeS3: + return newS3Archiver() default: panic(fmt.Errorf("exports: unknown storage provider %s", fsURL.Scheme)) } @@ -149,3 +152,84 @@ func (a *switfArchiver) RemoveArchives(exportDocs []*ExportDoc) error { } return nil } + +func newS3Archiver() Archiver { + client := config.GetS3Client() + bucket := config.GetS3BucketPrefix() + "-exports" + return &s3Archiver{ + client: client, + bucket: bucket, + ctx: context.Background(), + } +} + +type s3Archiver struct { + client *minio.Client + bucket string + ctx context.Context +} + +func (a *s3Archiver) ensureBucket() error { + err := a.client.MakeBucket(a.ctx, a.bucket, minio.MakeBucketOptions{}) + if err != nil { + code := minio.ToErrorResponse(err).Code + if code == "BucketAlreadyOwnedByYou" || code == "BucketAlreadyExists" { + return nil + } + return err + } + return nil +} + +func (a *s3Archiver) OpenArchive(inst *instance.Instance, exportDoc *ExportDoc) (io.ReadCloser, error) { + objectName := exportDoc.Domain + "/" + exportDoc.ID() + obj, err := a.client.GetObject(a.ctx, a.bucket, objectName, minio.GetObjectOptions{}) + if err != nil { + return nil, err + } + // Verify the object exists by calling Stat; GetObject itself does not + // perform a network request until Read is called, and Stat triggers + // a HEAD that surfaces NoSuchKey early. + if _, err := obj.Stat(); err != nil { + obj.Close() + return nil, err + } + return obj, nil +} + +func (a *s3Archiver) CreateArchive(exportDoc *ExportDoc) (io.WriteCloser, error) { + if err := a.ensureBucket(); err != nil { + return nil, err + } + + objectName := exportDoc.Domain + "/" + exportDoc.ID() + pr, pw := io.Pipe() + + go func() { + _, err := a.client.PutObject(a.ctx, a.bucket, objectName, + pr, -1, + minio.PutObjectOptions{ContentType: "application/tar+gzip"}) + // Close the read side so that any error is propagated to the writer. + pr.CloseWithError(err) + }() + + return pw, nil +} + +func (a *s3Archiver) RemoveArchives(exportDocs []*ExportDoc) error { + if len(exportDocs) == 0 { + return nil + } + + objectsCh := make(chan minio.ObjectInfo, len(exportDocs)) + for _, e := range exportDocs { + objectsCh <- minio.ObjectInfo{Key: e.Domain + "/" + e.ID()} + } + close(objectsCh) + + var errm error + for e := range a.client.RemoveObjects(a.ctx, a.bucket, objectsCh, minio.RemoveObjectsOptions{}) { + errm = multierror.Append(errm, e.Err) + } + return errm +} diff --git a/model/stack/main.go b/model/stack/main.go index 5ddb66f0660..f82bdbe87da 100644 --- a/model/stack/main.go +++ b/model/stack/main.go @@ -85,6 +85,11 @@ security features. Please do not use this binary as your production server. return nil, nil, fmt.Errorf("failed to init the swift connection: %w", err) } + // Init the main global connection to the S3 server + if err := config.InitDefaultS3Connection(); err != nil { + return nil, nil, fmt.Errorf("failed to init the S3 connection: %w", err) + } + workersList, err := job.GetWorkersList() if err != nil { return nil, nil, fmt.Errorf("failed to get the workers list: %w", err) diff --git a/model/vfs/vfs_test.go b/model/vfs/vfs_test.go index 7d20d308d3f..40d9f1e52b0 100644 --- a/model/vfs/vfs_test.go +++ b/model/vfs/vfs_test.go @@ -17,6 +17,7 @@ import ( "github.com/cozy/cozy-stack/model/vfs" "github.com/cozy/cozy-stack/model/vfs/vfsafero" + "github.com/cozy/cozy-stack/model/vfs/vfss3" "github.com/cozy/cozy-stack/model/vfs/vfsswift" "github.com/cozy/cozy-stack/pkg/config/config" "github.com/cozy/cozy-stack/pkg/consts" @@ -54,6 +55,7 @@ func TestVfs(t *testing.T) { aferoFS := makeAferoFS(t) swiftFS := makeSwiftFS(t) + s3FS := makeS3FS(t) var tests = []struct { name string @@ -61,6 +63,7 @@ func TestVfs(t *testing.T) { }{ {"afero", aferoFS}, {"swift", swiftFS}, + {"s3", s3FS}, } for _, tt := range tests { @@ -909,3 +912,31 @@ func makeSwiftFS(t *testing.T) vfs.VFS { return swiftFs } + +func makeS3FS(t *testing.T) vfs.VFS { + t.Helper() + + minioFixture := testutils.StartMinio(t) + db := &contexter{0, "io.cozy.vfs.s3.test", "io.cozy.vfs.s3.test", "cozy_beta"} + index := vfs.NewCouchdbIndexer(db) + + require.NoError(t, config.InitS3Connection(config.Fs{ + URL: minioFixture.FsURL("test"), + })) + + mutex = config.Lock().ReadWrite(db, "vfs-s3-test") + s3Fs, err := vfss3.New(db, index, &diskImpl{}, mutex) + require.NoError(t, err) + + require.NoError(t, couchdb.ResetDB(db, consts.Files)) + t.Cleanup(func() { _ = couchdb.DeleteDB(db, consts.Files) }) + + g, _ := errgroup.WithContext(context.Background()) + couchdb.DefineIndexes(g, db, couchdb.IndexesByDoctype(consts.Files)) + couchdb.DefineViews(g, db, couchdb.ViewsByDoctype(consts.Files)) + + require.NoError(t, g.Wait()) + require.NoError(t, s3Fs.InitFs()) + + return s3Fs +} diff --git a/model/vfs/vfss3/avatar.go b/model/vfs/vfss3/avatar.go new file mode 100644 index 00000000000..a164ef39c4b --- /dev/null +++ b/model/vfs/vfss3/avatar.go @@ -0,0 +1,106 @@ +package vfss3 + +import ( + "context" + "fmt" + "io" + "net/http" + "time" + + "github.com/cozy/cozy-stack/model/vfs" + "github.com/cozy/cozy-stack/pkg/s3util" + "github.com/minio/minio-go/v7" +) + +// NewAvatarFs creates a new avatar filesystem backed by S3. +func NewAvatarFs(client *minio.Client, bucket, keyPrefix string) vfs.Avatarer { + return &avatarS3{ + client: client, + bucket: bucket, + keyPrefix: keyPrefix, + ctx: context.Background(), + } +} + +type avatarS3 struct { + client *minio.Client + bucket string + keyPrefix string + ctx context.Context +} + +func (a *avatarS3) avatarKey() string { + return a.keyPrefix + "avatar" +} + +func (a *avatarS3) CreateAvatar(contentType string) (io.WriteCloser, error) { + key := a.avatarKey() + pr, pw := io.Pipe() + + meta := map[string]string{ + "created-at": time.Now().UTC().Format(time.RFC3339), + } + + errCh := make(chan error, 1) + go func() { + _, err := a.client.PutObject(a.ctx, a.bucket, key, pr, -1, minio.PutObjectOptions{ + ContentType: contentType, + UserMetadata: meta, + }) + errCh <- err + }() + + return &avatarWriter{pw: pw, errCh: errCh}, nil +} + +type avatarWriter struct { + pw *io.PipeWriter + errCh chan error +} + +func (w *avatarWriter) Write(p []byte) (int, error) { + return w.pw.Write(p) +} + +func (w *avatarWriter) Close() error { + if err := w.pw.Close(); err != nil { + return err + } + return <-w.errCh +} + +func (a *avatarS3) DeleteAvatar() error { + err := a.client.RemoveObject(a.ctx, a.bucket, a.avatarKey(), minio.RemoveObjectOptions{}) + if err != nil { + if minio.ToErrorResponse(err).Code == "NoSuchKey" { + return nil + } + return err + } + return nil +} + +func (a *avatarS3) ServeAvatarContent(w http.ResponseWriter, req *http.Request) error { + obj, err := a.client.GetObject(a.ctx, a.bucket, a.avatarKey(), minio.GetObjectOptions{}) + if err != nil { + return s3util.WrapNotFound(err) + } + defer obj.Close() + + info, err := obj.Stat() + if err != nil { + return s3util.WrapNotFound(err) + } + + t := time.Time{} + if createdAt, ok := info.UserMetadata["Created-At"]; ok && createdAt != "" { + if createdAtTime, err := time.Parse(time.RFC3339, createdAt); err == nil { + t = createdAtTime + } + } + + w.Header().Set("Etag", fmt.Sprintf(`"%s"`, info.ETag)) + w.Header().Set("Content-Type", info.ContentType) + http.ServeContent(w, req, "avatar", t, obj) + return nil +} diff --git a/model/vfs/vfss3/fsck.go b/model/vfs/vfss3/fsck.go new file mode 100644 index 00000000000..364f6c3bca1 --- /dev/null +++ b/model/vfs/vfss3/fsck.go @@ -0,0 +1,254 @@ +package vfss3 + +import ( + "bytes" + "encoding/hex" + "encoding/json" + "errors" + "path" + "strings" + + "github.com/cozy/cozy-stack/model/vfs" + "github.com/cozy/cozy-stack/pkg/consts" + "github.com/cozy/cozy-stack/pkg/couchdb" + "github.com/minio/minio-go/v7" +) + +func (sfs *s3VFS) Fsck(accumulate func(log *vfs.FsckLog), failFast bool) error { + entries := make(map[string]*vfs.TreeFile, 1024) + tree, err := sfs.BuildTree(func(f *vfs.TreeFile) { + if !f.IsDir { + entries[f.DocID+"/"+f.InternalID] = f + } + }) + if err != nil { + return err + } + if err = sfs.CheckTreeIntegrity(tree, accumulate, failFast); err != nil { + if errors.Is(err, vfs.ErrFsckFailFast) { + return nil + } + return err + } + return sfs.checkFiles(entries, accumulate, failFast) +} + +func (sfs *s3VFS) CheckFilesConsistency(accumulate func(log *vfs.FsckLog), failFast bool) error { + entries := make(map[string]*vfs.TreeFile, 1024) + _, err := sfs.BuildTree(func(f *vfs.TreeFile) { + if !f.IsDir { + entries[f.DocID+"/"+f.InternalID] = f + } + }) + if err != nil { + return err + } + return sfs.checkFiles(entries, accumulate, failFast) +} + +func (sfs *s3VFS) checkFiles( + entries map[string]*vfs.TreeFile, + accumulate func(log *vfs.FsckLog), + failFast bool, +) error { + versions := make(map[string]*vfs.Version, 1024) + err := couchdb.ForeachDocs(sfs, consts.FilesVersions, func(_ string, data json.RawMessage) error { + v := &vfs.Version{} + if erru := json.Unmarshal(data, v); erru != nil { + return erru + } + versions[v.DocID] = v + return nil + }) + if err != nil { + return err + } + + images := make(map[string]struct{}) + err = couchdb.ForeachDocs(sfs, consts.NotesImages, func(_ string, data json.RawMessage) error { + img := make(map[string]interface{}) + if erru := json.Unmarshal(data, &img); erru != nil { + return erru + } + id, _ := img["_id"].(string) + images[id] = struct{}{} + return nil + }) + if err != nil && !couchdb.IsNoDatabaseError(err) { + return err + } + + fileIDs := make(map[string]struct{}, len(entries)) + for _, f := range entries { + fileIDs[f.DocID] = struct{}{} + } + + // List all objects under our key prefix + for obj := range sfs.client.ListObjects(sfs.ctx, sfs.bucket, minio.ListObjectsOptions{ + Prefix: sfs.keyPrefix, + Recursive: true, + }) { + if obj.Err != nil { + return obj.Err + } + + // Strip the key prefix to get the object name + objName := strings.TrimPrefix(obj.Key, sfs.keyPrefix) + + if objName == "avatar" { + continue + } + if strings.HasPrefix(objName, "thumbs/") { + thumbName := strings.TrimPrefix(objName, "thumbs/") + idx := strings.LastIndex(thumbName, "-") + if idx < 0 { + continue + } + thumbName = thumbName[0:idx] // Remove -format suffix + fileID, _ := makeDocID(thumbName) + if _, ok := fileIDs[fileID]; !ok { + if _, ok := images[fileID]; !ok { + accumulate(&vfs.FsckLog{ + Type: vfs.ThumbnailWithNoFile, + IsFile: true, + FileDoc: &vfs.TreeFile{ + DirOrFileDoc: vfs.DirOrFileDoc{ + DirDoc: &vfs.DirDoc{ + Type: consts.FileType, + DocID: fileID, + DocName: objName, + }, + }, + }, + }) + if failFast { + return nil + } + } + } + continue + } + + docID, internalID := makeDocID(objName) + if v, ok := versions[docID+"/"+internalID]; ok { + // ETag from S3 may or may not be an MD5 (multipart uploads use composite ETags). + etag := strings.Trim(obj.ETag, "\"") + if !strings.Contains(etag, "-") { + md5sum, err := hex.DecodeString(etag) + if err == nil { + if !bytes.Equal(md5sum, v.MD5Sum) || v.ByteSize != obj.Size { + accumulate(&vfs.FsckLog{ + Type: vfs.ContentMismatch, + IsVersion: true, + VersionDoc: v, + ContentMismatch: &vfs.FsckContentMismatch{ + SizeFile: obj.Size, + SizeIndex: v.ByteSize, + MD5SumFile: md5sum, + MD5SumIndex: v.MD5Sum, + }, + }) + if failFast { + return nil + } + } + } + } + delete(versions, v.DocID) + continue + } + f, ok := entries[docID+"/"+internalID] + if !ok { + accumulate(&vfs.FsckLog{ + Type: vfs.IndexMissing, + IsFile: true, + FileDoc: objectToFileDoc(obj), + }) + if failFast { + return nil + } + } else { + etag := strings.Trim(obj.ETag, "\"") + if !strings.Contains(etag, "-") { + md5sum, err := hex.DecodeString(etag) + if err == nil { + if !bytes.Equal(md5sum, f.MD5Sum) || f.ByteSize != obj.Size { + accumulate(&vfs.FsckLog{ + Type: vfs.ContentMismatch, + IsFile: true, + FileDoc: f, + ContentMismatch: &vfs.FsckContentMismatch{ + SizeFile: obj.Size, + SizeIndex: f.ByteSize, + MD5SumFile: md5sum, + MD5SumIndex: f.MD5Sum, + }, + }) + if failFast { + return nil + } + } + } + } + delete(entries, docID+"/"+internalID) + } + } + + // entries should contain only data that does not contain an associated + // object in S3. + for _, f := range entries { + accumulate(&vfs.FsckLog{ + Type: vfs.FSMissing, + IsFile: true, + FileDoc: f, + }) + if failFast { + return nil + } + } + + for _, v := range versions { + accumulate(&vfs.FsckLog{ + Type: vfs.FSMissing, + IsVersion: true, + VersionDoc: v, + }) + if failFast { + return nil + } + } + + return nil +} + +func objectToFileDoc(obj minio.ObjectInfo) *vfs.TreeFile { + md5sum, _ := hex.DecodeString(strings.Trim(obj.ETag, "\"")) + name := "unknown" + mime, class := vfs.ExtractMimeAndClass(obj.ContentType) + // Strip any key prefix — we need to find the object name portion + // which is just the last segments of the key. + objName := obj.Key + if idx := strings.Index(objName, "/"); idx >= 0 { + // The first segment is the key prefix (db prefix); skip it + objName = objName[idx+1:] + } + fileID, internalID := makeDocID(objName) + return &vfs.TreeFile{ + DirOrFileDoc: vfs.DirOrFileDoc{ + DirDoc: &vfs.DirDoc{ + Type: consts.FileType, + DocID: fileID, + DocName: name, + DirID: "", + CreatedAt: obj.LastModified, + UpdatedAt: obj.LastModified, + Fullpath: path.Join(vfs.OrphansDirName, name), + }, + ByteSize: obj.Size, + Mime: mime, + Class: class, + MD5Sum: md5sum, + InternalID: internalID, + }, + } +} diff --git a/model/vfs/vfss3/impl.go b/model/vfs/vfss3/impl.go new file mode 100644 index 00000000000..caa2cbabfa5 --- /dev/null +++ b/model/vfs/vfss3/impl.go @@ -0,0 +1,1154 @@ +// Package vfss3 is the implementation of the Virtual File System by using +// an S3-compatible object storage. The file contents are saved in S3 buckets, +// and the metadata are indexed in CouchDB. +package vfss3 + +import ( + "bytes" + "context" + "crypto/md5" + + "errors" + "fmt" + "hash" + "io" + "os" + "regexp" + "strings" + + "github.com/cozy/cozy-stack/model/vfs" + "github.com/cozy/cozy-stack/pkg/config/config" + "github.com/cozy/cozy-stack/pkg/couchdb" + "github.com/cozy/cozy-stack/pkg/lock" + "github.com/cozy/cozy-stack/pkg/logger" + "github.com/cozy/cozy-stack/pkg/s3util" + "github.com/cozy/cozy-stack/pkg/utils" + "github.com/gofrs/uuid/v5" + "github.com/minio/minio-go/v7" +) + +type s3VFS struct { + vfs.Indexer + vfs.DiskThresholder + client *minio.Client + cluster int + domain string + prefix string // DBPrefix — used as key prefix in the bucket + contextName string + ctx context.Context + bucket string + keyPrefix string // prefix + "/" + region string + mu lock.ErrorRWLocker + log *logger.Entry +} + +var bucketNameCleaner = regexp.MustCompile(`[^a-z0-9-]`) + +// sanitizeBucketName produces a valid S3 bucket name component from an arbitrary string. +func sanitizeBucketName(s string) string { + s = strings.ToLower(s) + s = strings.ReplaceAll(s, "_", "-") + s = strings.ReplaceAll(s, ".", "-") + s = bucketNameCleaner.ReplaceAllString(s, "") + // Collapse consecutive hyphens + for strings.Contains(s, "--") { + s = strings.ReplaceAll(s, "--", "-") + } + s = strings.Trim(s, "-") + if len(s) > 37 { + s = s[:37] + } + return s +} + +// BucketName returns the S3 bucket name for a given orgID and bucket prefix. +func BucketName(orgID, bucketPrefix string) string { + if orgID == "" { + orgID = "default" + } + name := bucketPrefix + "-" + sanitizeBucketName(orgID) + if len(name) > 63 { + name = name[:63] + } + name = strings.TrimRight(name, "-") + if len(name) < 3 { + name += strings.Repeat("0", 3-len(name)) + } + return name +} + +// MakeObjectKey builds the S3 object key for a given file. +// It reuses the same virtual subfolder structure as Swift V3. +func MakeObjectKey(keyPrefix, docID, internalID string) string { + return keyPrefix + makeObjectName(docID, internalID) +} + +// makeObjectName builds the object name (without key prefix), identical to +// vfsswift.MakeObjectNameV3. +func makeObjectName(docID, internalID string) string { + if len(docID) != 32 || len(internalID) != 16 { + return docID + "/" + internalID + } + return docID[:22] + "/" + docID[22:27] + "/" + docID[27:] + "/" + internalID +} + +func makeDocID(objName string) (string, string) { + if len(objName) != 51 { + parts := strings.SplitN(objName, "/", 2) + if len(parts) < 2 { + return objName, "" + } + return parts[0], parts[1] + } + return objName[:22] + objName[23:28] + objName[29:34], objName[35:] +} + +// NewInternalID returns a random string that can be used as an internal_vfs_id. +func NewInternalID() string { + return utils.RandomString(16) +} + +// New returns a vfs.VFS instance backed by an S3-compatible object store. +func New(db vfs.Prefixer, index vfs.Indexer, disk vfs.DiskThresholder, mu lock.ErrorRWLocker) (vfs.VFS, error) { + client := config.GetS3Client() + bucketPrefix := config.GetS3BucketPrefix() + + orgID := "" + if inst, ok := db.(interface{ GetOrgID() string }); ok { + orgID = inst.GetOrgID() + } + bucket := BucketName(orgID, bucketPrefix) + dbPrefix := db.DBPrefix() + if dbPrefix == "" { + return nil, fmt.Errorf("vfss3: empty DBPrefix") + } + + return &s3VFS{ + Indexer: index, + DiskThresholder: disk, + client: client, + cluster: db.DBCluster(), + domain: db.DomainName(), + prefix: dbPrefix, + contextName: db.GetContextName(), + ctx: context.Background(), + bucket: bucket, + keyPrefix: dbPrefix + "/", + region: config.GetS3Region(), + mu: mu, + log: logger.WithDomain(db.DomainName()).WithNamespace("vfss3"), + }, nil +} + +func (sfs *s3VFS) MaxFileSize() int64 { + return -1 // no per-file limit — S3 multipart handles large files transparently +} + +func (sfs *s3VFS) DBCluster() int { + return sfs.cluster +} + +func (sfs *s3VFS) DBPrefix() string { + return sfs.prefix +} + +func (sfs *s3VFS) DomainName() string { + return sfs.domain +} + +func (sfs *s3VFS) GetContextName() string { + return sfs.contextName +} + +func (sfs *s3VFS) GetIndexer() vfs.Indexer { + return sfs.Indexer +} + +func (sfs *s3VFS) UseSharingIndexer(index vfs.Indexer) vfs.VFS { + return &s3VFS{ + Indexer: index, + DiskThresholder: sfs.DiskThresholder, + client: sfs.client, + cluster: sfs.cluster, + domain: sfs.domain, + prefix: sfs.prefix, + contextName: sfs.contextName, + ctx: context.Background(), + bucket: sfs.bucket, + keyPrefix: sfs.keyPrefix, + region: sfs.region, + mu: sfs.mu, + log: sfs.log, + } +} + +func (sfs *s3VFS) InitFs() error { + if lockerr := sfs.mu.Lock(); lockerr != nil { + return lockerr + } + defer sfs.mu.Unlock() + if err := sfs.Indexer.InitIndex(); err != nil { + return err + } + err := sfs.client.MakeBucket(sfs.ctx, sfs.bucket, minio.MakeBucketOptions{ + Region: sfs.region, + }) + if err != nil { + code := minio.ToErrorResponse(err).Code + if code == "BucketAlreadyOwnedByYou" || code == "BucketAlreadyExists" { + return nil + } + sfs.log.Errorf("Could not create bucket %q: %s", sfs.bucket, err.Error()) + return err + } + sfs.log.Infof("Created bucket %q", sfs.bucket) + return nil +} + +func (sfs *s3VFS) Delete() error { + sfs.log.Infof("Deleting all objects with prefix %q in bucket %q", sfs.keyPrefix, sfs.bucket) + return s3util.DeletePrefixObjects(sfs.ctx, sfs.client, sfs.bucket, sfs.keyPrefix) +} + +func (sfs *s3VFS) CreateDir(doc *vfs.DirDoc) error { + if lockerr := sfs.mu.Lock(); lockerr != nil { + return lockerr + } + defer sfs.mu.Unlock() + exists, err := sfs.Indexer.DirChildExists(doc.DirID, doc.DocName) + if err != nil { + return err + } + if exists { + return os.ErrExist + } + if doc.ID() == "" { + return sfs.Indexer.CreateDirDoc(doc) + } + return sfs.Indexer.CreateNamedDirDoc(doc) +} + +// putResult is the result sent back from the background PutObject goroutine. +type putResult struct { + info minio.UploadInfo + err error +} + +func (sfs *s3VFS) CreateFile(newdoc, olddoc *vfs.FileDoc, opts ...vfs.CreateOptions) (vfs.File, error) { + if lockerr := sfs.mu.Lock(); lockerr != nil { + return nil, lockerr + } + defer sfs.mu.Unlock() + + newsize, maxsize, capsize, err := vfs.CheckAvailableDiskSpace(sfs, newdoc) + if err != nil { + return nil, err + } + // CheckAvailableDiskSpace already returns ErrFileTooBig / ErrMaxFileSize + // when the upload would exceed the per-file or quota budget. The + // streaming-time check below (s3FileCreation.Write) re-checks against + // maxsize once the actual byte count is known. + + if olddoc != nil { + newdoc.SetID(olddoc.ID()) + newdoc.SetRev(olddoc.Rev()) + newdoc.CreatedAt = olddoc.CreatedAt + } + + newpath, err := sfs.Indexer.FilePath(newdoc) + if err != nil { + return nil, err + } + if strings.HasPrefix(newpath, vfs.TrashDirName+"/") { + if !vfs.OptionsAllowCreationInTrash(opts) { + return nil, vfs.ErrParentInTrash + } + } + + if olddoc == nil { + var exists bool + exists, err = sfs.Indexer.DirChildExists(newdoc.DirID, newdoc.DocName) + if err != nil { + return nil, err + } + if exists { + return nil, os.ErrExist + } + } + + if newdoc.DocID == "" { + uid, err := uuid.NewV7() + if err != nil { + return nil, err + } + newdoc.DocID = uid.String() + } + + newdoc.InternalID = NewInternalID() + objKey := MakeObjectKey(sfs.keyPrefix, newdoc.DocID, newdoc.InternalID) + + // Use a pipe: writes go into pw, the PutObject goroutine reads from pr. + pr, pw := io.Pipe() + + uploadSize := newdoc.ByteSize + if uploadSize < 0 { + uploadSize = -1 + } + + resultCh := make(chan putResult, 1) + go func() { + info, err := sfs.client.PutObject(sfs.ctx, sfs.bucket, objKey, pr, uploadSize, minio.PutObjectOptions{ + ContentType: newdoc.Mime, + PartSize: 5 * 1024 * 1024, // 5 MiB + NumThreads: 1, + }) + // Propagate the outcome to the writer side: if PutObject errored before + // draining the pipe, an in-flight Write would otherwise block forever. + _ = pr.CloseWithError(err) + resultCh <- putResult{info: info, err: err} + }() + + extractor := vfs.NewMetaExtractor(newdoc) + + return &s3FileCreation{ + fs: sfs, + pw: pw, + resultCh: resultCh, + newdoc: newdoc, + olddoc: olddoc, + objKey: objKey, + w: 0, + size: newsize, + maxsize: maxsize, + capsize: capsize, + meta: extractor, + md5H: md5.New(), + }, nil +} + +func (sfs *s3VFS) CopyFile(olddoc, newdoc *vfs.FileDoc) error { + if lockerr := sfs.mu.Lock(); lockerr != nil { + return lockerr + } + defer sfs.mu.Unlock() + + exists, err := sfs.Indexer.DirChildExists(newdoc.DirID, newdoc.DocName) + if err != nil { + return err + } + if exists { + return os.ErrExist + } + + newsize, _, capsize, err := vfs.CheckAvailableDiskSpace(sfs, olddoc) + if err != nil { + return err + } + + uid, err := uuid.NewV7() + if err != nil { + return err + } + newdoc.DocID = uid.String() + newdoc.InternalID = NewInternalID() + + srcKey := MakeObjectKey(sfs.keyPrefix, olddoc.DocID, olddoc.InternalID) + dstKey := MakeObjectKey(sfs.keyPrefix, newdoc.DocID, newdoc.InternalID) + + if _, err := sfs.client.CopyObject(sfs.ctx, + minio.CopyDestOptions{Bucket: sfs.bucket, Object: dstKey}, + minio.CopySrcOptions{Bucket: sfs.bucket, Object: srcKey}, + ); err != nil { + return err + } + if err := sfs.Indexer.CreateNamedFileDoc(newdoc); err != nil { + _ = sfs.client.RemoveObject(sfs.ctx, sfs.bucket, dstKey, minio.RemoveObjectOptions{}) + return err + } + + if capsize > 0 && newsize >= capsize { + vfs.PushDiskQuotaAlert(sfs, true) + } + + return nil +} + +func (sfs *s3VFS) DissociateFile(src, dst *vfs.FileDoc) error { + if lockerr := sfs.mu.Lock(); lockerr != nil { + return lockerr + } + defer sfs.mu.Unlock() + + if src.DirID != dst.DirID || src.DocName != dst.DocName { + exists, err := sfs.Indexer.DirChildExists(dst.DirID, dst.DocName) + if err != nil { + return err + } + if exists { + return os.ErrExist + } + } + + uid, err := uuid.NewV7() + if err != nil { + return err + } + dst.DocID = uid.String() + + srcKey := MakeObjectKey(sfs.keyPrefix, src.DocID, src.InternalID) + dstKey := MakeObjectKey(sfs.keyPrefix, dst.DocID, dst.InternalID) + + if _, err := sfs.client.CopyObject(sfs.ctx, + minio.CopyDestOptions{Bucket: sfs.bucket, Object: dstKey}, + minio.CopySrcOptions{Bucket: sfs.bucket, Object: srcKey}, + ); err != nil { + return err + } + if err := sfs.Indexer.CreateNamedFileDoc(dst); err != nil { + _ = sfs.client.RemoveObject(sfs.ctx, sfs.bucket, dstKey, minio.RemoveObjectOptions{}) + return err + } + + return sfs.destroyFileLocked(src) +} + +func (sfs *s3VFS) DissociateDir(src, dst *vfs.DirDoc) error { + if lockerr := sfs.mu.Lock(); lockerr != nil { + return lockerr + } + defer sfs.mu.Unlock() + + if dst.DirID != src.DirID || dst.DocName != src.DocName { + exists, err := sfs.Indexer.DirChildExists(dst.DirID, dst.DocName) + if err != nil { + return err + } + if exists { + return os.ErrExist + } + } + + if err := sfs.Indexer.CreateDirDoc(dst); err != nil { + return err + } + return sfs.Indexer.DeleteDirDoc(src) +} + +func (sfs *s3VFS) destroyDir(doc *vfs.DirDoc, push func(vfs.TrashJournal) error, onlyContent bool) error { + if lockerr := sfs.mu.Lock(); lockerr != nil { + return lockerr + } + defer sfs.mu.Unlock() + diskUsage, _ := sfs.Indexer.DiskUsage() + files, destroyed, err := sfs.Indexer.DeleteDirDocAndContent(doc, onlyContent) + if err != nil { + return err + } + if len(files) == 0 { + return nil + } + vfs.DiskQuotaAfterDestroy(sfs, diskUsage, destroyed) + ids := make([]string, len(files)) + objNames := make([]string, len(files)) + for i, file := range files { + ids[i] = file.DocID + objNames[i] = MakeObjectKey(sfs.keyPrefix, file.DocID, file.InternalID) + } + return push(vfs.TrashJournal{ + FileIDs: ids, + ObjectNames: objNames, + }) +} + +func (sfs *s3VFS) DestroyDirContent(doc *vfs.DirDoc, push func(vfs.TrashJournal) error) error { + return sfs.destroyDir(doc, push, true) +} + +func (sfs *s3VFS) DestroyDirAndContent(doc *vfs.DirDoc, push func(vfs.TrashJournal) error) error { + return sfs.destroyDir(doc, push, false) +} + +func (sfs *s3VFS) DestroyFile(doc *vfs.FileDoc) error { + if lockerr := sfs.mu.Lock(); lockerr != nil { + return lockerr + } + defer sfs.mu.Unlock() + return sfs.destroyFileLocked(doc) +} + +func (sfs *s3VFS) destroyFileLocked(doc *vfs.FileDoc) error { + diskUsage, _ := sfs.Indexer.DiskUsage() + objNames := []string{ + MakeObjectKey(sfs.keyPrefix, doc.DocID, doc.InternalID), + } + if err := sfs.Indexer.DeleteFileDoc(doc); err != nil { + return err + } + destroyed := doc.ByteSize + if versions, errv := vfs.VersionsFor(sfs, doc.DocID); errv == nil { + for _, v := range versions { + internalID := v.DocID + if parts := strings.SplitN(v.DocID, "/", 2); len(parts) > 1 { + internalID = parts[1] + } + objNames = append(objNames, MakeObjectKey(sfs.keyPrefix, doc.DocID, internalID)) + destroyed += v.ByteSize + } + if err := sfs.Indexer.BatchDeleteVersions(versions); err != nil { + sfs.log.Warnf("DestroyFile failed on BatchDeleteVersions: %s", err) + } + } + if err := s3util.DeleteObjects(sfs.ctx, sfs.client, sfs.bucket, objNames); err != nil { + sfs.log.Warnf("DestroyFile failed on deleteObjects: %s", err) + } + vfs.DiskQuotaAfterDestroy(sfs, diskUsage, destroyed) + return nil +} + +func (sfs *s3VFS) EnsureErased(journal vfs.TrashJournal) error { + diskUsage, _ := sfs.Indexer.DiskUsage() + objNames := journal.ObjectNames + var errm error + var destroyed int64 + var allVersions []*vfs.Version + for _, fileID := range journal.FileIDs { + versions, err := vfs.VersionsFor(sfs, fileID) + if err != nil { + if !couchdb.IsNoDatabaseError(err) { + sfs.log.Warnf("EnsureErased failed on VersionsFor(%s): %s", fileID, err) + errm = errors.Join(errm, err) + } + continue + } + for _, v := range versions { + internalID := v.DocID + if parts := strings.SplitN(v.DocID, "/", 2); len(parts) > 1 { + internalID = parts[1] + } + objNames = append(objNames, MakeObjectKey(sfs.keyPrefix, fileID, internalID)) + destroyed += v.ByteSize + } + allVersions = append(allVersions, versions...) + } + if err := sfs.Indexer.BatchDeleteVersions(allVersions); err != nil { + sfs.log.Warnf("EnsureErased failed on BatchDeleteVersions: %s", err) + errm = errors.Join(errm, err) + } + if err := s3util.DeleteObjects(sfs.ctx, sfs.client, sfs.bucket, objNames); err != nil { + sfs.log.Warnf("EnsureErased failed on deleteObjects: %s", err) + errm = errors.Join(errm, err) + } + vfs.DiskQuotaAfterDestroy(sfs, diskUsage, destroyed) + return errm +} + +func (sfs *s3VFS) OpenFile(doc *vfs.FileDoc) (vfs.File, error) { + if lockerr := sfs.mu.RLock(); lockerr != nil { + return nil, lockerr + } + defer sfs.mu.RUnlock() + objKey := MakeObjectKey(sfs.keyPrefix, doc.DocID, doc.InternalID) + obj, err := sfs.client.GetObject(sfs.ctx, sfs.bucket, objKey, minio.GetObjectOptions{}) + if err != nil { + return nil, err + } + // Stat the object to detect if it exists. + if _, err := obj.Stat(); err != nil { + _ = obj.Close() + if minio.ToErrorResponse(err).Code == "NoSuchKey" { + return nil, os.ErrNotExist + } + return nil, err + } + return &s3FileOpen{obj}, nil +} + +func (sfs *s3VFS) OpenFileVersion(doc *vfs.FileDoc, version *vfs.Version) (vfs.File, error) { + if lockerr := sfs.mu.RLock(); lockerr != nil { + return nil, lockerr + } + defer sfs.mu.RUnlock() + internalID := version.DocID + if parts := strings.SplitN(version.DocID, "/", 2); len(parts) > 1 { + internalID = parts[1] + } + objKey := MakeObjectKey(sfs.keyPrefix, doc.DocID, internalID) + obj, err := sfs.client.GetObject(sfs.ctx, sfs.bucket, objKey, minio.GetObjectOptions{}) + if err != nil { + return nil, err + } + if _, err := obj.Stat(); err != nil { + _ = obj.Close() + if minio.ToErrorResponse(err).Code == "NoSuchKey" { + return nil, os.ErrNotExist + } + return nil, err + } + return &s3FileOpen{obj}, nil +} + +func (sfs *s3VFS) ImportFileVersion(version *vfs.Version, content io.ReadCloser) error { + if lockerr := sfs.mu.Lock(); lockerr != nil { + return lockerr + } + defer sfs.mu.Unlock() + + diskQuota := sfs.DiskQuota() + if diskQuota > 0 { + diskUsage, err := sfs.DiskUsage() + if err != nil { + return err + } + if diskUsage+version.ByteSize > diskQuota { + return vfs.ErrFileTooBig + } + } + + parts := strings.SplitN(version.DocID, "/", 2) + if len(parts) != 2 { + return vfs.ErrIllegalFilename + } + objKey := MakeObjectKey(sfs.keyPrefix, parts[0], parts[1]) + + _, err := sfs.client.PutObject(sfs.ctx, sfs.bucket, objKey, content, version.ByteSize, minio.PutObjectOptions{ + ContentType: "application/octet-stream", + SendContentMd5: true, + }) + if errc := content.Close(); err == nil { + err = errc + } + if err != nil { + return err + } + + return sfs.Indexer.CreateVersion(version) +} + +func (sfs *s3VFS) RevertFileVersion(doc *vfs.FileDoc, version *vfs.Version) error { + if lockerr := sfs.mu.Lock(); lockerr != nil { + return lockerr + } + defer sfs.mu.Unlock() + + save := vfs.NewVersion(doc) + if err := sfs.Indexer.CreateVersion(save); err != nil { + return err + } + + newdoc := doc.Clone().(*vfs.FileDoc) + if parts := strings.SplitN(version.DocID, "/", 2); len(parts) > 1 { + newdoc.InternalID = parts[1] + } + vfs.SetMetaFromVersion(newdoc, version) + if err := sfs.Indexer.UpdateFileDoc(doc, newdoc); err != nil { + _ = sfs.Indexer.DeleteVersion(save) + return err + } + + return sfs.Indexer.DeleteVersion(version) +} + +func (sfs *s3VFS) CleanOldVersion(fileID string, v *vfs.Version) error { + if lockerr := sfs.mu.Lock(); lockerr != nil { + return lockerr + } + defer sfs.mu.Unlock() + return sfs.cleanOldVersion(fileID, v) +} + +func (sfs *s3VFS) cleanOldVersion(fileID string, v *vfs.Version) error { + if err := sfs.Indexer.DeleteVersion(v); err != nil { + return err + } + internalID := v.DocID + if parts := strings.SplitN(v.DocID, "/", 2); len(parts) > 1 { + internalID = parts[1] + } + objKey := MakeObjectKey(sfs.keyPrefix, fileID, internalID) + return sfs.client.RemoveObject(sfs.ctx, sfs.bucket, objKey, minio.RemoveObjectOptions{}) +} + +func (sfs *s3VFS) ClearOldVersions() error { + if lockerr := sfs.mu.Lock(); lockerr != nil { + return lockerr + } + defer sfs.mu.Unlock() + diskUsage, _ := sfs.Indexer.DiskUsage() + versions, err := sfs.Indexer.AllVersions() + if err != nil { + return err + } + var objNames []string + var destroyed int64 + for _, v := range versions { + if parts := strings.SplitN(v.DocID, "/", 2); len(parts) > 1 { + objNames = append(objNames, MakeObjectKey(sfs.keyPrefix, parts[0], parts[1])) + } + destroyed += v.ByteSize + } + if err := sfs.Indexer.BatchDeleteVersions(versions); err != nil { + return err + } + vfs.DiskQuotaAfterDestroy(sfs, diskUsage, destroyed) + return s3util.DeleteObjects(sfs.ctx, sfs.client, sfs.bucket, objNames) +} + +func (sfs *s3VFS) CopyFileFromOtherFS( + newdoc, olddoc *vfs.FileDoc, + srcFS vfs.Fs, + srcDoc *vfs.FileDoc, +) error { + if lockerr := sfs.mu.Lock(); lockerr != nil { + return lockerr + } + defer sfs.mu.Unlock() + + newsize, _, capsize, err := vfs.CheckAvailableDiskSpace(sfs, newdoc) + if err != nil { + return err + } + + newpath, err := sfs.Indexer.FilePath(newdoc) + if err != nil { + return err + } + if strings.HasPrefix(newpath, vfs.TrashDirName+"/") { + return vfs.ErrParentInTrash + } + + if olddoc == nil { + var exists bool + exists, err = sfs.Indexer.DirChildExists(newdoc.DirID, newdoc.DocName) + if err != nil { + return err + } + if exists { + return os.ErrExist + } + } + + if newdoc.DocID == "" { + uid, err := uuid.NewV7() + if err != nil { + return err + } + newdoc.DocID = uid.String() + } + + newdoc.InternalID = NewInternalID() + + dstKey := MakeObjectKey(sfs.keyPrefix, newdoc.DocID, newdoc.InternalID) + + // Try server-side copy if the source is also an s3VFS on the same client. + if srcS3, ok := srcFS.(*s3VFS); ok { + srcKey := MakeObjectKey(srcS3.keyPrefix, srcDoc.DocID, srcDoc.InternalID) + if _, err := sfs.client.CopyObject(sfs.ctx, + minio.CopyDestOptions{Bucket: sfs.bucket, Object: dstKey}, + minio.CopySrcOptions{Bucket: srcS3.bucket, Object: srcKey}, + ); err != nil { + return err + } + } else { + // Stream from the source FS. + srcFile, err := srcFS.OpenFile(srcDoc) + if err != nil { + return err + } + _, err = sfs.client.PutObject(sfs.ctx, sfs.bucket, dstKey, srcFile, srcDoc.ByteSize, minio.PutObjectOptions{ + ContentType: srcDoc.Mime, + }) + if errc := srcFile.Close(); err == nil { + err = errc + } + if err != nil { + return err + } + } + + var v *vfs.Version + if olddoc != nil { + v = vfs.NewVersion(olddoc) + err = sfs.Indexer.UpdateFileDoc(olddoc, newdoc) + } else { + err = sfs.Indexer.CreateNamedFileDoc(newdoc) + } + if err != nil { + return err + } + + if v != nil { + actionV, toClean, _ := vfs.FindVersionsToClean(sfs, newdoc.DocID, v) + if bytes.Equal(newdoc.MD5Sum, olddoc.MD5Sum) { + actionV = vfs.CleanCandidateVersion + } + if actionV == vfs.KeepCandidateVersion { + if errv := sfs.Indexer.CreateVersion(v); errv != nil { + actionV = vfs.CleanCandidateVersion + } + } + if actionV == vfs.CleanCandidateVersion { + internalID := v.DocID + if parts := strings.SplitN(v.DocID, "/", 2); len(parts) > 1 { + internalID = parts[1] + } + objKey := MakeObjectKey(sfs.keyPrefix, newdoc.DocID, internalID) + _ = sfs.client.RemoveObject(sfs.ctx, sfs.bucket, objKey, minio.RemoveObjectOptions{}) + } + for _, old := range toClean { + _ = sfs.cleanOldVersion(newdoc.DocID, old) + } + } + + if capsize > 0 && newsize >= capsize { + vfs.PushDiskQuotaAlert(sfs, true) + } + + return nil +} + +// UpdateFileDoc calls the indexer UpdateFileDoc function and adds a few checks +// before actually calling this method: +// - locks the filesystem for writing +// - checks in case we have a move operation that the new path is available +// +// @override Indexer.UpdateFileDoc +func (sfs *s3VFS) UpdateFileDoc(olddoc, newdoc *vfs.FileDoc) error { + if lockerr := sfs.mu.Lock(); lockerr != nil { + return lockerr + } + defer sfs.mu.Unlock() + if newdoc.DirID != olddoc.DirID || newdoc.DocName != olddoc.DocName { + exists, err := sfs.Indexer.DirChildExists(newdoc.DirID, newdoc.DocName) + if err != nil { + return err + } + if exists { + return os.ErrExist + } + } + return sfs.Indexer.UpdateFileDoc(olddoc, newdoc) +} + +// UpdateDirDoc calls the indexer UpdateDirDoc function and adds a few checks +// before actually calling this method: +// - locks the filesystem for writing +// - checks that we don't move a directory to one of its descendant +// - checks in case we have a move operation that the new path is available +// +// @override Indexer.UpdateDirDoc +func (sfs *s3VFS) UpdateDirDoc(olddoc, newdoc *vfs.DirDoc) error { + if lockerr := sfs.mu.Lock(); lockerr != nil { + return lockerr + } + defer sfs.mu.Unlock() + if newdoc.DirID != olddoc.DirID || newdoc.DocName != olddoc.DocName { + if strings.HasPrefix(newdoc.Fullpath, olddoc.Fullpath+"/") { + return vfs.ErrForbiddenDocMove + } + exists, err := sfs.Indexer.DirChildExists(newdoc.DirID, newdoc.DocName) + if err != nil { + return err + } + if exists { + return os.ErrExist + } + } + return sfs.Indexer.UpdateDirDoc(olddoc, newdoc) +} + +func (sfs *s3VFS) DirByID(fileID string) (*vfs.DirDoc, error) { + if lockerr := sfs.mu.RLock(); lockerr != nil { + return nil, lockerr + } + defer sfs.mu.RUnlock() + return sfs.Indexer.DirByID(fileID) +} + +func (sfs *s3VFS) DirByPath(name string) (*vfs.DirDoc, error) { + if lockerr := sfs.mu.RLock(); lockerr != nil { + return nil, lockerr + } + defer sfs.mu.RUnlock() + return sfs.Indexer.DirByPath(name) +} + +func (sfs *s3VFS) FileByID(fileID string) (*vfs.FileDoc, error) { + if lockerr := sfs.mu.RLock(); lockerr != nil { + return nil, lockerr + } + defer sfs.mu.RUnlock() + return sfs.Indexer.FileByID(fileID) +} + +func (sfs *s3VFS) FileByPath(name string) (*vfs.FileDoc, error) { + if lockerr := sfs.mu.RLock(); lockerr != nil { + return nil, lockerr + } + defer sfs.mu.RUnlock() + return sfs.Indexer.FileByPath(name) +} + +func (sfs *s3VFS) FilePath(doc *vfs.FileDoc) (string, error) { + if lockerr := sfs.mu.RLock(); lockerr != nil { + return "", lockerr + } + defer sfs.mu.RUnlock() + return sfs.Indexer.FilePath(doc) +} + +func (sfs *s3VFS) DirOrFileByID(fileID string) (*vfs.DirDoc, *vfs.FileDoc, error) { + if lockerr := sfs.mu.RLock(); lockerr != nil { + return nil, nil, lockerr + } + defer sfs.mu.RUnlock() + return sfs.Indexer.DirOrFileByID(fileID) +} + +func (sfs *s3VFS) DirOrFileByPath(name string) (*vfs.DirDoc, *vfs.FileDoc, error) { + if lockerr := sfs.mu.RLock(); lockerr != nil { + return nil, nil, lockerr + } + defer sfs.mu.RUnlock() + return sfs.Indexer.DirOrFileByPath(name) +} + +// s3FileCreation represents a file open for writing. It is used to create +// a file or to modify the content of a file. +type s3FileCreation struct { + fs *s3VFS + pw *io.PipeWriter + resultCh chan putResult + newdoc *vfs.FileDoc + olddoc *vfs.FileDoc + objKey string + w int64 + size int64 + maxsize int64 + capsize int64 + meta *vfs.MetaExtractor + md5H hash.Hash + err error +} + +func (f *s3FileCreation) Read(p []byte) (int, error) { + return 0, os.ErrInvalid +} + +func (f *s3FileCreation) ReadAt(p []byte, off int64) (int, error) { + return 0, os.ErrInvalid +} + +func (f *s3FileCreation) Seek(offset int64, whence int) (int64, error) { + return 0, os.ErrInvalid +} + +func (f *s3FileCreation) Write(p []byte) (int, error) { + if f.err != nil { + return 0, f.err + } + + if f.meta != nil { + if _, err := (*f.meta).Write(p); err != nil && !errors.Is(err, io.ErrClosedPipe) { + (*f.meta).Abort(err) + f.meta = nil + } + } + + // Write to local MD5 hash + _, _ = f.md5H.Write(p) + + n, err := f.pw.Write(p) + if err != nil { + f.err = err + return n, err + } + + f.w += int64(n) + if f.maxsize >= 0 && f.w > f.maxsize { + f.err = vfs.ErrFileTooBig + _ = f.pw.CloseWithError(f.err) + return n, f.err + } + + if f.size >= 0 && f.w > f.size { + f.err = vfs.ErrContentLengthMismatch + _ = f.pw.CloseWithError(f.err) + return n, f.err + } + + return n, nil +} + +func (f *s3FileCreation) Close() (err error) { + defer func() { + if err != nil { + // Remove the object from S3 if an error occurred + _ = f.fs.client.RemoveObject(f.fs.ctx, f.fs.bucket, f.objKey, minio.RemoveObjectOptions{}) + // If an error has occurred when creating a new file, we should + // also delete the file from the index. + if f.olddoc == nil { + _ = f.fs.Indexer.DeleteFileDoc(f.newdoc) + } + } + }() + + // Close the pipe writer to signal EOF to PutObject + if err = f.pw.Close(); err != nil { + if f.meta != nil { + (*f.meta).Abort(err) + f.meta = nil + } + if f.err == nil { + f.err = err + } + } + + // Wait for the PutObject goroutine to finish + result := <-f.resultCh + + if result.err != nil { + if f.meta != nil { + (*f.meta).Abort(result.err) + f.meta = nil + } + if f.err == nil { + f.err = result.err + } + } + + newdoc, olddoc, written := f.newdoc, f.olddoc, f.w + + if f.meta != nil { + if errc := (*f.meta).Close(); errc == nil { + vfs.MergeMetadata(newdoc, (*f.meta).Result()) + } + } + + if f.err != nil { + return f.err + } + + // Verify or compute MD5 checksum. + // The local md5H hash is always computed from the same data stream that + // goes to S3 (via the Write method), so it is authoritative. + localMD5 := f.md5H.Sum(nil) + if newdoc.MD5Sum != nil { + // The caller provided an expected hash — verify it matches what was written. + if !bytes.Equal(newdoc.MD5Sum, localMD5) { + return vfs.ErrInvalidHash + } + } else { + newdoc.MD5Sum = localMD5 + } + + if f.size < 0 { + newdoc.ByteSize = written + } + + if newdoc.ByteSize != written { + return vfs.ErrContentLengthMismatch + } + + lockerr := f.fs.mu.Lock() + if lockerr != nil { + return lockerr + } + defer f.fs.mu.Unlock() + + // Check again that a file with the same path does not exist. It can happen + // when the same file is uploaded twice in parallel. + if olddoc == nil { + exists, err := f.fs.Indexer.DirChildExists(newdoc.DirID, newdoc.DocName) + if err != nil { + return err + } + if exists { + return os.ErrExist + } + } + + var newpath string + newpath, err = f.fs.Indexer.FilePath(newdoc) + if err != nil { + return err + } + newdoc.Trashed = strings.HasPrefix(newpath, vfs.TrashDirName+"/") + + var v *vfs.Version + if olddoc != nil { + v = vfs.NewVersion(olddoc) + err = f.fs.Indexer.UpdateFileDoc(olddoc, newdoc) + } else if newdoc.ID() == "" { + err = f.fs.Indexer.CreateFileDoc(newdoc) + } else { + err = f.fs.Indexer.CreateNamedFileDoc(newdoc) + } + if err != nil { + return err + } + + if v != nil { + actionV, toClean, _ := vfs.FindVersionsToClean(f.fs, newdoc.DocID, v) + if bytes.Equal(newdoc.MD5Sum, olddoc.MD5Sum) { + actionV = vfs.CleanCandidateVersion + } + if actionV == vfs.KeepCandidateVersion { + if errv := f.fs.Indexer.CreateVersion(v); errv != nil { + actionV = vfs.CleanCandidateVersion + } + } + if actionV == vfs.CleanCandidateVersion { + internalID := v.DocID + if parts := strings.SplitN(v.DocID, "/", 2); len(parts) > 1 { + internalID = parts[1] + } + objKey := MakeObjectKey(f.fs.keyPrefix, newdoc.DocID, internalID) + if err := f.fs.client.RemoveObject(f.fs.ctx, f.fs.bucket, objKey, minio.RemoveObjectOptions{}); err != nil { + f.fs.log.Warnf("Could not delete previous version %q: %s", objKey, err.Error()) + } + } + for _, old := range toClean { + if err := f.fs.cleanOldVersion(newdoc.DocID, old); err != nil { + f.fs.log.Warnf("Could not delete old versions for %s: %s", newdoc.DocID, err.Error()) + } + } + } + + if f.capsize > 0 && f.size >= f.capsize { + vfs.PushDiskQuotaAlert(f.fs, true) + } + + return nil +} + +// s3FileOpen represents a file open for reading. +type s3FileOpen struct { + obj *minio.Object +} + +func (f *s3FileOpen) Read(p []byte) (int, error) { + return f.obj.Read(p) +} + +func (f *s3FileOpen) ReadAt(p []byte, off int64) (int, error) { + return f.obj.ReadAt(p, off) +} + +func (f *s3FileOpen) Seek(offset int64, whence int) (int64, error) { + return f.obj.Seek(offset, whence) +} + +func (f *s3FileOpen) Write(p []byte) (int, error) { + return 0, os.ErrInvalid +} + +func (f *s3FileOpen) Close() error { + return f.obj.Close() +} + +var ( + _ vfs.VFS = &s3VFS{} + _ vfs.File = &s3FileCreation{} + _ vfs.File = &s3FileOpen{} +) diff --git a/model/vfs/vfss3/naming_test.go b/model/vfs/vfss3/naming_test.go new file mode 100644 index 00000000000..1201dc82988 --- /dev/null +++ b/model/vfs/vfss3/naming_test.go @@ -0,0 +1,71 @@ +package vfss3 + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestSanitizeBucketName(t *testing.T) { + tests := []struct { + input string + expected string + }{ + {"production", "production"}, + {"my_context", "my-context"}, + {"My.Context", "my-context"}, + {"UPPERCASE", "uppercase"}, + {"with spaces!", "withspaces"}, + {"a--b--c", "a-b-c"}, + {"-leading-trailing-", "leading-trailing"}, + {"very-long-name-that-exceeds-the-maximum-allowed-length", "very-long-name-that-exceeds-the-maxim"}, + {"", ""}, + } + + for _, tt := range tests { + t.Run(tt.input, func(t *testing.T) { + assert.Equal(t, tt.expected, sanitizeBucketName(tt.input)) + }) + } +} + +func TestBucketName(t *testing.T) { + tests := []struct { + orgID string + bucketPrefix string + expected string + }{ + {"org-123", "cozy", "cozy-org-123"}, + {"", "cozy", "cozy-default"}, + {"My_Org", "cozy", "cozy-my-org"}, + {"org.example.com", "cozy", "cozy-org-example-com"}, + } + + for _, tt := range tests { + t.Run(tt.orgID, func(t *testing.T) { + assert.Equal(t, tt.expected, BucketName(tt.orgID, tt.bucketPrefix)) + }) + } +} + +func TestMakeObjectKey(t *testing.T) { + // Standard 32-char docID and 16-char internalID + key := MakeObjectKey("alice.example.com/", "abcdefghijklmnopqrstuvwxyz012345", "0123456789abcdef") + assert.Equal(t, "alice.example.com/abcdefghijklmnopqrstuv/wxyz0/12345/0123456789abcdef", key) + + // Non-standard lengths + key = MakeObjectKey("alice.example.com/", "short", "id") + assert.Equal(t, "alice.example.com/short/id", key) +} + +func TestMakeDocID(t *testing.T) { + // Standard 51-char object name + docID, internalID := makeDocID("abcdefghijklmnopqrstuv/wxyz0/12345/0123456789abcdef") + assert.Equal(t, "abcdefghijklmnopqrstuvwxyz012345", docID) + assert.Equal(t, "0123456789abcdef", internalID) + + // Non-standard + docID, internalID = makeDocID("short/id") + assert.Equal(t, "short", docID) + assert.Equal(t, "id", internalID) +} diff --git a/model/vfs/vfss3/thumbs.go b/model/vfs/vfss3/thumbs.go new file mode 100644 index 00000000000..ee9c064ad0d --- /dev/null +++ b/model/vfs/vfss3/thumbs.go @@ -0,0 +1,247 @@ +package vfss3 + +import ( + "bytes" + "context" + "encoding/hex" + "fmt" + "io" + "net/http" + "os" + "time" + + "github.com/cozy/cozy-stack/model/vfs" + "github.com/cozy/cozy-stack/pkg/consts" + "github.com/cozy/cozy-stack/pkg/logger" + "github.com/cozy/cozy-stack/pkg/s3util" + "github.com/labstack/echo/v4" + "github.com/minio/minio-go/v7" +) + +var unixEpochZero = time.Time{} + +// NewThumbsFs creates a new thumbnail filesystem backed by S3. +func NewThumbsFs(client *minio.Client, bucket, keyPrefix string) vfs.Thumbser { + return &thumbsS3{ + client: client, + bucket: bucket, + keyPrefix: keyPrefix, + ctx: context.Background(), + } +} + +type thumbsS3 struct { + client *minio.Client + bucket string + keyPrefix string + ctx context.Context +} + +type s3Thumb struct { + pw *io.PipeWriter + errCh chan error + client *minio.Client + bucket string + name string + ctx context.Context +} + +func (t *s3Thumb) Write(p []byte) (int, error) { + return t.pw.Write(p) +} + +func (t *s3Thumb) Commit() error { + if err := t.pw.Close(); err != nil { + return err + } + return <-t.errCh +} + +func (t *s3Thumb) Abort() error { + // Close the pipe with an error to cancel the PutObject goroutine. + errc := t.pw.CloseWithError(fmt.Errorf("thumbnail creation aborted")) + // Drain the errCh so the goroutine is not leaked. + <-t.errCh + // Try to remove the possibly partially written object. + errd := t.client.RemoveObject(t.ctx, t.bucket, t.name, minio.RemoveObjectOptions{}) + if errd != nil && minio.ToErrorResponse(errd).Code == "NoSuchKey" { + errd = nil + } + // Write an empty marker object to indicate that the thumbnail generation failed. + _, errp := t.client.PutObject(t.ctx, t.bucket, t.name, + bytes.NewReader(nil), 0, minio.PutObjectOptions{ + ContentType: echo.MIMEOctetStream, + }) + if errc != nil { + return errc + } + if errd != nil { + return errd + } + return errp +} + +func (ts *thumbsS3) createThumbFile(name, contentType string, meta map[string]string) (vfs.ThumbFiler, error) { + pr, pw := io.Pipe() + + errCh := make(chan error, 1) + go func() { + _, err := ts.client.PutObject(ts.ctx, ts.bucket, name, pr, -1, minio.PutObjectOptions{ + ContentType: contentType, + UserMetadata: meta, + }) + errCh <- err + }() + + return &s3Thumb{ + pw: pw, + errCh: errCh, + client: ts.client, + bucket: ts.bucket, + name: name, + ctx: ts.ctx, + }, nil +} + +func (ts *thumbsS3) CreateThumb(img *vfs.FileDoc, format string) (vfs.ThumbFiler, error) { + name := ts.makeName(img.ID(), format) + meta := map[string]string{ + "file-md5": hex.EncodeToString(img.MD5Sum), + } + return ts.createThumbFile(name, "image/jpeg", meta) +} + +func (ts *thumbsS3) ThumbExists(img *vfs.FileDoc, format string) (bool, error) { + name := ts.makeName(img.ID(), format) + info, err := ts.client.StatObject(ts.ctx, ts.bucket, name, minio.StatObjectOptions{}) + if err != nil { + if minio.ToErrorResponse(err).Code == "NoSuchKey" { + return false, nil + } + return false, err + } + if md5str, ok := info.UserMetadata["File-Md5"]; ok && md5str != "" { + md5sum, err := hex.DecodeString(md5str) + if err == nil && !bytes.Equal(md5sum, img.MD5Sum) { + return false, nil + } + } + return true, nil +} + +func (ts *thumbsS3) RemoveThumbs(img *vfs.FileDoc, formats []string) error { + objNames := make([]string, len(formats)) + for i, format := range formats { + objNames[i] = ts.makeName(img.ID(), format) + } + return s3util.DeleteObjects(ts.ctx, ts.client, ts.bucket, objNames) +} + +func (ts *thumbsS3) ServeThumbContent(w http.ResponseWriter, req *http.Request, img *vfs.FileDoc, format string) error { + name := ts.makeName(img.ID(), format) + obj, err := ts.client.GetObject(ts.ctx, ts.bucket, name, minio.GetObjectOptions{}) + if err != nil { + return s3util.WrapNotFound(err) + } + defer obj.Close() + + info, err := obj.Stat() + if err != nil { + return s3util.WrapNotFound(err) + } + + if info.ContentType == echo.MIMEOctetStream { + // We have some old images where the thumbnail has not been correctly + // saved. We should delete the thumbnail to allow another try. + if info.Size > 0 { + _ = ts.RemoveThumbs(img, vfs.ThumbnailFormatNames) + return os.ErrNotExist + } + // Image magick has failed to generate a thumbnail, and retrying would + // be useless. + return os.ErrInvalid + } + + w.Header().Set("Etag", fmt.Sprintf(`"%s"`, info.ETag)) + w.Header().Set("Content-Type", info.ContentType) + http.ServeContent(w, req, name, unixEpochZero, obj) + return nil +} + +func (ts *thumbsS3) CreateNoteThumb(id, mime, format string) (vfs.ThumbFiler, error) { + name := ts.makeName(id, format) + return ts.createThumbFile(name, mime, nil) +} + +func (ts *thumbsS3) OpenNoteThumb(id, format string) (io.ReadCloser, error) { + name := ts.makeName(id, format) + obj, err := ts.client.GetObject(ts.ctx, ts.bucket, name, minio.GetObjectOptions{}) + if err != nil { + return nil, s3util.WrapNotFound(err) + } + // Stat to verify the object actually exists (GetObject doesn't fail on missing keys). + if _, err := obj.Stat(); err != nil { + obj.Close() + if minio.ToErrorResponse(err).Code == "NoSuchKey" { + return nil, os.ErrNotExist + } + return nil, err + } + return obj, nil +} + +func (ts *thumbsS3) RemoveNoteThumb(id string, formats []string) error { + objNames := make([]string, len(formats)) + for i, format := range formats { + objNames[i] = ts.makeName(id, format) + } + err := s3util.DeleteObjects(ts.ctx, ts.client, ts.bucket, objNames) + if err != nil { + logger.WithNamespace("vfss3").Infof("Cannot remove note thumbs: %s", err) + } + return err +} + +func (ts *thumbsS3) ServeNoteThumbContent(w http.ResponseWriter, req *http.Request, id string) error { + name := ts.makeName(id, consts.NoteImageThumbFormat) + obj, err := ts.client.GetObject(ts.ctx, ts.bucket, name, minio.GetObjectOptions{}) + if err != nil { + return s3util.WrapNotFound(err) + } + + info, err := obj.Stat() + if err != nil { + obj.Close() + // Try the original format as fallback. + name = ts.makeName(id, consts.NoteImageOriginalFormat) + obj, err = ts.client.GetObject(ts.ctx, ts.bucket, name, minio.GetObjectOptions{}) + if err != nil { + return s3util.WrapNotFound(err) + } + info, err = obj.Stat() + if err != nil { + obj.Close() + return s3util.WrapNotFound(err) + } + } + defer obj.Close() + + w.Header().Set("Etag", fmt.Sprintf(`"%s"`, info.ETag)) + w.Header().Set("Content-Type", info.ContentType) + http.ServeContent(w, req, name, unixEpochZero, obj) + return nil +} + +func (ts *thumbsS3) makeName(imgID string, format string) string { + return ts.keyPrefix + fmt.Sprintf("thumbs/%s-%s", makeThumbObjectName(imgID), format) +} + +// makeThumbObjectName builds a virtual subfolder structure for thumbnails. +// It splits the 32-char ID into three parts to avoid a flat hierarchy. +// This is the same logic as vfsswift.MakeObjectName (without internalID). +func makeThumbObjectName(docID string) string { + if len(docID) != 32 { + return docID + } + return docID[:22] + "/" + docID[22:27] + "/" + docID[27:] +} diff --git a/pkg/appfs/s3.go b/pkg/appfs/s3.go new file mode 100644 index 00000000000..33e08f88794 --- /dev/null +++ b/pkg/appfs/s3.go @@ -0,0 +1,333 @@ +package appfs + +import ( + "bytes" + "compress/gzip" + "context" + "fmt" + "io" + "mime" + "net/http" + "os" + "path" + "strconv" + "strings" + + "github.com/andybalholm/brotli" + "github.com/cozy/cozy-stack/pkg/consts" + "github.com/cozy/cozy-stack/pkg/filetype" + "github.com/cozy/cozy-stack/pkg/s3util" + + web_utils "github.com/cozy/cozy-stack/pkg/utils" + "github.com/labstack/echo/v4" + "github.com/minio/minio-go/v7" +) + +// installedMarkerSuffix is appended to the app's base key to form the +// "installation complete" marker object. The suffix is chosen so the marker +// can't collide with any file under //... — S3 browsers +// would otherwise display the marker as a file sitting next to a folder of +// the same name. +const installedMarkerSuffix = ".cozy-installed" + +// s3Copier implements the Copier interface backed by S3. +type s3Copier struct { + client *minio.Client + bucket string + appObj string + started bool + objectNames []string + ctx context.Context +} + +// NewS3Copier creates a Copier that stores app files in S3. +func NewS3Copier(client *minio.Client, bucket string) Copier { + return &s3Copier{ + client: client, + bucket: bucket, + ctx: context.Background(), + } +} + +func (f *s3Copier) Exist(slug, version, shasum string) (bool, error) { + f.appObj = path.Join(slug, version) + if shasum != "" { + f.appObj += "-" + shasum + } + _, err := f.client.StatObject(f.ctx, f.bucket, f.appObj+installedMarkerSuffix, minio.StatObjectOptions{}) + if err == nil { + return true, nil + } + if s3util.IsNotFound(err) { + return false, nil + } + return false, err +} + +func (f *s3Copier) Start(slug, version, shasum string) (bool, error) { + exist, err := f.Exist(slug, version, shasum) + if err != nil || exist { + return exist, err + } + + if err := s3util.EnsureBucket(f.ctx, f.client, f.bucket, ""); err != nil { + return false, err + } + + f.objectNames = []string{} + f.started = true + return false, nil +} + +func (f *s3Copier) Copy(stat os.FileInfo, src io.Reader) error { + if !f.started { + return fmt.Errorf("appfs: copier must call Start() before Copy()") + } + + // Write directly to the final location (appObj/filename). + // Reject path traversal attempts in filenames. + name := stat.Name() + if strings.Contains(name, "..") { + return fmt.Errorf("appfs: invalid filename %q", name) + } + objName := path.Join(f.appObj, name) + + contentType := filetype.ByExtension(path.Ext(stat.Name())) + if contentType == "" { + contentType, src = filetype.FromReader(src) + } + + // Compress with brotli. + var buf bytes.Buffer + bw := brotli.NewWriter(&buf) + if _, err := io.Copy(bw, src); err != nil { + return err + } + if err := bw.Close(); err != nil { + return err + } + + meta := map[string]string{ + "X-Content-Encoding": "br", + "Original-Content-Length": strconv.FormatInt(stat.Size(), 10), + } + + f.objectNames = append(f.objectNames, objName) + _, err := f.client.PutObject(f.ctx, f.bucket, objName, + bytes.NewReader(buf.Bytes()), int64(buf.Len()), + minio.PutObjectOptions{ + ContentType: contentType, + UserMetadata: meta, + }) + return err +} + +func (f *s3Copier) Abort() error { + return s3util.DeleteObjects(f.ctx, f.client, f.bucket, f.objectNames) +} + +func (f *s3Copier) Commit() (err error) { + // Create the marker object that signals the version is complete. The + // suffix keeps it on a distinct key from the //... files, + // so S3 browsers don't render a folder and a file with the same name. + _, err = f.client.PutObject(f.ctx, f.bucket, f.appObj+installedMarkerSuffix, + bytes.NewReader(nil), 0, minio.PutObjectOptions{ + ContentType: "text/plain", + }) + return err +} + +// s3Server implements the FileServer interface backed by S3. +type s3Server struct { + client *minio.Client + bucket string + ctx context.Context +} + +// NewS3FileServer creates a FileServer that serves app files from S3. +func NewS3FileServer(client *minio.Client, bucket string) FileServer { + return &s3Server{ + client: client, + bucket: bucket, + ctx: context.Background(), + } +} + +func (s *s3Server) Open(slug, version, shasum, file string) (io.ReadCloser, error) { + objName := s.makeObjectName(slug, version, shasum, file) + obj, err := s.client.GetObject(s.ctx, s.bucket, objName, minio.GetObjectOptions{}) + if err != nil { + return nil, s3util.WrapNotFound(err) + } + info, err := obj.Stat() + if err != nil { + obj.Close() + return nil, s3util.WrapNotFound(err) + } + contentEncoding := info.UserMetadata["X-Content-Encoding"] + if contentEncoding == "br" { + return newBrotliReadCloser(obj) + } else if contentEncoding == "gzip" { + return newGzipReadCloser(obj) + } + return obj, nil +} + +func (s *s3Server) ServeFileContent(w http.ResponseWriter, req *http.Request, slug, version, shasum, file string) error { + objName := s.makeObjectName(slug, version, shasum, file) + obj, err := s.client.GetObject(s.ctx, s.bucket, objName, minio.GetObjectOptions{}) + if err != nil { + return s3util.WrapNotFound(err) + } + defer obj.Close() + + info, err := obj.Stat() + if err != nil { + return s3util.WrapNotFound(err) + } + + if checkETag := req.Header.Get("Cache-Control") == ""; checkETag { + etagVal := info.ETag + if len(etagVal) > 10 { + etagVal = etagVal[:10] + } + etag := fmt.Sprintf(`"%s"`, etagVal) + if web_utils.CheckPreconditions(w, req, etag) { + return nil + } + w.Header().Set("Etag", etag) + } + + // Read the full object to handle brotli decompression. + // Limit to 50 MiB to avoid unbounded memory allocation from corrupted objects. + const maxAppFileSize = 50 << 20 + content, err := io.ReadAll(io.LimitReader(obj, maxAppFileSize)) + if err != nil { + return err + } + + var r io.Reader = bytes.NewReader(content) + contentLength := info.Size + contentType := info.ContentType + + contentEncoding := info.UserMetadata["X-Content-Encoding"] + origContentLength := info.UserMetadata["Original-Content-Length"] + if contentEncoding == "br" { + if acceptBrotliEncoding(req) { + w.Header().Set(echo.HeaderContentEncoding, "br") + } else { + if origContentLength != "" { + contentLength, _ = strconv.ParseInt(origContentLength, 10, 64) + } + r = brotli.NewReader(bytes.NewReader(content)) + } + } else if contentEncoding == "gzip" { + if acceptGzipEncoding(req) { + w.Header().Set(echo.HeaderContentEncoding, "gzip") + } else { + if origContentLength != "" { + contentLength, _ = strconv.ParseInt(origContentLength, 10, 64) + } + gr, gerr := gzip.NewReader(bytes.NewReader(content)) + if gerr != nil { + return gerr + } + defer gr.Close() + r = gr + } + } + + ext := path.Ext(file) + if contentType == "" { + contentType = mime.TypeByExtension(ext) + } + if contentType == "text/xml" && ext == ".svg" { + contentType = "image/svg+xml" + } + + return serveContent(w, req, contentType, contentLength, r) +} + +func (s *s3Server) ServeCodeTarball(w http.ResponseWriter, req *http.Request, slug, version, shasum string) error { + objName := path.Join(slug, version) + if shasum != "" { + objName += "-" + shasum + } + objName += ".tgz" + + // Try to serve a pre-built tarball first. + obj, err := s.client.GetObject(s.ctx, s.bucket, objName, minio.GetObjectOptions{}) + if err == nil { + info, serr := obj.Stat() + if serr == nil { + defer obj.Close() + return serveContent(w, req, info.ContentType, info.Size, obj) + } + obj.Close() + } + + buf, err := prepareTarball(s, slug, version, shasum) + if err != nil { + return err + } + content, err := io.ReadAll(buf) + if err != nil { + return err + } + contentType := mime.TypeByExtension(".gz") + + // Store the tarball for future requests. + _, _ = s.client.PutObject(s.ctx, s.bucket, objName, + bytes.NewReader(content), int64(len(content)), + minio.PutObjectOptions{ContentType: contentType}) + + return serveContent(w, req, contentType, int64(len(content)), bytes.NewReader(content)) +} + +func (s *s3Server) makeObjectName(slug, version, shasum, file string) string { + basepath := path.Join(slug, version) + if shasum != "" { + basepath += "-" + shasum + } + // Prevent path traversal + if strings.Contains(file, "..") { + return basepath + "/invalid" + } + return path.Join(basepath, file) +} + +func (s *s3Server) FilesList(slug, version, shasum string) ([]string, error) { + prefix := s.makeObjectName(slug, version, shasum, "") + "/" + var names []string + for obj := range s.client.ListObjects(s.ctx, s.bucket, minio.ListObjectsOptions{ + Prefix: prefix, + Recursive: true, + }) { + if obj.Err != nil { + return nil, obj.Err + } + name := strings.TrimPrefix(obj.Key, prefix) + if name != "" { + names = append(names, name) + } + } + return names, nil +} + +// S3AppsBucket returns the S3 bucket name used for storing applications of a +// given type. The bucket is shared across all instances (like Swift containers). +func S3AppsBucket(bucketPrefix string, appsType consts.AppType) string { + switch appsType { + case consts.WebappType: + return bucketPrefix + "-apps-web" + case consts.KonnectorType: + return bucketPrefix + "-apps-konnectors" + } + panic("Unknown AppType") +} + +// prepareTarball is reused from server.go via the FileServer interface (it +// calls Open and FilesList). The function is defined in server.go. +// We reference it here to document that s3Server satisfies prepareTarball's +// requirements. +var _ FileServer = (*s3Server)(nil) diff --git a/pkg/assets/dynamic/fs.go b/pkg/assets/dynamic/fs.go index baf867a7a55..a0764c8a050 100644 --- a/pkg/assets/dynamic/fs.go +++ b/pkg/assets/dynamic/fs.go @@ -53,6 +53,12 @@ func InitDynamicAssetFS(fsURL string) error { return err } + case config.SchemeS3: + assetFS, err = NewS3FS() + if err != nil { + return err + } + default: return fmt.Errorf("Invalid scheme %s for dynamic assets FS", u.Scheme) } diff --git a/pkg/assets/dynamic/impl_s3.go b/pkg/assets/dynamic/impl_s3.go new file mode 100644 index 00000000000..5001c4faa88 --- /dev/null +++ b/pkg/assets/dynamic/impl_s3.go @@ -0,0 +1,126 @@ +package dynamic + +import ( + "bytes" + "context" + "fmt" + "io" + "os" + "path" + "strings" + "time" + + "github.com/cozy/cozy-stack/pkg/assets/model" + "github.com/cozy/cozy-stack/pkg/config/config" + "github.com/hashicorp/golang-lru/v2/expirable" + "github.com/minio/minio-go/v7" +) + +// S3FS is the S3 implementation of [AssetsFS]. +// +// It saves and fetches assets into/from any S3-compatible object store. +type S3FS struct { + client *minio.Client + bucket string + ctx context.Context +} + +// NewS3FS instantiates a new S3FS. +func NewS3FS() (*S3FS, error) { + initCacheOnce.Do(func() { + cache = expirable.NewLRU[string, cacheEntry](1024, nil, 1*time.Hour) + }) + + ctx := context.Background() + client := config.GetS3Client() + bucket := config.GetS3BucketPrefix() + "-assets" + + err := client.MakeBucket(ctx, bucket, minio.MakeBucketOptions{Region: config.GetS3Region()}) + if err != nil { + code := minio.ToErrorResponse(err).Code + if code != "BucketAlreadyOwnedByYou" && code != "BucketAlreadyExists" { + return nil, fmt.Errorf("Cannot create bucket for dynamic assets: %s", err) + } + } + + return &S3FS{client: client, bucket: bucket, ctx: ctx}, nil +} + +func (s *S3FS) Add(_ string, _ string, asset *model.Asset) error { + objectName := path.Join(asset.Context, asset.Name) + data := asset.GetData() + _, err := s.client.PutObject(s.ctx, s.bucket, objectName, + bytes.NewReader(data), int64(len(data)), + minio.PutObjectOptions{}) + return err +} + +func (s *S3FS) Get(ctx string, name string) ([]byte, error) { + objectName := path.Join(ctx, name) + if entry, ok := cache.Get(objectName); ok { + if !entry.found { + return nil, os.ErrNotExist + } + return entry.content, nil + } + + obj, err := s.client.GetObject(s.ctx, s.bucket, objectName, minio.GetObjectOptions{}) + if err != nil { + return nil, err + } + defer obj.Close() + + content, err := io.ReadAll(obj) + if err != nil { + if minio.ToErrorResponse(err).Code == "NoSuchKey" { + cache.Add(objectName, cacheEntry{found: false}) + return nil, os.ErrNotExist + } + return nil, err + } + + cache.Add(objectName, cacheEntry{found: true, content: content}) + return content, nil +} + +func (s *S3FS) Remove(context, name string) error { + objectName := path.Join(context, name) + return s.client.RemoveObject(s.ctx, s.bucket, objectName, minio.RemoveObjectOptions{}) +} + +func (s *S3FS) List() (map[string][]*model.Asset, error) { + objs := map[string][]*model.Asset{} + + for obj := range s.client.ListObjects(s.ctx, s.bucket, minio.ListObjectsOptions{ + Recursive: true, + }) { + if obj.Err != nil { + return nil, obj.Err + } + + splitted := strings.SplitN(obj.Key, "/", 2) + if len(splitted) < 2 { + continue + } + ctx := splitted[0] + assetName := model.NormalizeAssetName(splitted[1]) + + a, err := GetAsset(ctx, assetName) + if err != nil { + return nil, err + } + + objs[ctx] = append(objs[ctx], a) + } + + return objs, nil +} + +func (s *S3FS) CheckStatus(ctx context.Context) (time.Duration, error) { + before := time.Now() + _, err := s.client.ListBuckets(ctx) + if err != nil { + return 0, err + } + return time.Since(before), nil +} diff --git a/pkg/config/config/config.go b/pkg/config/config/config.go index 9614176108c..5826ebb885a 100644 --- a/pkg/config/config/config.go +++ b/pkg/config/config/config.go @@ -82,6 +82,8 @@ const ( // SchemeSwiftSecure is the URL scheme used to configure the swift filesystem // in secure mode (HTTPS). SchemeSwiftSecure = "swift+https" + // SchemeS3 is the URL scheme used to configure an S3-compatible filesystem. + SchemeS3 = "s3" ) // defaultAdminSecretFileName is the default name of the file containing the diff --git a/pkg/config/config/s3.go b/pkg/config/config/s3.go new file mode 100644 index 00000000000..506fd57afe2 --- /dev/null +++ b/pkg/config/config/s3.go @@ -0,0 +1,106 @@ +package config + +import ( + "context" + "fmt" + "strings" + + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" +) + +var s3Client *minio.Client +var s3BucketPrefix string +var s3Region string + +// InitDefaultS3Connection initializes the default S3 handler. +func InitDefaultS3Connection() error { + return InitS3Connection(config.Fs) +} + +// InitS3Connection initializes the global S3 client connection. This is +// not a thread-safe method. +func InitS3Connection(fs Fs) error { + fsURL := fs.URL + if fsURL.Scheme != SchemeS3 { + return nil + } + + q := fsURL.Query() + endpoint := fsURL.Host + accessKey := q.Get("access_key") + secretKey := q.Get("secret_key") + region := q.Get("region") + useSSL := q.Get("use_ssl") != "false" // default true + + s3BucketPrefix = q.Get("bucket_prefix") + if s3BucketPrefix == "" { + s3BucketPrefix = "cozy" + } + // Sanitize bucket prefix: lowercase, only alphanumeric and hyphens + s3BucketPrefix = strings.ToLower(s3BucketPrefix) + s3BucketPrefix = strings.Map(func(r rune) rune { + if (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9') || r == '-' { + return r + } + return '-' + }, s3BucketPrefix) + s3BucketPrefix = strings.Trim(s3BucketPrefix, "-") + s3Region = region + + var err error + opts := &minio.Options{ + Creds: credentials.NewStaticV4(accessKey, secretKey, ""), + Secure: useSSL, + Region: region, + } + if fs.Transport != nil { + opts.Transport = fs.Transport + } + + s3Client, err = minio.New(endpoint, opts) + if err != nil { + return fmt.Errorf("s3: could not create client: %w", err) + } + + // Verify connectivity by listing buckets + if _, err = s3Client.ListBuckets(context.Background()); err != nil { + log.Errorf("Could not connect to S3 endpoint %s: %s", endpoint, err) + return err + } + + log.Infof("Successfully connected to S3 endpoint %s", endpoint) + + // Pre-create the fixed buckets used by secondary storage (apps, assets, + // previews, exports). The per-org VFS bucket is created on instance init. + ctx := context.Background() + for _, suffix := range []string{"-apps-web", "-apps-konnectors", "-assets", "-previews", "-exports"} { + bucket := s3BucketPrefix + suffix + if err := s3Client.MakeBucket(ctx, bucket, minio.MakeBucketOptions{Region: region}); err != nil { + code := minio.ToErrorResponse(err).Code + if code != "BucketAlreadyOwnedByYou" && code != "BucketAlreadyExists" { + log.Warnf("Could not create bucket %s: %s", bucket, err) + } + } + } + + return nil +} + +// GetS3Client returns the global S3 client. +func GetS3Client() *minio.Client { + if s3Client == nil { + panic("Called GetS3Client() before InitS3Connection()") + } + return s3Client +} + +// GetS3BucketPrefix returns the configured bucket prefix. +func GetS3BucketPrefix() string { + return s3BucketPrefix +} + +// GetS3Region returns the configured S3 region. +func GetS3Region() string { + return s3Region +} diff --git a/pkg/previewfs/cache.go b/pkg/previewfs/cache.go index 3d62f95f5e5..04cb1e49a7d 100644 --- a/pkg/previewfs/cache.go +++ b/pkg/previewfs/cache.go @@ -13,6 +13,7 @@ import ( "time" "github.com/cozy/cozy-stack/pkg/config/config" + "github.com/minio/minio-go/v7" "github.com/ncw/swift/v2" "github.com/spf13/afero" ) @@ -41,6 +42,10 @@ func SystemCache() Cache { conn := config.GetSwiftConnection() ctx := context.Background() return swiftCache{conn, ctx} + case config.SchemeS3: + client := config.GetS3Client() + bucket := config.GetS3BucketPrefix() + "-previews" + return newS3Cache(client, bucket) default: panic(fmt.Errorf("previewfs: unknown storage provider %s", fsURL.Scheme)) } @@ -151,6 +156,81 @@ func (s swiftCache) SetPreview(md5sum []byte, buffer *bytes.Buffer) error { return err } +type s3Cache struct { + client *minio.Client + bucket string + ctx context.Context +} + +func newS3Cache(client *minio.Client, bucket string) s3Cache { + return s3Cache{client: client, bucket: bucket, ctx: context.Background()} +} + +func (s s3Cache) ensureBucket() error { + err := s.client.MakeBucket(s.ctx, s.bucket, minio.MakeBucketOptions{}) + if err != nil { + code := minio.ToErrorResponse(err).Code + if code == "BucketAlreadyOwnedByYou" || code == "BucketAlreadyExists" { + return nil + } + return err + } + return nil +} + +func (s s3Cache) getObject(name string) (*bytes.Buffer, error) { + obj, err := s.client.GetObject(s.ctx, s.bucket, name, minio.GetObjectOptions{}) + if err != nil { + return nil, err + } + defer obj.Close() + + buf := &bytes.Buffer{} + _, err = buf.ReadFrom(obj) + if err != nil { + if minio.ToErrorResponse(err).Code == "NoSuchKey" { + return nil, os.ErrNotExist + } + return nil, err + } + return buf, nil +} + +func (s s3Cache) putObject(name string, buffer *bytes.Buffer) error { + data := buffer.Bytes() + _, err := s.client.PutObject(s.ctx, s.bucket, name, + bytes.NewReader(data), int64(len(data)), + minio.PutObjectOptions{ContentType: "image/jpg"}) + if err != nil { + code := minio.ToErrorResponse(err).Code + if code == "NoSuchBucket" { + if berr := s.ensureBucket(); berr != nil { + return berr + } + _, err = s.client.PutObject(s.ctx, s.bucket, name, + bytes.NewReader(data), int64(len(data)), + minio.PutObjectOptions{ContentType: "image/jpg"}) + } + } + return err +} + +func (s s3Cache) GetIcon(md5sum []byte) (*bytes.Buffer, error) { + return s.getObject(iconFilename(md5sum)) +} + +func (s s3Cache) SetIcon(md5sum []byte, buffer *bytes.Buffer) error { + return s.putObject(iconFilename(md5sum), buffer) +} + +func (s s3Cache) GetPreview(md5sum []byte) (*bytes.Buffer, error) { + return s.getObject(previewFilename(md5sum)) +} + +func (s s3Cache) SetPreview(md5sum []byte, buffer *bytes.Buffer) error { + return s.putObject(previewFilename(md5sum), buffer) +} + func iconFilename(md5sum []byte) string { return "icon-" + hex.EncodeToString(md5sum) + ".jpg" } diff --git a/pkg/s3util/s3util.go b/pkg/s3util/s3util.go new file mode 100644 index 00000000000..592fe7cdaf0 --- /dev/null +++ b/pkg/s3util/s3util.go @@ -0,0 +1,91 @@ +// Package s3util provides shared helpers for interacting with S3-compatible +// object stores via the minio-go client. +package s3util + +import ( + "context" + "errors" + "fmt" + "os" + + "github.com/minio/minio-go/v7" +) + +// IsNotFound returns true when the error is an S3 "not found" response +// (NoSuchKey or NoSuchBucket). +func IsNotFound(err error) bool { + code := minio.ToErrorResponse(err).Code + return code == "NoSuchKey" || code == "NoSuchBucket" +} + +// WrapNotFound converts S3 not-found errors to os.ErrNotExist and sanitizes +// other S3 errors to avoid leaking internal bucket/key details. +func WrapNotFound(err error) error { + if IsNotFound(err) { + return os.ErrNotExist + } + code := minio.ToErrorResponse(err).Code + if code != "" { + return fmt.Errorf("s3 storage error: %s", code) + } + return err +} + +// EnsureBucket creates the bucket if it does not already exist. +func EnsureBucket(ctx context.Context, client *minio.Client, bucket, region string) error { + err := client.MakeBucket(ctx, bucket, minio.MakeBucketOptions{ + Region: region, + }) + if err != nil { + code := minio.ToErrorResponse(err).Code + if code == "BucketAlreadyOwnedByYou" || code == "BucketAlreadyExists" { + return nil + } + return err + } + return nil +} + +// DeleteObjects deletes a list of named objects from a bucket. +func DeleteObjects(ctx context.Context, client *minio.Client, bucket string, objNames []string) error { + if len(objNames) == 0 { + return nil + } + objectsCh := make(chan minio.ObjectInfo, len(objNames)) + for _, name := range objNames { + objectsCh <- minio.ObjectInfo{Key: name} + } + close(objectsCh) + var errm error + for e := range client.RemoveObjects(ctx, bucket, objectsCh, minio.RemoveObjectsOptions{}) { + errm = errors.Join(errm, e.Err) + } + return errm +} + +// DeletePrefixObjects deletes all objects in a bucket under a given prefix. +func DeletePrefixObjects(ctx context.Context, client *minio.Client, bucket, prefix string) error { + objectsCh := make(chan minio.ObjectInfo) + var listErr error + go func() { + defer close(objectsCh) + for obj := range client.ListObjects(ctx, bucket, minio.ListObjectsOptions{ + Prefix: prefix, + Recursive: true, + }) { + if obj.Err != nil { + listErr = obj.Err + return + } + objectsCh <- obj + } + }() + var errm error + for e := range client.RemoveObjects(ctx, bucket, objectsCh, minio.RemoveObjectsOptions{}) { + errm = errors.Join(errm, e.Err) + } + if listErr != nil { + return listErr + } + return errm +} diff --git a/pkg/s3util/s3util_test.go b/pkg/s3util/s3util_test.go new file mode 100644 index 00000000000..45789f05dd6 --- /dev/null +++ b/pkg/s3util/s3util_test.go @@ -0,0 +1,120 @@ +package s3util_test + +import ( + "bytes" + "context" + "io" + "os" + "testing" + + "github.com/cozy/cozy-stack/pkg/s3util" + "github.com/cozy/cozy-stack/tests/testutils" + "github.com/minio/minio-go/v7" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestS3Util(t *testing.T) { + if testing.Short() { + t.Skip("requires minio container: skipped with --short") + } + + fixture := testutils.StartMinio(t) + client := fixture.Client(t) + ctx := context.Background() + bucket := "s3util-test" + + require.NoError(t, client.MakeBucket(ctx, bucket, minio.MakeBucketOptions{})) + t.Cleanup(func() { + // Best-effort cleanup: remove all objects then the bucket. + for obj := range client.ListObjects(ctx, bucket, minio.ListObjectsOptions{Recursive: true}) { + _ = client.RemoveObject(ctx, bucket, obj.Key, minio.RemoveObjectOptions{}) + } + _ = client.RemoveBucket(ctx, bucket) + }) + + t.Run("IsNotFound", func(t *testing.T) { + _, err := client.GetObject(ctx, bucket, "does-not-exist", minio.GetObjectOptions{}) + require.NoError(t, err) // GetObject itself doesn't fail; Stat does. + + _, err = client.StatObject(ctx, bucket, "does-not-exist", minio.StatObjectOptions{}) + require.Error(t, err) + assert.True(t, s3util.IsNotFound(err)) + + // A non-not-found error should return false. + assert.False(t, s3util.IsNotFound(io.ErrUnexpectedEOF)) + }) + + t.Run("WrapNotFound", func(t *testing.T) { + _, err := client.StatObject(ctx, bucket, "does-not-exist", minio.StatObjectOptions{}) + require.Error(t, err) + + wrapped := s3util.WrapNotFound(err) + assert.ErrorIs(t, wrapped, os.ErrNotExist) + + // A non-S3 error passes through unchanged. + orig := io.ErrUnexpectedEOF + assert.Equal(t, orig, s3util.WrapNotFound(orig)) + }) + + t.Run("EnsureBucket", func(t *testing.T) { + newBucket := "s3util-ensure-test" + t.Cleanup(func() { _ = client.RemoveBucket(ctx, newBucket) }) + + // First call creates. + err := s3util.EnsureBucket(ctx, client, newBucket, "") + assert.NoError(t, err) + + // Second call is idempotent. + err = s3util.EnsureBucket(ctx, client, newBucket, "") + assert.NoError(t, err) + }) + + t.Run("DeleteObjects", func(t *testing.T) { + // Create a few objects. + for _, key := range []string{"del-a", "del-b", "del-c"} { + _, err := client.PutObject(ctx, bucket, key, + bytes.NewReader([]byte("x")), 1, + minio.PutObjectOptions{}) + require.NoError(t, err) + } + + err := s3util.DeleteObjects(ctx, client, bucket, []string{"del-a", "del-b", "del-c"}) + assert.NoError(t, err) + + // Verify they are gone. + for _, key := range []string{"del-a", "del-b", "del-c"} { + _, err := client.StatObject(ctx, bucket, key, minio.StatObjectOptions{}) + assert.True(t, s3util.IsNotFound(err), "object %s should be deleted", key) + } + }) + + t.Run("DeleteObjectsEmpty", func(t *testing.T) { + // Should be a no-op, not an error. + assert.NoError(t, s3util.DeleteObjects(ctx, client, bucket, nil)) + assert.NoError(t, s3util.DeleteObjects(ctx, client, bucket, []string{})) + }) + + t.Run("DeletePrefixObjects", func(t *testing.T) { + // Create objects under a prefix and one outside. + for _, key := range []string{"pfx/one", "pfx/two", "pfx/sub/three", "outside"} { + _, err := client.PutObject(ctx, bucket, key, + bytes.NewReader([]byte("x")), 1, + minio.PutObjectOptions{}) + require.NoError(t, err) + } + + err := s3util.DeletePrefixObjects(ctx, client, bucket, "pfx/") + assert.NoError(t, err) + + // Prefixed objects should be gone. + for _, key := range []string{"pfx/one", "pfx/two", "pfx/sub/three"} { + _, err := client.StatObject(ctx, bucket, key, minio.StatObjectOptions{}) + assert.True(t, s3util.IsNotFound(err), "object %s should be deleted", key) + } + + // Object outside the prefix should still exist. + _, err = client.StatObject(ctx, bucket, "outside", minio.StatObjectOptions{}) + assert.NoError(t, err) + }) +} diff --git a/scripts/docker/production/Dockerfile b/scripts/docker/production/Dockerfile index 98d41bcf726..dc26d9a9cba 100644 --- a/scripts/docker/production/Dockerfile +++ b/scripts/docker/production/Dockerfile @@ -14,7 +14,17 @@ RUN go mod download # Build cozy-stack COPY . . -RUN ./scripts/build.sh release ./cozy-stack +# Build directly instead of going through scripts/build.sh, which runs +# `git describe` / `git rev-parse` and trips on the COPY'd working tree +# inside the container. The version string can be overridden at build +# time via --build-arg VERSION_STRING=... +ARG VERSION_STRING=docker-build +RUN BUILD_TIME="$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ + && go build \ + -ldflags "-X github.com/cozy/cozy-stack/pkg/config.Version=${VERSION_STRING} \ + -X github.com/cozy/cozy-stack/pkg/config.BuildTime=${BUILD_TIME} \ + -X github.com/cozy/cozy-stack/pkg/config.BuildMode=production" \ + -o ./cozy-stack # Multi-stage image: the main image diff --git a/tests/testutils/minio_utils.go b/tests/testutils/minio_utils.go new file mode 100644 index 00000000000..8af8bf262ab --- /dev/null +++ b/tests/testutils/minio_utils.go @@ -0,0 +1,98 @@ +package testutils + +import ( + "context" + "fmt" + "net/url" + "testing" + "time" + + "github.com/docker/go-connections/nat" + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" + "github.com/stretchr/testify/require" + + c "github.com/docker/docker/api/types/container" + tc "github.com/testcontainers/testcontainers-go" + "github.com/testcontainers/testcontainers-go/wait" +) + +// MinioFixture holds the state for a running MinIO container. +type MinioFixture struct { + Container tc.Container + Endpoint string // host:port + AccessKey string + SecretKey string + t *testing.T +} + +// StartMinio starts a MinIO container for testing. +func StartMinio(t *testing.T) *MinioFixture { + t.Helper() + + accessKey := "minioadmin" + secretKey := "minioadmin" + hostPort := getFreePort(t) + + req := tc.ContainerRequest{ + Image: "minio/minio:RELEASE.2025-02-28T09-55-16Z", + ExposedPorts: []string{"9000/tcp"}, + Env: map[string]string{ + "MINIO_ROOT_USER": accessKey, + "MINIO_ROOT_PASSWORD": secretKey, + }, + Cmd: []string{"server", "/data"}, + HostConfigModifier: func(hc *c.HostConfig) { + hc.PortBindings = nat.PortMap{ + "9000/tcp": []nat.PortBinding{{HostIP: "0.0.0.0", HostPort: hostPort}}, + } + }, + WaitingFor: wait.ForHTTP("/minio/health/live"). + WithPort("9000/tcp"). + WithStartupTimeout(60 * time.Second), + } + + container, err := tc.GenericContainer(context.Background(), tc.GenericContainerRequest{ + ContainerRequest: req, + Started: true, + }) + require.NoError(t, err, "failed to start MinIO") + + host, err := container.Host(context.Background()) + require.NoError(t, err) + + endpoint := fmt.Sprintf("%s:%s", host, hostPort) + t.Logf("MinIO endpoint: %s", endpoint) + + t.Cleanup(func() { + _ = container.Terminate(context.Background()) + }) + + return &MinioFixture{ + Container: container, + Endpoint: endpoint, + AccessKey: accessKey, + SecretKey: secretKey, + t: t, + } +} + +// Client returns a minio.Client connected to this fixture. +func (f *MinioFixture) Client(t *testing.T) *minio.Client { + t.Helper() + client, err := minio.New(f.Endpoint, &minio.Options{ + Creds: credentials.NewStaticV4(f.AccessKey, f.SecretKey, ""), + Secure: false, + }) + require.NoError(t, err) + return client +} + +// FsURL returns a *url.URL suitable for config.InitS3Connection. +func (f *MinioFixture) FsURL(bucketPrefix string) *url.URL { + return &url.URL{ + Scheme: "s3", + Host: f.Endpoint, + RawQuery: fmt.Sprintf("access_key=%s&secret_key=%s&bucket_prefix=%s&use_ssl=false", f.AccessKey, f.SecretKey, bucketPrefix), + } +} diff --git a/web/settings/capabilities.go b/web/settings/capabilities.go index dcd2104294a..c76fcb1c86f 100644 --- a/web/settings/capabilities.go +++ b/web/settings/capabilities.go @@ -43,6 +43,8 @@ func NewCapabilities(inst *instance.Instance) jsonapi.Object { switch config.FsURL().Scheme { case config.SchemeSwift, config.SchemeSwiftSecure: versioning = inst.SwiftLayout >= 2 + case config.SchemeS3: + versioning = true } flat := config.GetConfig().Subdomains == config.FlatSubdomains