feat: add S3-compatible storage backend

.github/workflows/docker-s3-test.yml

@@ -0,0 +1,61 @@
+name: Build S3-test Docker image
+
+on:
+  push:
+    branches:
+      - feat/s3-vfs-backend
+  workflow_dispatch:
+    inputs:
+      tag:
+        description: "Image tag to publish under ghcr.io/<repo>"
+        required: false
+        default: "s3-test"
+
+permissions:
+  contents: read
+  packages: write
+
+jobs:
+  build-and-push:
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Log in to GHCR
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Compute image metadata
+        id: meta
+        run: |
+          repo_lc="$(echo "${{ github.repository }}" | tr '[:upper:]' '[:lower:]')"
+          tag="${{ github.event.inputs.tag }}"
+          if [ -z "$tag" ]; then
+            tag="s3-test"
+          fi
+          tag_lc="$(echo "$tag" | tr '[:upper:]' '[:lower:]')"
+          sha_tag="${tag_lc}-${GITHUB_SHA::7}"
+          echo "image=ghcr.io/${repo_lc}" >> "$GITHUB_OUTPUT"
+          echo "tag=${tag_lc}" >> "$GITHUB_OUTPUT"
+          echo "sha_tag=${sha_tag}" >> "$GITHUB_OUTPUT"
+
+      - name: Build and push
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          file: scripts/docker/production/Dockerfile
+          push: true
+          tags: |
+            ${{ steps.meta.outputs.image }}:${{ steps.meta.outputs.tag }}
+            ${{ steps.meta.outputs.image }}:${{ steps.meta.outputs.sha_tag }}
+          build-args: |
+            VERSION_STRING=${{ steps.meta.outputs.tag }}-${{ github.sha }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max

.github/workflows/go-tests.yml

@@ -64,4 +64,4 @@ jobs:
           cache: true
 
       - name: Run tests
-        run: go test -p 1 -timeout 5m ./...
+        run: go test -p 1 -timeout 10m ./...

cozy.example.yaml

@@ -67,6 +67,7 @@ fs:
 
   # url: file://localhost/var/lib/cozy
   # url: swift://openstack/?UserName={{ .Env.OS_USERNAME }}&Password={{ .Env.OS_PASSWORD }}&ProjectName={{ .Env.OS_PROJECT_NAME }}&UserDomainName={{ .Env.OS_USER_DOMAIN_NAME }}&Timeout={{ .Env.GOSWIFT_TIMEOUT }}
+  # url: s3://{{ .Env.S3_ENDPOINT }}?access_key={{ .Env.S3_ACCESS_KEY }}&secret_key={{ .Env.S3_SECRET_KEY }}&region={{ .Env.S3_REGION }}&bucket_prefix=cozy&use_ssl=true
 
   # Swift FS can be used with advanced parameters to activate TLS properties.
   # For using swift with https, you must use the "swift+https" scheme.

docs/s3.md

@@ -0,0 +1,248 @@
+[Table of contents](README.md#table-of-contents)
+
+# S3 Storage Backend
+
+cozy-stack supports S3-compatible object storage as a file system backend,
+alongside the existing local filesystem (afero) and OpenStack Swift backends.
+It has been designed to work with any S3-compatible provider (OVH, MinIO,
+Scaleway, etc.) and does not depend on the AWS SDK.
+
+## Configuration
+
+The S3 backend is configured via the `fs.url` parameter using the `s3://`
+scheme. All connection parameters are passed as query parameters:
+
+```yaml
+fs:
+  url: s3://s3.rbx.io.cloud.ovh.net?access_key=ACCESS&secret_key=SECRET&region=rbx&bucket_prefix=cozy&use_ssl=true
+```
+
+| Parameter       | Description                          | Default |
+|-----------------|--------------------------------------|---------|
+| `access_key`    | S3 access key ID                     | —       |
+| `secret_key`    | S3 secret access key                 | —       |
+| `region`        | S3 region                            | —       |
+| `bucket_prefix` | Prefix for all bucket names          | `cozy`  |
+| `use_ssl`       | Use HTTPS for S3 connections         | `true`  |
+
+The host part of the URL is the S3 endpoint (e.g. `s3.rbx.io.cloud.ovh.net`
+for OVH, `localhost:9000` for MinIO).
+
+### Local development with MinIO
+
+This tutorial explains how to set up a local S3 backend using MinIO for
+development and testing.
+
+**1. Start MinIO with Docker:**
+
+```bash
+docker run -d --name minio \
+  -p 9000:9000 \
+  -p 9001:9001 \
+  -e MINIO_ROOT_USER=minioadmin \
+  -e MINIO_ROOT_PASSWORD=minioadmin \
+  minio/minio server /data --console-address ":9001"
+```
+
+MinIO is now running:
+- S3 API: `http://localhost:9000`
+- Web console: `http://localhost:9001` (login: `minioadmin` / `minioadmin`)
+
+**2. Configure cozy-stack:**
+
+Buckets are created automatically at startup. No manual bucket creation
+is needed.
+
+Edit your `~/.cozy/cozy.yaml`:
+
+```yaml
+fs:
+  url: s3://localhost:9000?access_key=minioadmin&secret_key=minioadmin&bucket_prefix=cozy&use_ssl=false
+```
+
+**3. Build and start:**
+
+```bash
+go build -o ~/go/bin/cozy-stack .
+~/go/bin/cozy-stack serve
+```
+
+You should see in the logs:
+
+```
+Successfully connected to S3 endpoint localhost:9000
+```
+
+**4. (Re)install your apps:**
+
+When switching from a different storage backend (e.g. `file://`), you need
+to reinstall the apps so their assets are stored in S3:
+
+```bash
+cozy-stack apps uninstall drive --domain your.domain.localhost:8080
+cozy-stack apps install drive --domain your.domain.localhost:8080
+cozy-stack apps uninstall home --domain your.domain.localhost:8080
+cozy-stack apps install home --domain your.domain.localhost:8080
+```
+
+**5. Verify:**
+
+Check that objects appear in MinIO:
+
+```bash
+docker exec minio mc ls --recursive local/cozy-apps-web/
+```
+
+Upload a file via the Drive UI or the API:
+
+```bash
+TOKEN=$(cozy-stack instances token-cli your.domain.localhost:8080 io.cozy.files)
+curl -X POST \
+  -H "Authorization: Bearer $TOKEN" \
+  -H "Content-Type: text/plain" \
+  "http://your.domain.localhost:8080/files/io.cozy.files.root-dir?Type=file&Name=test.txt" \
+  -d "Hello S3!"
+```
+
+Verify the file is in MinIO:
+
+```bash
+docker exec minio mc ls --recursive local/cozy-default/
+```
+
+**6. Switching back to local filesystem:**
+
+Comment out the S3 URL in your config and restart cozy-stack:
+
+```yaml
+fs:
+  # url: s3://localhost:9000?access_key=minioadmin&secret_key=minioadmin&bucket_prefix=cozy&use_ssl=false
+```
+
+Note: files uploaded to S3 won't be accessible when using the local
+filesystem backend, and vice versa. Each backend has its own storage.
+
+## Bucket strategy
+
+### Design rationale
+
+Swift uses one container per instance (`cozy-v3-<DBPrefix>`). This doesn't
+scale well for S3 where bucket creation can be limited (AWS limits to 100
+buckets per account by default, OVH to 100 as well). Instead, the S3 backend
+uses a **shared bucket per organization** with **key prefixes per instance**.
+
+### Bucket naming
+
+Each bucket name is derived from the instance's `OrgID` field:
+
+```
+<bucket_prefix>-<sanitized_org_id>
+```
+
+- If `OrgID` is empty, `"default"` is used as fallback
+- The org ID is sanitized: lowercased, underscores/dots replaced by hyphens,
+  non-alphanumeric characters stripped, consecutive hyphens collapsed,
+  truncated to respect the 63-character S3 bucket name limit
+- Examples: `cozy-default`, `cozy-acme-corp`, `cozy-org-12345`
+
+### Dedicated buckets for secondary storage
+
+In addition to the main VFS bucket, the S3 backend uses dedicated buckets for
+other storage needs:
+
+| Bucket                        | Content                                 |
+|-------------------------------|-----------------------------------------|
+| `<prefix>-<orgId>`           | Main VFS data (files, versions)         |
+| `<prefix>-apps-web`          | Web application assets (drive, etc.)    |
+| `<prefix>-apps-konnectors`   | Konnector assets                        |
+| `<prefix>-assets`            | Dynamic assets                          |
+| `<prefix>-previews`          | PDF preview and icon cache              |
+| `<prefix>-exports`           | Instance export archives                |
+
+Buckets are created automatically on first use.
+
+## Object key structure
+
+Within a bucket, each instance's data is isolated by a key prefix derived
+from `DBPrefix()` (typically the instance domain or a CouchDB prefix).
+
+### VFS files
+
+```
+<DBPrefix>/<docID_part1>/<docID_part2>/<docID_part3>/<internalID>
+```
+
+The document ID (a 32-character UUID v7 hex string) is split into virtual
+subfolders to avoid flat hierarchies:
+
+```
+cozy218def.../019d35b1-9dc3-78ec-994d-f5/44336/7f1b6/e0AbCdEfGh123456
+              ^^^^^^^^^^^^^^^^^^^^^^       ^^^^^  ^^^^^  ^^^^^^^^^^^^^^^^
+              first 22 chars               5 ch   5 ch   16-char internalID
+```
+
+This structure mirrors the Swift V3 layout (`MakeObjectNameV3`).
+
+### Thumbnails
+
+```
+<DBPrefix>/thumbs/<docID_split>-<format>
+```
+
+Formats: `small`, `medium`, `large`.
+
+### Avatar
+
+```
+<DBPrefix>/avatar
+```
+
+## Memory consumption
+
+The S3 backend is designed to have comparable memory usage to Swift:
+
+| Scenario                    | Memory per upload |
+|-----------------------------|-------------------|
+| Known size, file < 5 GiB   | ~32 KB (single PUT, stream) |
+| Unknown size (rare)         | ~5 MiB (multipart, PartSize=5MiB, NumThreads=1) |
+
+When `ByteSize` is known on the file document (the common case for drive
+uploads), the backend passes the exact size to `PutObject`, which uses a
+single PUT request that streams directly to S3 with minimal buffering — the
+same behavior as Swift's `ObjectCreate`.
+
+Multipart upload is only used for files with unknown size or exceeding 5 GiB,
+with `PartSize=5MiB` and `NumThreads=1` to limit memory.
+
+## Encryption at rest
+
+The S3 backend does not implement client-side encryption. Encryption should
+be configured at the infrastructure level (S3 bucket default encryption /
+SSE-S3), the same approach used for the Swift backend.
+
+## Differences from Swift
+
+| Aspect             | Swift                         | S3                                     |
+|--------------------|-------------------------------|----------------------------------------|
+| Container/Bucket   | One per instance              | One per organization (shared)          |
+| Instance isolation | Container name                | Key prefix within bucket               |
+| Delete instance    | Delete entire container       | Delete all objects with key prefix     |
+| File streaming     | Native `io.WriteCloser`       | `io.Pipe` + `PutObject` goroutine      |
+| Bulk delete        | `BulkDelete` API              | `RemoveObjects` channel API            |
+| Server-side copy   | `ObjectCopy`                  | `CopyObject` (same endpoint only)      |
+
+## Testing
+
+The VFS integration tests run against all three backends (afero, swift, s3)
+using a table-driven approach. The S3 tests use
+[testcontainers-go](https://testcontainers.com/guides/getting-started-with-testcontainers-for-go/)
+with a MinIO container that is started automatically.
+
+```bash
+# Run VFS tests (requires CouchDB + Docker)
+COZY_COUCHDB_URL=http://admin:admin@localhost:5984/ \
+  go test ./model/vfs/ -run TestVfs -v -count=1 -timeout 300s
+
+# Run naming unit tests (no external deps)
+go test ./model/vfs/vfss3/ -run "TestSanitize|TestBucketName|TestMakeObjectKey|TestMakeDocID" -v
+```

go.mod

@@ -31,6 +31,7 @@ require (
 	github.com/justincampbell/bigduration v0.0.0-20160531141349-e45bf03c0666
 	github.com/labstack/echo/v4 v4.15.1
 	github.com/leonelquinteros/gotext v1.7.2
+	github.com/minio/minio-go/v7 v7.0.99
 	github.com/mitchellh/mapstructure v1.5.0
 	github.com/mssola/user_agent v0.6.0
 	github.com/ncw/swift/v2 v2.0.3
@@ -103,6 +104,7 @@ require (
 	github.com/fatih/structs v1.1.0 // indirect
 	github.com/felixge/httpsnoop v1.0.4 // indirect
 	github.com/fsnotify/fsnotify v1.7.0 // indirect
+	github.com/go-ini/ini v1.67.0 // indirect
 	github.com/go-jose/go-jose/v4 v4.1.3 // indirect
 	github.com/go-logr/logr v1.4.3 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
@@ -121,13 +123,17 @@ require (
 	github.com/imkira/go-interpol v1.1.0 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/jonas-p/go-shp v0.1.1 // indirect
-	github.com/klauspost/compress v1.18.0 // indirect
+	github.com/klauspost/compress v1.18.2 // indirect
+	github.com/klauspost/cpuid/v2 v2.2.11 // indirect
+	github.com/klauspost/crc32 v1.3.0 // indirect
 	github.com/labstack/gommon v0.4.2 // indirect
 	github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect
 	github.com/magiconair/properties v1.8.10 // indirect
 	github.com/mattn/go-colorable v0.1.14 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
 	github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect
+	github.com/minio/crc64nvme v1.1.1 // indirect
+	github.com/minio/md5-simd v1.1.2 // indirect
 	github.com/mitchellh/go-wordwrap v1.0.1 // indirect
 	github.com/moby/docker-image-spec v1.3.1 // indirect
 	github.com/moby/go-archive v0.1.0 // indirect
@@ -142,13 +148,15 @@ require (
 	github.com/opencontainers/go-digest v1.0.0 // indirect
 	github.com/opencontainers/image-spec v1.1.1 // indirect
 	github.com/pelletier/go-toml/v2 v2.2.2 // indirect
+	github.com/philhofer/fwd v1.2.0 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
 	github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect
 	github.com/prometheus/client_model v0.6.2 // indirect
 	github.com/prometheus/common v0.45.0 // indirect
 	github.com/prometheus/procfs v0.12.0 // indirect
+	github.com/rs/xid v1.6.0 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	github.com/sagikazarmark/locafero v0.4.0 // indirect
 	github.com/sagikazarmark/slog-shim v0.1.0 // indirect
@@ -161,6 +169,7 @@ require (
 	github.com/spiffe/go-spiffe/v2 v2.6.0 // indirect
 	github.com/stretchr/objx v0.5.2 // indirect
 	github.com/subosito/gotenv v1.6.0 // indirect
+	github.com/tinylib/msgp v1.6.1 // indirect
 	github.com/tklauser/go-sysconf v0.3.12 // indirect
 	github.com/tklauser/numcpus v0.6.1 // indirect
 	github.com/valyala/bytebufferpool v1.0.0 // indirect
@@ -185,6 +194,7 @@ require (
 	go.opentelemetry.io/otel/trace v1.39.0 // indirect
 	go.uber.org/atomic v1.9.0 // indirect
 	go.uber.org/multierr v1.9.0 // indirect
+	go.yaml.in/yaml/v3 v3.0.4 // indirect
 	golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect
 	golang.org/x/sys v0.41.0 // indirect
 	golang.org/x/time v0.14.0 // indirect