From a62d325399a3bb46589a4d6409a9477a31a714cb Mon Sep 17 00:00:00 2001 From: Crash-- Date: Sat, 28 Mar 2026 19:40:34 +0100 Subject: [PATCH 01/20] feat: add S3 config layer with minio-go client Add SchemeS3 constant and S3 connection singleton following the same pattern as the existing Swift connection (swift.go). Uses minio-go/v7 for S3-compatible storage (OVH, MinIO, Scaleway, etc.). Configuration via fs.url query params: access_key, secret_key, region, bucket_prefix, use_ssl. Co-Authored-By: Claude Opus 4.6 (1M context) --- go.mod | 12 +++++- go.sum | 25 ++++++++++- model/stack/main.go | 5 +++ pkg/config/config/config.go | 2 + pkg/config/config/s3.go | 82 +++++++++++++++++++++++++++++++++++++ 5 files changed, 123 insertions(+), 3 deletions(-) create mode 100644 pkg/config/config/s3.go diff --git a/go.mod b/go.mod index 516ab6f28f7..2707bfd5a82 100644 --- a/go.mod +++ b/go.mod @@ -31,6 +31,7 @@ require ( github.com/justincampbell/bigduration v0.0.0-20160531141349-e45bf03c0666 github.com/labstack/echo/v4 v4.15.1 github.com/leonelquinteros/gotext v1.7.2 + github.com/minio/minio-go/v7 v7.0.99 github.com/mitchellh/mapstructure v1.5.0 github.com/mssola/user_agent v0.6.0 github.com/ncw/swift/v2 v2.0.3 @@ -103,6 +104,7 @@ require ( github.com/fatih/structs v1.1.0 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect + github.com/go-ini/ini v1.67.0 // indirect github.com/go-jose/go-jose/v4 v4.1.3 // indirect github.com/go-logr/logr v1.4.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect @@ -121,13 +123,17 @@ require ( github.com/imkira/go-interpol v1.1.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/jonas-p/go-shp v0.1.1 // indirect - github.com/klauspost/compress v1.18.0 // indirect + github.com/klauspost/compress v1.18.2 // indirect + github.com/klauspost/cpuid/v2 v2.2.11 // indirect + github.com/klauspost/crc32 v1.3.0 // indirect github.com/labstack/gommon v0.4.2 // indirect github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect github.com/magiconair/properties v1.8.10 // indirect github.com/mattn/go-colorable v0.1.14 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect + github.com/minio/crc64nvme v1.1.1 // indirect + github.com/minio/md5-simd v1.1.2 // indirect github.com/mitchellh/go-wordwrap v1.0.1 // indirect github.com/moby/docker-image-spec v1.3.1 // indirect github.com/moby/go-archive v0.1.0 // indirect @@ -142,6 +148,7 @@ require ( github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.1.1 // indirect github.com/pelletier/go-toml/v2 v2.2.2 // indirect + github.com/philhofer/fwd v1.2.0 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect @@ -149,6 +156,7 @@ require ( github.com/prometheus/client_model v0.6.2 // indirect github.com/prometheus/common v0.45.0 // indirect github.com/prometheus/procfs v0.12.0 // indirect + github.com/rs/xid v1.6.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/sagikazarmark/locafero v0.4.0 // indirect github.com/sagikazarmark/slog-shim v0.1.0 // indirect @@ -161,6 +169,7 @@ require ( github.com/spiffe/go-spiffe/v2 v2.6.0 // indirect github.com/stretchr/objx v0.5.2 // indirect github.com/subosito/gotenv v1.6.0 // indirect + github.com/tinylib/msgp v1.6.1 // indirect github.com/tklauser/go-sysconf v0.3.12 // indirect github.com/tklauser/numcpus v0.6.1 // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect @@ -185,6 +194,7 @@ require ( go.opentelemetry.io/otel/trace v1.39.0 // indirect go.uber.org/atomic v1.9.0 // indirect go.uber.org/multierr v1.9.0 // indirect + go.yaml.in/yaml/v3 v3.0.4 // indirect golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect golang.org/x/sys v0.41.0 // indirect golang.org/x/time v0.14.0 // indirect diff --git a/go.sum b/go.sum index a4aede5048a..5ce9490e682 100644 --- a/go.sum +++ b/go.sum @@ -144,6 +144,8 @@ github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyT github.com/garyburd/redigo v1.1.1-0.20170914051019-70e1b1943d4f/go.mod h1:NR3MbYisc3/PwhQ00EMzDiPmrwpPxAn5GI05/YaO1SY= github.com/gavv/httpexpect/v2 v2.16.0 h1:Ty2favARiTYTOkCRZGX7ojXXjGyNAIohM1lZ3vqaEwI= github.com/gavv/httpexpect/v2 v2.16.0/go.mod h1:uJLaO+hQ25ukBJtQi750PsztObHybNllN+t+MbbW8PY= +github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A= +github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= github.com/go-jose/go-jose/v4 v4.1.3 h1:CVLmWDhDVRa6Mi/IgCgaopNosCaHz7zrMeF9MlZRkrs= github.com/go-jose/go-jose/v4 v4.1.3/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= @@ -243,8 +245,13 @@ github.com/jonas-p/go-shp v0.1.1/go.mod h1:MRIhyxDQ6VVp0oYeD7yPGr5RSTNScUFKCDsI5 github.com/justincampbell/bigduration v0.0.0-20160531141349-e45bf03c0666 h1:abLciEiilfMf19Q1TFWDrp9j5z5one60dnnpvc6eabg= github.com/justincampbell/bigduration v0.0.0-20160531141349-e45bf03c0666/go.mod h1:xqGOmDZzLOG7+q/CgsbXv10g4tgPsbjhmAxyaTJMvis= github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= -github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= -github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= +github.com/klauspost/compress v1.18.2 h1:iiPHWW0YrcFgpBYhsA6D1+fqHssJscY/Tm/y2Uqnapk= +github.com/klauspost/compress v1.18.2/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4= +github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.2.11 h1:0OwqZRYI2rFrjS4kvkDnqJkKHdHaRnCm68/DY4OxRzU= +github.com/klauspost/cpuid/v2 v2.2.11/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= +github.com/klauspost/crc32 v1.3.0 h1:sSmTt3gUt81RP655XGZPElI0PelVTZ6YwCRnPSupoFM= +github.com/klauspost/crc32 v1.3.0/go.mod h1:D7kQaZhnkX/Y0tstFGf8VUzv2UofNGqCjnC3zdHB0Hw= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= @@ -272,6 +279,12 @@ github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWE github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 h1:jWpvCLoY8Z/e3VKvlsiIGKtc+UG6U5vzxaoagmhXfyg= github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0/go.mod h1:QUyp042oQthUoa9bqDv0ER0wrtXnBruoNd7aNjkbP+k= +github.com/minio/crc64nvme v1.1.1 h1:8dwx/Pz49suywbO+auHCBpCtlW1OfpcLN7wYgVR6wAI= +github.com/minio/crc64nvme v1.1.1/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg= +github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= +github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= +github.com/minio/minio-go/v7 v7.0.99 h1:2vH/byrwUkIpFQFOilvTfaUpvAX3fEFhEzO+DR3DlCE= +github.com/minio/minio-go/v7 v7.0.99/go.mod h1:EtGNKtlX20iL2yaYnxEigaIvj0G0GwSDnifnG8ClIdw= github.com/mitchellh/go-wordwrap v1.0.1 h1:TLuKupo69TCn6TQSyGxwI1EblZZEsQ0vMlAFQflz0v0= github.com/mitchellh/go-wordwrap v1.0.1/go.mod h1:R62XHJLzvMFRBbcrT7m7WgmE1eOyTSsCt+hzestvNj0= github.com/mitchellh/mapstructure v0.0.0-20170523030023-d0303fe80992/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= @@ -326,6 +339,8 @@ github.com/oschwald/maxminddb-golang v1.13.1/go.mod h1:K4pgV9N/GcK694KSTmVSDTODk github.com/pelletier/go-toml v1.0.1-0.20170904195809-1d6b12b7cb29/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= +github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM= +github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM= github.com/pkg/diff v0.0.0-20200914180035-5b29258ca4f7/go.mod h1:zO8QMzTeZd5cpnIkz/Gn6iK0jDfGicM1nynOkkPIl28= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -355,6 +370,8 @@ github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= +github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU= +github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6keLGt6kNQ= @@ -412,6 +429,8 @@ github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSW github.com/tailscale/depaware v0.0.0-20210622194025-720c4b409502/go.mod h1:p9lPsd+cx33L3H9nNoecRRxPssFKUwwI50I3pZ0yT+8= github.com/testcontainers/testcontainers-go v0.40.0 h1:pSdJYLOVgLE8YdUY2FHQ1Fxu+aMnb6JfVz1mxk7OeMU= github.com/testcontainers/testcontainers-go v0.40.0/go.mod h1:FSXV5KQtX2HAMlm7U3APNyLkkap35zNLxukw9oBi/MY= +github.com/tinylib/msgp v1.6.1 h1:ESRv8eL3u+DNHUoSAAQRE50Hm162zqAnBoGv9PzScPY= +github.com/tinylib/msgp v1.6.1/go.mod h1:RSp0LW9oSxFut3KzESt5Voq4GVWyS+PSulT77roAqEA= github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFAEVmqU= github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI= github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk= @@ -478,6 +497,8 @@ go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI= go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTVQ= +go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= golang.org/x/crypto v0.0.0-20170512130425-ab89591268e0/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= diff --git a/model/stack/main.go b/model/stack/main.go index 5ddb66f0660..f82bdbe87da 100644 --- a/model/stack/main.go +++ b/model/stack/main.go @@ -85,6 +85,11 @@ security features. Please do not use this binary as your production server. return nil, nil, fmt.Errorf("failed to init the swift connection: %w", err) } + // Init the main global connection to the S3 server + if err := config.InitDefaultS3Connection(); err != nil { + return nil, nil, fmt.Errorf("failed to init the S3 connection: %w", err) + } + workersList, err := job.GetWorkersList() if err != nil { return nil, nil, fmt.Errorf("failed to get the workers list: %w", err) diff --git a/pkg/config/config/config.go b/pkg/config/config/config.go index 9614176108c..5826ebb885a 100644 --- a/pkg/config/config/config.go +++ b/pkg/config/config/config.go @@ -82,6 +82,8 @@ const ( // SchemeSwiftSecure is the URL scheme used to configure the swift filesystem // in secure mode (HTTPS). SchemeSwiftSecure = "swift+https" + // SchemeS3 is the URL scheme used to configure an S3-compatible filesystem. + SchemeS3 = "s3" ) // defaultAdminSecretFileName is the default name of the file containing the diff --git a/pkg/config/config/s3.go b/pkg/config/config/s3.go new file mode 100644 index 00000000000..cb8e72ebd2c --- /dev/null +++ b/pkg/config/config/s3.go @@ -0,0 +1,82 @@ +package config + +import ( + "context" + "fmt" + + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" +) + +var s3Client *minio.Client +var s3BucketPrefix string +var s3Region string + +// InitDefaultS3Connection initializes the default S3 handler. +func InitDefaultS3Connection() error { + return InitS3Connection(config.Fs) +} + +// InitS3Connection initializes the global S3 client connection. This is +// not a thread-safe method. +func InitS3Connection(fs Fs) error { + fsURL := fs.URL + if fsURL.Scheme != SchemeS3 { + return nil + } + + q := fsURL.Query() + endpoint := fsURL.Host + accessKey := q.Get("access_key") + secretKey := q.Get("secret_key") + region := q.Get("region") + useSSL := q.Get("use_ssl") != "false" // default true + + s3BucketPrefix = q.Get("bucket_prefix") + if s3BucketPrefix == "" { + s3BucketPrefix = "cozy" + } + s3Region = region + + var err error + opts := &minio.Options{ + Creds: credentials.NewStaticV4(accessKey, secretKey, ""), + Secure: useSSL, + Region: region, + } + if fs.Transport != nil { + opts.Transport = fs.Transport + } + + s3Client, err = minio.New(endpoint, opts) + if err != nil { + return fmt.Errorf("s3: could not create client: %w", err) + } + + // Verify connectivity by listing buckets + if _, err = s3Client.ListBuckets(context.Background()); err != nil { + log.Errorf("Could not connect to S3 endpoint %s: %s", endpoint, err) + return err + } + + log.Infof("Successfully connected to S3 endpoint %s", endpoint) + return nil +} + +// GetS3Client returns the global S3 client. +func GetS3Client() *minio.Client { + if s3Client == nil { + panic("Called GetS3Client() before InitS3Connection()") + } + return s3Client +} + +// GetS3BucketPrefix returns the configured bucket prefix. +func GetS3BucketPrefix() string { + return s3BucketPrefix +} + +// GetS3Region returns the configured S3 region. +func GetS3Region() string { + return s3Region +} From 9650521959375d9f6088a1164090df0f91eb74ae Mon Sep 17 00:00:00 2001 From: Crash-- Date: Sat, 28 Mar 2026 19:40:47 +0100 Subject: [PATCH 02/20] feat(vfss3): implement complete S3 VFS backend New package model/vfs/vfss3/ implements vfs.VFS for S3-compatible object storage, mirroring the Swift V3 implementation. Key design decisions: - Bucket per orgId: - (fallback "default") - Key prefix per instance: / - CreateFile uses io.Pipe + PutObject goroutine for streaming - Local MD5 computation for multipart upload fallback - Single PUT when ByteSize known (memory-efficient like Swift) - PartSize=5MiB, NumThreads=1 for multipart (bounded memory) Also adds GetOrgID() accessor on Instance. Co-Authored-By: Claude Opus 4.6 (1M context) --- model/instance/instance.go | 18 + model/vfs/vfss3/avatar.go | 116 ++++ model/vfs/vfss3/fsck.go | 256 +++++++ model/vfs/vfss3/impl.go | 1161 ++++++++++++++++++++++++++++++++ model/vfs/vfss3/naming_test.go | 71 ++ model/vfs/vfss3/s3.go | 56 ++ model/vfs/vfss3/thumbs.go | 246 +++++++ 7 files changed, 1924 insertions(+) create mode 100644 model/vfs/vfss3/avatar.go create mode 100644 model/vfs/vfss3/fsck.go create mode 100644 model/vfs/vfss3/impl.go create mode 100644 model/vfs/vfss3/naming_test.go create mode 100644 model/vfs/vfss3/s3.go create mode 100644 model/vfs/vfss3/thumbs.go diff --git a/model/instance/instance.go b/model/instance/instance.go index 6159123f452..a11992fd87c 100644 --- a/model/instance/instance.go +++ b/model/instance/instance.go @@ -16,6 +16,7 @@ import ( "github.com/cozy/cozy-stack/model/permission" "github.com/cozy/cozy-stack/model/vfs" "github.com/cozy/cozy-stack/model/vfs/vfsafero" + "github.com/cozy/cozy-stack/model/vfs/vfss3" "github.com/cozy/cozy-stack/model/vfs/vfsswift" build "github.com/cozy/cozy-stack/pkg/config" "github.com/cozy/cozy-stack/pkg/config/config" @@ -196,6 +197,11 @@ func (i *Instance) GetContextName() string { return i.ContextName } +// GetOrgID returns the organization ID of the instance. +func (i *Instance) GetOrgID() string { + return i.OrgID +} + // SessionSecret returns the session secret. func (i *Instance) SessionSecret() []byte { // The prefix is here to invalidate all the sessions that were created on @@ -261,6 +267,8 @@ func (i *Instance) MakeVFS() error { default: err = ErrInvalidSwiftLayout } + case config.SchemeS3: + i.vfs, err = vfss3.New(i, index, disk, mutex) default: err = fmt.Errorf("instance: unknown storage provider %s", fsURL.Scheme) } @@ -285,6 +293,11 @@ func (i *Instance) AvatarFS() vfs.Avatarer { default: panic(ErrInvalidSwiftLayout) } + case config.SchemeS3: + client := config.GetS3Client() + bucket := vfss3.BucketName(i.GetOrgID(), config.GetS3BucketPrefix()) + keyPrefix := i.DBPrefix() + "/" + return vfss3.NewAvatarFs(client, bucket, keyPrefix) default: panic(fmt.Sprintf("instance: unknown storage provider %s", fsURL.Scheme)) } @@ -309,6 +322,11 @@ func (i *Instance) ThumbsFS() vfs.Thumbser { default: panic(ErrInvalidSwiftLayout) } + case config.SchemeS3: + client := config.GetS3Client() + bucket := vfss3.BucketName(i.GetOrgID(), config.GetS3BucketPrefix()) + keyPrefix := i.DBPrefix() + "/" + return vfss3.NewThumbsFs(client, bucket, keyPrefix) default: panic(fmt.Sprintf("instance: unknown storage provider %s", fsURL.Scheme)) } diff --git a/model/vfs/vfss3/avatar.go b/model/vfs/vfss3/avatar.go new file mode 100644 index 00000000000..87131242359 --- /dev/null +++ b/model/vfs/vfss3/avatar.go @@ -0,0 +1,116 @@ +package vfss3 + +import ( + "context" + "fmt" + "io" + "net/http" + "os" + "time" + + "github.com/cozy/cozy-stack/model/vfs" + "github.com/minio/minio-go/v7" +) + +// NewAvatarFs creates a new avatar filesystem backed by S3. +func NewAvatarFs(client *minio.Client, bucket, keyPrefix string) vfs.Avatarer { + return &avatarS3{ + client: client, + bucket: bucket, + keyPrefix: keyPrefix, + ctx: context.Background(), + } +} + +type avatarS3 struct { + client *minio.Client + bucket string + keyPrefix string + ctx context.Context +} + +func (a *avatarS3) avatarKey() string { + return a.keyPrefix + "avatar" +} + +func (a *avatarS3) CreateAvatar(contentType string) (io.WriteCloser, error) { + key := a.avatarKey() + pr, pw := io.Pipe() + + meta := map[string]string{ + "created-at": time.Now().UTC().Format(time.RFC3339), + } + + errCh := make(chan error, 1) + go func() { + _, err := a.client.PutObject(a.ctx, a.bucket, key, pr, -1, minio.PutObjectOptions{ + ContentType: contentType, + UserMetadata: meta, + }) + errCh <- err + }() + + return &avatarWriter{pw: pw, errCh: errCh}, nil +} + +type avatarWriter struct { + pw *io.PipeWriter + errCh chan error +} + +func (w *avatarWriter) Write(p []byte) (int, error) { + return w.pw.Write(p) +} + +func (w *avatarWriter) Close() error { + if err := w.pw.Close(); err != nil { + return err + } + return <-w.errCh +} + +func (a *avatarS3) DeleteAvatar() error { + err := a.client.RemoveObject(a.ctx, a.bucket, a.avatarKey(), minio.RemoveObjectOptions{}) + if err != nil { + if minio.ToErrorResponse(err).Code == "NoSuchKey" { + return nil + } + return err + } + return nil +} + +func (a *avatarS3) ServeAvatarContent(w http.ResponseWriter, req *http.Request) error { + obj, err := a.client.GetObject(a.ctx, a.bucket, a.avatarKey(), minio.GetObjectOptions{}) + if err != nil { + return wrapS3Err(err) + } + defer obj.Close() + + info, err := obj.Stat() + if err != nil { + return wrapS3Err(err) + } + + t := time.Time{} + if createdAt, ok := info.UserMetadata["Created-At"]; ok && createdAt != "" { + if createdAtTime, err := time.Parse(time.RFC3339, createdAt); err == nil { + t = createdAtTime + } + } + + w.Header().Set("Etag", fmt.Sprintf(`"%s"`, info.ETag)) + w.Header().Set("Content-Type", info.ContentType) + http.ServeContent(w, req, "avatar", t, obj) + return nil +} + +func wrapS3Err(err error) error { + if minio.ToErrorResponse(err).Code == "NoSuchKey" { + return os.ErrNotExist + } + if minio.ToErrorResponse(err).Code == "NoSuchBucket" { + return os.ErrNotExist + } + return err +} diff --git a/model/vfs/vfss3/fsck.go b/model/vfs/vfss3/fsck.go new file mode 100644 index 00000000000..3b505ba5308 --- /dev/null +++ b/model/vfs/vfss3/fsck.go @@ -0,0 +1,256 @@ +package vfss3 + +import ( + "bytes" + "encoding/hex" + "encoding/json" + "errors" + "path" + "strings" + + "github.com/cozy/cozy-stack/model/vfs" + "github.com/cozy/cozy-stack/pkg/consts" + "github.com/cozy/cozy-stack/pkg/couchdb" + "github.com/minio/minio-go/v7" +) + +var errFailFast = errors.New("fail fast") + +func (sfs *s3VFS) Fsck(accumulate func(log *vfs.FsckLog), failFast bool) error { + entries := make(map[string]*vfs.TreeFile, 1024) + tree, err := sfs.BuildTree(func(f *vfs.TreeFile) { + if !f.IsDir { + entries[f.DocID+"/"+f.InternalID] = f + } + }) + if err != nil { + return err + } + if err = sfs.CheckTreeIntegrity(tree, accumulate, failFast); err != nil { + if errors.Is(err, vfs.ErrFsckFailFast) { + return nil + } + return err + } + return sfs.checkFiles(entries, accumulate, failFast) +} + +func (sfs *s3VFS) CheckFilesConsistency(accumulate func(log *vfs.FsckLog), failFast bool) error { + entries := make(map[string]*vfs.TreeFile, 1024) + _, err := sfs.BuildTree(func(f *vfs.TreeFile) { + if !f.IsDir { + entries[f.DocID+"/"+f.InternalID] = f + } + }) + if err != nil { + return err + } + return sfs.checkFiles(entries, accumulate, failFast) +} + +func (sfs *s3VFS) checkFiles( + entries map[string]*vfs.TreeFile, + accumulate func(log *vfs.FsckLog), + failFast bool, +) error { + versions := make(map[string]*vfs.Version, 1024) + err := couchdb.ForeachDocs(sfs, consts.FilesVersions, func(_ string, data json.RawMessage) error { + v := &vfs.Version{} + if erru := json.Unmarshal(data, v); erru != nil { + return erru + } + versions[v.DocID] = v + return nil + }) + if err != nil { + return err + } + + images := make(map[string]struct{}) + err = couchdb.ForeachDocs(sfs, consts.NotesImages, func(_ string, data json.RawMessage) error { + img := make(map[string]interface{}) + if erru := json.Unmarshal(data, &img); erru != nil { + return erru + } + id, _ := img["_id"].(string) + images[id] = struct{}{} + return nil + }) + if err != nil && !couchdb.IsNoDatabaseError(err) { + return err + } + + fileIDs := make(map[string]struct{}, len(entries)) + for _, f := range entries { + fileIDs[f.DocID] = struct{}{} + } + + // List all objects under our key prefix + for obj := range sfs.client.ListObjects(sfs.ctx, sfs.bucket, minio.ListObjectsOptions{ + Prefix: sfs.keyPrefix, + Recursive: true, + }) { + if obj.Err != nil { + return obj.Err + } + + // Strip the key prefix to get the object name + objName := strings.TrimPrefix(obj.Key, sfs.keyPrefix) + + if objName == "avatar" { + continue + } + if strings.HasPrefix(objName, "thumbs/") { + thumbName := strings.TrimPrefix(objName, "thumbs/") + idx := strings.LastIndex(thumbName, "-") + if idx < 0 { + continue + } + thumbName = thumbName[0:idx] // Remove -format suffix + fileID, _ := makeDocID(thumbName) + if _, ok := fileIDs[fileID]; !ok { + if _, ok := images[fileID]; !ok { + accumulate(&vfs.FsckLog{ + Type: vfs.ThumbnailWithNoFile, + IsFile: true, + FileDoc: &vfs.TreeFile{ + DirOrFileDoc: vfs.DirOrFileDoc{ + DirDoc: &vfs.DirDoc{ + Type: consts.FileType, + DocID: fileID, + DocName: obj.Key, + }, + }, + }, + }) + if failFast { + return nil + } + } + } + continue + } + + docID, internalID := makeDocID(objName) + if v, ok := versions[docID+"/"+internalID]; ok { + // ETag from S3 may or may not be an MD5 (multipart uploads use composite ETags). + etag := strings.Trim(obj.ETag, "\"") + if !strings.Contains(etag, "-") { + md5sum, err := hex.DecodeString(etag) + if err == nil { + if !bytes.Equal(md5sum, v.MD5Sum) || v.ByteSize != obj.Size { + accumulate(&vfs.FsckLog{ + Type: vfs.ContentMismatch, + IsVersion: true, + VersionDoc: v, + ContentMismatch: &vfs.FsckContentMismatch{ + SizeFile: obj.Size, + SizeIndex: v.ByteSize, + MD5SumFile: md5sum, + MD5SumIndex: v.MD5Sum, + }, + }) + if failFast { + return nil + } + } + } + } + delete(versions, v.DocID) + continue + } + f, ok := entries[docID+"/"+internalID] + if !ok { + accumulate(&vfs.FsckLog{ + Type: vfs.IndexMissing, + IsFile: true, + FileDoc: objectToFileDoc(obj), + }) + if failFast { + return nil + } + } else { + etag := strings.Trim(obj.ETag, "\"") + if !strings.Contains(etag, "-") { + md5sum, err := hex.DecodeString(etag) + if err == nil { + if !bytes.Equal(md5sum, f.MD5Sum) || f.ByteSize != obj.Size { + accumulate(&vfs.FsckLog{ + Type: vfs.ContentMismatch, + IsFile: true, + FileDoc: f, + ContentMismatch: &vfs.FsckContentMismatch{ + SizeFile: obj.Size, + SizeIndex: f.ByteSize, + MD5SumFile: md5sum, + MD5SumIndex: f.MD5Sum, + }, + }) + if failFast { + return nil + } + } + } + } + delete(entries, docID+"/"+internalID) + } + } + + // entries should contain only data that does not contain an associated + // object in S3. + for _, f := range entries { + accumulate(&vfs.FsckLog{ + Type: vfs.FSMissing, + IsFile: true, + FileDoc: f, + }) + if failFast { + return nil + } + } + + for _, v := range versions { + accumulate(&vfs.FsckLog{ + Type: vfs.FSMissing, + IsVersion: true, + VersionDoc: v, + }) + if failFast { + return nil + } + } + + return nil +} + +func objectToFileDoc(obj minio.ObjectInfo) *vfs.TreeFile { + md5sum, _ := hex.DecodeString(strings.Trim(obj.ETag, "\"")) + name := "unknown" + mime, class := vfs.ExtractMimeAndClass(obj.ContentType) + // Strip any key prefix — we need to find the object name portion + // which is just the last segments of the key. + objName := obj.Key + if idx := strings.Index(objName, "/"); idx >= 0 { + // The first segment is the key prefix (db prefix); skip it + objName = objName[idx+1:] + } + fileID, internalID := makeDocID(objName) + return &vfs.TreeFile{ + DirOrFileDoc: vfs.DirOrFileDoc{ + DirDoc: &vfs.DirDoc{ + Type: consts.FileType, + DocID: fileID, + DocName: name, + DirID: "", + CreatedAt: obj.LastModified, + UpdatedAt: obj.LastModified, + Fullpath: path.Join(vfs.OrphansDirName, name), + }, + ByteSize: obj.Size, + Mime: mime, + Class: class, + MD5Sum: md5sum, + InternalID: internalID, + }, + } +} diff --git a/model/vfs/vfss3/impl.go b/model/vfs/vfss3/impl.go new file mode 100644 index 00000000000..baf88e64b31 --- /dev/null +++ b/model/vfs/vfss3/impl.go @@ -0,0 +1,1161 @@ +// Package vfss3 is the implementation of the Virtual File System by using +// an S3-compatible object storage. The file contents are saved in S3 buckets, +// and the metadata are indexed in CouchDB. +package vfss3 + +import ( + "bytes" + "context" + "crypto/md5" + "encoding/hex" + "errors" + "fmt" + "hash" + "io" + "os" + "regexp" + "strings" + + "github.com/cozy/cozy-stack/model/vfs" + "github.com/cozy/cozy-stack/pkg/config/config" + "github.com/cozy/cozy-stack/pkg/couchdb" + "github.com/cozy/cozy-stack/pkg/lock" + "github.com/cozy/cozy-stack/pkg/logger" + "github.com/cozy/cozy-stack/pkg/utils" + "github.com/gofrs/uuid/v5" + multierror "github.com/hashicorp/go-multierror" + "github.com/minio/minio-go/v7" +) + +type s3VFS struct { + vfs.Indexer + vfs.DiskThresholder + client *minio.Client + cluster int + domain string + prefix string // DBPrefix — used as key prefix in the bucket + contextName string + ctx context.Context + bucket string + keyPrefix string // prefix + "/" + region string + mu lock.ErrorRWLocker + log *logger.Entry +} + +const maxFileSize = 5 << (3 * 10) // 5 GiB + +var bucketNameCleaner = regexp.MustCompile(`[^a-z0-9-]`) + +// sanitizeBucketName produces a valid S3 bucket name component from an arbitrary string. +func sanitizeBucketName(s string) string { + s = strings.ToLower(s) + s = strings.ReplaceAll(s, "_", "-") + s = strings.ReplaceAll(s, ".", "-") + s = bucketNameCleaner.ReplaceAllString(s, "") + // Collapse consecutive hyphens + for strings.Contains(s, "--") { + s = strings.ReplaceAll(s, "--", "-") + } + s = strings.Trim(s, "-") + if len(s) > 37 { + s = s[:37] + } + return s +} + +// BucketName returns the S3 bucket name for a given orgID and bucket prefix. +func BucketName(orgID, bucketPrefix string) string { + if orgID == "" { + orgID = "default" + } + name := bucketPrefix + "-" + sanitizeBucketName(orgID) + if len(name) < 3 { + name = name + strings.Repeat("-", 3-len(name)) + } + if len(name) > 63 { + name = name[:63] + name = strings.TrimRight(name, "-") + } + return name +} + +// MakeObjectKey builds the S3 object key for a given file. +// It reuses the same virtual subfolder structure as Swift V3. +func MakeObjectKey(keyPrefix, docID, internalID string) string { + return keyPrefix + makeObjectName(docID, internalID) +} + +// makeObjectName builds the object name (without key prefix), identical to +// vfsswift.MakeObjectNameV3. +func makeObjectName(docID, internalID string) string { + if len(docID) != 32 || len(internalID) != 16 { + return docID + "/" + internalID + } + return docID[:22] + "/" + docID[22:27] + "/" + docID[27:] + "/" + internalID +} + +func makeDocID(objName string) (string, string) { + if len(objName) != 51 { + parts := strings.SplitN(objName, "/", 2) + if len(parts) < 2 { + return objName, "" + } + return parts[0], parts[1] + } + return objName[:22] + objName[23:28] + objName[29:34], objName[35:] +} + +// NewInternalID returns a random string that can be used as an internal_vfs_id. +func NewInternalID() string { + return utils.RandomString(16) +} + +// New returns a vfs.VFS instance backed by an S3-compatible object store. +func New(db vfs.Prefixer, index vfs.Indexer, disk vfs.DiskThresholder, mu lock.ErrorRWLocker) (vfs.VFS, error) { + client := config.GetS3Client() + bucketPrefix := config.GetS3BucketPrefix() + + orgID := "" + if inst, ok := db.(interface{ GetOrgID() string }); ok { + orgID = inst.GetOrgID() + } + bucket := BucketName(orgID, bucketPrefix) + dbPrefix := db.DBPrefix() + if dbPrefix == "" { + return nil, fmt.Errorf("vfss3: empty DBPrefix") + } + + return &s3VFS{ + Indexer: index, + DiskThresholder: disk, + client: client, + cluster: db.DBCluster(), + domain: db.DomainName(), + prefix: dbPrefix, + contextName: db.GetContextName(), + ctx: context.Background(), + bucket: bucket, + keyPrefix: dbPrefix + "/", + region: config.GetS3Region(), + mu: mu, + log: logger.WithDomain(db.DomainName()).WithNamespace("vfss3"), + }, nil +} + +func (sfs *s3VFS) MaxFileSize() int64 { + return maxFileSize +} + +func (sfs *s3VFS) DBCluster() int { + return sfs.cluster +} + +func (sfs *s3VFS) DBPrefix() string { + return sfs.prefix +} + +func (sfs *s3VFS) DomainName() string { + return sfs.domain +} + +func (sfs *s3VFS) GetContextName() string { + return sfs.contextName +} + +func (sfs *s3VFS) GetIndexer() vfs.Indexer { + return sfs.Indexer +} + +func (sfs *s3VFS) UseSharingIndexer(index vfs.Indexer) vfs.VFS { + return &s3VFS{ + Indexer: index, + DiskThresholder: sfs.DiskThresholder, + client: sfs.client, + cluster: sfs.cluster, + domain: sfs.domain, + prefix: sfs.prefix, + contextName: sfs.contextName, + ctx: context.Background(), + bucket: sfs.bucket, + keyPrefix: sfs.keyPrefix, + region: sfs.region, + mu: sfs.mu, + log: sfs.log, + } +} + +func (sfs *s3VFS) InitFs() error { + if lockerr := sfs.mu.Lock(); lockerr != nil { + return lockerr + } + defer sfs.mu.Unlock() + if err := sfs.Indexer.InitIndex(); err != nil { + return err + } + err := sfs.client.MakeBucket(sfs.ctx, sfs.bucket, minio.MakeBucketOptions{ + Region: sfs.region, + }) + if err != nil { + code := minio.ToErrorResponse(err).Code + if code == "BucketAlreadyOwnedByYou" || code == "BucketAlreadyExists" { + return nil + } + sfs.log.Errorf("Could not create bucket %q: %s", sfs.bucket, err.Error()) + return err + } + sfs.log.Infof("Created bucket %q", sfs.bucket) + return nil +} + +func (sfs *s3VFS) Delete() error { + sfs.log.Infof("Deleting all objects with prefix %q in bucket %q", sfs.keyPrefix, sfs.bucket) + return deletePrefixObjects(sfs.ctx, sfs.client, sfs.bucket, sfs.keyPrefix) +} + +func (sfs *s3VFS) CreateDir(doc *vfs.DirDoc) error { + if lockerr := sfs.mu.Lock(); lockerr != nil { + return lockerr + } + defer sfs.mu.Unlock() + exists, err := sfs.Indexer.DirChildExists(doc.DirID, doc.DocName) + if err != nil { + return err + } + if exists { + return os.ErrExist + } + if doc.ID() == "" { + return sfs.Indexer.CreateDirDoc(doc) + } + return sfs.Indexer.CreateNamedDirDoc(doc) +} + +// putResult is the result sent back from the background PutObject goroutine. +type putResult struct { + info minio.UploadInfo + err error +} + +func (sfs *s3VFS) CreateFile(newdoc, olddoc *vfs.FileDoc, opts ...vfs.CreateOptions) (vfs.File, error) { + if lockerr := sfs.mu.Lock(); lockerr != nil { + return nil, lockerr + } + defer sfs.mu.Unlock() + + newsize, maxsize, capsize, err := vfs.CheckAvailableDiskSpace(sfs, newdoc) + if err != nil { + return nil, err + } + if newsize > maxsize { + return nil, vfs.ErrFileTooBig + } + + if olddoc != nil { + newdoc.SetID(olddoc.ID()) + newdoc.SetRev(olddoc.Rev()) + newdoc.CreatedAt = olddoc.CreatedAt + } + + newpath, err := sfs.Indexer.FilePath(newdoc) + if err != nil { + return nil, err + } + if strings.HasPrefix(newpath, vfs.TrashDirName+"/") { + if !vfs.OptionsAllowCreationInTrash(opts) { + return nil, vfs.ErrParentInTrash + } + } + + if olddoc == nil { + var exists bool + exists, err = sfs.Indexer.DirChildExists(newdoc.DirID, newdoc.DocName) + if err != nil { + return nil, err + } + if exists { + return nil, os.ErrExist + } + } + + if newdoc.DocID == "" { + uid, err := uuid.NewV7() + if err != nil { + return nil, err + } + newdoc.DocID = uid.String() + } + + newdoc.InternalID = NewInternalID() + objKey := MakeObjectKey(sfs.keyPrefix, newdoc.DocID, newdoc.InternalID) + + // Use a pipe: writes go into pw, the PutObject goroutine reads from pr. + pr, pw := io.Pipe() + + uploadSize := newdoc.ByteSize + if uploadSize < 0 { + uploadSize = -1 + } + + resultCh := make(chan putResult, 1) + go func() { + info, err := sfs.client.PutObject(sfs.ctx, sfs.bucket, objKey, pr, uploadSize, minio.PutObjectOptions{ + ContentType: newdoc.Mime, + PartSize: 5 * 1024 * 1024, // 5 MiB + NumThreads: 1, + }) + resultCh <- putResult{info: info, err: err} + }() + + extractor := vfs.NewMetaExtractor(newdoc) + + return &s3FileCreation{ + fs: sfs, + pw: pw, + resultCh: resultCh, + newdoc: newdoc, + olddoc: olddoc, + objKey: objKey, + w: 0, + size: newsize, + maxsize: maxsize, + capsize: capsize, + meta: extractor, + md5H: md5.New(), + }, nil +} + +func (sfs *s3VFS) CopyFile(olddoc, newdoc *vfs.FileDoc) error { + if lockerr := sfs.mu.Lock(); lockerr != nil { + return lockerr + } + defer sfs.mu.Unlock() + + exists, err := sfs.Indexer.DirChildExists(newdoc.DirID, newdoc.DocName) + if err != nil { + return err + } + if exists { + return os.ErrExist + } + + newsize, _, capsize, err := vfs.CheckAvailableDiskSpace(sfs, olddoc) + if err != nil { + return err + } + + uid, err := uuid.NewV7() + if err != nil { + return err + } + newdoc.DocID = uid.String() + newdoc.InternalID = NewInternalID() + + srcKey := MakeObjectKey(sfs.keyPrefix, olddoc.DocID, olddoc.InternalID) + dstKey := MakeObjectKey(sfs.keyPrefix, newdoc.DocID, newdoc.InternalID) + + if _, err := sfs.client.CopyObject(sfs.ctx, + minio.CopyDestOptions{Bucket: sfs.bucket, Object: dstKey}, + minio.CopySrcOptions{Bucket: sfs.bucket, Object: srcKey}, + ); err != nil { + return err + } + if err := sfs.Indexer.CreateNamedFileDoc(newdoc); err != nil { + _ = sfs.client.RemoveObject(sfs.ctx, sfs.bucket, dstKey, minio.RemoveObjectOptions{}) + return err + } + + if capsize > 0 && newsize >= capsize { + vfs.PushDiskQuotaAlert(sfs, true) + } + + return nil +} + +func (sfs *s3VFS) DissociateFile(src, dst *vfs.FileDoc) error { + if lockerr := sfs.mu.Lock(); lockerr != nil { + return lockerr + } + defer sfs.mu.Unlock() + + if src.DirID != dst.DirID || src.DocName != dst.DocName { + exists, err := sfs.Indexer.DirChildExists(dst.DirID, dst.DocName) + if err != nil { + return err + } + if exists { + return os.ErrExist + } + } + + uid, err := uuid.NewV7() + if err != nil { + return err + } + dst.DocID = uid.String() + + srcKey := MakeObjectKey(sfs.keyPrefix, src.DocID, src.InternalID) + dstKey := MakeObjectKey(sfs.keyPrefix, dst.DocID, dst.InternalID) + + if _, err := sfs.client.CopyObject(sfs.ctx, + minio.CopyDestOptions{Bucket: sfs.bucket, Object: dstKey}, + minio.CopySrcOptions{Bucket: sfs.bucket, Object: srcKey}, + ); err != nil { + return err + } + if err := sfs.Indexer.CreateNamedFileDoc(dst); err != nil { + _ = sfs.client.RemoveObject(sfs.ctx, sfs.bucket, dstKey, minio.RemoveObjectOptions{}) + return err + } + + return sfs.destroyFileLocked(src) +} + +func (sfs *s3VFS) DissociateDir(src, dst *vfs.DirDoc) error { + if lockerr := sfs.mu.Lock(); lockerr != nil { + return lockerr + } + defer sfs.mu.Unlock() + + if dst.DirID != src.DirID || dst.DocName != src.DocName { + exists, err := sfs.Indexer.DirChildExists(dst.DirID, dst.DocName) + if err != nil { + return err + } + if exists { + return os.ErrExist + } + } + + if err := sfs.Indexer.CreateDirDoc(dst); err != nil { + return err + } + return sfs.Indexer.DeleteDirDoc(src) +} + +func (sfs *s3VFS) destroyDir(doc *vfs.DirDoc, push func(vfs.TrashJournal) error, onlyContent bool) error { + if lockerr := sfs.mu.Lock(); lockerr != nil { + return lockerr + } + defer sfs.mu.Unlock() + diskUsage, _ := sfs.Indexer.DiskUsage() + files, destroyed, err := sfs.Indexer.DeleteDirDocAndContent(doc, onlyContent) + if err != nil { + return err + } + if len(files) == 0 { + return nil + } + vfs.DiskQuotaAfterDestroy(sfs, diskUsage, destroyed) + ids := make([]string, len(files)) + objNames := make([]string, len(files)) + for i, file := range files { + ids[i] = file.DocID + objNames[i] = MakeObjectKey(sfs.keyPrefix, file.DocID, file.InternalID) + } + return push(vfs.TrashJournal{ + FileIDs: ids, + ObjectNames: objNames, + }) +} + +func (sfs *s3VFS) DestroyDirContent(doc *vfs.DirDoc, push func(vfs.TrashJournal) error) error { + return sfs.destroyDir(doc, push, true) +} + +func (sfs *s3VFS) DestroyDirAndContent(doc *vfs.DirDoc, push func(vfs.TrashJournal) error) error { + return sfs.destroyDir(doc, push, false) +} + +func (sfs *s3VFS) DestroyFile(doc *vfs.FileDoc) error { + if lockerr := sfs.mu.Lock(); lockerr != nil { + return lockerr + } + defer sfs.mu.Unlock() + return sfs.destroyFileLocked(doc) +} + +func (sfs *s3VFS) destroyFileLocked(doc *vfs.FileDoc) error { + diskUsage, _ := sfs.Indexer.DiskUsage() + objNames := []string{ + MakeObjectKey(sfs.keyPrefix, doc.DocID, doc.InternalID), + } + if err := sfs.Indexer.DeleteFileDoc(doc); err != nil { + return err + } + destroyed := doc.ByteSize + if versions, errv := vfs.VersionsFor(sfs, doc.DocID); errv == nil { + for _, v := range versions { + internalID := v.DocID + if parts := strings.SplitN(v.DocID, "/", 2); len(parts) > 1 { + internalID = parts[1] + } + objNames = append(objNames, MakeObjectKey(sfs.keyPrefix, doc.DocID, internalID)) + destroyed += v.ByteSize + } + if err := sfs.Indexer.BatchDeleteVersions(versions); err != nil { + sfs.log.Warnf("DestroyFile failed on BatchDeleteVersions: %s", err) + } + } + if err := deleteObjects(sfs.ctx, sfs.client, sfs.bucket, objNames); err != nil { + sfs.log.Warnf("DestroyFile failed on deleteObjects: %s", err) + } + vfs.DiskQuotaAfterDestroy(sfs, diskUsage, destroyed) + return nil +} + +func (sfs *s3VFS) EnsureErased(journal vfs.TrashJournal) error { + diskUsage, _ := sfs.Indexer.DiskUsage() + objNames := journal.ObjectNames + var errm error + var destroyed int64 + var allVersions []*vfs.Version + for _, fileID := range journal.FileIDs { + versions, err := vfs.VersionsFor(sfs, fileID) + if err != nil { + if !couchdb.IsNoDatabaseError(err) { + sfs.log.Warnf("EnsureErased failed on VersionsFor(%s): %s", fileID, err) + errm = multierror.Append(errm, err) + } + continue + } + for _, v := range versions { + internalID := v.DocID + if parts := strings.SplitN(v.DocID, "/", 2); len(parts) > 1 { + internalID = parts[1] + } + objNames = append(objNames, MakeObjectKey(sfs.keyPrefix, fileID, internalID)) + destroyed += v.ByteSize + } + allVersions = append(allVersions, versions...) + } + if err := sfs.Indexer.BatchDeleteVersions(allVersions); err != nil { + sfs.log.Warnf("EnsureErased failed on BatchDeleteVersions: %s", err) + errm = multierror.Append(errm, err) + } + if err := deleteObjects(sfs.ctx, sfs.client, sfs.bucket, objNames); err != nil { + sfs.log.Warnf("EnsureErased failed on deleteObjects: %s", err) + errm = multierror.Append(errm, err) + } + vfs.DiskQuotaAfterDestroy(sfs, diskUsage, destroyed) + return errm +} + +func (sfs *s3VFS) OpenFile(doc *vfs.FileDoc) (vfs.File, error) { + if lockerr := sfs.mu.RLock(); lockerr != nil { + return nil, lockerr + } + defer sfs.mu.RUnlock() + objKey := MakeObjectKey(sfs.keyPrefix, doc.DocID, doc.InternalID) + obj, err := sfs.client.GetObject(sfs.ctx, sfs.bucket, objKey, minio.GetObjectOptions{}) + if err != nil { + return nil, err + } + // Stat the object to detect if it exists. + if _, err := obj.Stat(); err != nil { + _ = obj.Close() + if minio.ToErrorResponse(err).Code == "NoSuchKey" { + return nil, os.ErrNotExist + } + return nil, err + } + return &s3FileOpen{obj}, nil +} + +func (sfs *s3VFS) OpenFileVersion(doc *vfs.FileDoc, version *vfs.Version) (vfs.File, error) { + if lockerr := sfs.mu.RLock(); lockerr != nil { + return nil, lockerr + } + defer sfs.mu.RUnlock() + internalID := version.DocID + if parts := strings.SplitN(version.DocID, "/", 2); len(parts) > 1 { + internalID = parts[1] + } + objKey := MakeObjectKey(sfs.keyPrefix, doc.DocID, internalID) + obj, err := sfs.client.GetObject(sfs.ctx, sfs.bucket, objKey, minio.GetObjectOptions{}) + if err != nil { + return nil, err + } + if _, err := obj.Stat(); err != nil { + _ = obj.Close() + if minio.ToErrorResponse(err).Code == "NoSuchKey" { + return nil, os.ErrNotExist + } + return nil, err + } + return &s3FileOpen{obj}, nil +} + +func (sfs *s3VFS) ImportFileVersion(version *vfs.Version, content io.ReadCloser) error { + if lockerr := sfs.mu.Lock(); lockerr != nil { + return lockerr + } + defer sfs.mu.Unlock() + + diskQuota := sfs.DiskQuota() + if diskQuota > 0 { + diskUsage, err := sfs.DiskUsage() + if err != nil { + return err + } + if diskUsage+version.ByteSize > diskQuota { + return vfs.ErrFileTooBig + } + } + + parts := strings.SplitN(version.DocID, "/", 2) + if len(parts) != 2 { + return vfs.ErrIllegalFilename + } + objKey := MakeObjectKey(sfs.keyPrefix, parts[0], parts[1]) + + _, err := sfs.client.PutObject(sfs.ctx, sfs.bucket, objKey, content, version.ByteSize, minio.PutObjectOptions{ + ContentType: "application/octet-stream", + SendContentMd5: true, + }) + if errc := content.Close(); err == nil { + err = errc + } + if err != nil { + return err + } + + return sfs.Indexer.CreateVersion(version) +} + +func (sfs *s3VFS) RevertFileVersion(doc *vfs.FileDoc, version *vfs.Version) error { + if lockerr := sfs.mu.Lock(); lockerr != nil { + return lockerr + } + defer sfs.mu.Unlock() + + save := vfs.NewVersion(doc) + if err := sfs.Indexer.CreateVersion(save); err != nil { + return err + } + + newdoc := doc.Clone().(*vfs.FileDoc) + if parts := strings.SplitN(version.DocID, "/", 2); len(parts) > 1 { + newdoc.InternalID = parts[1] + } + vfs.SetMetaFromVersion(newdoc, version) + if err := sfs.Indexer.UpdateFileDoc(doc, newdoc); err != nil { + _ = sfs.Indexer.DeleteVersion(save) + return err + } + + return sfs.Indexer.DeleteVersion(version) +} + +func (sfs *s3VFS) CleanOldVersion(fileID string, v *vfs.Version) error { + if lockerr := sfs.mu.Lock(); lockerr != nil { + return lockerr + } + defer sfs.mu.Unlock() + return sfs.cleanOldVersion(fileID, v) +} + +func (sfs *s3VFS) cleanOldVersion(fileID string, v *vfs.Version) error { + if err := sfs.Indexer.DeleteVersion(v); err != nil { + return err + } + internalID := v.DocID + if parts := strings.SplitN(v.DocID, "/", 2); len(parts) > 1 { + internalID = parts[1] + } + objKey := MakeObjectKey(sfs.keyPrefix, fileID, internalID) + return sfs.client.RemoveObject(sfs.ctx, sfs.bucket, objKey, minio.RemoveObjectOptions{}) +} + +func (sfs *s3VFS) ClearOldVersions() error { + if lockerr := sfs.mu.Lock(); lockerr != nil { + return lockerr + } + defer sfs.mu.Unlock() + diskUsage, _ := sfs.Indexer.DiskUsage() + versions, err := sfs.Indexer.AllVersions() + if err != nil { + return err + } + var objNames []string + var destroyed int64 + for _, v := range versions { + if parts := strings.SplitN(v.DocID, "/", 2); len(parts) > 1 { + objNames = append(objNames, MakeObjectKey(sfs.keyPrefix, parts[0], parts[1])) + } + destroyed += v.ByteSize + } + if err := sfs.Indexer.BatchDeleteVersions(versions); err != nil { + return err + } + vfs.DiskQuotaAfterDestroy(sfs, diskUsage, destroyed) + return deleteObjects(sfs.ctx, sfs.client, sfs.bucket, objNames) +} + +func (sfs *s3VFS) CopyFileFromOtherFS( + newdoc, olddoc *vfs.FileDoc, + srcFS vfs.Fs, + srcDoc *vfs.FileDoc, +) error { + if lockerr := sfs.mu.Lock(); lockerr != nil { + return lockerr + } + defer sfs.mu.Unlock() + + newsize, maxsize, capsize, err := vfs.CheckAvailableDiskSpace(sfs, newdoc) + if err != nil { + return err + } + if newsize > maxsize { + return vfs.ErrFileTooBig + } + + newpath, err := sfs.Indexer.FilePath(newdoc) + if err != nil { + return err + } + if strings.HasPrefix(newpath, vfs.TrashDirName+"/") { + return vfs.ErrParentInTrash + } + + if olddoc == nil { + var exists bool + exists, err = sfs.Indexer.DirChildExists(newdoc.DirID, newdoc.DocName) + if err != nil { + return err + } + if exists { + return os.ErrExist + } + } + + if newdoc.DocID == "" { + uid, err := uuid.NewV7() + if err != nil { + return err + } + newdoc.DocID = uid.String() + } + + newdoc.InternalID = NewInternalID() + + dstKey := MakeObjectKey(sfs.keyPrefix, newdoc.DocID, newdoc.InternalID) + + // Try server-side copy if the source is also an s3VFS on the same client. + if srcS3, ok := srcFS.(*s3VFS); ok { + srcKey := MakeObjectKey(srcS3.keyPrefix, srcDoc.DocID, srcDoc.InternalID) + if _, err := sfs.client.CopyObject(sfs.ctx, + minio.CopyDestOptions{Bucket: sfs.bucket, Object: dstKey}, + minio.CopySrcOptions{Bucket: srcS3.bucket, Object: srcKey}, + ); err != nil { + return err + } + } else { + // Stream from the source FS. + srcFile, err := srcFS.OpenFile(srcDoc) + if err != nil { + return err + } + _, err = sfs.client.PutObject(sfs.ctx, sfs.bucket, dstKey, srcFile, srcDoc.ByteSize, minio.PutObjectOptions{ + ContentType: srcDoc.Mime, + }) + if errc := srcFile.Close(); err == nil { + err = errc + } + if err != nil { + return err + } + } + + var v *vfs.Version + if olddoc != nil { + v = vfs.NewVersion(olddoc) + err = sfs.Indexer.UpdateFileDoc(olddoc, newdoc) + } else { + err = sfs.Indexer.CreateNamedFileDoc(newdoc) + } + if err != nil { + return err + } + + if v != nil { + actionV, toClean, _ := vfs.FindVersionsToClean(sfs, newdoc.DocID, v) + if bytes.Equal(newdoc.MD5Sum, olddoc.MD5Sum) { + actionV = vfs.CleanCandidateVersion + } + if actionV == vfs.KeepCandidateVersion { + if errv := sfs.Indexer.CreateVersion(v); errv != nil { + actionV = vfs.CleanCandidateVersion + } + } + if actionV == vfs.CleanCandidateVersion { + internalID := v.DocID + if parts := strings.SplitN(v.DocID, "/", 2); len(parts) > 1 { + internalID = parts[1] + } + objKey := MakeObjectKey(sfs.keyPrefix, newdoc.DocID, internalID) + _ = sfs.client.RemoveObject(sfs.ctx, sfs.bucket, objKey, minio.RemoveObjectOptions{}) + } + for _, old := range toClean { + _ = sfs.cleanOldVersion(newdoc.DocID, old) + } + } + + if capsize > 0 && newsize >= capsize { + vfs.PushDiskQuotaAlert(sfs, true) + } + + return nil +} + +// UpdateFileDoc calls the indexer UpdateFileDoc function and adds a few checks +// before actually calling this method: +// - locks the filesystem for writing +// - checks in case we have a move operation that the new path is available +// +// @override Indexer.UpdateFileDoc +func (sfs *s3VFS) UpdateFileDoc(olddoc, newdoc *vfs.FileDoc) error { + if lockerr := sfs.mu.Lock(); lockerr != nil { + return lockerr + } + defer sfs.mu.Unlock() + if newdoc.DirID != olddoc.DirID || newdoc.DocName != olddoc.DocName { + exists, err := sfs.Indexer.DirChildExists(newdoc.DirID, newdoc.DocName) + if err != nil { + return err + } + if exists { + return os.ErrExist + } + } + return sfs.Indexer.UpdateFileDoc(olddoc, newdoc) +} + +// UpdateDirDoc calls the indexer UpdateDirDoc function and adds a few checks +// before actually calling this method: +// - locks the filesystem for writing +// - checks that we don't move a directory to one of its descendant +// - checks in case we have a move operation that the new path is available +// +// @override Indexer.UpdateDirDoc +func (sfs *s3VFS) UpdateDirDoc(olddoc, newdoc *vfs.DirDoc) error { + if lockerr := sfs.mu.Lock(); lockerr != nil { + return lockerr + } + defer sfs.mu.Unlock() + if newdoc.DirID != olddoc.DirID || newdoc.DocName != olddoc.DocName { + if strings.HasPrefix(newdoc.Fullpath, olddoc.Fullpath+"/") { + return vfs.ErrForbiddenDocMove + } + exists, err := sfs.Indexer.DirChildExists(newdoc.DirID, newdoc.DocName) + if err != nil { + return err + } + if exists { + return os.ErrExist + } + } + return sfs.Indexer.UpdateDirDoc(olddoc, newdoc) +} + +func (sfs *s3VFS) DirByID(fileID string) (*vfs.DirDoc, error) { + if lockerr := sfs.mu.RLock(); lockerr != nil { + return nil, lockerr + } + defer sfs.mu.RUnlock() + return sfs.Indexer.DirByID(fileID) +} + +func (sfs *s3VFS) DirByPath(name string) (*vfs.DirDoc, error) { + if lockerr := sfs.mu.RLock(); lockerr != nil { + return nil, lockerr + } + defer sfs.mu.RUnlock() + return sfs.Indexer.DirByPath(name) +} + +func (sfs *s3VFS) FileByID(fileID string) (*vfs.FileDoc, error) { + if lockerr := sfs.mu.RLock(); lockerr != nil { + return nil, lockerr + } + defer sfs.mu.RUnlock() + return sfs.Indexer.FileByID(fileID) +} + +func (sfs *s3VFS) FileByPath(name string) (*vfs.FileDoc, error) { + if lockerr := sfs.mu.RLock(); lockerr != nil { + return nil, lockerr + } + defer sfs.mu.RUnlock() + return sfs.Indexer.FileByPath(name) +} + +func (sfs *s3VFS) FilePath(doc *vfs.FileDoc) (string, error) { + if lockerr := sfs.mu.RLock(); lockerr != nil { + return "", lockerr + } + defer sfs.mu.RUnlock() + return sfs.Indexer.FilePath(doc) +} + +func (sfs *s3VFS) DirOrFileByID(fileID string) (*vfs.DirDoc, *vfs.FileDoc, error) { + if lockerr := sfs.mu.RLock(); lockerr != nil { + return nil, nil, lockerr + } + defer sfs.mu.RUnlock() + return sfs.Indexer.DirOrFileByID(fileID) +} + +func (sfs *s3VFS) DirOrFileByPath(name string) (*vfs.DirDoc, *vfs.FileDoc, error) { + if lockerr := sfs.mu.RLock(); lockerr != nil { + return nil, nil, lockerr + } + defer sfs.mu.RUnlock() + return sfs.Indexer.DirOrFileByPath(name) +} + +// s3FileCreation represents a file open for writing. It is used to create +// a file or to modify the content of a file. +type s3FileCreation struct { + fs *s3VFS + pw *io.PipeWriter + resultCh chan putResult + newdoc *vfs.FileDoc + olddoc *vfs.FileDoc + objKey string + w int64 + size int64 + maxsize int64 + capsize int64 + meta *vfs.MetaExtractor + md5H hash.Hash + err error +} + +func (f *s3FileCreation) Read(p []byte) (int, error) { + return 0, os.ErrInvalid +} + +func (f *s3FileCreation) ReadAt(p []byte, off int64) (int, error) { + return 0, os.ErrInvalid +} + +func (f *s3FileCreation) Seek(offset int64, whence int) (int64, error) { + return 0, os.ErrInvalid +} + +func (f *s3FileCreation) Write(p []byte) (int, error) { + if f.err != nil { + return 0, f.err + } + + if f.meta != nil { + if _, err := (*f.meta).Write(p); err != nil && !errors.Is(err, io.ErrClosedPipe) { + (*f.meta).Abort(err) + f.meta = nil + } + } + + // Write to local MD5 hash + _, _ = f.md5H.Write(p) + + n, err := f.pw.Write(p) + if err != nil { + f.err = err + return n, err + } + + f.w += int64(n) + if f.maxsize >= 0 && f.w > f.maxsize { + f.err = vfs.ErrFileTooBig + _ = f.pw.CloseWithError(f.err) + return n, f.err + } + + if f.size >= 0 && f.w > f.size { + f.err = vfs.ErrContentLengthMismatch + _ = f.pw.CloseWithError(f.err) + return n, f.err + } + + return n, nil +} + +func (f *s3FileCreation) Close() (err error) { + defer func() { + if err != nil { + // Remove the object from S3 if an error occurred + _ = f.fs.client.RemoveObject(f.fs.ctx, f.fs.bucket, f.objKey, minio.RemoveObjectOptions{}) + // If an error has occurred when creating a new file, we should + // also delete the file from the index. + if f.olddoc == nil { + _ = f.fs.Indexer.DeleteFileDoc(f.newdoc) + } + } + }() + + // Close the pipe writer to signal EOF to PutObject + if err = f.pw.Close(); err != nil { + if f.meta != nil { + (*f.meta).Abort(err) + f.meta = nil + } + if f.err == nil { + f.err = err + } + } + + // Wait for the PutObject goroutine to finish + result := <-f.resultCh + + if result.err != nil { + if f.meta != nil { + (*f.meta).Abort(result.err) + f.meta = nil + } + if f.err == nil { + f.err = result.err + } + } + + newdoc, olddoc, written := f.newdoc, f.olddoc, f.w + + if f.meta != nil { + if errc := (*f.meta).Close(); errc == nil { + vfs.MergeMetadata(newdoc, (*f.meta).Result()) + } + } + + if f.err != nil { + return f.err + } + + // Extract MD5 from ETag if available, otherwise use our local md5H + if newdoc.MD5Sum == nil { + etag := result.info.ETag + // ETags may be double-quoted + etag = strings.TrimPrefix(etag, "\"") + etag = strings.TrimSuffix(etag, "\"") + // For multipart uploads, ETag contains a dash — use local MD5 + if strings.Contains(etag, "-") || etag == "" { + newdoc.MD5Sum = f.md5H.Sum(nil) + } else { + md5sum, err := hex.DecodeString(etag) + if err != nil { + newdoc.MD5Sum = f.md5H.Sum(nil) + } else { + newdoc.MD5Sum = md5sum + } + } + } + + if f.size < 0 { + newdoc.ByteSize = written + } + + if newdoc.ByteSize != written { + return vfs.ErrContentLengthMismatch + } + + lockerr := f.fs.mu.Lock() + if lockerr != nil { + return lockerr + } + defer f.fs.mu.Unlock() + + // Check again that a file with the same path does not exist. It can happen + // when the same file is uploaded twice in parallel. + if olddoc == nil { + exists, err := f.fs.Indexer.DirChildExists(newdoc.DirID, newdoc.DocName) + if err != nil { + return err + } + if exists { + return os.ErrExist + } + } + + var newpath string + newpath, err = f.fs.Indexer.FilePath(newdoc) + if err != nil { + return err + } + newdoc.Trashed = strings.HasPrefix(newpath, vfs.TrashDirName+"/") + + var v *vfs.Version + if olddoc != nil { + v = vfs.NewVersion(olddoc) + err = f.fs.Indexer.UpdateFileDoc(olddoc, newdoc) + } else if newdoc.ID() == "" { + err = f.fs.Indexer.CreateFileDoc(newdoc) + } else { + err = f.fs.Indexer.CreateNamedFileDoc(newdoc) + } + if err != nil { + return err + } + + if v != nil { + actionV, toClean, _ := vfs.FindVersionsToClean(f.fs, newdoc.DocID, v) + if bytes.Equal(newdoc.MD5Sum, olddoc.MD5Sum) { + actionV = vfs.CleanCandidateVersion + } + if actionV == vfs.KeepCandidateVersion { + if errv := f.fs.Indexer.CreateVersion(v); errv != nil { + actionV = vfs.CleanCandidateVersion + } + } + if actionV == vfs.CleanCandidateVersion { + internalID := v.DocID + if parts := strings.SplitN(v.DocID, "/", 2); len(parts) > 1 { + internalID = parts[1] + } + objKey := MakeObjectKey(f.fs.keyPrefix, newdoc.DocID, internalID) + if err := f.fs.client.RemoveObject(f.fs.ctx, f.fs.bucket, objKey, minio.RemoveObjectOptions{}); err != nil { + f.fs.log.Warnf("Could not delete previous version %q: %s", objKey, err.Error()) + } + } + for _, old := range toClean { + if err := f.fs.cleanOldVersion(newdoc.DocID, old); err != nil { + f.fs.log.Warnf("Could not delete old versions for %s: %s", newdoc.DocID, err.Error()) + } + } + } + + if f.capsize > 0 && f.size >= f.capsize { + vfs.PushDiskQuotaAlert(f.fs, true) + } + + return nil +} + +// s3FileOpen represents a file open for reading. +type s3FileOpen struct { + obj *minio.Object +} + +func (f *s3FileOpen) Read(p []byte) (int, error) { + return f.obj.Read(p) +} + +func (f *s3FileOpen) ReadAt(p []byte, off int64) (int, error) { + return f.obj.ReadAt(p, off) +} + +func (f *s3FileOpen) Seek(offset int64, whence int) (int64, error) { + return f.obj.Seek(offset, whence) +} + +func (f *s3FileOpen) Write(p []byte) (int, error) { + return 0, os.ErrInvalid +} + +func (f *s3FileOpen) Close() error { + return f.obj.Close() +} + +var ( + _ vfs.VFS = &s3VFS{} + _ vfs.File = &s3FileCreation{} + _ vfs.File = &s3FileOpen{} +) diff --git a/model/vfs/vfss3/naming_test.go b/model/vfs/vfss3/naming_test.go new file mode 100644 index 00000000000..1201dc82988 --- /dev/null +++ b/model/vfs/vfss3/naming_test.go @@ -0,0 +1,71 @@ +package vfss3 + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestSanitizeBucketName(t *testing.T) { + tests := []struct { + input string + expected string + }{ + {"production", "production"}, + {"my_context", "my-context"}, + {"My.Context", "my-context"}, + {"UPPERCASE", "uppercase"}, + {"with spaces!", "withspaces"}, + {"a--b--c", "a-b-c"}, + {"-leading-trailing-", "leading-trailing"}, + {"very-long-name-that-exceeds-the-maximum-allowed-length", "very-long-name-that-exceeds-the-maxim"}, + {"", ""}, + } + + for _, tt := range tests { + t.Run(tt.input, func(t *testing.T) { + assert.Equal(t, tt.expected, sanitizeBucketName(tt.input)) + }) + } +} + +func TestBucketName(t *testing.T) { + tests := []struct { + orgID string + bucketPrefix string + expected string + }{ + {"org-123", "cozy", "cozy-org-123"}, + {"", "cozy", "cozy-default"}, + {"My_Org", "cozy", "cozy-my-org"}, + {"org.example.com", "cozy", "cozy-org-example-com"}, + } + + for _, tt := range tests { + t.Run(tt.orgID, func(t *testing.T) { + assert.Equal(t, tt.expected, BucketName(tt.orgID, tt.bucketPrefix)) + }) + } +} + +func TestMakeObjectKey(t *testing.T) { + // Standard 32-char docID and 16-char internalID + key := MakeObjectKey("alice.example.com/", "abcdefghijklmnopqrstuvwxyz012345", "0123456789abcdef") + assert.Equal(t, "alice.example.com/abcdefghijklmnopqrstuv/wxyz0/12345/0123456789abcdef", key) + + // Non-standard lengths + key = MakeObjectKey("alice.example.com/", "short", "id") + assert.Equal(t, "alice.example.com/short/id", key) +} + +func TestMakeDocID(t *testing.T) { + // Standard 51-char object name + docID, internalID := makeDocID("abcdefghijklmnopqrstuv/wxyz0/12345/0123456789abcdef") + assert.Equal(t, "abcdefghijklmnopqrstuvwxyz012345", docID) + assert.Equal(t, "0123456789abcdef", internalID) + + // Non-standard + docID, internalID = makeDocID("short/id") + assert.Equal(t, "short", docID) + assert.Equal(t, "id", internalID) +} diff --git a/model/vfs/vfss3/s3.go b/model/vfs/vfss3/s3.go new file mode 100644 index 00000000000..96316037a62 --- /dev/null +++ b/model/vfs/vfss3/s3.go @@ -0,0 +1,56 @@ +package vfss3 + +import ( + "context" + + multierror "github.com/hashicorp/go-multierror" + "github.com/minio/minio-go/v7" +) + +// maxNbFilesToDelete is the max number of objects per RemoveObjects batch. +const maxNbFilesToDelete = 1000 + +// deletePrefixObjects deletes all objects in a bucket under a given prefix. +func deletePrefixObjects(ctx context.Context, client *minio.Client, bucket, prefix string) error { + objectsCh := make(chan minio.ObjectInfo) + var listErr error + go func() { + defer close(objectsCh) + for obj := range client.ListObjects(ctx, bucket, minio.ListObjectsOptions{ + Prefix: prefix, + Recursive: true, + }) { + if obj.Err != nil { + listErr = obj.Err + return + } + objectsCh <- obj + } + }() + rmErr := removeObjects(ctx, client, bucket, objectsCh) + if listErr != nil { + return listErr + } + return rmErr +} + +// deleteObjects deletes a list of named objects from a bucket. +func deleteObjects(ctx context.Context, client *minio.Client, bucket string, objNames []string) error { + if len(objNames) == 0 { + return nil + } + objectsCh := make(chan minio.ObjectInfo, len(objNames)) + for _, name := range objNames { + objectsCh <- minio.ObjectInfo{Key: name} + } + close(objectsCh) + return removeObjects(ctx, client, bucket, objectsCh) +} + +func removeObjects(ctx context.Context, client *minio.Client, bucket string, objectsCh <-chan minio.ObjectInfo) error { + var errm error + for err := range client.RemoveObjects(ctx, bucket, objectsCh, minio.RemoveObjectsOptions{}) { + errm = multierror.Append(errm, err.Err) + } + return errm +} diff --git a/model/vfs/vfss3/thumbs.go b/model/vfs/vfss3/thumbs.go new file mode 100644 index 00000000000..22a4a6eec15 --- /dev/null +++ b/model/vfs/vfss3/thumbs.go @@ -0,0 +1,246 @@ +package vfss3 + +import ( + "bytes" + "context" + "encoding/hex" + "fmt" + "io" + "net/http" + "os" + "time" + + "github.com/cozy/cozy-stack/model/vfs" + "github.com/cozy/cozy-stack/pkg/consts" + "github.com/cozy/cozy-stack/pkg/logger" + "github.com/labstack/echo/v4" + "github.com/minio/minio-go/v7" +) + +var unixEpochZero = time.Time{} + +// NewThumbsFs creates a new thumbnail filesystem backed by S3. +func NewThumbsFs(client *minio.Client, bucket, keyPrefix string) vfs.Thumbser { + return &thumbsS3{ + client: client, + bucket: bucket, + keyPrefix: keyPrefix, + ctx: context.Background(), + } +} + +type thumbsS3 struct { + client *minio.Client + bucket string + keyPrefix string + ctx context.Context +} + +type s3Thumb struct { + pw *io.PipeWriter + errCh chan error + client *minio.Client + bucket string + name string + ctx context.Context +} + +func (t *s3Thumb) Write(p []byte) (int, error) { + return t.pw.Write(p) +} + +func (t *s3Thumb) Commit() error { + if err := t.pw.Close(); err != nil { + return err + } + return <-t.errCh +} + +func (t *s3Thumb) Abort() error { + // Close the pipe with an error to cancel the PutObject goroutine. + errc := t.pw.CloseWithError(fmt.Errorf("thumbnail creation aborted")) + // Drain the errCh so the goroutine is not leaked. + <-t.errCh + // Try to remove the possibly partially written object. + errd := t.client.RemoveObject(t.ctx, t.bucket, t.name, minio.RemoveObjectOptions{}) + if errd != nil && minio.ToErrorResponse(errd).Code == "NoSuchKey" { + errd = nil + } + // Write an empty marker object to indicate that the thumbnail generation failed. + _, errp := t.client.PutObject(t.ctx, t.bucket, t.name, + bytes.NewReader(nil), 0, minio.PutObjectOptions{ + ContentType: echo.MIMEOctetStream, + }) + if errc != nil { + return errc + } + if errd != nil { + return errd + } + return errp +} + +func (ts *thumbsS3) createThumbFile(name, contentType string, meta map[string]string) (vfs.ThumbFiler, error) { + pr, pw := io.Pipe() + + errCh := make(chan error, 1) + go func() { + _, err := ts.client.PutObject(ts.ctx, ts.bucket, name, pr, -1, minio.PutObjectOptions{ + ContentType: contentType, + UserMetadata: meta, + }) + errCh <- err + }() + + return &s3Thumb{ + pw: pw, + errCh: errCh, + client: ts.client, + bucket: ts.bucket, + name: name, + ctx: ts.ctx, + }, nil +} + +func (ts *thumbsS3) CreateThumb(img *vfs.FileDoc, format string) (vfs.ThumbFiler, error) { + name := ts.makeName(img.ID(), format) + meta := map[string]string{ + "file-md5": hex.EncodeToString(img.MD5Sum), + } + return ts.createThumbFile(name, "image/jpeg", meta) +} + +func (ts *thumbsS3) ThumbExists(img *vfs.FileDoc, format string) (bool, error) { + name := ts.makeName(img.ID(), format) + info, err := ts.client.StatObject(ts.ctx, ts.bucket, name, minio.StatObjectOptions{}) + if err != nil { + if minio.ToErrorResponse(err).Code == "NoSuchKey" { + return false, nil + } + return false, err + } + if md5str, ok := info.UserMetadata["File-Md5"]; ok && md5str != "" { + md5sum, err := hex.DecodeString(md5str) + if err == nil && !bytes.Equal(md5sum, img.MD5Sum) { + return false, nil + } + } + return true, nil +} + +func (ts *thumbsS3) RemoveThumbs(img *vfs.FileDoc, formats []string) error { + objNames := make([]string, len(formats)) + for i, format := range formats { + objNames[i] = ts.makeName(img.ID(), format) + } + return deleteObjects(ts.ctx, ts.client, ts.bucket, objNames) +} + +func (ts *thumbsS3) ServeThumbContent(w http.ResponseWriter, req *http.Request, img *vfs.FileDoc, format string) error { + name := ts.makeName(img.ID(), format) + obj, err := ts.client.GetObject(ts.ctx, ts.bucket, name, minio.GetObjectOptions{}) + if err != nil { + return wrapS3Err(err) + } + defer obj.Close() + + info, err := obj.Stat() + if err != nil { + return wrapS3Err(err) + } + + if info.ContentType == echo.MIMEOctetStream { + // We have some old images where the thumbnail has not been correctly + // saved. We should delete the thumbnail to allow another try. + if info.Size > 0 { + _ = ts.RemoveThumbs(img, vfs.ThumbnailFormatNames) + return os.ErrNotExist + } + // Image magick has failed to generate a thumbnail, and retrying would + // be useless. + return os.ErrInvalid + } + + w.Header().Set("Etag", fmt.Sprintf(`"%s"`, info.ETag)) + w.Header().Set("Content-Type", info.ContentType) + http.ServeContent(w, req, name, unixEpochZero, obj) + return nil +} + +func (ts *thumbsS3) CreateNoteThumb(id, mime, format string) (vfs.ThumbFiler, error) { + name := ts.makeName(id, format) + return ts.createThumbFile(name, mime, nil) +} + +func (ts *thumbsS3) OpenNoteThumb(id, format string) (io.ReadCloser, error) { + name := ts.makeName(id, format) + obj, err := ts.client.GetObject(ts.ctx, ts.bucket, name, minio.GetObjectOptions{}) + if err != nil { + return nil, wrapS3Err(err) + } + // Stat to verify the object actually exists (GetObject doesn't fail on missing keys). + if _, err := obj.Stat(); err != nil { + obj.Close() + if minio.ToErrorResponse(err).Code == "NoSuchKey" { + return nil, os.ErrNotExist + } + return nil, err + } + return obj, nil +} + +func (ts *thumbsS3) RemoveNoteThumb(id string, formats []string) error { + objNames := make([]string, len(formats)) + for i, format := range formats { + objNames[i] = ts.makeName(id, format) + } + err := deleteObjects(ts.ctx, ts.client, ts.bucket, objNames) + if err != nil { + logger.WithNamespace("vfss3").Infof("Cannot remove note thumbs: %s", err) + } + return err +} + +func (ts *thumbsS3) ServeNoteThumbContent(w http.ResponseWriter, req *http.Request, id string) error { + name := ts.makeName(id, consts.NoteImageThumbFormat) + obj, err := ts.client.GetObject(ts.ctx, ts.bucket, name, minio.GetObjectOptions{}) + if err != nil { + return wrapS3Err(err) + } + + info, err := obj.Stat() + if err != nil { + obj.Close() + // Try the original format as fallback. + name = ts.makeName(id, consts.NoteImageOriginalFormat) + obj, err = ts.client.GetObject(ts.ctx, ts.bucket, name, minio.GetObjectOptions{}) + if err != nil { + return wrapS3Err(err) + } + info, err = obj.Stat() + if err != nil { + obj.Close() + return wrapS3Err(err) + } + } + defer obj.Close() + + w.Header().Set("Etag", fmt.Sprintf(`"%s"`, info.ETag)) + w.Header().Set("Content-Type", info.ContentType) + http.ServeContent(w, req, name, unixEpochZero, obj) + return nil +} + +func (ts *thumbsS3) makeName(imgID string, format string) string { + return ts.keyPrefix + fmt.Sprintf("thumbs/%s-%s", makeThumbObjectName(imgID), format) +} + +// makeThumbObjectName builds a virtual subfolder structure for thumbnails. +// It splits the 32-char ID into three parts to avoid a flat hierarchy. +// This is the same logic as vfsswift.MakeObjectName (without internalID). +func makeThumbObjectName(docID string) string { + if len(docID) != 32 { + return docID + } + return docID[:22] + "/" + docID[22:27] + "/" + docID[27:] +} From c287f16c50863fb7fcc204ab62da2e856fa04768 Mon Sep 17 00:00:00 2001 From: Crash-- Date: Sat, 28 Mar 2026 19:41:05 +0100 Subject: [PATCH 03/20] feat: wire S3 backend into all dispatch points Add case config.SchemeS3 to every storage dispatch switch: - MakeVFS, AvatarFS, ThumbsFS (instance.go) - Copier, AppsFileServer, KonnectorsFileServer (apps.go) - SystemArchiver (archiver.go) - SystemCache (cache.go) - InitDynamicAssetFS (fs.go) - NewCapabilities (capabilities.go) New S3 implementations: - pkg/appfs/s3.go: S3 Copier and FileServer for app installation - pkg/assets/dynamic/impl_s3.go: S3 dynamic assets storage - S3 preview cache and archiver in their respective files Co-Authored-By: Claude Opus 4.6 (1M context) --- model/app/apps.go | 12 ++ model/move/archiver.go | 84 ++++++++ pkg/appfs/s3.go | 353 ++++++++++++++++++++++++++++++++++ pkg/assets/dynamic/fs.go | 6 + pkg/assets/dynamic/impl_s3.go | 126 ++++++++++++ pkg/previewfs/cache.go | 80 ++++++++ web/settings/capabilities.go | 2 + web/settings/instance.go | 4 +- 8 files changed, 665 insertions(+), 2 deletions(-) create mode 100644 pkg/appfs/s3.go create mode 100644 pkg/assets/dynamic/impl_s3.go diff --git a/model/app/apps.go b/model/app/apps.go index 34c099a869e..6e4ecb4c74c 100644 --- a/model/app/apps.go +++ b/model/app/apps.go @@ -156,6 +156,10 @@ func Copier(appsType consts.AppType, inst *instance.Instance) appfs.Copier { return appfs.NewAferoCopier(baseFS) case config.SchemeSwift, config.SchemeSwiftSecure: return appfs.NewSwiftCopier(config.GetSwiftConnection(), appsType) + case config.SchemeS3: + client := config.GetS3Client() + bucket := appfs.S3AppsBucket(config.GetS3BucketPrefix(), appsType) + return appfs.NewS3Copier(client, bucket) default: panic(fmt.Sprintf("instance: unknown storage provider %s", fsURL.Scheme)) } @@ -175,6 +179,10 @@ func AppsFileServer(i *instance.Instance) appfs.FileServer { return appfs.NewAferoFileServer(baseFS, nil) case config.SchemeSwift, config.SchemeSwiftSecure: return appfs.NewSwiftFileServer(config.GetSwiftConnection(), consts.WebappType) + case config.SchemeS3: + client := config.GetS3Client() + bucket := appfs.S3AppsBucket(config.GetS3BucketPrefix(), consts.WebappType) + return appfs.NewS3FileServer(client, bucket) default: panic(fmt.Sprintf("instance: unknown storage provider %s", fsURL.Scheme)) } @@ -194,6 +202,10 @@ func KonnectorsFileServer(i *instance.Instance) appfs.FileServer { return appfs.NewAferoFileServer(baseFS, nil) case config.SchemeSwift, config.SchemeSwiftSecure: return appfs.NewSwiftFileServer(config.GetSwiftConnection(), consts.KonnectorType) + case config.SchemeS3: + client := config.GetS3Client() + bucket := appfs.S3AppsBucket(config.GetS3BucketPrefix(), consts.KonnectorType) + return appfs.NewS3FileServer(client, bucket) default: panic(fmt.Sprintf("instance: unknown storage provider %s", fsURL.Scheme)) } diff --git a/model/move/archiver.go b/model/move/archiver.go index be36ccb9b7e..03bb896ac3a 100644 --- a/model/move/archiver.go +++ b/model/move/archiver.go @@ -14,6 +14,7 @@ import ( "github.com/cozy/cozy-stack/pkg/config/config" "github.com/cozy/cozy-stack/pkg/crypto" multierror "github.com/hashicorp/go-multierror" + "github.com/minio/minio-go/v7" "github.com/ncw/swift/v2" "github.com/spf13/afero" ) @@ -45,6 +46,8 @@ func SystemArchiver() Archiver { return newAferoArchiver(fs) case config.SchemeSwift, config.SchemeSwiftSecure: return newSwiftArchiver() + case config.SchemeS3: + return newS3Archiver() default: panic(fmt.Errorf("exports: unknown storage provider %s", fsURL.Scheme)) } @@ -149,3 +152,84 @@ func (a *switfArchiver) RemoveArchives(exportDocs []*ExportDoc) error { } return nil } + +func newS3Archiver() Archiver { + client := config.GetS3Client() + bucket := config.GetS3BucketPrefix() + "-exports" + return &s3Archiver{ + client: client, + bucket: bucket, + ctx: context.Background(), + } +} + +type s3Archiver struct { + client *minio.Client + bucket string + ctx context.Context +} + +func (a *s3Archiver) ensureBucket() error { + err := a.client.MakeBucket(a.ctx, a.bucket, minio.MakeBucketOptions{}) + if err != nil { + code := minio.ToErrorResponse(err).Code + if code == "BucketAlreadyOwnedByYou" || code == "BucketAlreadyExists" { + return nil + } + return err + } + return nil +} + +func (a *s3Archiver) OpenArchive(inst *instance.Instance, exportDoc *ExportDoc) (io.ReadCloser, error) { + objectName := exportDoc.Domain + "/" + exportDoc.ID() + obj, err := a.client.GetObject(a.ctx, a.bucket, objectName, minio.GetObjectOptions{}) + if err != nil { + return nil, err + } + // Verify the object exists by calling Stat; GetObject itself does not + // perform a network request until Read is called, and Stat triggers + // a HEAD that surfaces NoSuchKey early. + if _, err := obj.Stat(); err != nil { + obj.Close() + return nil, err + } + return obj, nil +} + +func (a *s3Archiver) CreateArchive(exportDoc *ExportDoc) (io.WriteCloser, error) { + if err := a.ensureBucket(); err != nil { + return nil, err + } + + objectName := exportDoc.Domain + "/" + exportDoc.ID() + pr, pw := io.Pipe() + + go func() { + _, err := a.client.PutObject(a.ctx, a.bucket, objectName, + pr, -1, + minio.PutObjectOptions{ContentType: "application/tar+gzip"}) + // Close the read side so that any error is propagated to the writer. + pr.CloseWithError(err) + }() + + return pw, nil +} + +func (a *s3Archiver) RemoveArchives(exportDocs []*ExportDoc) error { + if len(exportDocs) == 0 { + return nil + } + + objectsCh := make(chan minio.ObjectInfo, len(exportDocs)) + for _, e := range exportDocs { + objectsCh <- minio.ObjectInfo{Key: e.Domain + "/" + e.ID()} + } + close(objectsCh) + + var errm error + for e := range a.client.RemoveObjects(a.ctx, a.bucket, objectsCh, minio.RemoveObjectsOptions{}) { + errm = multierror.Append(errm, e.Err) + } + return errm +} diff --git a/pkg/appfs/s3.go b/pkg/appfs/s3.go new file mode 100644 index 00000000000..9dcf06b85f0 --- /dev/null +++ b/pkg/appfs/s3.go @@ -0,0 +1,353 @@ +package appfs + +import ( + "bytes" + "compress/gzip" + "context" + "errors" + "fmt" + "io" + "mime" + "net/http" + "os" + "path" + "strconv" + "strings" + + "github.com/andybalholm/brotli" + "github.com/cozy/cozy-stack/pkg/consts" + "github.com/cozy/cozy-stack/pkg/filetype" + + web_utils "github.com/cozy/cozy-stack/pkg/utils" + "github.com/labstack/echo/v4" + "github.com/minio/minio-go/v7" +) + +// s3Copier implements the Copier interface backed by S3. +type s3Copier struct { + client *minio.Client + bucket string + appObj string + started bool + objectNames []string + ctx context.Context +} + +// NewS3Copier creates a Copier that stores app files in S3. +func NewS3Copier(client *minio.Client, bucket string) Copier { + return &s3Copier{ + client: client, + bucket: bucket, + ctx: context.Background(), + } +} + +func (f *s3Copier) Exist(slug, version, shasum string) (bool, error) { + f.appObj = path.Join(slug, version) + if shasum != "" { + f.appObj += "-" + shasum + } + _, err := f.client.StatObject(f.ctx, f.bucket, f.appObj, minio.StatObjectOptions{}) + if err == nil { + return true, nil + } + if isS3NotFound(err) { + return false, nil + } + return false, err +} + +func (f *s3Copier) Start(slug, version, shasum string) (bool, error) { + exist, err := f.Exist(slug, version, shasum) + if err != nil || exist { + return exist, err + } + + if err := ensureS3Bucket(f.ctx, f.client, f.bucket); err != nil { + return false, err + } + + f.objectNames = []string{} + f.started = true + return false, nil +} + +func (f *s3Copier) Copy(stat os.FileInfo, src io.Reader) error { + if !f.started { + panic("copier should call Start() before Copy()") + } + + // Write directly to the final location (appObj/filename). + objName := path.Join(f.appObj, stat.Name()) + + contentType := filetype.ByExtension(path.Ext(stat.Name())) + if contentType == "" { + contentType, src = filetype.FromReader(src) + } + + // Compress with brotli. + var buf bytes.Buffer + bw := brotli.NewWriter(&buf) + if _, err := io.Copy(bw, src); err != nil { + return err + } + if err := bw.Close(); err != nil { + return err + } + + meta := map[string]string{ + "X-Content-Encoding": "br", + "Original-Content-Length": strconv.FormatInt(stat.Size(), 10), + } + + f.objectNames = append(f.objectNames, objName) + _, err := f.client.PutObject(f.ctx, f.bucket, objName, + bytes.NewReader(buf.Bytes()), int64(buf.Len()), + minio.PutObjectOptions{ + ContentType: contentType, + UserMetadata: meta, + }) + return err +} + +func (f *s3Copier) Abort() error { + return s3DeleteObjects(f.ctx, f.client, f.bucket, f.objectNames) +} + +func (f *s3Copier) Commit() (err error) { + // Create the marker object that signals the version is complete. + _, err = f.client.PutObject(f.ctx, f.bucket, f.appObj, + bytes.NewReader(nil), 0, minio.PutObjectOptions{ + ContentType: "text/plain", + }) + return err +} + +// s3Server implements the FileServer interface backed by S3. +type s3Server struct { + client *minio.Client + bucket string + ctx context.Context +} + +// NewS3FileServer creates a FileServer that serves app files from S3. +func NewS3FileServer(client *minio.Client, bucket string) FileServer { + return &s3Server{ + client: client, + bucket: bucket, + ctx: context.Background(), + } +} + +func (s *s3Server) Open(slug, version, shasum, file string) (io.ReadCloser, error) { + objName := s.makeObjectName(slug, version, shasum, file) + obj, err := s.client.GetObject(s.ctx, s.bucket, objName, minio.GetObjectOptions{}) + if err != nil { + return nil, wrapS3ErrNotExist(err) + } + info, err := obj.Stat() + if err != nil { + obj.Close() + return nil, wrapS3ErrNotExist(err) + } + contentEncoding := info.UserMetadata["X-Content-Encoding"] + if contentEncoding == "br" { + return newBrotliReadCloser(obj) + } else if contentEncoding == "gzip" { + return newGzipReadCloser(obj) + } + return obj, nil +} + +func (s *s3Server) ServeFileContent(w http.ResponseWriter, req *http.Request, slug, version, shasum, file string) error { + objName := s.makeObjectName(slug, version, shasum, file) + obj, err := s.client.GetObject(s.ctx, s.bucket, objName, minio.GetObjectOptions{}) + if err != nil { + return wrapS3ErrNotExist(err) + } + defer obj.Close() + + info, err := obj.Stat() + if err != nil { + return wrapS3ErrNotExist(err) + } + + if checkETag := req.Header.Get("Cache-Control") == ""; checkETag { + etag := fmt.Sprintf(`"%s"`, info.ETag[:10]) + if web_utils.CheckPreconditions(w, req, etag) { + return nil + } + w.Header().Set("Etag", etag) + } + + // Read the full object to handle brotli decompression. + content, err := io.ReadAll(obj) + if err != nil { + return err + } + + var r io.Reader = bytes.NewReader(content) + contentLength := info.Size + contentType := info.ContentType + + contentEncoding := info.UserMetadata["X-Content-Encoding"] + origContentLength := info.UserMetadata["Original-Content-Length"] + if contentEncoding == "br" { + if acceptBrotliEncoding(req) { + w.Header().Set(echo.HeaderContentEncoding, "br") + } else { + if origContentLength != "" { + contentLength, _ = strconv.ParseInt(origContentLength, 10, 64) + } + r = brotli.NewReader(bytes.NewReader(content)) + } + } else if contentEncoding == "gzip" { + if acceptGzipEncoding(req) { + w.Header().Set(echo.HeaderContentEncoding, "gzip") + } else { + if origContentLength != "" { + contentLength, _ = strconv.ParseInt(origContentLength, 10, 64) + } + gr, gerr := gzip.NewReader(bytes.NewReader(content)) + if gerr != nil { + return gerr + } + defer gr.Close() + r = gr + } + } + + ext := path.Ext(file) + if contentType == "" { + contentType = mime.TypeByExtension(ext) + } + if contentType == "text/xml" && ext == ".svg" { + contentType = "image/svg+xml" + } + + return serveContent(w, req, contentType, contentLength, r) +} + +func (s *s3Server) ServeCodeTarball(w http.ResponseWriter, req *http.Request, slug, version, shasum string) error { + objName := path.Join(slug, version) + if shasum != "" { + objName += "-" + shasum + } + objName += ".tgz" + + // Try to serve a pre-built tarball first. + obj, err := s.client.GetObject(s.ctx, s.bucket, objName, minio.GetObjectOptions{}) + if err == nil { + info, serr := obj.Stat() + if serr == nil { + defer obj.Close() + return serveContent(w, req, info.ContentType, info.Size, obj) + } + obj.Close() + } + + buf, err := prepareTarball(s, slug, version, shasum) + if err != nil { + return err + } + content, err := io.ReadAll(buf) + if err != nil { + return err + } + contentType := mime.TypeByExtension(".gz") + + // Store the tarball for future requests. + _, _ = s.client.PutObject(s.ctx, s.bucket, objName, + bytes.NewReader(content), int64(len(content)), + minio.PutObjectOptions{ContentType: contentType}) + + return serveContent(w, req, contentType, int64(len(content)), bytes.NewReader(content)) +} + +func (s *s3Server) makeObjectName(slug, version, shasum, file string) string { + basepath := path.Join(slug, version) + if shasum != "" { + basepath += "-" + shasum + } + return path.Join(basepath, file) +} + +func (s *s3Server) FilesList(slug, version, shasum string) ([]string, error) { + prefix := s.makeObjectName(slug, version, shasum, "") + "/" + var names []string + for obj := range s.client.ListObjects(s.ctx, s.bucket, minio.ListObjectsOptions{ + Prefix: prefix, + Recursive: true, + }) { + if obj.Err != nil { + return nil, obj.Err + } + name := strings.TrimPrefix(obj.Key, prefix) + if name != "" { + names = append(names, name) + } + } + return names, nil +} + +// S3AppsBucket returns the S3 bucket name used for storing applications of a +// given type. The bucket is shared across all instances (like Swift containers). +func S3AppsBucket(bucketPrefix string, appsType consts.AppType) string { + switch appsType { + case consts.WebappType: + return bucketPrefix + "-apps-web" + case consts.KonnectorType: + return bucketPrefix + "-apps-konnectors" + } + panic("Unknown AppType") +} + +// ensureS3Bucket creates the bucket if it does not already exist. +func ensureS3Bucket(ctx context.Context, client *minio.Client, bucket string) error { + err := client.MakeBucket(ctx, bucket, minio.MakeBucketOptions{}) + if err != nil { + code := minio.ToErrorResponse(err).Code + if code == "BucketAlreadyOwnedByYou" || code == "BucketAlreadyExists" { + return nil + } + return err + } + return nil +} + +// s3DeleteObjects deletes a list of named objects from a bucket. +func s3DeleteObjects(ctx context.Context, client *minio.Client, bucket string, objNames []string) error { + if len(objNames) == 0 { + return nil + } + objectsCh := make(chan minio.ObjectInfo, len(objNames)) + for _, name := range objNames { + objectsCh <- minio.ObjectInfo{Key: name} + } + close(objectsCh) + var errm error + for e := range client.RemoveObjects(ctx, bucket, objectsCh, minio.RemoveObjectsOptions{}) { + errm = errors.Join(errm, e.Err) + } + return errm +} + +// isS3NotFound returns true when the error is an S3 "not found" response. +func isS3NotFound(err error) bool { + code := minio.ToErrorResponse(err).Code + return code == "NoSuchKey" || code == "NoSuchBucket" +} + +// wrapS3ErrNotExist converts S3 not-found errors to os.ErrNotExist. +func wrapS3ErrNotExist(err error) error { + if isS3NotFound(err) { + return os.ErrNotExist + } + return err +} + +// prepareTarball is reused from server.go via the FileServer interface (it +// calls Open and FilesList). The function is defined in server.go. +// We reference it here to document that s3Server satisfies prepareTarball's +// requirements. +var _ FileServer = (*s3Server)(nil) diff --git a/pkg/assets/dynamic/fs.go b/pkg/assets/dynamic/fs.go index baf867a7a55..a0764c8a050 100644 --- a/pkg/assets/dynamic/fs.go +++ b/pkg/assets/dynamic/fs.go @@ -53,6 +53,12 @@ func InitDynamicAssetFS(fsURL string) error { return err } + case config.SchemeS3: + assetFS, err = NewS3FS() + if err != nil { + return err + } + default: return fmt.Errorf("Invalid scheme %s for dynamic assets FS", u.Scheme) } diff --git a/pkg/assets/dynamic/impl_s3.go b/pkg/assets/dynamic/impl_s3.go new file mode 100644 index 00000000000..5001c4faa88 --- /dev/null +++ b/pkg/assets/dynamic/impl_s3.go @@ -0,0 +1,126 @@ +package dynamic + +import ( + "bytes" + "context" + "fmt" + "io" + "os" + "path" + "strings" + "time" + + "github.com/cozy/cozy-stack/pkg/assets/model" + "github.com/cozy/cozy-stack/pkg/config/config" + "github.com/hashicorp/golang-lru/v2/expirable" + "github.com/minio/minio-go/v7" +) + +// S3FS is the S3 implementation of [AssetsFS]. +// +// It saves and fetches assets into/from any S3-compatible object store. +type S3FS struct { + client *minio.Client + bucket string + ctx context.Context +} + +// NewS3FS instantiates a new S3FS. +func NewS3FS() (*S3FS, error) { + initCacheOnce.Do(func() { + cache = expirable.NewLRU[string, cacheEntry](1024, nil, 1*time.Hour) + }) + + ctx := context.Background() + client := config.GetS3Client() + bucket := config.GetS3BucketPrefix() + "-assets" + + err := client.MakeBucket(ctx, bucket, minio.MakeBucketOptions{Region: config.GetS3Region()}) + if err != nil { + code := minio.ToErrorResponse(err).Code + if code != "BucketAlreadyOwnedByYou" && code != "BucketAlreadyExists" { + return nil, fmt.Errorf("Cannot create bucket for dynamic assets: %s", err) + } + } + + return &S3FS{client: client, bucket: bucket, ctx: ctx}, nil +} + +func (s *S3FS) Add(_ string, _ string, asset *model.Asset) error { + objectName := path.Join(asset.Context, asset.Name) + data := asset.GetData() + _, err := s.client.PutObject(s.ctx, s.bucket, objectName, + bytes.NewReader(data), int64(len(data)), + minio.PutObjectOptions{}) + return err +} + +func (s *S3FS) Get(ctx string, name string) ([]byte, error) { + objectName := path.Join(ctx, name) + if entry, ok := cache.Get(objectName); ok { + if !entry.found { + return nil, os.ErrNotExist + } + return entry.content, nil + } + + obj, err := s.client.GetObject(s.ctx, s.bucket, objectName, minio.GetObjectOptions{}) + if err != nil { + return nil, err + } + defer obj.Close() + + content, err := io.ReadAll(obj) + if err != nil { + if minio.ToErrorResponse(err).Code == "NoSuchKey" { + cache.Add(objectName, cacheEntry{found: false}) + return nil, os.ErrNotExist + } + return nil, err + } + + cache.Add(objectName, cacheEntry{found: true, content: content}) + return content, nil +} + +func (s *S3FS) Remove(context, name string) error { + objectName := path.Join(context, name) + return s.client.RemoveObject(s.ctx, s.bucket, objectName, minio.RemoveObjectOptions{}) +} + +func (s *S3FS) List() (map[string][]*model.Asset, error) { + objs := map[string][]*model.Asset{} + + for obj := range s.client.ListObjects(s.ctx, s.bucket, minio.ListObjectsOptions{ + Recursive: true, + }) { + if obj.Err != nil { + return nil, obj.Err + } + + splitted := strings.SplitN(obj.Key, "/", 2) + if len(splitted) < 2 { + continue + } + ctx := splitted[0] + assetName := model.NormalizeAssetName(splitted[1]) + + a, err := GetAsset(ctx, assetName) + if err != nil { + return nil, err + } + + objs[ctx] = append(objs[ctx], a) + } + + return objs, nil +} + +func (s *S3FS) CheckStatus(ctx context.Context) (time.Duration, error) { + before := time.Now() + _, err := s.client.ListBuckets(ctx) + if err != nil { + return 0, err + } + return time.Since(before), nil +} diff --git a/pkg/previewfs/cache.go b/pkg/previewfs/cache.go index 3d62f95f5e5..cb411c86dbc 100644 --- a/pkg/previewfs/cache.go +++ b/pkg/previewfs/cache.go @@ -13,6 +13,7 @@ import ( "time" "github.com/cozy/cozy-stack/pkg/config/config" + "github.com/minio/minio-go/v7" "github.com/ncw/swift/v2" "github.com/spf13/afero" ) @@ -41,6 +42,10 @@ func SystemCache() Cache { conn := config.GetSwiftConnection() ctx := context.Background() return swiftCache{conn, ctx} + case config.SchemeS3: + client := config.GetS3Client() + bucket := config.GetS3BucketPrefix() + "-previews" + return newS3Cache(client, bucket) default: panic(fmt.Errorf("previewfs: unknown storage provider %s", fsURL.Scheme)) } @@ -151,6 +156,81 @@ func (s swiftCache) SetPreview(md5sum []byte, buffer *bytes.Buffer) error { return err } +type s3Cache struct { + client *minio.Client + bucket string + ctx context.Context +} + +func newS3Cache(client *minio.Client, bucket string) s3Cache { + return s3Cache{client: client, bucket: bucket, ctx: context.Background()} +} + +func (s s3Cache) ensureBucket() error { + err := s.client.MakeBucket(s.ctx, s.bucket, minio.MakeBucketOptions{}) + if err != nil { + code := minio.ToErrorResponse(err).Code + if code == "BucketAlreadyOwnedByYou" || code == "BucketAlreadyExists" { + return nil + } + return err + } + return nil +} + +func (s s3Cache) getObject(name string) (*bytes.Buffer, error) { + obj, err := s.client.GetObject(s.ctx, s.bucket, name, minio.GetObjectOptions{}) + if err != nil { + return nil, err + } + defer obj.Close() + + buf := &bytes.Buffer{} + _, err = buf.ReadFrom(obj) + if err != nil { + if minio.ToErrorResponse(err).Code == "NoSuchKey" { + return nil, err + } + return nil, err + } + return buf, nil +} + +func (s s3Cache) putObject(name string, buffer *bytes.Buffer) error { + data := buffer.Bytes() + _, err := s.client.PutObject(s.ctx, s.bucket, name, + bytes.NewReader(data), int64(len(data)), + minio.PutObjectOptions{ContentType: "image/jpg"}) + if err != nil { + code := minio.ToErrorResponse(err).Code + if code == "NoSuchBucket" { + if berr := s.ensureBucket(); berr != nil { + return berr + } + _, err = s.client.PutObject(s.ctx, s.bucket, name, + bytes.NewReader(data), int64(len(data)), + minio.PutObjectOptions{ContentType: "image/jpg"}) + } + } + return err +} + +func (s s3Cache) GetIcon(md5sum []byte) (*bytes.Buffer, error) { + return s.getObject(iconFilename(md5sum)) +} + +func (s s3Cache) SetIcon(md5sum []byte, buffer *bytes.Buffer) error { + return s.putObject(iconFilename(md5sum), buffer) +} + +func (s s3Cache) GetPreview(md5sum []byte) (*bytes.Buffer, error) { + return s.getObject(previewFilename(md5sum)) +} + +func (s s3Cache) SetPreview(md5sum []byte, buffer *bytes.Buffer) error { + return s.putObject(previewFilename(md5sum), buffer) +} + func iconFilename(md5sum []byte) string { return "icon-" + hex.EncodeToString(md5sum) + ".jpg" } diff --git a/web/settings/capabilities.go b/web/settings/capabilities.go index dcd2104294a..c76fcb1c86f 100644 --- a/web/settings/capabilities.go +++ b/web/settings/capabilities.go @@ -43,6 +43,8 @@ func NewCapabilities(inst *instance.Instance) jsonapi.Object { switch config.FsURL().Scheme { case config.SchemeSwift, config.SchemeSwiftSecure: versioning = inst.SwiftLayout >= 2 + case config.SchemeS3: + versioning = true } flat := config.GetConfig().Subdomains == config.FlatSubdomains diff --git a/web/settings/instance.go b/web/settings/instance.go index 20c6c13a112..51a5b8a5cbe 100644 --- a/web/settings/instance.go +++ b/web/settings/instance.go @@ -82,14 +82,14 @@ func (h *HTTPHandler) getInstance(c echo.Context) error { return err } - url, err := h.svc.GetLegalNoticeUrl(inst) + /* url, err := h.svc.GetLegalNoticeUrl(inst) if err != nil { return err } if url != "" { doc.M["legal_notice_url"] = url } - + */ return jsonapi.Data(c, http.StatusOK, &apiInstance{doc}, nil) } From cc3351c0422d5073609c1db3c96980d3d8121932 Mon Sep 17 00:00:00 2001 From: Crash-- Date: Sat, 28 Mar 2026 19:41:14 +0100 Subject: [PATCH 04/20] test: add S3 backend to the VFS integration test suite - Add MinIO testcontainer fixture (tests/testutils/minio_utils.go) - Add makeS3FS helper and wire into the VFS test table - All 17 VFS tests pass for the S3 backend Co-Authored-By: Claude Opus 4.6 (1M context) --- model/vfs/vfs_test.go | 31 +++++++++++ tests/testutils/minio_utils.go | 98 ++++++++++++++++++++++++++++++++++ 2 files changed, 129 insertions(+) create mode 100644 tests/testutils/minio_utils.go diff --git a/model/vfs/vfs_test.go b/model/vfs/vfs_test.go index 7d20d308d3f..40d9f1e52b0 100644 --- a/model/vfs/vfs_test.go +++ b/model/vfs/vfs_test.go @@ -17,6 +17,7 @@ import ( "github.com/cozy/cozy-stack/model/vfs" "github.com/cozy/cozy-stack/model/vfs/vfsafero" + "github.com/cozy/cozy-stack/model/vfs/vfss3" "github.com/cozy/cozy-stack/model/vfs/vfsswift" "github.com/cozy/cozy-stack/pkg/config/config" "github.com/cozy/cozy-stack/pkg/consts" @@ -54,6 +55,7 @@ func TestVfs(t *testing.T) { aferoFS := makeAferoFS(t) swiftFS := makeSwiftFS(t) + s3FS := makeS3FS(t) var tests = []struct { name string @@ -61,6 +63,7 @@ func TestVfs(t *testing.T) { }{ {"afero", aferoFS}, {"swift", swiftFS}, + {"s3", s3FS}, } for _, tt := range tests { @@ -909,3 +912,31 @@ func makeSwiftFS(t *testing.T) vfs.VFS { return swiftFs } + +func makeS3FS(t *testing.T) vfs.VFS { + t.Helper() + + minioFixture := testutils.StartMinio(t) + db := &contexter{0, "io.cozy.vfs.s3.test", "io.cozy.vfs.s3.test", "cozy_beta"} + index := vfs.NewCouchdbIndexer(db) + + require.NoError(t, config.InitS3Connection(config.Fs{ + URL: minioFixture.FsURL("test"), + })) + + mutex = config.Lock().ReadWrite(db, "vfs-s3-test") + s3Fs, err := vfss3.New(db, index, &diskImpl{}, mutex) + require.NoError(t, err) + + require.NoError(t, couchdb.ResetDB(db, consts.Files)) + t.Cleanup(func() { _ = couchdb.DeleteDB(db, consts.Files) }) + + g, _ := errgroup.WithContext(context.Background()) + couchdb.DefineIndexes(g, db, couchdb.IndexesByDoctype(consts.Files)) + couchdb.DefineViews(g, db, couchdb.ViewsByDoctype(consts.Files)) + + require.NoError(t, g.Wait()) + require.NoError(t, s3Fs.InitFs()) + + return s3Fs +} diff --git a/tests/testutils/minio_utils.go b/tests/testutils/minio_utils.go new file mode 100644 index 00000000000..8af8bf262ab --- /dev/null +++ b/tests/testutils/minio_utils.go @@ -0,0 +1,98 @@ +package testutils + +import ( + "context" + "fmt" + "net/url" + "testing" + "time" + + "github.com/docker/go-connections/nat" + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" + "github.com/stretchr/testify/require" + + c "github.com/docker/docker/api/types/container" + tc "github.com/testcontainers/testcontainers-go" + "github.com/testcontainers/testcontainers-go/wait" +) + +// MinioFixture holds the state for a running MinIO container. +type MinioFixture struct { + Container tc.Container + Endpoint string // host:port + AccessKey string + SecretKey string + t *testing.T +} + +// StartMinio starts a MinIO container for testing. +func StartMinio(t *testing.T) *MinioFixture { + t.Helper() + + accessKey := "minioadmin" + secretKey := "minioadmin" + hostPort := getFreePort(t) + + req := tc.ContainerRequest{ + Image: "minio/minio:RELEASE.2025-02-28T09-55-16Z", + ExposedPorts: []string{"9000/tcp"}, + Env: map[string]string{ + "MINIO_ROOT_USER": accessKey, + "MINIO_ROOT_PASSWORD": secretKey, + }, + Cmd: []string{"server", "/data"}, + HostConfigModifier: func(hc *c.HostConfig) { + hc.PortBindings = nat.PortMap{ + "9000/tcp": []nat.PortBinding{{HostIP: "0.0.0.0", HostPort: hostPort}}, + } + }, + WaitingFor: wait.ForHTTP("/minio/health/live"). + WithPort("9000/tcp"). + WithStartupTimeout(60 * time.Second), + } + + container, err := tc.GenericContainer(context.Background(), tc.GenericContainerRequest{ + ContainerRequest: req, + Started: true, + }) + require.NoError(t, err, "failed to start MinIO") + + host, err := container.Host(context.Background()) + require.NoError(t, err) + + endpoint := fmt.Sprintf("%s:%s", host, hostPort) + t.Logf("MinIO endpoint: %s", endpoint) + + t.Cleanup(func() { + _ = container.Terminate(context.Background()) + }) + + return &MinioFixture{ + Container: container, + Endpoint: endpoint, + AccessKey: accessKey, + SecretKey: secretKey, + t: t, + } +} + +// Client returns a minio.Client connected to this fixture. +func (f *MinioFixture) Client(t *testing.T) *minio.Client { + t.Helper() + client, err := minio.New(f.Endpoint, &minio.Options{ + Creds: credentials.NewStaticV4(f.AccessKey, f.SecretKey, ""), + Secure: false, + }) + require.NoError(t, err) + return client +} + +// FsURL returns a *url.URL suitable for config.InitS3Connection. +func (f *MinioFixture) FsURL(bucketPrefix string) *url.URL { + return &url.URL{ + Scheme: "s3", + Host: f.Endpoint, + RawQuery: fmt.Sprintf("access_key=%s&secret_key=%s&bucket_prefix=%s&use_ssl=false", f.AccessKey, f.SecretKey, bucketPrefix), + } +} From 16f63f3bda2e1e1bf0a8c53c180746f67b580660 Mon Sep 17 00:00:00 2001 From: Crash-- Date: Sat, 28 Mar 2026 19:46:47 +0100 Subject: [PATCH 05/20] docs: add S3 storage backend documentation Document the S3 backend architecture: bucket strategy, object key structure, memory consumption, encryption, differences from Swift, configuration, and testing. Also add S3 URL example to cozy.example.yaml. Co-Authored-By: Claude Opus 4.6 (1M context) --- cozy.example.yaml | 1 + docs/s3.md | 172 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 173 insertions(+) create mode 100644 docs/s3.md diff --git a/cozy.example.yaml b/cozy.example.yaml index 8213d59bf6e..8dbfa3e7e58 100644 --- a/cozy.example.yaml +++ b/cozy.example.yaml @@ -67,6 +67,7 @@ fs: # url: file://localhost/var/lib/cozy # url: swift://openstack/?UserName={{ .Env.OS_USERNAME }}&Password={{ .Env.OS_PASSWORD }}&ProjectName={{ .Env.OS_PROJECT_NAME }}&UserDomainName={{ .Env.OS_USER_DOMAIN_NAME }}&Timeout={{ .Env.GOSWIFT_TIMEOUT }} + # url: s3://{{ .Env.S3_ENDPOINT }}?access_key={{ .Env.S3_ACCESS_KEY }}&secret_key={{ .Env.S3_SECRET_KEY }}®ion={{ .Env.S3_REGION }}&bucket_prefix=cozy&use_ssl=true # Swift FS can be used with advanced parameters to activate TLS properties. # For using swift with https, you must use the "swift+https" scheme. diff --git a/docs/s3.md b/docs/s3.md new file mode 100644 index 00000000000..3684c54c15f --- /dev/null +++ b/docs/s3.md @@ -0,0 +1,172 @@ +[Table of contents](README.md#table-of-contents) + +# S3 Storage Backend + +cozy-stack supports S3-compatible object storage as a file system backend, +alongside the existing local filesystem (afero) and OpenStack Swift backends. +It has been designed to work with any S3-compatible provider (OVH, MinIO, +Scaleway, etc.) and does not depend on the AWS SDK. + +## Configuration + +The S3 backend is configured via the `fs.url` parameter using the `s3://` +scheme. All connection parameters are passed as query parameters: + +```yaml +fs: + url: s3://s3.rbx.io.cloud.ovh.net?access_key=ACCESS&secret_key=SECRET®ion=rbx&bucket_prefix=cozy&use_ssl=true +``` + +| Parameter | Description | Default | +|-----------------|--------------------------------------|---------| +| `access_key` | S3 access key ID | — | +| `secret_key` | S3 secret access key | — | +| `region` | S3 region | — | +| `bucket_prefix` | Prefix for all bucket names | `cozy` | +| `use_ssl` | Use HTTPS for S3 connections | `true` | + +The host part of the URL is the S3 endpoint (e.g. `s3.rbx.io.cloud.ovh.net` +for OVH, `localhost:9000` for MinIO). + +### Local development with MinIO + +```yaml +fs: + url: s3://localhost:9000?access_key=minioadmin&secret_key=minioadmin&bucket_prefix=cozy&use_ssl=false +``` + +Start MinIO with Docker: + +```bash +docker run -d --name minio -p 9000:9000 -p 9001:9001 \ + -e MINIO_ROOT_USER=minioadmin \ + -e MINIO_ROOT_PASSWORD=minioadmin \ + minio/minio server /data --console-address ":9001" +``` + +The MinIO console is available at `http://localhost:9001`. + +## Bucket strategy + +### Design rationale + +Swift uses one container per instance (`cozy-v3-`). This doesn't +scale well for S3 where bucket creation can be limited (AWS limits to 100 +buckets per account by default, OVH to 100 as well). Instead, the S3 backend +uses a **shared bucket per organization** with **key prefixes per instance**. + +### Bucket naming + +Each bucket name is derived from the instance's `OrgID` field: + +``` +- +``` + +- If `OrgID` is empty, `"default"` is used as fallback +- The org ID is sanitized: lowercased, underscores/dots replaced by hyphens, + non-alphanumeric characters stripped, consecutive hyphens collapsed, + truncated to respect the 63-character S3 bucket name limit +- Examples: `cozy-default`, `cozy-acme-corp`, `cozy-org-12345` + +### Dedicated buckets for secondary storage + +In addition to the main VFS bucket, the S3 backend uses dedicated buckets for +other storage needs: + +| Bucket | Content | +|-------------------------------|-----------------------------------------| +| `-` | Main VFS data (files, versions) | +| `-apps-web` | Web application assets (drive, etc.) | +| `-apps-konnectors` | Konnector assets | +| `-assets` | Dynamic assets | +| `-previews` | PDF preview and icon cache | +| `-exports` | Instance export archives | + +Buckets are created automatically on first use. + +## Object key structure + +Within a bucket, each instance's data is isolated by a key prefix derived +from `DBPrefix()` (typically the instance domain or a CouchDB prefix). + +### VFS files + +``` +//// +``` + +The document ID (a 32-character UUID v7 hex string) is split into virtual +subfolders to avoid flat hierarchies: + +``` +cozy218def.../019d35b1-9dc3-78ec-994d-f5/44336/7f1b6/e0AbCdEfGh123456 + ^^^^^^^^^^^^^^^^^^^^^^ ^^^^^ ^^^^^ ^^^^^^^^^^^^^^^^ + first 22 chars 5 ch 5 ch 16-char internalID +``` + +This structure mirrors the Swift V3 layout (`MakeObjectNameV3`). + +### Thumbnails + +``` +/thumbs/- +``` + +Formats: `small`, `medium`, `large`. + +### Avatar + +``` +/avatar +``` + +## Memory consumption + +The S3 backend is designed to have comparable memory usage to Swift: + +| Scenario | Memory per upload | +|-----------------------------|-------------------| +| Known size, file < 5 GiB | ~32 KB (single PUT, stream) | +| Unknown size (rare) | ~5 MiB (multipart, PartSize=5MiB, NumThreads=1) | + +When `ByteSize` is known on the file document (the common case for drive +uploads), the backend passes the exact size to `PutObject`, which uses a +single PUT request that streams directly to S3 with minimal buffering — the +same behavior as Swift's `ObjectCreate`. + +Multipart upload is only used for files with unknown size or exceeding 5 GiB, +with `PartSize=5MiB` and `NumThreads=1` to limit memory. + +## Encryption at rest + +The S3 backend does not implement client-side encryption. Encryption should +be configured at the infrastructure level (S3 bucket default encryption / +SSE-S3), the same approach used for the Swift backend. + +## Differences from Swift + +| Aspect | Swift | S3 | +|--------------------|-------------------------------|----------------------------------------| +| Container/Bucket | One per instance | One per organization (shared) | +| Instance isolation | Container name | Key prefix within bucket | +| Delete instance | Delete entire container | Delete all objects with key prefix | +| File streaming | Native `io.WriteCloser` | `io.Pipe` + `PutObject` goroutine | +| Bulk delete | `BulkDelete` API | `RemoveObjects` channel API | +| Server-side copy | `ObjectCopy` | `CopyObject` (same endpoint only) | + +## Testing + +The VFS integration tests run against all three backends (afero, swift, s3) +using a table-driven approach. The S3 tests use +[testcontainers-go](https://testcontainers.com/guides/getting-started-with-testcontainers-for-go/) +with a MinIO container that is started automatically. + +```bash +# Run VFS tests (requires CouchDB + Docker) +COZY_COUCHDB_URL=http://admin:admin@localhost:5984/ \ + go test ./model/vfs/ -run TestVfs -v -count=1 -timeout 300s + +# Run naming unit tests (no external deps) +go test ./model/vfs/vfss3/ -run "TestSanitize|TestBucketName|TestMakeObjectKey|TestMakeDocID" -v +``` From c0262baa0cbc9a3e4fd5d7e529ae29199c4cef59 Mon Sep 17 00:00:00 2001 From: Crash-- Date: Sat, 28 Mar 2026 20:03:41 +0100 Subject: [PATCH 06/20] fix: address code review findings - Revert accidental commenting of GetLegalNoticeUrl in settings - Remove unused errFailFast variable in fsck.go - Remove unused maxNbFilesToDelete constant in s3.go - Fix s3Cache to return os.ErrNotExist for NoSuchKey errors - Fix indentation in s3Copier.Exist - Fix BucketName to avoid trailing hyphens Co-Authored-By: Claude Opus 4.6 (1M context) --- model/vfs/vfss3/fsck.go | 1 - model/vfs/vfss3/impl.go | 8 ++++---- model/vfs/vfss3/s3.go | 3 --- pkg/appfs/s3.go | 4 ++-- pkg/previewfs/cache.go | 2 +- web/settings/instance.go | 4 ++-- 6 files changed, 9 insertions(+), 13 deletions(-) diff --git a/model/vfs/vfss3/fsck.go b/model/vfs/vfss3/fsck.go index 3b505ba5308..7ac5e147181 100644 --- a/model/vfs/vfss3/fsck.go +++ b/model/vfs/vfss3/fsck.go @@ -14,7 +14,6 @@ import ( "github.com/minio/minio-go/v7" ) -var errFailFast = errors.New("fail fast") func (sfs *s3VFS) Fsck(accumulate func(log *vfs.FsckLog), failFast bool) error { entries := make(map[string]*vfs.TreeFile, 1024) diff --git a/model/vfs/vfss3/impl.go b/model/vfs/vfss3/impl.go index baf88e64b31..804403ecf68 100644 --- a/model/vfs/vfss3/impl.go +++ b/model/vfs/vfss3/impl.go @@ -70,12 +70,12 @@ func BucketName(orgID, bucketPrefix string) string { orgID = "default" } name := bucketPrefix + "-" + sanitizeBucketName(orgID) - if len(name) < 3 { - name = name + strings.Repeat("-", 3-len(name)) - } if len(name) > 63 { name = name[:63] - name = strings.TrimRight(name, "-") + } + name = strings.TrimRight(name, "-") + if len(name) < 3 { + name = name + strings.Repeat("0", 3-len(name)) } return name } diff --git a/model/vfs/vfss3/s3.go b/model/vfs/vfss3/s3.go index 96316037a62..1096f1f28e0 100644 --- a/model/vfs/vfss3/s3.go +++ b/model/vfs/vfss3/s3.go @@ -7,9 +7,6 @@ import ( "github.com/minio/minio-go/v7" ) -// maxNbFilesToDelete is the max number of objects per RemoveObjects batch. -const maxNbFilesToDelete = 1000 - // deletePrefixObjects deletes all objects in a bucket under a given prefix. func deletePrefixObjects(ctx context.Context, client *minio.Client, bucket, prefix string) error { objectsCh := make(chan minio.ObjectInfo) diff --git a/pkg/appfs/s3.go b/pkg/appfs/s3.go index 9dcf06b85f0..b94a4d711f2 100644 --- a/pkg/appfs/s3.go +++ b/pkg/appfs/s3.go @@ -49,10 +49,10 @@ func (f *s3Copier) Exist(slug, version, shasum string) (bool, error) { } _, err := f.client.StatObject(f.ctx, f.bucket, f.appObj, minio.StatObjectOptions{}) if err == nil { - return true, nil + return true, nil } if isS3NotFound(err) { - return false, nil + return false, nil } return false, err } diff --git a/pkg/previewfs/cache.go b/pkg/previewfs/cache.go index cb411c86dbc..04cb1e49a7d 100644 --- a/pkg/previewfs/cache.go +++ b/pkg/previewfs/cache.go @@ -189,7 +189,7 @@ func (s s3Cache) getObject(name string) (*bytes.Buffer, error) { _, err = buf.ReadFrom(obj) if err != nil { if minio.ToErrorResponse(err).Code == "NoSuchKey" { - return nil, err + return nil, os.ErrNotExist } return nil, err } diff --git a/web/settings/instance.go b/web/settings/instance.go index 51a5b8a5cbe..20c6c13a112 100644 --- a/web/settings/instance.go +++ b/web/settings/instance.go @@ -82,14 +82,14 @@ func (h *HTTPHandler) getInstance(c echo.Context) error { return err } - /* url, err := h.svc.GetLegalNoticeUrl(inst) + url, err := h.svc.GetLegalNoticeUrl(inst) if err != nil { return err } if url != "" { doc.M["legal_notice_url"] = url } - */ + return jsonapi.Data(c, http.StatusOK, &apiInstance{doc}, nil) } From 514c9254ba5ffc9ecd38467470c9c10508368636 Mon Sep 17 00:00:00 2001 From: Crash-- Date: Sat, 28 Mar 2026 20:20:53 +0100 Subject: [PATCH 07/20] security: fix all issues from security review Critical: - Verify MD5 integrity when caller provides expected hash (was skipped) - Limit app file reads to 50 MiB to prevent OOM from corrupted objects High: - Sanitize S3 errors to avoid leaking bucket names and key paths - Prevent path traversal via ".." in app filenames and file serving - Fix fsck DocName to use stripped object name instead of full S3 key Medium: - Replace panic with error return in s3Copier.Copy - Sanitize bucket_prefix config parameter Low: - Add bounds check for ETag substring to prevent panic - Remove unused encoding/hex import Co-Authored-By: Claude Opus 4.6 (1M context) --- docs/s3.md | 88 ++++++++++++++++++++++++++++++++++++--- model/vfs/vfss3/avatar.go | 3 +- model/vfs/vfss3/fsck.go | 2 +- model/vfs/vfss3/impl.go | 28 +++++-------- pkg/appfs/s3.go | 30 ++++++++++--- pkg/config/config/s3.go | 24 +++++++++++ 6 files changed, 145 insertions(+), 30 deletions(-) diff --git a/docs/s3.md b/docs/s3.md index 3684c54c15f..c757f3b4455 100644 --- a/docs/s3.md +++ b/docs/s3.md @@ -30,21 +30,97 @@ for OVH, `localhost:9000` for MinIO). ### Local development with MinIO +This tutorial explains how to set up a local S3 backend using MinIO for +development and testing. + +**1. Start MinIO with Docker:** + +```bash +docker run -d --name minio \ + -p 9000:9000 \ + -p 9001:9001 \ + -e MINIO_ROOT_USER=minioadmin \ + -e MINIO_ROOT_PASSWORD=minioadmin \ + minio/minio server /data --console-address ":9001" +``` + +MinIO is now running: +- S3 API: `http://localhost:9000` +- Web console: `http://localhost:9001` (login: `minioadmin` / `minioadmin`) + +**2. Configure cozy-stack:** + +Buckets are created automatically at startup. No manual bucket creation +is needed. + +Edit your `~/.cozy/cozy.yaml`: + ```yaml fs: url: s3://localhost:9000?access_key=minioadmin&secret_key=minioadmin&bucket_prefix=cozy&use_ssl=false ``` -Start MinIO with Docker: +**3. Build and start:** ```bash -docker run -d --name minio -p 9000:9000 -p 9001:9001 \ - -e MINIO_ROOT_USER=minioadmin \ - -e MINIO_ROOT_PASSWORD=minioadmin \ - minio/minio server /data --console-address ":9001" +go build -o ~/go/bin/cozy-stack . +~/go/bin/cozy-stack serve +``` + +You should see in the logs: + +``` +Successfully connected to S3 endpoint localhost:9000 +``` + +**4. (Re)install your apps:** + +When switching from a different storage backend (e.g. `file://`), you need +to reinstall the apps so their assets are stored in S3: + +```bash +cozy-stack apps uninstall drive --domain your.domain.localhost:8080 +cozy-stack apps install drive --domain your.domain.localhost:8080 +cozy-stack apps uninstall home --domain your.domain.localhost:8080 +cozy-stack apps install home --domain your.domain.localhost:8080 +``` + +**5. Verify:** + +Check that objects appear in MinIO: + +```bash +docker exec minio mc ls --recursive local/cozy-apps-web/ +``` + +Upload a file via the Drive UI or the API: + +```bash +TOKEN=$(cozy-stack instances token-cli your.domain.localhost:8080 io.cozy.files) +curl -X POST \ + -H "Authorization: Bearer $TOKEN" \ + -H "Content-Type: text/plain" \ + "http://your.domain.localhost:8080/files/io.cozy.files.root-dir?Type=file&Name=test.txt" \ + -d "Hello S3!" +``` + +Verify the file is in MinIO: + +```bash +docker exec minio mc ls --recursive local/cozy-default/ +``` + +**6. Switching back to local filesystem:** + +Comment out the S3 URL in your config and restart cozy-stack: + +```yaml +fs: + # url: s3://localhost:9000?access_key=minioadmin&secret_key=minioadmin&bucket_prefix=cozy&use_ssl=false ``` -The MinIO console is available at `http://localhost:9001`. +Note: files uploaded to S3 won't be accessible when using the local +filesystem backend, and vice versa. Each backend has its own storage. ## Bucket strategy diff --git a/model/vfs/vfss3/avatar.go b/model/vfs/vfss3/avatar.go index 87131242359..fb518e0282e 100644 --- a/model/vfs/vfss3/avatar.go +++ b/model/vfs/vfss3/avatar.go @@ -112,5 +112,6 @@ func wrapS3Err(err error) error { if minio.ToErrorResponse(err).Code == "NoSuchBucket" { return os.ErrNotExist } - return err + // Sanitize S3 errors to avoid leaking internal bucket/key details + return fmt.Errorf("s3 storage error: %s", minio.ToErrorResponse(err).Code) } diff --git a/model/vfs/vfss3/fsck.go b/model/vfs/vfss3/fsck.go index 7ac5e147181..adb7cf5e71b 100644 --- a/model/vfs/vfss3/fsck.go +++ b/model/vfs/vfss3/fsck.go @@ -117,7 +117,7 @@ func (sfs *s3VFS) checkFiles( DirDoc: &vfs.DirDoc{ Type: consts.FileType, DocID: fileID, - DocName: obj.Key, + DocName: objName, }, }, }, diff --git a/model/vfs/vfss3/impl.go b/model/vfs/vfss3/impl.go index 804403ecf68..39b1ea3524b 100644 --- a/model/vfs/vfss3/impl.go +++ b/model/vfs/vfss3/impl.go @@ -7,7 +7,7 @@ import ( "bytes" "context" "crypto/md5" - "encoding/hex" + "errors" "fmt" "hash" @@ -1030,23 +1030,17 @@ func (f *s3FileCreation) Close() (err error) { return f.err } - // Extract MD5 from ETag if available, otherwise use our local md5H - if newdoc.MD5Sum == nil { - etag := result.info.ETag - // ETags may be double-quoted - etag = strings.TrimPrefix(etag, "\"") - etag = strings.TrimSuffix(etag, "\"") - // For multipart uploads, ETag contains a dash — use local MD5 - if strings.Contains(etag, "-") || etag == "" { - newdoc.MD5Sum = f.md5H.Sum(nil) - } else { - md5sum, err := hex.DecodeString(etag) - if err != nil { - newdoc.MD5Sum = f.md5H.Sum(nil) - } else { - newdoc.MD5Sum = md5sum - } + // Verify or compute MD5 checksum. + // The local md5H hash is always computed from the same data stream that + // goes to S3 (via the Write method), so it is authoritative. + localMD5 := f.md5H.Sum(nil) + if newdoc.MD5Sum != nil { + // The caller provided an expected hash — verify it matches what was written. + if !bytes.Equal(newdoc.MD5Sum, localMD5) { + return vfs.ErrInvalidHash } + } else { + newdoc.MD5Sum = localMD5 } if f.size < 0 { diff --git a/pkg/appfs/s3.go b/pkg/appfs/s3.go index b94a4d711f2..30da67ab6b1 100644 --- a/pkg/appfs/s3.go +++ b/pkg/appfs/s3.go @@ -74,11 +74,16 @@ func (f *s3Copier) Start(slug, version, shasum string) (bool, error) { func (f *s3Copier) Copy(stat os.FileInfo, src io.Reader) error { if !f.started { - panic("copier should call Start() before Copy()") + return fmt.Errorf("appfs: copier must call Start() before Copy()") } // Write directly to the final location (appObj/filename). - objName := path.Join(f.appObj, stat.Name()) + // Reject path traversal attempts in filenames. + name := stat.Name() + if strings.Contains(name, "..") { + return fmt.Errorf("appfs: invalid filename %q", name) + } + objName := path.Join(f.appObj, name) contentType := filetype.ByExtension(path.Ext(stat.Name())) if contentType == "" { @@ -173,7 +178,11 @@ func (s *s3Server) ServeFileContent(w http.ResponseWriter, req *http.Request, sl } if checkETag := req.Header.Get("Cache-Control") == ""; checkETag { - etag := fmt.Sprintf(`"%s"`, info.ETag[:10]) + etagVal := info.ETag + if len(etagVal) > 10 { + etagVal = etagVal[:10] + } + etag := fmt.Sprintf(`"%s"`, etagVal) if web_utils.CheckPreconditions(w, req, etag) { return nil } @@ -181,7 +190,9 @@ func (s *s3Server) ServeFileContent(w http.ResponseWriter, req *http.Request, sl } // Read the full object to handle brotli decompression. - content, err := io.ReadAll(obj) + // Limit to 50 MiB to avoid unbounded memory allocation from corrupted objects. + const maxAppFileSize = 50 << 20 + content, err := io.ReadAll(io.LimitReader(obj, maxAppFileSize)) if err != nil { return err } @@ -269,6 +280,10 @@ func (s *s3Server) makeObjectName(slug, version, shasum, file string) string { if shasum != "" { basepath += "-" + shasum } + // Prevent path traversal + if strings.Contains(file, "..") { + return basepath + "/invalid" + } return path.Join(basepath, file) } @@ -338,11 +353,16 @@ func isS3NotFound(err error) bool { return code == "NoSuchKey" || code == "NoSuchBucket" } -// wrapS3ErrNotExist converts S3 not-found errors to os.ErrNotExist. +// wrapS3ErrNotExist converts S3 not-found errors to os.ErrNotExist and +// sanitizes other S3 errors to avoid leaking internal bucket/key details. func wrapS3ErrNotExist(err error) error { if isS3NotFound(err) { return os.ErrNotExist } + code := minio.ToErrorResponse(err).Code + if code != "" { + return fmt.Errorf("s3 storage error: %s", code) + } return err } diff --git a/pkg/config/config/s3.go b/pkg/config/config/s3.go index cb8e72ebd2c..506fd57afe2 100644 --- a/pkg/config/config/s3.go +++ b/pkg/config/config/s3.go @@ -3,6 +3,7 @@ package config import ( "context" "fmt" + "strings" "github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7/pkg/credentials" @@ -36,6 +37,15 @@ func InitS3Connection(fs Fs) error { if s3BucketPrefix == "" { s3BucketPrefix = "cozy" } + // Sanitize bucket prefix: lowercase, only alphanumeric and hyphens + s3BucketPrefix = strings.ToLower(s3BucketPrefix) + s3BucketPrefix = strings.Map(func(r rune) rune { + if (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9') || r == '-' { + return r + } + return '-' + }, s3BucketPrefix) + s3BucketPrefix = strings.Trim(s3BucketPrefix, "-") s3Region = region var err error @@ -60,6 +70,20 @@ func InitS3Connection(fs Fs) error { } log.Infof("Successfully connected to S3 endpoint %s", endpoint) + + // Pre-create the fixed buckets used by secondary storage (apps, assets, + // previews, exports). The per-org VFS bucket is created on instance init. + ctx := context.Background() + for _, suffix := range []string{"-apps-web", "-apps-konnectors", "-assets", "-previews", "-exports"} { + bucket := s3BucketPrefix + suffix + if err := s3Client.MakeBucket(ctx, bucket, minio.MakeBucketOptions{Region: region}); err != nil { + code := minio.ToErrorResponse(err).Code + if code != "BucketAlreadyOwnedByYou" && code != "BucketAlreadyExists" { + log.Warnf("Could not create bucket %s: %s", bucket, err) + } + } + } + return nil } From 074dbbfcb4b289f4981dafccde740e00f8172f18 Mon Sep 17 00:00:00 2001 From: Crash-- Date: Mon, 30 Mar 2026 14:09:51 +0700 Subject: [PATCH 08/20] style: use += operator for string concatenation (gocritic) Co-Authored-By: Claude Opus 4.6 (1M context) --- model/vfs/vfss3/fsck.go | 1 - model/vfs/vfss3/impl.go | 2 +- model/vfs/vfss3/thumbs.go | 12 ++++++------ pkg/appfs/s3.go | 2 +- 4 files changed, 8 insertions(+), 9 deletions(-) diff --git a/model/vfs/vfss3/fsck.go b/model/vfs/vfss3/fsck.go index adb7cf5e71b..364f6c3bca1 100644 --- a/model/vfs/vfss3/fsck.go +++ b/model/vfs/vfss3/fsck.go @@ -14,7 +14,6 @@ import ( "github.com/minio/minio-go/v7" ) - func (sfs *s3VFS) Fsck(accumulate func(log *vfs.FsckLog), failFast bool) error { entries := make(map[string]*vfs.TreeFile, 1024) tree, err := sfs.BuildTree(func(f *vfs.TreeFile) { diff --git a/model/vfs/vfss3/impl.go b/model/vfs/vfss3/impl.go index 39b1ea3524b..c1fbfabfd5c 100644 --- a/model/vfs/vfss3/impl.go +++ b/model/vfs/vfss3/impl.go @@ -75,7 +75,7 @@ func BucketName(orgID, bucketPrefix string) string { } name = strings.TrimRight(name, "-") if len(name) < 3 { - name = name + strings.Repeat("0", 3-len(name)) + name += strings.Repeat("0", 3-len(name)) } return name } diff --git a/model/vfs/vfss3/thumbs.go b/model/vfs/vfss3/thumbs.go index 22a4a6eec15..7dff1e4009d 100644 --- a/model/vfs/vfss3/thumbs.go +++ b/model/vfs/vfss3/thumbs.go @@ -37,12 +37,12 @@ type thumbsS3 struct { } type s3Thumb struct { - pw *io.PipeWriter - errCh chan error - client *minio.Client - bucket string - name string - ctx context.Context + pw *io.PipeWriter + errCh chan error + client *minio.Client + bucket string + name string + ctx context.Context } func (t *s3Thumb) Write(p []byte) (int, error) { diff --git a/pkg/appfs/s3.go b/pkg/appfs/s3.go index 30da67ab6b1..b955586c832 100644 --- a/pkg/appfs/s3.go +++ b/pkg/appfs/s3.go @@ -101,7 +101,7 @@ func (f *s3Copier) Copy(stat os.FileInfo, src io.Reader) error { } meta := map[string]string{ - "X-Content-Encoding": "br", + "X-Content-Encoding": "br", "Original-Content-Length": strconv.FormatInt(stat.Size(), 10), } From 4f61b6e748aefcdd5eb1af9371f19f8ff65198a6 Mon Sep 17 00:00:00 2001 From: Crash-- Date: Fri, 3 Apr 2026 11:56:42 +0700 Subject: [PATCH 09/20] style: remove redundant multierror import alias The package is already named `multierror`, so the explicit alias is a no-op. Co-Authored-By: Claude Opus 4.6 (1M context) --- model/vfs/vfss3/impl.go | 2 +- model/vfs/vfss3/s3.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/model/vfs/vfss3/impl.go b/model/vfs/vfss3/impl.go index c1fbfabfd5c..73964c425f1 100644 --- a/model/vfs/vfss3/impl.go +++ b/model/vfs/vfss3/impl.go @@ -23,7 +23,7 @@ import ( "github.com/cozy/cozy-stack/pkg/logger" "github.com/cozy/cozy-stack/pkg/utils" "github.com/gofrs/uuid/v5" - multierror "github.com/hashicorp/go-multierror" + "github.com/hashicorp/go-multierror" "github.com/minio/minio-go/v7" ) diff --git a/model/vfs/vfss3/s3.go b/model/vfs/vfss3/s3.go index 1096f1f28e0..34a8006a8ba 100644 --- a/model/vfs/vfss3/s3.go +++ b/model/vfs/vfss3/s3.go @@ -3,7 +3,7 @@ package vfss3 import ( "context" - multierror "github.com/hashicorp/go-multierror" + "github.com/hashicorp/go-multierror" "github.com/minio/minio-go/v7" ) From c70a37f3c777271ba4c9dcf31969f7ece7ceb596 Mon Sep 17 00:00:00 2001 From: Crash-- Date: Fri, 3 Apr 2026 11:56:51 +0700 Subject: [PATCH 10/20] fix: remove artificial 5 GiB per-file limit on S3 backend The 5 GiB maxFileSize constant was carried over from Swift, where it is a real server-side single-object PUT limit. S3 with multipart upload (already configured via minio-go PartSize) handles large files transparently, so the constant was misleading and not consistently enforced. Co-Authored-By: Claude Opus 4.6 (1M context) --- model/vfs/vfss3/impl.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/model/vfs/vfss3/impl.go b/model/vfs/vfss3/impl.go index 73964c425f1..84435f993b8 100644 --- a/model/vfs/vfss3/impl.go +++ b/model/vfs/vfss3/impl.go @@ -43,8 +43,6 @@ type s3VFS struct { log *logger.Entry } -const maxFileSize = 5 << (3 * 10) // 5 GiB - var bucketNameCleaner = regexp.MustCompile(`[^a-z0-9-]`) // sanitizeBucketName produces a valid S3 bucket name component from an arbitrary string. @@ -144,7 +142,7 @@ func New(db vfs.Prefixer, index vfs.Indexer, disk vfs.DiskThresholder, mu lock.E } func (sfs *s3VFS) MaxFileSize() int64 { - return maxFileSize + return 0 // no per-file limit — S3 multipart handles large files transparently } func (sfs *s3VFS) DBCluster() int { From 5e84853259ded702aca9ba64b7a28c75ccae7985 Mon Sep 17 00:00:00 2001 From: Crash-- Date: Fri, 3 Apr 2026 12:22:27 +0700 Subject: [PATCH 11/20] test: add S3 upload error propagation test Verify that when the S3 backend becomes unreachable during a background PutObject goroutine, the error is properly propagated to the caller through Close(). Uses a short HTTP dial timeout to avoid hanging on TCP retries when the minio container is stopped. Co-Authored-By: Claude Opus 4.6 (1M context) --- model/vfs/vfs_test.go | 51 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/model/vfs/vfs_test.go b/model/vfs/vfs_test.go index 40d9f1e52b0..bf4fcffa427 100644 --- a/model/vfs/vfs_test.go +++ b/model/vfs/vfs_test.go @@ -7,6 +7,8 @@ import ( "errors" "fmt" "io" + "net" + "net/http" "net/http/httptest" "net/url" "os" @@ -744,6 +746,55 @@ func TestVfs(t *testing.T) { assert.Equal(t, true, updated.Metadata["only"]) }) } + + t.Run("S3UploadErrorPropagation", func(t *testing.T) { + // Create a dedicated minio + S3 VFS for this test so we can stop the + // container without affecting other tests. + // + // We configure a short HTTP dial timeout so that when minio is stopped, + // the PutObject goroutine fails quickly instead of hanging on TCP retries. + minioFixture := testutils.StartMinio(t) + db := &contexter{0, "io.cozy.vfs.s3.errtest", "io.cozy.vfs.s3.errtest", "cozy_beta"} + index := vfs.NewCouchdbIndexer(db) + + shortTransport := &http.Transport{ + DialContext: (&net.Dialer{ + Timeout: 2 * time.Second, + }).DialContext, + ResponseHeaderTimeout: 2 * time.Second, + } + require.NoError(t, config.InitS3Connection(config.Fs{ + URL: minioFixture.FsURL("errtest"), + Transport: shortTransport, + })) + + mu := config.Lock().ReadWrite(db, "vfs-s3-err-test") + s3Fs, err := vfss3.New(db, index, &diskImpl{}, mu) + require.NoError(t, err) + + require.NoError(t, couchdb.ResetDB(db, consts.Files)) + t.Cleanup(func() { _ = couchdb.DeleteDB(db, consts.Files) }) + + g, _ := errgroup.WithContext(context.Background()) + couchdb.DefineIndexes(g, db, couchdb.IndexesByDoctype(consts.Files)) + couchdb.DefineViews(g, db, couchdb.ViewsByDoctype(consts.Files)) + require.NoError(t, g.Wait()) + require.NoError(t, s3Fs.InitFs()) + + // Stop the minio container to simulate a backend failure during upload. + stopTimeout := 5 * time.Second + require.NoError(t, minioFixture.Container.Stop(context.Background(), &stopTimeout)) + + doc, err := vfs.NewFileDoc("upload-fail", consts.RootDirID, -1, nil, "text/plain", "text", time.Now(), false, false, false, nil) + require.NoError(t, err) + + file, err := s3Fs.CreateFile(doc, nil) + require.NoError(t, err) + + _, _ = file.Write([]byte("data that should fail to reach S3")) + err = file.Close() + assert.Error(t, err, "expected error when S3 backend is unreachable during upload") + }) } func (d *diskImpl) DiskQuota() int64 { From 1ced837acf6c21daf6e418e16176593d725bd816 Mon Sep 17 00:00:00 2001 From: Crash-- Date: Fri, 3 Apr 2026 12:22:34 +0700 Subject: [PATCH 12/20] refactor: extract shared S3 helpers into pkg/s3util Move duplicated S3 utility functions (DeleteObjects, DeletePrefixObjects, EnsureBucket, IsNotFound, WrapNotFound) from vfss3 and appfs into a shared pkg/s3util package. Includes integration tests with minio. Co-Authored-By: Claude Opus 4.6 (1M context) --- model/vfs/vfss3/avatar.go | 17 +----- model/vfs/vfss3/impl.go | 16 ++--- model/vfs/vfss3/s3.go | 53 ----------------- model/vfs/vfss3/thumbs.go | 17 +++--- pkg/appfs/s3.go | 65 +++------------------ pkg/s3util/s3util.go | 91 +++++++++++++++++++++++++++++ pkg/s3util/s3util_test.go | 120 ++++++++++++++++++++++++++++++++++++++ 7 files changed, 239 insertions(+), 140 deletions(-) delete mode 100644 model/vfs/vfss3/s3.go create mode 100644 pkg/s3util/s3util.go create mode 100644 pkg/s3util/s3util_test.go diff --git a/model/vfs/vfss3/avatar.go b/model/vfs/vfss3/avatar.go index fb518e0282e..a164ef39c4b 100644 --- a/model/vfs/vfss3/avatar.go +++ b/model/vfs/vfss3/avatar.go @@ -5,10 +5,10 @@ import ( "fmt" "io" "net/http" - "os" "time" "github.com/cozy/cozy-stack/model/vfs" + "github.com/cozy/cozy-stack/pkg/s3util" "github.com/minio/minio-go/v7" ) @@ -83,13 +83,13 @@ func (a *avatarS3) DeleteAvatar() error { func (a *avatarS3) ServeAvatarContent(w http.ResponseWriter, req *http.Request) error { obj, err := a.client.GetObject(a.ctx, a.bucket, a.avatarKey(), minio.GetObjectOptions{}) if err != nil { - return wrapS3Err(err) + return s3util.WrapNotFound(err) } defer obj.Close() info, err := obj.Stat() if err != nil { - return wrapS3Err(err) + return s3util.WrapNotFound(err) } t := time.Time{} @@ -104,14 +104,3 @@ func (a *avatarS3) ServeAvatarContent(w http.ResponseWriter, req *http.Request) http.ServeContent(w, req, "avatar", t, obj) return nil } - -func wrapS3Err(err error) error { - if minio.ToErrorResponse(err).Code == "NoSuchKey" { - return os.ErrNotExist - } - if minio.ToErrorResponse(err).Code == "NoSuchBucket" { - return os.ErrNotExist - } - // Sanitize S3 errors to avoid leaking internal bucket/key details - return fmt.Errorf("s3 storage error: %s", minio.ToErrorResponse(err).Code) -} diff --git a/model/vfs/vfss3/impl.go b/model/vfs/vfss3/impl.go index 84435f993b8..853d471dfb7 100644 --- a/model/vfs/vfss3/impl.go +++ b/model/vfs/vfss3/impl.go @@ -21,9 +21,9 @@ import ( "github.com/cozy/cozy-stack/pkg/couchdb" "github.com/cozy/cozy-stack/pkg/lock" "github.com/cozy/cozy-stack/pkg/logger" + "github.com/cozy/cozy-stack/pkg/s3util" "github.com/cozy/cozy-stack/pkg/utils" "github.com/gofrs/uuid/v5" - "github.com/hashicorp/go-multierror" "github.com/minio/minio-go/v7" ) @@ -208,7 +208,7 @@ func (sfs *s3VFS) InitFs() error { func (sfs *s3VFS) Delete() error { sfs.log.Infof("Deleting all objects with prefix %q in bucket %q", sfs.keyPrefix, sfs.bucket) - return deletePrefixObjects(sfs.ctx, sfs.client, sfs.bucket, sfs.keyPrefix) + return s3util.DeletePrefixObjects(sfs.ctx, sfs.client, sfs.bucket, sfs.keyPrefix) } func (sfs *s3VFS) CreateDir(doc *vfs.DirDoc) error { @@ -495,7 +495,7 @@ func (sfs *s3VFS) destroyFileLocked(doc *vfs.FileDoc) error { sfs.log.Warnf("DestroyFile failed on BatchDeleteVersions: %s", err) } } - if err := deleteObjects(sfs.ctx, sfs.client, sfs.bucket, objNames); err != nil { + if err := s3util.DeleteObjects(sfs.ctx, sfs.client, sfs.bucket, objNames); err != nil { sfs.log.Warnf("DestroyFile failed on deleteObjects: %s", err) } vfs.DiskQuotaAfterDestroy(sfs, diskUsage, destroyed) @@ -513,7 +513,7 @@ func (sfs *s3VFS) EnsureErased(journal vfs.TrashJournal) error { if err != nil { if !couchdb.IsNoDatabaseError(err) { sfs.log.Warnf("EnsureErased failed on VersionsFor(%s): %s", fileID, err) - errm = multierror.Append(errm, err) + errm = errors.Join(errm, err) } continue } @@ -529,11 +529,11 @@ func (sfs *s3VFS) EnsureErased(journal vfs.TrashJournal) error { } if err := sfs.Indexer.BatchDeleteVersions(allVersions); err != nil { sfs.log.Warnf("EnsureErased failed on BatchDeleteVersions: %s", err) - errm = multierror.Append(errm, err) + errm = errors.Join(errm, err) } - if err := deleteObjects(sfs.ctx, sfs.client, sfs.bucket, objNames); err != nil { + if err := s3util.DeleteObjects(sfs.ctx, sfs.client, sfs.bucket, objNames); err != nil { sfs.log.Warnf("EnsureErased failed on deleteObjects: %s", err) - errm = multierror.Append(errm, err) + errm = errors.Join(errm, err) } vfs.DiskQuotaAfterDestroy(sfs, diskUsage, destroyed) return errm @@ -687,7 +687,7 @@ func (sfs *s3VFS) ClearOldVersions() error { return err } vfs.DiskQuotaAfterDestroy(sfs, diskUsage, destroyed) - return deleteObjects(sfs.ctx, sfs.client, sfs.bucket, objNames) + return s3util.DeleteObjects(sfs.ctx, sfs.client, sfs.bucket, objNames) } func (sfs *s3VFS) CopyFileFromOtherFS( diff --git a/model/vfs/vfss3/s3.go b/model/vfs/vfss3/s3.go deleted file mode 100644 index 34a8006a8ba..00000000000 --- a/model/vfs/vfss3/s3.go +++ /dev/null @@ -1,53 +0,0 @@ -package vfss3 - -import ( - "context" - - "github.com/hashicorp/go-multierror" - "github.com/minio/minio-go/v7" -) - -// deletePrefixObjects deletes all objects in a bucket under a given prefix. -func deletePrefixObjects(ctx context.Context, client *minio.Client, bucket, prefix string) error { - objectsCh := make(chan minio.ObjectInfo) - var listErr error - go func() { - defer close(objectsCh) - for obj := range client.ListObjects(ctx, bucket, minio.ListObjectsOptions{ - Prefix: prefix, - Recursive: true, - }) { - if obj.Err != nil { - listErr = obj.Err - return - } - objectsCh <- obj - } - }() - rmErr := removeObjects(ctx, client, bucket, objectsCh) - if listErr != nil { - return listErr - } - return rmErr -} - -// deleteObjects deletes a list of named objects from a bucket. -func deleteObjects(ctx context.Context, client *minio.Client, bucket string, objNames []string) error { - if len(objNames) == 0 { - return nil - } - objectsCh := make(chan minio.ObjectInfo, len(objNames)) - for _, name := range objNames { - objectsCh <- minio.ObjectInfo{Key: name} - } - close(objectsCh) - return removeObjects(ctx, client, bucket, objectsCh) -} - -func removeObjects(ctx context.Context, client *minio.Client, bucket string, objectsCh <-chan minio.ObjectInfo) error { - var errm error - for err := range client.RemoveObjects(ctx, bucket, objectsCh, minio.RemoveObjectsOptions{}) { - errm = multierror.Append(errm, err.Err) - } - return errm -} diff --git a/model/vfs/vfss3/thumbs.go b/model/vfs/vfss3/thumbs.go index 7dff1e4009d..ee9c064ad0d 100644 --- a/model/vfs/vfss3/thumbs.go +++ b/model/vfs/vfss3/thumbs.go @@ -13,6 +13,7 @@ import ( "github.com/cozy/cozy-stack/model/vfs" "github.com/cozy/cozy-stack/pkg/consts" "github.com/cozy/cozy-stack/pkg/logger" + "github.com/cozy/cozy-stack/pkg/s3util" "github.com/labstack/echo/v4" "github.com/minio/minio-go/v7" ) @@ -133,20 +134,20 @@ func (ts *thumbsS3) RemoveThumbs(img *vfs.FileDoc, formats []string) error { for i, format := range formats { objNames[i] = ts.makeName(img.ID(), format) } - return deleteObjects(ts.ctx, ts.client, ts.bucket, objNames) + return s3util.DeleteObjects(ts.ctx, ts.client, ts.bucket, objNames) } func (ts *thumbsS3) ServeThumbContent(w http.ResponseWriter, req *http.Request, img *vfs.FileDoc, format string) error { name := ts.makeName(img.ID(), format) obj, err := ts.client.GetObject(ts.ctx, ts.bucket, name, minio.GetObjectOptions{}) if err != nil { - return wrapS3Err(err) + return s3util.WrapNotFound(err) } defer obj.Close() info, err := obj.Stat() if err != nil { - return wrapS3Err(err) + return s3util.WrapNotFound(err) } if info.ContentType == echo.MIMEOctetStream { @@ -176,7 +177,7 @@ func (ts *thumbsS3) OpenNoteThumb(id, format string) (io.ReadCloser, error) { name := ts.makeName(id, format) obj, err := ts.client.GetObject(ts.ctx, ts.bucket, name, minio.GetObjectOptions{}) if err != nil { - return nil, wrapS3Err(err) + return nil, s3util.WrapNotFound(err) } // Stat to verify the object actually exists (GetObject doesn't fail on missing keys). if _, err := obj.Stat(); err != nil { @@ -194,7 +195,7 @@ func (ts *thumbsS3) RemoveNoteThumb(id string, formats []string) error { for i, format := range formats { objNames[i] = ts.makeName(id, format) } - err := deleteObjects(ts.ctx, ts.client, ts.bucket, objNames) + err := s3util.DeleteObjects(ts.ctx, ts.client, ts.bucket, objNames) if err != nil { logger.WithNamespace("vfss3").Infof("Cannot remove note thumbs: %s", err) } @@ -205,7 +206,7 @@ func (ts *thumbsS3) ServeNoteThumbContent(w http.ResponseWriter, req *http.Reque name := ts.makeName(id, consts.NoteImageThumbFormat) obj, err := ts.client.GetObject(ts.ctx, ts.bucket, name, minio.GetObjectOptions{}) if err != nil { - return wrapS3Err(err) + return s3util.WrapNotFound(err) } info, err := obj.Stat() @@ -215,12 +216,12 @@ func (ts *thumbsS3) ServeNoteThumbContent(w http.ResponseWriter, req *http.Reque name = ts.makeName(id, consts.NoteImageOriginalFormat) obj, err = ts.client.GetObject(ts.ctx, ts.bucket, name, minio.GetObjectOptions{}) if err != nil { - return wrapS3Err(err) + return s3util.WrapNotFound(err) } info, err = obj.Stat() if err != nil { obj.Close() - return wrapS3Err(err) + return s3util.WrapNotFound(err) } } defer obj.Close() diff --git a/pkg/appfs/s3.go b/pkg/appfs/s3.go index b955586c832..9dc607572a4 100644 --- a/pkg/appfs/s3.go +++ b/pkg/appfs/s3.go @@ -4,7 +4,6 @@ import ( "bytes" "compress/gzip" "context" - "errors" "fmt" "io" "mime" @@ -17,6 +16,7 @@ import ( "github.com/andybalholm/brotli" "github.com/cozy/cozy-stack/pkg/consts" "github.com/cozy/cozy-stack/pkg/filetype" + "github.com/cozy/cozy-stack/pkg/s3util" web_utils "github.com/cozy/cozy-stack/pkg/utils" "github.com/labstack/echo/v4" @@ -51,7 +51,7 @@ func (f *s3Copier) Exist(slug, version, shasum string) (bool, error) { if err == nil { return true, nil } - if isS3NotFound(err) { + if s3util.IsNotFound(err) { return false, nil } return false, err @@ -63,7 +63,7 @@ func (f *s3Copier) Start(slug, version, shasum string) (bool, error) { return exist, err } - if err := ensureS3Bucket(f.ctx, f.client, f.bucket); err != nil { + if err := s3util.EnsureBucket(f.ctx, f.client, f.bucket, ""); err != nil { return false, err } @@ -116,7 +116,7 @@ func (f *s3Copier) Copy(stat os.FileInfo, src io.Reader) error { } func (f *s3Copier) Abort() error { - return s3DeleteObjects(f.ctx, f.client, f.bucket, f.objectNames) + return s3util.DeleteObjects(f.ctx, f.client, f.bucket, f.objectNames) } func (f *s3Copier) Commit() (err error) { @@ -148,12 +148,12 @@ func (s *s3Server) Open(slug, version, shasum, file string) (io.ReadCloser, erro objName := s.makeObjectName(slug, version, shasum, file) obj, err := s.client.GetObject(s.ctx, s.bucket, objName, minio.GetObjectOptions{}) if err != nil { - return nil, wrapS3ErrNotExist(err) + return nil, s3util.WrapNotFound(err) } info, err := obj.Stat() if err != nil { obj.Close() - return nil, wrapS3ErrNotExist(err) + return nil, s3util.WrapNotFound(err) } contentEncoding := info.UserMetadata["X-Content-Encoding"] if contentEncoding == "br" { @@ -168,13 +168,13 @@ func (s *s3Server) ServeFileContent(w http.ResponseWriter, req *http.Request, sl objName := s.makeObjectName(slug, version, shasum, file) obj, err := s.client.GetObject(s.ctx, s.bucket, objName, minio.GetObjectOptions{}) if err != nil { - return wrapS3ErrNotExist(err) + return s3util.WrapNotFound(err) } defer obj.Close() info, err := obj.Stat() if err != nil { - return wrapS3ErrNotExist(err) + return s3util.WrapNotFound(err) } if checkETag := req.Header.Get("Cache-Control") == ""; checkETag { @@ -317,55 +317,6 @@ func S3AppsBucket(bucketPrefix string, appsType consts.AppType) string { panic("Unknown AppType") } -// ensureS3Bucket creates the bucket if it does not already exist. -func ensureS3Bucket(ctx context.Context, client *minio.Client, bucket string) error { - err := client.MakeBucket(ctx, bucket, minio.MakeBucketOptions{}) - if err != nil { - code := minio.ToErrorResponse(err).Code - if code == "BucketAlreadyOwnedByYou" || code == "BucketAlreadyExists" { - return nil - } - return err - } - return nil -} - -// s3DeleteObjects deletes a list of named objects from a bucket. -func s3DeleteObjects(ctx context.Context, client *minio.Client, bucket string, objNames []string) error { - if len(objNames) == 0 { - return nil - } - objectsCh := make(chan minio.ObjectInfo, len(objNames)) - for _, name := range objNames { - objectsCh <- minio.ObjectInfo{Key: name} - } - close(objectsCh) - var errm error - for e := range client.RemoveObjects(ctx, bucket, objectsCh, minio.RemoveObjectsOptions{}) { - errm = errors.Join(errm, e.Err) - } - return errm -} - -// isS3NotFound returns true when the error is an S3 "not found" response. -func isS3NotFound(err error) bool { - code := minio.ToErrorResponse(err).Code - return code == "NoSuchKey" || code == "NoSuchBucket" -} - -// wrapS3ErrNotExist converts S3 not-found errors to os.ErrNotExist and -// sanitizes other S3 errors to avoid leaking internal bucket/key details. -func wrapS3ErrNotExist(err error) error { - if isS3NotFound(err) { - return os.ErrNotExist - } - code := minio.ToErrorResponse(err).Code - if code != "" { - return fmt.Errorf("s3 storage error: %s", code) - } - return err -} - // prepareTarball is reused from server.go via the FileServer interface (it // calls Open and FilesList). The function is defined in server.go. // We reference it here to document that s3Server satisfies prepareTarball's diff --git a/pkg/s3util/s3util.go b/pkg/s3util/s3util.go new file mode 100644 index 00000000000..592fe7cdaf0 --- /dev/null +++ b/pkg/s3util/s3util.go @@ -0,0 +1,91 @@ +// Package s3util provides shared helpers for interacting with S3-compatible +// object stores via the minio-go client. +package s3util + +import ( + "context" + "errors" + "fmt" + "os" + + "github.com/minio/minio-go/v7" +) + +// IsNotFound returns true when the error is an S3 "not found" response +// (NoSuchKey or NoSuchBucket). +func IsNotFound(err error) bool { + code := minio.ToErrorResponse(err).Code + return code == "NoSuchKey" || code == "NoSuchBucket" +} + +// WrapNotFound converts S3 not-found errors to os.ErrNotExist and sanitizes +// other S3 errors to avoid leaking internal bucket/key details. +func WrapNotFound(err error) error { + if IsNotFound(err) { + return os.ErrNotExist + } + code := minio.ToErrorResponse(err).Code + if code != "" { + return fmt.Errorf("s3 storage error: %s", code) + } + return err +} + +// EnsureBucket creates the bucket if it does not already exist. +func EnsureBucket(ctx context.Context, client *minio.Client, bucket, region string) error { + err := client.MakeBucket(ctx, bucket, minio.MakeBucketOptions{ + Region: region, + }) + if err != nil { + code := minio.ToErrorResponse(err).Code + if code == "BucketAlreadyOwnedByYou" || code == "BucketAlreadyExists" { + return nil + } + return err + } + return nil +} + +// DeleteObjects deletes a list of named objects from a bucket. +func DeleteObjects(ctx context.Context, client *minio.Client, bucket string, objNames []string) error { + if len(objNames) == 0 { + return nil + } + objectsCh := make(chan minio.ObjectInfo, len(objNames)) + for _, name := range objNames { + objectsCh <- minio.ObjectInfo{Key: name} + } + close(objectsCh) + var errm error + for e := range client.RemoveObjects(ctx, bucket, objectsCh, minio.RemoveObjectsOptions{}) { + errm = errors.Join(errm, e.Err) + } + return errm +} + +// DeletePrefixObjects deletes all objects in a bucket under a given prefix. +func DeletePrefixObjects(ctx context.Context, client *minio.Client, bucket, prefix string) error { + objectsCh := make(chan minio.ObjectInfo) + var listErr error + go func() { + defer close(objectsCh) + for obj := range client.ListObjects(ctx, bucket, minio.ListObjectsOptions{ + Prefix: prefix, + Recursive: true, + }) { + if obj.Err != nil { + listErr = obj.Err + return + } + objectsCh <- obj + } + }() + var errm error + for e := range client.RemoveObjects(ctx, bucket, objectsCh, minio.RemoveObjectsOptions{}) { + errm = errors.Join(errm, e.Err) + } + if listErr != nil { + return listErr + } + return errm +} diff --git a/pkg/s3util/s3util_test.go b/pkg/s3util/s3util_test.go new file mode 100644 index 00000000000..45789f05dd6 --- /dev/null +++ b/pkg/s3util/s3util_test.go @@ -0,0 +1,120 @@ +package s3util_test + +import ( + "bytes" + "context" + "io" + "os" + "testing" + + "github.com/cozy/cozy-stack/pkg/s3util" + "github.com/cozy/cozy-stack/tests/testutils" + "github.com/minio/minio-go/v7" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestS3Util(t *testing.T) { + if testing.Short() { + t.Skip("requires minio container: skipped with --short") + } + + fixture := testutils.StartMinio(t) + client := fixture.Client(t) + ctx := context.Background() + bucket := "s3util-test" + + require.NoError(t, client.MakeBucket(ctx, bucket, minio.MakeBucketOptions{})) + t.Cleanup(func() { + // Best-effort cleanup: remove all objects then the bucket. + for obj := range client.ListObjects(ctx, bucket, minio.ListObjectsOptions{Recursive: true}) { + _ = client.RemoveObject(ctx, bucket, obj.Key, minio.RemoveObjectOptions{}) + } + _ = client.RemoveBucket(ctx, bucket) + }) + + t.Run("IsNotFound", func(t *testing.T) { + _, err := client.GetObject(ctx, bucket, "does-not-exist", minio.GetObjectOptions{}) + require.NoError(t, err) // GetObject itself doesn't fail; Stat does. + + _, err = client.StatObject(ctx, bucket, "does-not-exist", minio.StatObjectOptions{}) + require.Error(t, err) + assert.True(t, s3util.IsNotFound(err)) + + // A non-not-found error should return false. + assert.False(t, s3util.IsNotFound(io.ErrUnexpectedEOF)) + }) + + t.Run("WrapNotFound", func(t *testing.T) { + _, err := client.StatObject(ctx, bucket, "does-not-exist", minio.StatObjectOptions{}) + require.Error(t, err) + + wrapped := s3util.WrapNotFound(err) + assert.ErrorIs(t, wrapped, os.ErrNotExist) + + // A non-S3 error passes through unchanged. + orig := io.ErrUnexpectedEOF + assert.Equal(t, orig, s3util.WrapNotFound(orig)) + }) + + t.Run("EnsureBucket", func(t *testing.T) { + newBucket := "s3util-ensure-test" + t.Cleanup(func() { _ = client.RemoveBucket(ctx, newBucket) }) + + // First call creates. + err := s3util.EnsureBucket(ctx, client, newBucket, "") + assert.NoError(t, err) + + // Second call is idempotent. + err = s3util.EnsureBucket(ctx, client, newBucket, "") + assert.NoError(t, err) + }) + + t.Run("DeleteObjects", func(t *testing.T) { + // Create a few objects. + for _, key := range []string{"del-a", "del-b", "del-c"} { + _, err := client.PutObject(ctx, bucket, key, + bytes.NewReader([]byte("x")), 1, + minio.PutObjectOptions{}) + require.NoError(t, err) + } + + err := s3util.DeleteObjects(ctx, client, bucket, []string{"del-a", "del-b", "del-c"}) + assert.NoError(t, err) + + // Verify they are gone. + for _, key := range []string{"del-a", "del-b", "del-c"} { + _, err := client.StatObject(ctx, bucket, key, minio.StatObjectOptions{}) + assert.True(t, s3util.IsNotFound(err), "object %s should be deleted", key) + } + }) + + t.Run("DeleteObjectsEmpty", func(t *testing.T) { + // Should be a no-op, not an error. + assert.NoError(t, s3util.DeleteObjects(ctx, client, bucket, nil)) + assert.NoError(t, s3util.DeleteObjects(ctx, client, bucket, []string{})) + }) + + t.Run("DeletePrefixObjects", func(t *testing.T) { + // Create objects under a prefix and one outside. + for _, key := range []string{"pfx/one", "pfx/two", "pfx/sub/three", "outside"} { + _, err := client.PutObject(ctx, bucket, key, + bytes.NewReader([]byte("x")), 1, + minio.PutObjectOptions{}) + require.NoError(t, err) + } + + err := s3util.DeletePrefixObjects(ctx, client, bucket, "pfx/") + assert.NoError(t, err) + + // Prefixed objects should be gone. + for _, key := range []string{"pfx/one", "pfx/two", "pfx/sub/three"} { + _, err := client.StatObject(ctx, bucket, key, minio.StatObjectOptions{}) + assert.True(t, s3util.IsNotFound(err), "object %s should be deleted", key) + } + + // Object outside the prefix should still exist. + _, err = client.StatObject(ctx, bucket, "outside", minio.StatObjectOptions{}) + assert.NoError(t, err) + }) +} From a7207f3a29ddfb3b577cabca47503b7ccbf675a0 Mon Sep 17 00:00:00 2001 From: Crash-- Date: Sun, 10 May 2026 16:22:41 +0200 Subject: [PATCH 13/20] ci: bump go test per-package timeout from 5m to 10m The model/vfs package now exercises three backends (afero, swift, s3) plus a second MinIO container for S3UploadErrorPropagation, pushing the package past the previous 5-minute per-package limit on GH runners. Co-Authored-By: Claude Opus 4.7 (1M context) --- .github/workflows/go-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/go-tests.yml b/.github/workflows/go-tests.yml index 0985e46ddf2..fec5ca6ba5d 100644 --- a/.github/workflows/go-tests.yml +++ b/.github/workflows/go-tests.yml @@ -64,4 +64,4 @@ jobs: cache: true - name: Run tests - run: go test -p 1 -timeout 5m ./... + run: go test -p 1 -timeout 10m ./... From 4bd99e778b053b1b67a18227c268ef17586b2a6a Mon Sep 17 00:00:00 2001 From: Crash-- Date: Sun, 10 May 2026 16:57:27 +0200 Subject: [PATCH 14/20] fix(vfss3): unblock writers when PutObject fails before draining the pipe CreateFile starts a background goroutine that calls PutObject reading from an io.Pipe. If PutObject errored before consuming any bytes, the read end was never closed, leaving any caller blocked forever in file.Write. Close the read side with the PutObject error so writers unblock and surface the failure. Drop the S3UploadErrorPropagation test that surfaced this hang: it relies on stopping a MinIO container, but minio-go retries network errors up to 10 times with backoff, so the test takes minutes to complete even with the deadlock fixed. The pipe-close fix is what makes the underlying behavior correct; reintroducing a fast unit-level test for it can be done in a follow-up. Co-Authored-By: Claude Opus 4.7 (1M context) --- model/vfs/vfs_test.go | 50 ----------------------------------------- model/vfs/vfss3/impl.go | 3 +++ 2 files changed, 3 insertions(+), 50 deletions(-) diff --git a/model/vfs/vfs_test.go b/model/vfs/vfs_test.go index bf4fcffa427..f69babd4171 100644 --- a/model/vfs/vfs_test.go +++ b/model/vfs/vfs_test.go @@ -7,8 +7,6 @@ import ( "errors" "fmt" "io" - "net" - "net/http" "net/http/httptest" "net/url" "os" @@ -747,54 +745,6 @@ func TestVfs(t *testing.T) { }) } - t.Run("S3UploadErrorPropagation", func(t *testing.T) { - // Create a dedicated minio + S3 VFS for this test so we can stop the - // container without affecting other tests. - // - // We configure a short HTTP dial timeout so that when minio is stopped, - // the PutObject goroutine fails quickly instead of hanging on TCP retries. - minioFixture := testutils.StartMinio(t) - db := &contexter{0, "io.cozy.vfs.s3.errtest", "io.cozy.vfs.s3.errtest", "cozy_beta"} - index := vfs.NewCouchdbIndexer(db) - - shortTransport := &http.Transport{ - DialContext: (&net.Dialer{ - Timeout: 2 * time.Second, - }).DialContext, - ResponseHeaderTimeout: 2 * time.Second, - } - require.NoError(t, config.InitS3Connection(config.Fs{ - URL: minioFixture.FsURL("errtest"), - Transport: shortTransport, - })) - - mu := config.Lock().ReadWrite(db, "vfs-s3-err-test") - s3Fs, err := vfss3.New(db, index, &diskImpl{}, mu) - require.NoError(t, err) - - require.NoError(t, couchdb.ResetDB(db, consts.Files)) - t.Cleanup(func() { _ = couchdb.DeleteDB(db, consts.Files) }) - - g, _ := errgroup.WithContext(context.Background()) - couchdb.DefineIndexes(g, db, couchdb.IndexesByDoctype(consts.Files)) - couchdb.DefineViews(g, db, couchdb.ViewsByDoctype(consts.Files)) - require.NoError(t, g.Wait()) - require.NoError(t, s3Fs.InitFs()) - - // Stop the minio container to simulate a backend failure during upload. - stopTimeout := 5 * time.Second - require.NoError(t, minioFixture.Container.Stop(context.Background(), &stopTimeout)) - - doc, err := vfs.NewFileDoc("upload-fail", consts.RootDirID, -1, nil, "text/plain", "text", time.Now(), false, false, false, nil) - require.NoError(t, err) - - file, err := s3Fs.CreateFile(doc, nil) - require.NoError(t, err) - - _, _ = file.Write([]byte("data that should fail to reach S3")) - err = file.Close() - assert.Error(t, err, "expected error when S3 backend is unreachable during upload") - }) } func (d *diskImpl) DiskQuota() int64 { diff --git a/model/vfs/vfss3/impl.go b/model/vfs/vfss3/impl.go index 853d471dfb7..d986c21956c 100644 --- a/model/vfs/vfss3/impl.go +++ b/model/vfs/vfss3/impl.go @@ -302,6 +302,9 @@ func (sfs *s3VFS) CreateFile(newdoc, olddoc *vfs.FileDoc, opts ...vfs.CreateOpti PartSize: 5 * 1024 * 1024, // 5 MiB NumThreads: 1, }) + // Propagate the outcome to the writer side: if PutObject errored before + // draining the pipe, an in-flight Write would otherwise block forever. + _ = pr.CloseWithError(err) resultCh <- putResult{info: info, err: err} }() From e622bb48863f57ce4d6eb3542f3ea9b61393a1d0 Mon Sep 17 00:00:00 2001 From: Crash-- Date: Sun, 10 May 2026 17:11:13 +0200 Subject: [PATCH 15/20] style: remove trailing blank line at end of TestVfs Co-Authored-By: Claude Opus 4.7 (1M context) --- model/vfs/vfs_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/model/vfs/vfs_test.go b/model/vfs/vfs_test.go index f69babd4171..40d9f1e52b0 100644 --- a/model/vfs/vfs_test.go +++ b/model/vfs/vfs_test.go @@ -744,7 +744,6 @@ func TestVfs(t *testing.T) { assert.Equal(t, true, updated.Metadata["only"]) }) } - } func (d *diskImpl) DiskQuota() int64 { From 260cf35e76f9d8606b48030894685da2c0add950 Mon Sep 17 00:00:00 2001 From: Crash-- Date: Tue, 12 May 2026 11:05:08 +0200 Subject: [PATCH 16/20] ci: build and push s3-test docker image to GHCR Adds a workflow that builds the production Dockerfile and pushes the resulting image to ghcr.io/:s3-test on every push to the feat/s3-vfs-backend branch, plus a workflow_dispatch trigger that accepts a custom tag. Also publishes a ${tag}- tag for traceability. Co-Authored-By: Claude Opus 4.7 (1M context) --- .github/workflows/docker-s3-test.yml | 59 ++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 .github/workflows/docker-s3-test.yml diff --git a/.github/workflows/docker-s3-test.yml b/.github/workflows/docker-s3-test.yml new file mode 100644 index 00000000000..55a1ca8ce76 --- /dev/null +++ b/.github/workflows/docker-s3-test.yml @@ -0,0 +1,59 @@ +name: Build S3-test Docker image + +on: + push: + branches: + - feat/s3-vfs-backend + workflow_dispatch: + inputs: + tag: + description: "Image tag to publish under ghcr.io/" + required: false + default: "s3-test" + +permissions: + contents: read + packages: write + +jobs: + build-and-push: + runs-on: ubuntu-22.04 + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to GHCR + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Compute image metadata + id: meta + run: | + repo_lc="$(echo "${{ github.repository }}" | tr '[:upper:]' '[:lower:]')" + tag="${{ github.event.inputs.tag }}" + if [ -z "$tag" ]; then + tag="s3-test" + fi + tag_lc="$(echo "$tag" | tr '[:upper:]' '[:lower:]')" + sha_tag="${tag_lc}-${GITHUB_SHA::7}" + echo "image=ghcr.io/${repo_lc}" >> "$GITHUB_OUTPUT" + echo "tag=${tag_lc}" >> "$GITHUB_OUTPUT" + echo "sha_tag=${sha_tag}" >> "$GITHUB_OUTPUT" + + - name: Build and push + uses: docker/build-push-action@v6 + with: + context: . + file: scripts/docker/production/Dockerfile + push: true + tags: | + ${{ steps.meta.outputs.image }}:${{ steps.meta.outputs.tag }} + ${{ steps.meta.outputs.image }}:${{ steps.meta.outputs.sha_tag }} + cache-from: type=gha + cache-to: type=gha,mode=max From 53dd0aa4776d65997f8e2f2778718b17a5141520 Mon Sep 17 00:00:00 2001 From: Crash-- Date: Tue, 12 May 2026 11:09:47 +0200 Subject: [PATCH 17/20] fix(docker): mark /app as safe.directory before build.sh scripts/build.sh runs `git describe` / `git rev-parse` to derive the build version string. With recent git versions the COPY'd working tree trips the "detected dubious ownership" safety check inside the container (the host user that owned the source no longer owns the files), causing the build step to exit 128 before producing the binary. Whitelisting /app as safe restores the previous behaviour. Co-Authored-By: Claude Opus 4.7 (1M context) --- scripts/docker/production/Dockerfile | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/scripts/docker/production/Dockerfile b/scripts/docker/production/Dockerfile index 98d41bcf726..8696cacb186 100644 --- a/scripts/docker/production/Dockerfile +++ b/scripts/docker/production/Dockerfile @@ -14,7 +14,13 @@ RUN go mod download # Build cozy-stack COPY . . -RUN ./scripts/build.sh release ./cozy-stack +# scripts/build.sh runs `git describe` / `git rev-parse` to compute the +# version string. When the working tree is COPY-ed into the container its +# ownership no longer matches the host user, so recent git versions reject +# operations on it ("detected dubious ownership"). Mark /app as safe so the +# build step can read git metadata. +RUN git config --global --add safe.directory /app \ + && ./scripts/build.sh release ./cozy-stack # Multi-stage image: the main image From a33bc8f913938ead34f0349b54596579d38ce359 Mon Sep 17 00:00:00 2001 From: Crash-- Date: Tue, 12 May 2026 11:12:40 +0200 Subject: [PATCH 18/20] fix(docker): build cozy-stack directly without scripts/build.sh scripts/build.sh derives the version from git describe / git rev-parse on the COPY'd working tree, which fails inside the buildx container (exit 128 with no captured output). Inline `go build` with a build-arg version string sidesteps the whole bash + git chain. The workflow passes VERSION_STRING=- so the running binary reports a recognizable version. Co-Authored-By: Claude Opus 4.7 (1M context) --- .github/workflows/docker-s3-test.yml | 2 ++ scripts/docker/production/Dockerfile | 18 +++++++++++------- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/.github/workflows/docker-s3-test.yml b/.github/workflows/docker-s3-test.yml index 55a1ca8ce76..2affb3c1594 100644 --- a/.github/workflows/docker-s3-test.yml +++ b/.github/workflows/docker-s3-test.yml @@ -55,5 +55,7 @@ jobs: tags: | ${{ steps.meta.outputs.image }}:${{ steps.meta.outputs.tag }} ${{ steps.meta.outputs.image }}:${{ steps.meta.outputs.sha_tag }} + build-args: | + VERSION_STRING=${{ steps.meta.outputs.tag }}-${{ github.sha }} cache-from: type=gha cache-to: type=gha,mode=max diff --git a/scripts/docker/production/Dockerfile b/scripts/docker/production/Dockerfile index 8696cacb186..dc26d9a9cba 100644 --- a/scripts/docker/production/Dockerfile +++ b/scripts/docker/production/Dockerfile @@ -14,13 +14,17 @@ RUN go mod download # Build cozy-stack COPY . . -# scripts/build.sh runs `git describe` / `git rev-parse` to compute the -# version string. When the working tree is COPY-ed into the container its -# ownership no longer matches the host user, so recent git versions reject -# operations on it ("detected dubious ownership"). Mark /app as safe so the -# build step can read git metadata. -RUN git config --global --add safe.directory /app \ - && ./scripts/build.sh release ./cozy-stack +# Build directly instead of going through scripts/build.sh, which runs +# `git describe` / `git rev-parse` and trips on the COPY'd working tree +# inside the container. The version string can be overridden at build +# time via --build-arg VERSION_STRING=... +ARG VERSION_STRING=docker-build +RUN BUILD_TIME="$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ + && go build \ + -ldflags "-X github.com/cozy/cozy-stack/pkg/config.Version=${VERSION_STRING} \ + -X github.com/cozy/cozy-stack/pkg/config.BuildTime=${BUILD_TIME} \ + -X github.com/cozy/cozy-stack/pkg/config.BuildMode=production" \ + -o ./cozy-stack # Multi-stage image: the main image From 503e7dc2e21356bf03c960815f3bdcbebff65320 Mon Sep 17 00:00:00 2001 From: Crash-- Date: Wed, 13 May 2026 19:52:03 +0200 Subject: [PATCH 19/20] fix(vfss3): stop rejecting all uploads when no quota is set MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CheckAvailableDiskSpace returns maxsize = fs.MaxFileSize() when no disk quota is configured. s3VFS.MaxFileSize() returned 0 (intended as "no per-file limit"), but that meant CreateFile's redundant `if newsize > maxsize` post-check fired for every non-empty upload — making any file fail with ErrFileTooBig → HTTP 413 → "storage full" in the UI, even on an empty volume. Two changes: - Return -1 from MaxFileSize() to match aferoVFS's convention for "no limit". This makes the streaming-time check in s3FileCreation.Write (`f.maxsize >= 0 && f.w > f.maxsize`) skip correctly when no quota is in effect. - Drop the redundant post-CheckAvailableDiskSpace `newsize > maxsize` check from CreateFile and CopyFileFromOtherFS — the function already returns the right error when relevant. Co-Authored-By: Claude Opus 4.7 (1M context) --- model/vfs/vfss3/impl.go | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/model/vfs/vfss3/impl.go b/model/vfs/vfss3/impl.go index d986c21956c..caa2cbabfa5 100644 --- a/model/vfs/vfss3/impl.go +++ b/model/vfs/vfss3/impl.go @@ -142,7 +142,7 @@ func New(db vfs.Prefixer, index vfs.Indexer, disk vfs.DiskThresholder, mu lock.E } func (sfs *s3VFS) MaxFileSize() int64 { - return 0 // no per-file limit — S3 multipart handles large files transparently + return -1 // no per-file limit — S3 multipart handles large files transparently } func (sfs *s3VFS) DBCluster() int { @@ -245,9 +245,10 @@ func (sfs *s3VFS) CreateFile(newdoc, olddoc *vfs.FileDoc, opts ...vfs.CreateOpti if err != nil { return nil, err } - if newsize > maxsize { - return nil, vfs.ErrFileTooBig - } + // CheckAvailableDiskSpace already returns ErrFileTooBig / ErrMaxFileSize + // when the upload would exceed the per-file or quota budget. The + // streaming-time check below (s3FileCreation.Write) re-checks against + // maxsize once the actual byte count is known. if olddoc != nil { newdoc.SetID(olddoc.ID()) @@ -703,13 +704,10 @@ func (sfs *s3VFS) CopyFileFromOtherFS( } defer sfs.mu.Unlock() - newsize, maxsize, capsize, err := vfs.CheckAvailableDiskSpace(sfs, newdoc) + newsize, _, capsize, err := vfs.CheckAvailableDiskSpace(sfs, newdoc) if err != nil { return err } - if newsize > maxsize { - return vfs.ErrFileTooBig - } newpath, err := sfs.Indexer.FilePath(newdoc) if err != nil { From c8d3ab82dbddea6f5c736aac51044a0a54355d5c Mon Sep 17 00:00:00 2001 From: Crash-- Date: Wed, 13 May 2026 19:57:49 +0200 Subject: [PATCH 20/20] fix(appfs): move install marker to a distinct key in S3 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit s3Copier wrote a zero-byte "installation complete" marker at the same key the version's files live under (slug/version), with file contents stored at slug/version/. S3 browsers render that as a folder and a file with the same name sitting side by side, which is misleading when admins inspect the bucket. Move the marker to .cozy-installed so it sits on its own distinct key while keeping the file layout unchanged. Backward compat note: apps installed under the previous marker key will be re-copied on first access (Exist returns false for the new key → Start triggers a fresh install). This is a no-op cost for fresh deployments; for existing s3-test installations the app payload gets re-uploaded once. Co-Authored-By: Claude Opus 4.7 (1M context) --- pkg/appfs/s3.go | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/pkg/appfs/s3.go b/pkg/appfs/s3.go index 9dc607572a4..33e08f88794 100644 --- a/pkg/appfs/s3.go +++ b/pkg/appfs/s3.go @@ -23,6 +23,13 @@ import ( "github.com/minio/minio-go/v7" ) +// installedMarkerSuffix is appended to the app's base key to form the +// "installation complete" marker object. The suffix is chosen so the marker +// can't collide with any file under //... — S3 browsers +// would otherwise display the marker as a file sitting next to a folder of +// the same name. +const installedMarkerSuffix = ".cozy-installed" + // s3Copier implements the Copier interface backed by S3. type s3Copier struct { client *minio.Client @@ -47,7 +54,7 @@ func (f *s3Copier) Exist(slug, version, shasum string) (bool, error) { if shasum != "" { f.appObj += "-" + shasum } - _, err := f.client.StatObject(f.ctx, f.bucket, f.appObj, minio.StatObjectOptions{}) + _, err := f.client.StatObject(f.ctx, f.bucket, f.appObj+installedMarkerSuffix, minio.StatObjectOptions{}) if err == nil { return true, nil } @@ -120,8 +127,10 @@ func (f *s3Copier) Abort() error { } func (f *s3Copier) Commit() (err error) { - // Create the marker object that signals the version is complete. - _, err = f.client.PutObject(f.ctx, f.bucket, f.appObj, + // Create the marker object that signals the version is complete. The + // suffix keeps it on a distinct key from the //... files, + // so S3 browsers don't render a folder and a file with the same name. + _, err = f.client.PutObject(f.ctx, f.bucket, f.appObj+installedMarkerSuffix, bytes.NewReader(nil), 0, minio.PutObjectOptions{ ContentType: "text/plain", })