Reduce GCS memory footprint (#717)

AlCutter · web-flow · commit d9fbef2aaf44 · 2025-07-16T15:54:35.000+01:00
diff --git a/storage/gcp/gcp.go b/storage/gcp/gcp.go
@@ -1153,6 +1153,8 @@ func (s *gcsStorage) setObject(ctx context.Context, objName string, data []byte,
 	}
 	w.ContentType = contType
 	w.CacheControl = cacheCtl
+	// Limit the amount of memory used for buffers, see https://pkg.go.dev/cloud.google.com/go/storage#Writer
+	w.ChunkSize = len(data) + 1024
 	if _, err := w.Write(data); err != nil {
 		return fmt.Errorf("failed to write object %q to bucket %q: %w", objName, s.bucket, err)
 	}

Original file line number	Diff line number	Diff line change
`@@ -1153,6 +1153,8 @@ func (s *gcsStorage) setObject(ctx context.Context, objName string, data []byte,`
`1153`	`1153`	`}`
`1154`	`1154`	`w.ContentType = contType`
`1155`	`1155`	`w.CacheControl = cacheCtl`
	`1156`	`+ // Limit the amount of memory used for buffers, see https://pkg.go.dev/cloud.google.com/go/storage#Writer`
	`1157`	`+ w.ChunkSize = len(data) + 1024`
`1156`	`1158`	`if _, err := w.Write(data); err != nil {`
`1157`	`1159`	`return fmt.Errorf("failed to write object %q to bucket %q: %w", objName, s.bucket, err)`
`1158`	`1160`	`}`