Skip to content

Commit d597639

Browse files
ChengyuZhu6ChengyuZhu6
committed
image: align erofs raw/zstd convert behavior with containerd
Fixes: #4861 Signed-off-by: Chengyu Zhu <hudson@cyzhu.com>
1 parent 04f63fe commit d597639

6 files changed

Lines changed: 119 additions & 9 deletions

File tree

‎cmd/nerdctl/image/image_convert.go‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -96,6 +96,12 @@ func convertCommand() *cobra.Command {
9696
cmd.Flags().Int64("soci-span-size", -1, "The size of SOCI spans")
9797
// #endregion
9898

99+
// #region erofs flags
100+
cmd.Flags().String("erofs", "", "Convert image layers to EROFS media type. Supported values: raw, zstd")
101+
cmd.Flags().String("erofs-compressors", "", "Specify mkfs.erofs compressor options (e.g. 'lz4hc,12')")
102+
cmd.Flags().String("erofs-mkfs-options", "", "Specify extra mkfs.erofs options (e.g. '-T0 --mkfs-time')")
103+
// #endregion
104+
99105
// #region generic flags
100106
cmd.Flags().Bool("uncompress", false, "Convert tar.gz layers to uncompressed tar layers")
101107
cmd.Flags().Bool("oci", false, "Convert Docker media types to OCI media types")
@@ -243,6 +249,21 @@ func convertOptions(cmd *cobra.Command) (types.ImageConvertOptions, error) {
243249
}
244250
// #endregion
245251

252+
// #region erofs flags
253+
erofs, err := cmd.Flags().GetString("erofs")
254+
if err != nil {
255+
return types.ImageConvertOptions{}, err
256+
}
257+
erofsCompressors, err := cmd.Flags().GetString("erofs-compressors")
258+
if err != nil {
259+
return types.ImageConvertOptions{}, err
260+
}
261+
erofsMkfsOptions, err := cmd.Flags().GetString("erofs-mkfs-options")
262+
if err != nil {
263+
return types.ImageConvertOptions{}, err
264+
}
265+
// #endregion
266+
246267
// #region generic flags
247268
uncompress, err := cmd.Flags().GetBool("uncompress")
248269
if err != nil {
@@ -317,6 +338,11 @@ func convertOptions(cmd *cobra.Command) (types.ImageConvertOptions, error) {
317338
AllPlatforms: allPlatforms,
318339
},
319340
},
341+
ErofsOptions: types.ErofsOptions{
342+
Erofs: erofs,
343+
ErofsCompressors: erofsCompressors,
344+
ErofsMkfsOptions: erofsMkfsOptions,
345+
},
320346
ProgressOutput: progressOutput,
321347
Stdout: cmd.OutOrStdout(),
322348
}, nil

‎cmd/nerdctl/image/image_convert_linux_test.go‎

Lines changed: 36 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ import (
2121
"testing"
2222
"time"
2323

24+
"github.com/containerd/nerdctl/mod/tigron/expect"
2425
"github.com/containerd/nerdctl/mod/tigron/require"
2526
"github.com/containerd/nerdctl/mod/tigron/test"
2627

@@ -52,7 +53,7 @@ func TestImageConvert(t *testing.T) {
5253
return helpers.Command("image", "convert", "--oci", "--estargz",
5354
testutil.CommonImage, data.Identifier("converted-image"))
5455
},
55-
Expected: test.Expects(0, nil, nil),
56+
Expected: test.Expects(expect.ExitCodeSuccess, nil, nil),
5657
},
5758
{
5859
Description: "nydus",
@@ -66,7 +67,7 @@ func TestImageConvert(t *testing.T) {
6667
return helpers.Command("image", "convert", "--oci", "--nydus",
6768
testutil.CommonImage, data.Identifier("converted-image"))
6869
},
69-
Expected: test.Expects(0, nil, nil),
70+
Expected: test.Expects(expect.ExitCodeSuccess, nil, nil),
7071
},
7172
{
7273
Description: "zstd",
@@ -77,7 +78,7 @@ func TestImageConvert(t *testing.T) {
7778
return helpers.Command("image", "convert", "--oci", "--zstd", "--zstd-compression-level", "3",
7879
testutil.CommonImage, data.Identifier("converted-image"))
7980
},
80-
Expected: test.Expects(0, nil, nil),
81+
Expected: test.Expects(expect.ExitCodeSuccess, nil, nil),
8182
},
8283
{
8384
Description: "zstdchunked",
@@ -88,7 +89,35 @@ func TestImageConvert(t *testing.T) {
8889
return helpers.Command("image", "convert", "--oci", "--zstdchunked", "--zstdchunked-compression-level", "3",
8990
testutil.CommonImage, data.Identifier("converted-image"))
9091
},
91-
Expected: test.Expects(0, nil, nil),
92+
Expected: test.Expects(expect.ExitCodeSuccess, nil, nil),
93+
},
94+
{
95+
Description: "erofs raw",
96+
Require: require.All(
97+
require.Binary("mkfs.erofs"),
98+
),
99+
Cleanup: func(data test.Data, helpers test.Helpers) {
100+
helpers.Anyhow("rmi", "-f", data.Identifier("converted-image"))
101+
},
102+
Command: func(data test.Data, helpers test.Helpers) test.TestableCommand {
103+
return helpers.Command("image", "convert", "--oci", "--erofs", "raw",
104+
testutil.CommonImage, data.Identifier("converted-image"))
105+
},
106+
Expected: test.Expects(expect.ExitCodeSuccess, nil, nil),
107+
},
108+
{
109+
Description: "erofs zstd",
110+
Require: require.All(
111+
require.Binary("mkfs.erofs"),
112+
),
113+
Cleanup: func(data test.Data, helpers test.Helpers) {
114+
helpers.Anyhow("rmi", "-f", data.Identifier("converted-image"))
115+
},
116+
Command: func(data test.Data, helpers test.Helpers) test.TestableCommand {
117+
return helpers.Command("image", "convert", "--oci", "--erofs", "zstd",
118+
testutil.CommonImage, data.Identifier("converted-image"))
119+
},
120+
Expected: test.Expects(expect.ExitCodeSuccess, nil, nil),
92121
},
93122
{
94123
Description: "soci",
@@ -107,7 +136,7 @@ func TestImageConvert(t *testing.T) {
107136
"--soci-min-layer-size", "0",
108137
testutil.CommonImage, data.Identifier("converted-image"))
109138
},
110-
Expected: test.Expects(0, nil, nil),
139+
Expected: test.Expects(expect.ExitCodeSuccess, nil, nil),
111140
},
112141
{
113142
Description: "soci with all-platforms",
@@ -126,7 +155,7 @@ func TestImageConvert(t *testing.T) {
126155
"--soci-min-layer-size", "0",
127156
testutil.CommonImage, data.Identifier("converted-image"))
128157
},
129-
Expected: test.Expects(0, nil, nil),
158+
Expected: test.Expects(expect.ExitCodeSuccess, nil, nil),
130159
},
131160
},
132161
}
@@ -188,7 +217,7 @@ func TestImageConvertNydusVerify(t *testing.T) {
188217
cmd.WithTimeout(30 * time.Second)
189218
return cmd
190219
},
191-
Expected: test.Expects(0, nil, nil),
220+
Expected: test.Expects(expect.ExitCodeSuccess, nil, nil),
192221
}
193222

194223
testCase.Run(t)

‎docs/command-reference.md‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1012,6 +1012,9 @@ Flags:
10121012
- `--zstdchunked-record-in=<FILE>` : read `ctr-remote optimize --record-out=<FILE>` record file. :warning: This flag is experimental and subject to change.
10131013
- `--zstdchunked-compression-level=<LEVEL>`: zstd:chunked compression level (default: 3)
10141014
- `--zstdchunked-chunk-size=<SIZE>`: zstd:chunked chunk size
1015+
- `--erofs=<MODE>` : convert image layers to EROFS media type. Supported values: `raw`, `zstd`
1016+
- `--erofs-compressors=<COMPRESSORS>` : specify mkfs.erofs compressor options, e.g. `lz4hc,12`
1017+
- `--erofs-mkfs-options=<OPTIONS>` : specify extra mkfs.erofs options, e.g. `-T0 --mkfs-time`
10151018
- `--uncompress` : convert tar.gz layers to uncompressed tar layers
10161019
- `--oci` : convert Docker media types to OCI media types
10171020
- `--platform=<PLATFORM>` : convert content for a specific platform

‎pkg/api/types/image_types.go‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,7 @@ type ImageConvertOptions struct {
7575
NydusOptions
7676
OverlaybdOptions
7777
SociConvertOptions
78+
ErofsOptions
7879
}
7980

8081
// EstargzOptions contains eStargz conversion options
@@ -150,6 +151,16 @@ type SociConvertOptions struct {
150151
// #endregion
151152
}
152153

154+
// ErofsOptions contains EROFS conversion options
155+
type ErofsOptions struct {
156+
// Erofs convert image layers to EROFS media type. Supported values: "raw" and "zstd"
157+
Erofs string
158+
// ErofsCompressors specifies mkfs compressor options, e.g. "lz4hc,12"
159+
ErofsCompressors string
160+
// ErofsMkfsOptions specifies extra options for mkfs.erofs, e.g. "-T0 --mkfs-time"
161+
ErofsMkfsOptions string
162+
}
163+
153164
// ImageCryptOptions specifies options for `nerdctl image encrypt` and `nerdctl image decrypt`.
154165
type ImageCryptOptions struct {
155166
Stdout io.Writer

‎pkg/cmd/image/convert.go‎

Lines changed: 36 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@ import (
3333
"github.com/containerd/containerd/v2/core/content"
3434
"github.com/containerd/containerd/v2/core/images"
3535
"github.com/containerd/containerd/v2/core/images/converter"
36+
erofsconvert "github.com/containerd/containerd/v2/core/images/converter/erofs"
3637
"github.com/containerd/containerd/v2/core/images/converter/uncompress"
3738
"github.com/containerd/log"
3839
nydusconvert "github.com/containerd/nydus-snapshotter/pkg/converter"
@@ -92,8 +93,9 @@ func Convert(ctx context.Context, client *containerd.Client, srcRawRef, targetRa
9293
overlaybd := options.Overlaybd
9394
nydus := options.Nydus
9495
soci := options.Soci
96+
erofs := options.Erofs != ""
9597
var finalize func(ctx context.Context, cs content.Store, ref string, desc *ocispec.Descriptor) (*images.Image, error)
96-
if estargz || zstd || zstdchunked || overlaybd || nydus || soci {
98+
if estargz || zstd || zstdchunked || overlaybd || nydus || soci || erofs {
9799
convertCount := 0
98100
if estargz {
99101
convertCount++
@@ -113,12 +115,16 @@ func Convert(ctx context.Context, client *containerd.Client, srcRawRef, targetRa
113115
if soci {
114116
convertCount++
115117
}
118+
if erofs {
119+
convertCount++
120+
}
116121

117122
if convertCount > 1 {
118-
return errors.New("options --estargz, --zstdchunked, --overlaybd, --nydus and --soci lead to conflict, only one of them can be used")
123+
return errors.New("options --estargz, --zstdchunked, --overlaybd, --nydus, --soci and --erofs lead to conflict, only one of them can be used")
119124
}
120125

121126
var convertFunc converter.ConvertFunc
127+
var updateManifestFunc converter.UpdateManifestFunc
122128
var convertType string
123129
switch {
124130
case estargz:
@@ -149,6 +155,12 @@ func Convert(ctx context.Context, client *containerd.Client, srcRawRef, targetRa
149155
convertFunc = overlaybdconvert.IndexConvertFunc(obdOpts...)
150156
convertOpts = append(convertOpts, converter.WithIndexConvertFunc(convertFunc))
151157
convertType = "overlaybd"
158+
case erofs:
159+
convertFunc, updateManifestFunc, err = getErofsConverter(options)
160+
if err != nil {
161+
return err
162+
}
163+
convertType = "erofs"
152164
case nydus:
153165
nydusOpts, err := getNydusConvertOpts(options)
154166
if err != nil {
@@ -188,6 +200,9 @@ func Convert(ctx context.Context, client *containerd.Client, srcRawRef, targetRa
188200
if convertType != "overlaybd" {
189201
convertOpts = append(convertOpts, converter.WithLayerConvertFunc(convertFunc))
190202
}
203+
if updateManifestFunc != nil {
204+
convertOpts = append(convertOpts, converter.WithUpdateManifest(updateManifestFunc))
205+
}
191206
if !options.Oci {
192207
if nydus || overlaybd {
193208
log.G(ctx).Warnf("option --%s should be used in conjunction with --oci, forcibly enabling on oci mediatype for %s conversion", convertType, convertType)
@@ -369,6 +384,25 @@ func getZstdchunkedConverter(options types.ImageConvertOptions) (converter.Conve
369384
return zstdchunkedconvert.LayerConvertFuncWithCompressionLevel(zstd.EncoderLevelFromZstd(options.ZstdChunkedCompressionLevel), esgzOpts...), nil
370385
}
371386

387+
func getErofsConverter(options types.ImageConvertOptions) (converter.ConvertFunc, converter.UpdateManifestFunc, error) {
388+
var convertOpts []erofsconvert.ConvertOpt
389+
switch options.Erofs {
390+
case "raw":
391+
// no-op: raw keeps native EROFS blob without extra stream compression
392+
case "zstd":
393+
convertOpts = append(convertOpts, erofsconvert.WithBlobCompression("zstd"))
394+
default:
395+
return nil, nil, fmt.Errorf("invalid value %q for --erofs, supported values are: raw, zstd", options.Erofs)
396+
}
397+
if options.ErofsCompressors != "" {
398+
convertOpts = append(convertOpts, erofsconvert.WithCompressors(options.ErofsCompressors))
399+
}
400+
if options.ErofsMkfsOptions != "" {
401+
convertOpts = append(convertOpts, erofsconvert.WithMkfsOptions(strings.Fields(options.ErofsMkfsOptions)))
402+
}
403+
return erofsconvert.LayerConvertFunc(convertOpts...), erofsconvert.UpdateManifestPlatform, nil
404+
}
405+
372406
func getNydusConvertOpts(options types.ImageConvertOptions) (*nydusconvert.PackOption, error) {
373407
workDir := options.NydusWorkDir
374408
if workDir == "" {

‎pkg/imgutil/push/push.go‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,7 @@ func Push(ctx context.Context, client *containerd.Client, resolver remotes.Resol
4848
}
4949
desc := img.Target
5050

51+
ctx = withErofsLayerRefKeyPrefixes(ctx)
5152
ongoing := newPushJobs(pushTracker)
5253

5354
eg, ctx := errgroup.WithContext(ctx)
@@ -172,3 +173,9 @@ func (j *pushjobs) status() []jobs.StatusInfo {
172173

173174
return statuses
174175
}
176+
177+
func withErofsLayerRefKeyPrefixes(ctx context.Context) context.Context {
178+
ctx = remotes.WithMediaTypeKeyPrefix(ctx, images.MediaTypeErofsLayer, "layer")
179+
ctx = remotes.WithMediaTypeKeyPrefix(ctx, images.MediaTypeErofsLayer+"+zstd", "layer")
180+
return ctx
181+
}

0 commit comments

Comments
 (0)