[deckhouse-cli] mirror: support multiple version constraints for the same module/package (#379)

Signed-off-by: Pavel Okhlopkov <pavel.okhlopkov@flant.com>

Author: ldmonster

internal/mirror/modules/constraints.go

@@ -257,3 +257,101 @@ func (e *ExactTagConstraint) IsExact() bool {
 func (e *ExactTagConstraint) HasChannelAlias() bool {
 	return e.channel != ""
 }
+
+// MultiConstraint is the OR-combination of several constraints declared for
+// the same name. It is produced when a user repeats a name on the command
+// line, e.g. `--include-package test@=v0.0.2 --include-package test@=v0.0.3`,
+// so that all of the named versions are pulled instead of only the last one.
+//
+// A value Match-es when ANY sub-constraint matches. It is considered exact
+// only when EVERY sub-constraint is exact (so a set of pinned tags keeps the
+// "no release-channel discovery, no tag listing" fast paths), and it advertises
+// a channel alias when ANY sub-constraint does.
+type MultiConstraint struct {
+	constraints []VersionConstraint
+}
+
+// Constraints returns the sub-constraints in declaration order.
+func (m *MultiConstraint) Constraints() []VersionConstraint {
+	return m.constraints
+}
+
+func (m *MultiConstraint) Match(version interface{}) bool {
+	for _, c := range m.constraints {
+		if c.Match(version) {
+			return true
+		}
+	}
+
+	return false
+}
+
+func (m *MultiConstraint) IsExact() bool {
+	for _, c := range m.constraints {
+		if !c.IsExact() {
+			return false
+		}
+	}
+
+	return true
+}
+
+func (m *MultiConstraint) HasChannelAlias() bool {
+	for _, c := range m.constraints {
+		if c.HasChannelAlias() {
+			return true
+		}
+	}
+
+	return false
+}
+
+// mergeConstraints OR-combines an already-registered constraint with an
+// additional one declared for the same name, flattening nested
+// MultiConstraints so repeated declarations stay a single flat list.
+func mergeConstraints(existing, additional VersionConstraint) VersionConstraint {
+	if multi, ok := existing.(*MultiConstraint); ok {
+		multi.constraints = append(multi.constraints, additional)
+		return multi
+	}
+
+	return &MultiConstraint{constraints: []VersionConstraint{existing, additional}}
+}
+
+// ExactConstraintsOf flattens a constraint into the exact-tag constraints it
+// contains. A plain ExactTagConstraint yields itself; a MultiConstraint yields
+// every exact sub-constraint; anything else yields nothing.
+func ExactConstraintsOf(c VersionConstraint) []*ExactTagConstraint {
+	switch t := c.(type) {
+	case *ExactTagConstraint:
+		return []*ExactTagConstraint{t}
+	case *MultiConstraint:
+		var out []*ExactTagConstraint
+		for _, sub := range t.constraints {
+			out = append(out, ExactConstraintsOf(sub)...)
+		}
+
+		return out
+	default:
+		return nil
+	}
+}
+
+// SemverConstraintsOf flattens a constraint into the semantic-version
+// constraints it contains, used by the proxy-registry probe which can only
+// walk semver ranges.
+func SemverConstraintsOf(c VersionConstraint) []*SemanticVersionConstraint {
+	switch t := c.(type) {
+	case *SemanticVersionConstraint:
+		return []*SemanticVersionConstraint{t}
+	case *MultiConstraint:
+		var out []*SemanticVersionConstraint
+		for _, sub := range t.constraints {
+			out = append(out, SemverConstraintsOf(sub)...)
+		}
+
+		return out
+	default:
+		return nil
+	}
+}

internal/mirror/modules/filter.go

@@ -58,30 +58,35 @@ func NewFilter(filterExpressions []string, filterType FilterType) (*Filter, erro
 	}
 
 	for _, filterExpr := range filterExpressions {
-		moduleName, versionStr, hasVersion := strings.Cut(strings.TrimSpace(filterExpr), "@")
+		name, versionStr, hasVersion := strings.Cut(strings.TrimSpace(filterExpr), "@")
 
-		moduleName = strings.TrimSpace(moduleName)
-		if moduleName == "" {
-			return nil, fmt.Errorf("Malformed filter expression %q: empty module name", filterExpr)
+		name = strings.TrimSpace(name)
+		if name == "" {
+			return nil, fmt.Errorf("Malformed filter expression %q: empty name", filterExpr)
 		}
 
-		if _, moduleRedeclared := filter.modules[moduleName]; moduleRedeclared {
-			return nil, fmt.Errorf("Malformed filter expression: module %s is declared multiple times", moduleName)
+		var constraint VersionConstraint
+		if !hasVersion {
+			constraint, _ = NewSemanticVersionConstraint(">=0.0.0")
+		} else {
+			var err error
+
+			constraint, err = parseVersionConstraint(versionStr)
+			if err != nil {
+				return nil, err
+			}
 		}
 
-		if !hasVersion {
-			constraint, _ := NewSemanticVersionConstraint(">=0.0.0")
-			filter.modules[moduleName] = constraint
+		// Repeating a name is allowed and OR-combines the constraints, so a
+		// user can pull several pinned versions at once, e.g.
+		// `--include-package test@=v0.0.2 --include-package test@=v0.0.3`.
+		if existing, redeclared := filter.modules[name]; redeclared {
+			filter.modules[name] = mergeConstraints(existing, constraint)
 
 			continue
 		}
 
-		constraint, err := parseVersionConstraint(versionStr)
-		if err != nil {
-			return nil, err
-		}
-
-		filter.modules[moduleName] = constraint
+		filter.modules[name] = constraint
 	}
 
 	return filter, nil
@@ -237,6 +242,21 @@ func (f *Filter) VersionsToMirror(mod *Module) []string {
 		return nil
 	}
 
+	return deduplicateTags(versionsForConstraint(constraint, mod))
+}
+
+// versionsForConstraint resolves a single constraint (or, recursively, every
+// sub-constraint of a MultiConstraint) into concrete tags to pull.
+func versionsForConstraint(constraint VersionConstraint, mod *Module) []string {
+	if multi, ok := constraint.(*MultiConstraint); ok {
+		var tags = make([]string, 0, len(multi.constraints))
+		for _, sub := range multi.constraints {
+			tags = append(tags, versionsForConstraint(sub, mod)...)
+		}
+
+		return tags
+	}
+
 	if constraint.IsExact() {
 		exact, isExactTag := constraint.(*ExactTagConstraint)
 		if !isExactTag {
@@ -270,6 +290,28 @@ func (f *Filter) VersionsToMirror(mod *Module) []string {
 	return tags
 }
 
+// deduplicateTags removes duplicate tags while preserving first-seen order,
+// so overlapping sub-constraints of a MultiConstraint don't yield repeats.
+func deduplicateTags(tags []string) []string {
+	if len(tags) == 0 {
+		return tags
+	}
+
+	seen := make(map[string]struct{}, len(tags))
+	out := make([]string, 0, len(tags))
+
+	for _, tag := range tags {
+		if _, dup := seen[tag]; dup {
+			continue
+		}
+
+		seen[tag] = struct{}{}
+		out = append(out, tag)
+	}
+
+	return out
+}
+
 // restoreInclusiveAnchors re-introduces any anchor versions (named via >=/<=
 // in the user's constraint string) that were dropped by filterOnlyLatestPatches.
 //

internal/mirror/modules/filter_test.go

@@ -61,9 +61,9 @@ func TestNewFilter(t *testing.T) {
 			wantErr:     true,
 		},
 		{
-			name:        "duplicate module",
+			name:        "duplicate module is merged, not rejected",
 			expressions: []string{"module@1.2.3", "module@2.3.4"},
-			wantErr:     true,
+			wantErr:     false,
 		},
 	}
 
@@ -555,6 +555,33 @@ func TestFilter_VersionsToMirror(t *testing.T) {
 			},
 			want: []string{"v1.3.0"},
 		},
+		{
+			// Several exact tags pinned for the same name (the user repeated
+			// --include-package test@=vX). They merge into a MultiConstraint
+			// of exacts, which stays exact so no release channels are added,
+			// and every pinned tag is pulled.
+			name: "multiple exact tags pinned for the same name pull all of them",
+			filter: Filter{
+				logger: logger,
+				modules: map[string]VersionConstraint{
+					"module1": mergeConstraints(
+						mergeConstraints(
+							NewExactTagConstraint("v0.0.2"),
+							NewExactTagConstraint("v0.0.3"),
+						),
+						NewExactTagConstraint("v0.0.8"),
+					),
+				},
+			},
+			mod: &Module{
+				Name: "module1",
+				Releases: []string{
+					internal.AlphaChannel,
+					internal.StableChannel,
+					"v0.0.1", "v0.0.2", "v0.0.3", "v0.0.4", "v0.0.8"},
+			},
+			want: []string{"v0.0.2", "v0.0.3", "v0.0.8"},
+		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {

internal/mirror/modules/modules.go

@@ -509,8 +509,8 @@ func (svc *Service) listTagsIfConstrained(ctx context.Context, moduleName string
 // this function, and a future constraint type would need its own
 // proxy-aware code path.
 func (svc *Service) probeModuleTags(ctx context.Context, moduleName string, constraint VersionConstraint) ([]string, error) {
-	semverConstraint, ok := constraint.(*SemanticVersionConstraint)
-	if !ok {
+	semverConstraints := SemverConstraintsOf(constraint)
+	if len(semverConstraints) == 0 {
 		// Be loud — silently falling back to ListTags here would defeat
 		// the whole purpose of --proxy-registry on a registry that
 		// refuses catalog access.
@@ -532,14 +532,17 @@ func (svc *Service) probeModuleTags(ctx context.Context, moduleName string, cons
 		return false, fmt.Errorf("check module %s tag %q: %w", moduleName, tag, err)
 	}
 
-	versions, err := ProbeAvailableVersions(ctx, semverConstraint, check)
-	if err != nil {
-		return nil, fmt.Errorf("probe tags for module %s: %w", moduleName, err)
-	}
+	tags := make([]string, 0)
+
+	for _, semverConstraint := range semverConstraints {
+		versions, err := ProbeAvailableVersions(ctx, semverConstraint, check)
+		if err != nil {
+			return nil, fmt.Errorf("probe tags for module %s: %w", moduleName, err)
+		}
 
-	tags := make([]string, 0, len(versions))
-	for _, v := range versions {
-		tags = append(tags, "v"+v.String())
+		for _, v := range versions {
+			tags = append(tags, "v"+v.String())
+		}
 	}
 
 	return tags, nil
@@ -1028,32 +1031,40 @@ func (svc *Service) applyChannelAliases(moduleName string) error {
 		return nil
 	}
 
-	exact, ok := constraint.(*ExactTagConstraint)
-	if !ok {
-		return nil
-	}
-
 	moduleLayout := svc.layout.Module(moduleName)
 	if moduleLayout == nil || moduleLayout.ModulesReleaseChannels == nil {
 		return nil
 	}
 
-	desc, err := layouts.FindImageDescriptorByTag(moduleLayout.ModulesReleaseChannels.Path(), exact.Tag())
-	if err != nil {
-		if errors.Is(err, layouts.ErrImageNotFound) {
-			return nil
-		}
+	exacts := ExactConstraintsOf(constraint)
 
-		return err
-	}
+	// A single pinned tag without an explicit +channel suffix is published to
+	// every release channel (the historical --deckhouse-tag-like behaviour).
+	// When several tags are pinned at once, propagating each to all channels
+	// would just clobber one another, so only tags that name their own channel
+	// (=vX.Y.Z+stable) are aliased; the rest are simply pulled as-is.
+	propagateToAllChannels := len(exacts) == 1 && !exacts[0].HasChannelAlias()
+
+	for _, exact := range exacts {
+		desc, err := layouts.FindImageDescriptorByTag(moduleLayout.ModulesReleaseChannels.Path(), exact.Tag())
+		if err != nil {
+			if errors.Is(err, layouts.ErrImageNotFound) {
+				continue
+			}
 
-	if exact.HasChannelAlias() {
-		if err := layouts.TagImage(moduleLayout.ModulesReleaseChannels.Path(), desc.Digest, exact.Channel()); err != nil {
 			return err
 		}
-	} else {
-		// Tag all channels with this version
-		for _, channel := range append(internal.GetAllDefaultReleaseChannels(), internal.LTSChannel) {
+
+		var channels []string
+
+		switch {
+		case exact.HasChannelAlias():
+			channels = []string{exact.Channel()}
+		case propagateToAllChannels:
+			channels = append(internal.GetAllDefaultReleaseChannels(), internal.LTSChannel)
+		}
+
+		for _, channel := range channels {
 			if err := layouts.TagImage(moduleLayout.ModulesReleaseChannels.Path(), desc.Digest, channel); err != nil {
 				return err
 			}