New command: 'vendir baseline'

This command updates the git & hg 'ref' field based on the actual checked-out commit of the corresponding content paths

Author: cdevienne

go.mod

@@ -22,6 +22,7 @@ require (
 	golang.org/x/oauth2 v0.30.0
 	golang.org/x/tools v0.34.0
 	gopkg.in/inf.v0 v0.9.1
+	gopkg.in/yaml.v3 v3.0.1
 	k8s.io/apimachinery v0.24.3
 	k8s.io/code-generator v0.17.2
 	sigs.k8s.io/yaml v1.4.0
@@ -103,7 +104,6 @@ require (
 	golang.org/x/text v0.26.0 // indirect
 	gonum.org/v1/gonum v0.0.0-20190331200053-3d26580ed485 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
-	gopkg.in/yaml.v3 v3.0.1 // indirect
 	k8s.io/gengo v0.0.0-20210813121822-485abfe95c7c // indirect
 	k8s.io/klog v1.0.0 // indirect
 	k8s.io/klog/v2 v2.70.1 // indirect

pkg/vendir/cmd/baseline.go

@@ -0,0 +1,222 @@
+package cmd
+
+import (
+	"fmt"
+	"os"
+	"path"
+
+	"github.com/cppforlife/go-cli-ui/ui"
+	"github.com/spf13/cobra"
+	"gopkg.in/yaml.v3"
+
+	ctlconf "carvel.dev/vendir/pkg/vendir/config"
+	ctldir "carvel.dev/vendir/pkg/vendir/directory"
+	ctlcache "carvel.dev/vendir/pkg/vendir/fetch/cache"
+)
+
+func NewBaselineOptions(ui ui.UI) *BaselineOptions {
+	return &BaselineOptions{ui: ui}
+}
+
+func NewBaselineCmd(o *BaselineOptions) *cobra.Command {
+	cmd := cobra.Command{
+		Use:   "baseline",
+		Short: "Update git/hg repositories 'ref' to match the current commits",
+		RunE:  func(_ *cobra.Command, _ []string) error { return o.Run() },
+	}
+
+	cmd.Flags().BoolVarP(
+		&o.Yes, "yes", "y", false,
+		"If true, automatically answer 'yes' to all the questions")
+
+	cmd.Flags().StringSliceVarP(&o.Files, "file", "f", []string{defaultConfigName}, "Set configuration file")
+	cmd.Flags().StringVar(&o.LockFile, "lock-file", defaultLockName, "Set lock file")
+	cmd.Flags().StringVar(&o.Chdir, "chdir", "", "Set current directory for process")
+	cmd.Flags().BoolVar(&o.PreferSHA, "prefer-sha", false, "Prefer sha instead of tags")
+	cmd.Flags().BoolVar(&o.DryRun, "dry-run", false, "List what would be done")
+
+	return &cmd
+}
+
+type BaselineOptions struct {
+	ui ui.UI
+
+	Files    []string
+	LockFile string
+
+	Chdir    string
+	ExitCode bool
+
+	PreferSHA bool
+	Yes       bool
+	DryRun    bool
+}
+
+func (o *BaselineOptions) Run() error {
+	if len(o.Chdir) > 0 {
+		err := os.Chdir(o.Chdir)
+		if err != nil {
+			return fmt.Errorf("Running chdir: %s", err)
+		}
+	}
+
+	conf, secrets, configMaps, err := ctlconf.NewConfigFromFiles(o.Files)
+	if err != nil {
+		return (*SyncOptions)(nil).configReadHintErrMsg(err, o.Files)
+	}
+
+	existingLockConfig, err := ctlconf.NewLockConfigFromFile(o.LockFile)
+	if err != nil {
+		return err
+	}
+
+	cache, err := ctlcache.NewCache(os.Getenv("VENDIR_CACHE_DIR"), "0Mi")
+	if err != nil {
+		return fmt.Errorf("Unable to create cache: %s", err)
+	}
+	syncOpts := ctldir.SyncOpts{
+		RefFetcher:     ctldir.NewNamedRefFetcher(secrets, configMaps),
+		GithubAPIToken: os.Getenv("VENDIR_GITHUB_API_TOKEN"),
+		HelmBinary:     os.Getenv("VENDIR_HELM_BINARY"),
+		Cache:          cache,
+		Lazy:           false,
+		Partial:        false,
+	}
+
+	statusMap, err := fullStatus(conf, syncOpts, existingLockConfig, o.ui)
+	if err != nil {
+		return err
+	}
+
+	newRefs := make(map[string]string)
+
+	for _, status := range statusMap {
+		if !status.MatchTarget() || !o.PreferSHA && !status.MatchTargetTag() && len(status.Ref.Tags) != 0 {
+			var newRef string
+			if o.PreferSHA || len(status.Ref.Tags) == 0 {
+				newRef = status.Ref.SHA
+			} else {
+				newRef = status.Ref.Tags[0]
+			}
+			newRefs[status.Path()] = newRef
+		}
+	}
+
+	if len(newRefs) == 0 {
+		o.ui.PrintLinef("All references already match current state, no update needed")
+
+		return nil
+	}
+
+	o.ui.PrintLinef("New baseline:")
+	for _, status := range statusMap {
+		newRef := newRefs[status.Path()]
+		if newRef != "" {
+			newRef = " -> " + newRef
+		}
+		o.ui.PrintLinef("%s/%s: %s%s", status.DirectoryPath, status.ContentPath, status.TargetRef, newRef)
+	}
+
+	if !o.DryRun {
+		for _, fname := range o.Files {
+			if err := updateRefs(fname, newRefs); err != nil {
+				return err
+			}
+		}
+	}
+
+	return nil
+}
+
+func loadFile(fname string) (*yaml.Node, error) {
+	f, err := os.Open(fname)
+	if err != nil {
+		return nil, err
+	}
+
+	defer f.Close()
+
+	var node yaml.Node
+
+	if err := yaml.NewDecoder(f).Decode(&node); err != nil {
+		return nil, err
+	}
+
+	return &node, err
+}
+
+func saveFile(fname string, doc *yaml.Node) error {
+	f, err := os.Create(fname)
+	if err != nil {
+		return err
+	}
+
+	enc := yaml.NewEncoder(f)
+	enc.SetIndent(2)
+	if err := enc.Encode(doc); err != nil {
+		_ = f.Close()
+
+		return err
+	}
+
+	return f.Close()
+}
+
+func updateRefs(fname string, newRefs map[string]string) error {
+	doc, err := loadFile(fname)
+	if err != nil {
+		return err
+	}
+
+	if doc.Kind != yaml.DocumentNode {
+		panic("expects the root node")
+	}
+
+	top := doc.Content[0]
+	if top.Kind != yaml.MappingNode {
+		panic("top content must be a mapping")
+	}
+
+	directories := getMappingNodeChild(top, "directories")
+
+	for _, d := range directories.Content {
+		dirPath := getMappingNodeChild(d, "path").Value
+
+		contents := getMappingNodeChild(d, "contents")
+
+		for _, content := range contents.Content {
+			contentPath := getMappingNodeChild(content, "path").Value
+
+			fullPath := path.Join(dirPath, contentPath)
+
+			if newRef, ok := newRefs[fullPath]; ok {
+				if hg := getMappingNodeChild(content, "hg"); hg != nil {
+					ref := getMappingNodeChild(hg, "ref")
+					if ref == nil {
+						return fmt.Errorf("could not find 'ref' for '%s'", fullPath)
+					}
+					ref.Value = newRef
+				}
+				if git := getMappingNodeChild(content, "git"); git != nil {
+					ref := getMappingNodeChild(git, "ref")
+					if ref == nil {
+						return fmt.Errorf("could not find 'ref' for '%s'", fullPath)
+					}
+					ref.Value = newRef
+				}
+			}
+		}
+	}
+
+	return saveFile(fname, doc)
+}
+
+func getMappingNodeChild(node *yaml.Node, name string) *yaml.Node {
+	for i := 0; i < len(node.Content); i += 2 {
+		if node.Content[i].Value == name {
+			return node.Content[i+1]
+		}
+	}
+
+	return nil
+}

pkg/vendir/cmd/vendir.go

@@ -44,6 +44,7 @@ func NewVendirCmd(o *VendirOptions) *cobra.Command {
 	cmd.AddCommand(NewSyncCmd(NewSyncOptions(o.ui)))
 	cmd.AddCommand(NewStatusCmd(NewStatusOptions(o.ui)))
 	cmd.AddCommand(NewVersionCmd(NewVersionOptions(o.ui)))
+	cmd.AddCommand(NewBaselineCmd(NewBaselineOptions(o.ui)))
 
 	toolsCmd := NewToolsCmd()
 	toolsCmd.AddCommand(NewSortSemverCmd(NewSortSemverOptions(o.ui)))

pkg/vendir/fetch/hg/status.go

@@ -6,6 +6,7 @@ import (
 	"os"
 	"os/exec"
 	"path"
+	"slices"
 	"strings"
 
 	ctlstatus "carvel.dev/vendir/pkg/vendir/status"
@@ -50,6 +51,8 @@ func (d Sync) Status(target string) (*ctlstatus.Status, error) {
 	}
 	if tags != "" {
 		status.Ref.Tags = strings.Split(tags, " ")
+		status.Ref.Tags = slices.DeleteFunc(
+			status.Ref.Tags, func(t string) bool { return t == "tip" })
 	}
 	if branch != "" {
 		status.Ref.Others = append(status.Ref.Others, branch)

pkg/vendir/status/status.go

@@ -5,6 +5,7 @@ package status
 
 import (
 	"fmt"
+	"path"
 	"slices"
 	"strings"
 )
@@ -24,10 +25,24 @@ type Status struct {
 	LocalCsets        []string
 }
 
+func (s Status) Path() string {
+	return path.Join(s.DirectoryPath, s.ContentPath)
+}
+
 func (s Status) IsSafe() bool {
 	return len(s.UncommitedChanges) == 0 && len(s.LocalCsets) == 0
 }
 
+func (s Status) MatchTarget() bool {
+	return strings.HasPrefix(s.Ref.SHA, s.TargetRef) ||
+		slices.Contains(s.Ref.Tags, s.TargetRef) ||
+		slices.Contains(s.Ref.Others, s.TargetRef)
+}
+
+func (s Status) MatchTargetTag() bool {
+	return slices.Contains(s.Ref.Tags, s.TargetRef)
+}
+
 func (s Status) String() string {
 	messages := make([]string, 0, 3)
 
@@ -42,9 +57,7 @@ func (s Status) String() string {
 		messages = append(messages, fmt.Sprintf("%d unpushed commits", len(s.LocalCsets)))
 	}
 
-	if !strings.HasPrefix(s.Ref.SHA, s.TargetRef) &&
-		!slices.Contains(s.Ref.Tags, s.TargetRef) &&
-		!slices.Contains(s.Ref.Others, s.TargetRef) {
+	if !s.MatchTarget() {
 		messages = append(messages, "ref mismatch")
 	}