Accelerate reconcile using Perforce digests (#7)

Instead of copying every file from the source repo to the destination repo and then running `p4 revert -a`, switch to comparing the Perforce-provided digest and only copying the files whose digest, type, or case has changed.

1. Replaces `p4 -ztag files -e` with `p4 fstat -T depotFile,headAction,headChange,headType,digest -Ol -F '^(headAction=move/delete | headAction=purge | headAction=archive | headAction=delete)'` (which contains the same information, though it also includes the digest)
2. Updates `p4.runAndParseDepotFiles` to support the `head*` names of fields and also parse `digest`
3. Modifies `Reconcile` to compare digests and only adding files to `Match` if they're actually different in some way

This also has the benefit of not returning that there are no changes if only existing file contents has changed, not paths or types.

Author: jorgenpt

cmd/p4harmonize/update.go

@@ -323,8 +323,13 @@ func Reconcile(src []p4.DepotFile, dst []p4.DepotFile) DepotFileDiff {
 		cmp := strings.Compare(srcCmp, dstCmp)
 		switch {
 		case cmp == 0:
-			out.Match = append(out.Match, [2]p4.DepotFile{src[is], dst[id]})
-			if src[is].Path != dst[id].Path || src[is].Type != dst[id].Type {
+			caseDifference := src[is].Path != dst[id].Path
+			typeDifference := src[is].Type != dst[id].Type
+			// If we don't have a digest, assume it's different (since we'll use `p4 revert -a`),
+			// otherwise compare digests to see if there's a difference.
+			contentDifference := len(src[is].Digest) == 0 || src[is].Digest != dst[id].Digest
+			if caseDifference || typeDifference || contentDifference {
+				out.Match = append(out.Match, [2]p4.DepotFile{src[is], dst[id]})
 				out.HasDifference = true
 			}
 			is++

cmd/p4harmonize/update_test.go

@@ -17,16 +17,18 @@ func Test_ReconcileJustPaths(t *testing.T) {
 		SrcOnly string
 		DstOnly string
 	}{
-		{"simple match", "foo", "foo", "foo:foo", "", ""},
+		{"simple match (no digest)", "foo", "foo", "foo:foo", "", ""},
 		{"missing src", "", "foo", "", "", "foo"},
 		{"missing dst", "foo", "", "", "foo", ""},
 		{"complex match", "a,b,c,f", "b,c,d,f,g", "b:b,c:c,f:f", "a", "d,g"},
+		{"same digest", "foo+somedigest", "foo+somedigest", "", "", ""},
+		{"differing digest", "foo+somedigest", "foo+otherdigest", "foo:foo", "", ""},
 		{"mismatched case", "a,b,c", "a,B,", "a:a,b:B", "c", ""},
 	}
 
 	for _, tc := range cases {
 		t.Run(tc.Name, func(t *testing.T) {
-			src, dst := makeDepotFilesFromStrings(tc.Src, tc.Dst)
+			src, dst := makeDepotFilesFromString(tc.Src), makeDepotFilesFromString(tc.Dst)
 			actual := Reconcile(src, dst)
 
 			var actualMatch string
@@ -72,16 +74,18 @@ func Test_ReconcileHasDifference(t *testing.T) {
 		Dst      string
 		Expected bool
 	}{
-		{"simple match", "foo", "foo", false},
+		{"simple match (no digest)", "foo", "foo", true},
 		{"missing src", "", "foo", true},
 		{"missing dst", "foo", "", true},
 		{"complex match", "a,b,c,f", "b,c,d,f,g", true},
 		{"mismatched case", "a,b,c", "a,B,", true},
+		{"same digest", "foo+somedigest", "foo+somedigest", false},
+		{"differing digest", "foo+somedigest", "foo+otherdigest", true},
 	}
 
 	for _, tc := range cases {
 		t.Run(tc.Name, func(t *testing.T) {
-			src, dst := makeDepotFilesFromStrings(tc.Src, tc.Dst)
+			src, dst := makeDepotFilesFromString(tc.Src), makeDepotFilesFromString(tc.Dst)
 			actual := Reconcile(src, dst)
 
 			if tc.Expected != actual.HasDifference {
@@ -91,18 +95,18 @@ func Test_ReconcileHasDifference(t *testing.T) {
 	}
 }
 
-func makeDepotFilesFromStrings(s, d string) (src []p4.DepotFile, dst []p4.DepotFile) {
-	for _, path := range strings.Split(s, ",") {
+func makeDepotFilesFromString(paths string) (depotFiles []p4.DepotFile) {
+	for _, path := range strings.Split(paths, ",") {
 		if len(path) == 0 {
 			continue
 		}
-		src = append(src, p4.DepotFile{Path: path})
-	}
-	for _, path := range strings.Split(d, ",") {
-		if len(path) == 0 {
-			continue
+
+		pathAndDigest := strings.SplitN(path, "+", 2)
+		if len(pathAndDigest) == 2 {
+			depotFiles = append(depotFiles, p4.DepotFile{Path: pathAndDigest[0], Digest: pathAndDigest[1]})
+		} else {
+			depotFiles = append(depotFiles, p4.DepotFile{Path: path})
 		}
-		dst = append(dst, p4.DepotFile{Path: path})
 	}
-	return src, dst
+	return depotFiles
 }

internal/p4/files.go

@@ -4,8 +4,8 @@ import (
 	"fmt"
 )
 
-// ListDepotFiles runs "p4 files -e" and parses the results into a slice of DepotFile structs.
+// ListDepotFiles runs "p4 fstat" and parses the results into a slice of DepotFile structs.
 // Order of resulting slice is alphabetical by Path, ignoring case.
 func (p *P4) ListDepotFiles() ([]DepotFile, error) {
-	return p.runAndParseDepotFiles(fmt.Sprintf(`%s -z tag files -e //%s/...`, p.cmd(), p.Client))
+	return p.runAndParseDepotFiles(fmt.Sprintf(`%s fstat -T depotFile,headAction,headChange,headType,digest -Ol -F '^(headAction=move/delete | headAction=purge | headAction=archive | headAction=delete)' //%s/...`, p.cmd(), p.Client))
 }

internal/p4/p4.go

@@ -140,6 +140,7 @@ type DepotFile struct {
 	Action string
 	CL     string
 	Type   string
+	Digest string
 }
 
 // DepotFileCaseInsensitive allows sorting slices of DepotFile by path, but ignoring case.
@@ -152,11 +153,12 @@ func (x DepotFileCaseInsensitive) Less(i, j int) bool {
 func (x DepotFileCaseInsensitive) Swap(i, j int) { x[i], x[j] = x[j], x[i] }
 
 // runAndParseDepotFiles calls the given command, which is expected to return a list of records, each
-// with at least a depotFile, and optionally also a type, change, and action.
+// with at least a depotFile, and optionally also a type, change, action, digest, headType, headChange,
+// and headAction.
 // The results are then sorted by Path (case-insensitive) and returned.
 func (p *P4) runAndParseDepotFiles(cmd string) ([]DepotFile, error) {
-	if !strings.Contains(cmd, "-ztag") && !strings.Contains(cmd, "-z tag") {
-		return nil, fmt.Errorf("missing '-z tag' in cmd: %s", cmd)
+	if !strings.Contains(cmd, "-ztag") && !strings.Contains(cmd, "-z tag")  && !strings.Contains(cmd, "fstat") {
+		return nil, fmt.Errorf("missing '-z tag' in non-fstat cmd: %s", cmd)
 	}
 
 	_, streamDepth, err := p.StreamInfo()
@@ -198,10 +200,18 @@ func (p *P4) runAndParseDepotFiles(cmd string) ([]DepotFile, error) {
 				cur.Path = raw[len(prefix):]
 			case strings.HasPrefix(line[4:], "action"):
 				cur.Action = strings.TrimSpace(line[10:])
+			case strings.HasPrefix(line[4:], "headAction"):
+				cur.Action = strings.TrimSpace(line[14:])
 			case strings.HasPrefix(line[4:], "change"):
 				cur.CL = strings.TrimSpace(line[10:])
+			case strings.HasPrefix(line[4:], "headChange"):
+				cur.CL = strings.TrimSpace(line[14:])
 			case strings.HasPrefix(line[4:], "type"):
 				cur.Type = strings.TrimSpace(line[8:])
+			case strings.HasPrefix(line[4:], "headType"):
+				cur.Type = strings.TrimSpace(line[12:])
+			case strings.HasPrefix(line[4:], "digest"):
+				cur.Digest = strings.TrimSpace(line[10:])
 			}
 
 			return nil