backup: accept unknown manifest fields for same-major evolution (PR #712, round 3)

Codex P1 #205 (round 2): DisallowUnknownFields broke the
documented same-major minor-evolution path. A newer producer
that adds an optional field at the same format_version turns
older readers into hard read failures during mixed-version
operation, even though both nodes are on the "same major" by
contract.

The format_version field IS the breaking-change signal. Major
bumps are gated by ErrUnsupportedFormatVersion; same-major
minor evolution must silently tolerate optional new fields.

Removed dec.DisallowUnknownFields() from ReadManifest. Replaced
TestReadManifest_RejectsUnknownFields with
TestReadManifest_AcceptsUnknownFieldsForSameMajorMinorEvolution.
The trailing-bytes guard (dec.More() check) is preserved -- it
catches concatenation bugs which is a different concern from
optional-field tolerance.

Author: bootjp

internal/backup/manifest.go

@@ -202,7 +202,17 @@ func WriteManifest(w io.Writer, m Manifest) error {
 func ReadManifest(r io.Reader) (Manifest, error) {
 	var m Manifest
 	dec := json.NewDecoder(r)
-	dec.DisallowUnknownFields() // surface format drift loudly
+	// We intentionally do NOT call DisallowUnknownFields here.
+	// The format-version contract (Codex P1, follow-up) is:
+	//   - format_version > CurrentFormatVersion -> hard refuse
+	//     (the major break signal)
+	//   - format_version == CurrentFormatVersion AND extra unknown
+	//     fields appear -> a newer minor version added them; the
+	//     older reader silently ignores. That's the documented
+	//     same-major minor-evolution path.
+	// Rejecting unknown fields outright would turn every minor
+	// optional-field addition into a hard read failure during
+	// mixed-version operation.
 	if err := dec.Decode(&m); err != nil {
 		return Manifest{}, errors.Wrap(ErrInvalidManifest, err.Error())
 	}

internal/backup/manifest_test.go

@@ -141,10 +141,13 @@ func TestReadManifest_RejectsZeroFormatVersion(t *testing.T) {
 	}
 }
 
-func TestReadManifest_RejectsUnknownFields(t *testing.T) {
+func TestReadManifest_AcceptsUnknownFieldsForSameMajorMinorEvolution(t *testing.T) {
 	t.Parallel()
-	// Format drift safety: an unknown field surfaces loudly rather than
-	// being silently ignored.
+	// Same-major minor evolution: a newer producer adds an optional
+	// field; older readers must silently ignore it rather than fail
+	// the read. Codex P1 #205 (round 2) caught the earlier
+	// DisallowUnknownFields strictness which broke the documented
+	// same-major compatibility model.
 	body := `{
 		"format_version": 1,
 		"phase": "phase0-snapshot-decode",
@@ -158,11 +161,14 @@ func TestReadManifest_RejectsUnknownFields(t *testing.T) {
 		"s3_meta_suffix": ".elastickv-meta.json",
 		"s3_collision_strategy": "leaf-data-suffix",
 		"dynamodb_layout": "per-item",
-		"unknown_field": "ahoy"
+		"future_optional_field": "added in v1.minor"
 	}`
-	_, err := ReadManifest(strings.NewReader(body))
-	if !errors.Is(err, ErrInvalidManifest) {
-		t.Fatalf("err=%v want ErrInvalidManifest", err)
+	got, err := ReadManifest(strings.NewReader(body))
+	if err != nil {
+		t.Fatalf("unknown optional field must be silently accepted: %v", err)
+	}
+	if got.FormatVersion != 1 {
+		t.Fatalf("format_version = %d", got.FormatVersion)
 	}
 }