Fix cross-ROM duplicate save counting

Author: joeblack2k

backend/cmd/server/agent_api_handlers.go

@@ -484,6 +484,7 @@ func (a *app) aggregatedSaveSummaries(romSHA1, romMD5 string, systemID int) []sa
 		saveCount      int
 		totalSizeBytes int
 		projectionLine bool
+		revisionKeys   map[string]struct{}
 	}
 	aggregates := make(map[string]saveAggregate, len(filteredRecords))
 	for _, record := range filteredRecords {
@@ -500,6 +501,9 @@ func (a *app) aggregatedSaveSummaries(romSHA1, romMD5 string, systemID int) []sa
 		}
 		key := canonicalListKeyForRecord(record)
 		agg := aggregates[key]
+		if agg.revisionKeys == nil {
+			agg.revisionKeys = map[string]struct{}{}
+		}
 		if _, _, _, ok := playStationProjectionInfoFromRecord(record); ok {
 			agg.projectionLine = true
 		}
@@ -514,8 +518,12 @@ func (a *app) aggregatedSaveSummaries(romSHA1, romMD5 string, systemID int) []sa
 				agg.totalSizeBytes = record.Summary.FileSize
 			}
 		} else {
-			agg.saveCount++
-			agg.totalSizeBytes += record.Summary.FileSize
+			revisionKey := saveRecordRevisionIdentity(record)
+			if _, seen := agg.revisionKeys[revisionKey]; !seen {
+				agg.revisionKeys[revisionKey] = struct{}{}
+				agg.saveCount++
+				agg.totalSizeBytes += record.Summary.FileSize
+			}
 		}
 		aggregates[key] = agg
 	}

backend/cmd/server/save_cleanup.go

@@ -101,15 +101,15 @@ func planGenericSaveDuplicateCleanup(records []saveRecord, existingRemovals map[
 		if _, _, _, ok := n64ControllerPakProjectionInfoFromRecord(record); ok {
 			continue
 		}
-		key := strings.TrimSpace(canonicalVersionKeyForRecord(record))
+		key := strings.TrimSpace(genericDuplicateCleanupKeyForRecord(record))
 		if key == "" {
 			continue
 		}
 		latest, ok := latestByKey[key]
 		if !ok || saveRecordSortsAfter(record, latest) {
 			latestByKey[key] = record
 		}
-		groupKey := key + "::" + strings.TrimSpace(record.Summary.SHA256)
+		groupKey := key + "::sha:" + strings.TrimSpace(record.Summary.SHA256)
 		groups[groupKey] = append(groups[groupKey], record)
 	}
 
@@ -120,7 +120,7 @@ func planGenericSaveDuplicateCleanup(records []saveRecord, existingRemovals map[
 			continue
 		}
 		duplicateGroups++
-		latest := latestByKey[strings.TrimSpace(canonicalVersionKeyForRecord(group[0]))]
+		latest := latestByKey[strings.TrimSpace(genericDuplicateCleanupKeyForRecord(group[0]))]
 		survivorID := selectSaveRecordDuplicateSurvivor(group, latest)
 		for _, record := range group {
 			if strings.TrimSpace(record.Summary.ID) == survivorID {
@@ -133,6 +133,13 @@ func planGenericSaveDuplicateCleanup(records []saveRecord, existingRemovals map[
 	return duplicateGroups, duplicateRemoved
 }
 
+func genericDuplicateCleanupKeyForRecord(record saveRecord) string {
+	if key := strings.TrimSpace(canonicalDuplicateTrackKeyForRecord(record)); key != "" {
+		return "track:" + key
+	}
+	return strings.TrimSpace(canonicalVersionKeyForRecord(record))
+}
+
 func selectSaveRecordDuplicateSurvivor(group []saveRecord, latest saveRecord) string {
 	latestID := strings.TrimSpace(latest.Summary.ID)
 	for _, record := range group {

backend/cmd/server/save_dedupe.go

@@ -27,11 +27,18 @@ func payloadSHA256Hex(payload []byte) string {
 }
 
 func latestSaveRecordForCanonicalKey(records []saveRecord, key string) (saveRecord, bool) {
+	return latestSaveRecordForKey(records, key, canonicalVersionKeyForRecord)
+}
+
+func latestSaveRecordForKey(records []saveRecord, key string, keyForRecord func(saveRecord) string) (saveRecord, bool) {
 	cleanKey := strings.TrimSpace(key)
+	if cleanKey == "" || keyForRecord == nil {
+		return saveRecord{}, false
+	}
 	var latest saveRecord
 	found := false
 	for _, record := range records {
-		if strings.TrimSpace(canonicalVersionKeyForRecord(record)) != cleanKey {
+		if strings.TrimSpace(keyForRecord(record)) != cleanKey {
 			continue
 		}
 		if !saveRecordPayloadExists(record) {
@@ -45,6 +52,25 @@ func latestSaveRecordForCanonicalKey(records []saveRecord, key string) (saveReco
 	return latest, found
 }
 
+func canonicalDuplicateTrackKeyForRecord(record saveRecord) string {
+	track := canonicalTrackFromRecord(record)
+	if canonicalTrackTitleKey(track.DisplayTitle) == "unknown game" {
+		return ""
+	}
+	return canonicalTrackKey(track)
+}
+
+func canonicalDuplicateTrackKeyForInput(input saveCreateInput, filename string) string {
+	if strings.TrimSpace(input.Filename) == "" {
+		input.Filename = filename
+	}
+	track := canonicalTrackFromInput(input)
+	if canonicalTrackTitleKey(track.DisplayTitle) == "unknown game" {
+		return ""
+	}
+	return canonicalTrackKey(track)
+}
+
 func saveRecordSortsAfter(left, right saveRecord) bool {
 	if left.Summary.Version != right.Summary.Version {
 		return left.Summary.Version > right.Summary.Version
@@ -57,37 +83,69 @@ func saveRecordSortsAfter(left, right saveRecord) bool {
 
 func checkUploadDuplicate(records []saveRecord, input saveCreateInput) uploadDuplicateCheck {
 	filename := safeFilename(firstNonEmpty(input.Filename, "save.bin"))
-	key := canonicalVersionKeyForInput(input, filename)
 	shaHex := payloadSHA256Hex(input.Payload)
-	latest, hasLatest := latestSaveRecordForCanonicalKey(records, key)
-	if !hasLatest {
-		return uploadDuplicateCheck{}
+
+	type duplicateScope struct {
+		name   string
+		key    string
+		keyFor func(saveRecord) string
+	}
+	scopes := []duplicateScope{{
+		name:   "version",
+		key:    canonicalVersionKeyForInput(input, filename),
+		keyFor: canonicalVersionKeyForRecord,
+	}}
+	if trackKey := canonicalDuplicateTrackKeyForInput(input, filename); trackKey != "" {
+		scopes = append(scopes, duplicateScope{
+			name:   "track",
+			key:    trackKey,
+			keyFor: canonicalDuplicateTrackKeyForRecord,
+		})
 	}
 
-	var matching saveRecord
-	foundMatch := false
-	for _, record := range records {
-		if strings.TrimSpace(canonicalVersionKeyForRecord(record)) != strings.TrimSpace(key) {
+	seenScopes := map[string]struct{}{}
+	for _, scope := range scopes {
+		scopeKey := strings.TrimSpace(scope.key)
+		if scopeKey == "" {
 			continue
 		}
-		if !saveRecordPayloadExists(record) {
+		seenKey := scope.name + ":" + scopeKey
+		if _, seen := seenScopes[seenKey]; seen {
 			continue
 		}
-		if strings.TrimSpace(record.Summary.SHA256) != shaHex {
+		seenScopes[seenKey] = struct{}{}
+
+		latest, hasLatest := latestSaveRecordForKey(records, scopeKey, scope.keyFor)
+		if !hasLatest {
 			continue
 		}
-		if !foundMatch || saveRecordSortsAfter(record, matching) {
-			matching = record
-			foundMatch = true
+
+		var matching saveRecord
+		foundMatch := false
+		for _, record := range records {
+			if strings.TrimSpace(scope.keyFor(record)) != scopeKey {
+				continue
+			}
+			if !saveRecordPayloadExists(record) {
+				continue
+			}
+			if strings.TrimSpace(record.Summary.SHA256) != shaHex {
+				continue
+			}
+			if !foundMatch || saveRecordSortsAfter(record, matching) {
+				matching = record
+				foundMatch = true
+			}
 		}
+		if !foundMatch {
+			continue
+		}
+		if strings.TrimSpace(latest.Summary.SHA256) == shaHex {
+			return uploadDuplicateCheck{Disposition: uploadDuplicateIgnoredLatest, Latest: latest, Matching: latest, Found: true}
+		}
+		return uploadDuplicateCheck{Disposition: uploadDuplicateStaleHistorical, Latest: latest, Matching: matching, Found: true}
 	}
-	if !foundMatch {
-		return uploadDuplicateCheck{}
-	}
-	if strings.TrimSpace(latest.Summary.SHA256) == shaHex {
-		return uploadDuplicateCheck{Disposition: uploadDuplicateIgnoredLatest, Latest: latest, Matching: latest, Found: true}
-	}
-	return uploadDuplicateCheck{Disposition: uploadDuplicateStaleHistorical, Latest: latest, Matching: matching, Found: true}
+	return uploadDuplicateCheck{}
 }
 
 func duplicateUploadResponse(record saveRecord, disposition uploadDuplicateDisposition) map[string]any {
@@ -131,6 +189,45 @@ func saveRecordIsRollback(record saveRecord) bool {
 	return metadataHasRollbackAudit(record.Summary.Metadata)
 }
 
+func saveRecordRevisionIdentity(record saveRecord) string {
+	id := strings.TrimSpace(record.Summary.ID)
+	if saveRecordIsRollback(record) {
+		return "rollback:" + id
+	}
+	if sha := strings.TrimSpace(record.Summary.SHA256); sha != "" {
+		return "sha:" + sha
+	}
+	return "id:" + id
+}
+
+func saveSummaryRevisionIdentity(summary saveSummary) string {
+	id := strings.TrimSpace(summary.ID)
+	if metadataHasRollbackAudit(summary.Metadata) {
+		return "rollback:" + id
+	}
+	if sha := strings.TrimSpace(summary.SHA256); sha != "" {
+		return "sha:" + sha
+	}
+	return "id:" + id
+}
+
+func dedupeSaveSummaryRevisions(summaries []saveSummary) []saveSummary {
+	if len(summaries) <= 1 {
+		return summaries
+	}
+	seen := map[string]struct{}{}
+	deduped := make([]saveSummary, 0, len(summaries))
+	for _, summary := range summaries {
+		key := saveSummaryRevisionIdentity(summary)
+		if _, exists := seen[key]; exists {
+			continue
+		}
+		seen[key] = struct{}{}
+		deduped = append(deduped, summary)
+	}
+	return deduped
+}
+
 func psLogicalRevisionIsRollback(revision psLogicalSaveRevision) bool {
 	return strings.HasPrefix(strings.TrimSpace(revision.ImportID), "rollback:")
 }

backend/cmd/server/save_dedupe_test.go

@@ -82,6 +82,59 @@ func TestGenericUploadHistoricalDuplicateReturnsConflict(t *testing.T) {
 	}
 }
 
+func TestGenericUploadDuplicateAcrossROMVariantsIsIgnored(t *testing.T) {
+	h := newContractHarness(t)
+	payload := buildTestN64Payload("eep", "cross-rom-duplicate")
+	first := uploadSave(t, h, "/saves", map[string]string{
+		"rom_sha1": "star-fox-retail-rom",
+		"slotName": "slot-a",
+		"system":   "n64",
+	}, "Star Fox 64 (USA).eep", payload)
+	firstID := mustString(t, mustObject(t, first["save"], "save")["id"], "save.id")
+
+	second := h.multipart("/saves", map[string]string{
+		"rom_sha1": "star-fox-rev1-rom",
+		"slotName": "slot-a",
+		"system":   "n64",
+	}, "file", "Star Fox 64 (USA) (Rev 1).eep", payload)
+	assertStatus(t, second, http.StatusOK)
+	body := decodeJSONMap(t, second.Body)
+	if !mustBool(t, body["duplicate"], "duplicate") {
+		t.Fatalf("expected cross-ROM duplicate response, got %s", prettyJSON(body))
+	}
+	if got := mustString(t, body["duplicateDisposition"], "duplicateDisposition"); got != string(uploadDuplicateIgnoredLatest) {
+		t.Fatalf("expected ignored-latest duplicate disposition, got %q", got)
+	}
+	if got := mustString(t, mustObject(t, body["save"], "save")["id"], "save.id"); got != firstID {
+		t.Fatalf("expected cross-ROM duplicate upload to return %q, got %q", firstID, got)
+	}
+
+	list := h.request(http.MethodGet, "/saves?limit=50&offset=0", nil)
+	assertStatus(t, list, http.StatusOK)
+	saves := mustArray(t, decodeJSONMap(t, list.Body)["saves"], "saves")
+	matches := 0
+	for _, item := range saves {
+		summary := mustObject(t, item, "saves[n]")
+		if mustString(t, summary["displayTitle"], "displayTitle") != "Star Fox 64" {
+			continue
+		}
+		matches++
+		if got := mustNumber(t, summary["saveCount"], "saveCount"); got != 1 {
+			t.Fatalf("expected one visible save after cross-ROM duplicate upload, got %s", prettyJSON(summary))
+		}
+	}
+	if matches != 1 {
+		t.Fatalf("expected one Star Fox row, got %d", matches)
+	}
+
+	history := h.request(http.MethodGet, "/save?saveId="+url.QueryEscape(firstID), nil)
+	assertStatus(t, history, http.StatusOK)
+	versions := mustArray(t, decodeJSONMap(t, history.Body)["versions"], "versions")
+	if len(versions) != 1 {
+		t.Fatalf("expected one history version after cross-ROM duplicate upload, got %d", len(versions))
+	}
+}
+
 func TestPlayStationHistoricalDuplicateImportIsStale(t *testing.T) {
 	h := newContractHarness(t)
 	firstPayload := makeTestPS1Card(t, "SCUS_941.63", "Final Fantasy VII Save")
@@ -273,3 +326,74 @@ func TestRescanCleanupRemovesDuplicateVersionsButKeepsRollbackAudit(t *testing.T
 		t.Fatalf("expected contiguous renumbered versions [3 2 1], got %v", versions)
 	}
 }
+
+func TestRescanCleanupRemovesDuplicateAcrossROMVariants(t *testing.T) {
+	h := newContractHarness(t)
+	payload := buildTestN64Payload("eep", "cleanup-cross-rom")
+	createdAt := time.Date(2026, 4, 24, 12, 0, 0, 0, time.UTC)
+
+	first, err := h.app.createSave(saveCreateInput{
+		Filename:   "Star Fox 64 (USA).eep",
+		Payload:    payload,
+		Game:       fallbackGameFromFilename("Star Fox 64 (USA).eep"),
+		Format:     "eep",
+		ROMSHA1:    "cleanup-starfox-retail",
+		SlotName:   "slot-a",
+		SystemSlug: "n64",
+		CreatedAt:  createdAt,
+	})
+	if err != nil {
+		t.Fatalf("create first cross-ROM save: %v", err)
+	}
+	_, err = h.app.createSave(saveCreateInput{
+		Filename:   "Star Fox 64 (USA) (Rev 1).eep",
+		Payload:    payload,
+		Game:       fallbackGameFromFilename("Star Fox 64 (USA) (Rev 1).eep"),
+		Format:     "eep",
+		ROMSHA1:    "cleanup-starfox-rev1",
+		SlotName:   "slot-a",
+		SystemSlug: "n64",
+		CreatedAt:  createdAt.Add(time.Minute),
+	})
+	if err != nil {
+		t.Fatalf("create second cross-ROM save: %v", err)
+	}
+	if got := countRecordsByDuplicateTrackAndSHA(h.app.snapshotSaveRecords(), canonicalDuplicateTrackKeyForRecord(first), first.Summary.SHA256); got != 2 {
+		t.Fatalf("expected 2 cross-ROM duplicate records before cleanup, got %d", got)
+	}
+
+	result, err := h.app.rescanSaves(saveRescanOptions{DryRun: false, PruneUnsupported: true})
+	if err != nil {
+		t.Fatalf("rescan with cross-ROM duplicate cleanup: %v", err)
+	}
+	if result.DuplicateGroups < 1 || result.DuplicateVersionsRemoved < 1 {
+		t.Fatalf("expected duplicate cleanup to report cross-ROM duplicate, got %+v", result)
+	}
+	records := h.app.snapshotSaveRecords()
+	matching := make([]saveRecord, 0, 1)
+	for _, record := range records {
+		if canonicalDuplicateTrackKeyForRecord(record) == canonicalDuplicateTrackKeyForRecord(first) && record.Summary.SHA256 == first.Summary.SHA256 {
+			matching = append(matching, record)
+		}
+	}
+	if len(matching) != 1 {
+		t.Fatalf("expected one surviving cross-ROM duplicate record after cleanup, got %d", len(matching))
+	}
+	if matching[0].Summary.SHA256 != first.Summary.SHA256 {
+		t.Fatalf("expected surviving record to keep the duplicate payload")
+	}
+}
+
+func countRecordsByDuplicateTrackAndSHA(records []saveRecord, key, sha string) int {
+	count := 0
+	for _, record := range records {
+		if canonicalDuplicateTrackKeyForRecord(record) != key {
+			continue
+		}
+		if strings.TrimSpace(record.Summary.SHA256) != strings.TrimSpace(sha) {
+			continue
+		}
+		count++
+	}
+	return count
+}